Clean up remains of restricted cold classes

tcmalloc/cpu_cache.h

@@ -161,10 +161,6 @@ class StaticForwarder : private Parameters {
     return state_.sizemap().class_to_size(size_class);
   }
 
-  absl::Span<const size_t> cold_size_classes() const {
-    return state_.sizemap().ColdSizeClasses();
-  }
-
   size_t num_objects_to_move(int size_class) const {
     return state_.sizemap().num_objects_to_move(size_class);
   }
@@ -935,6 +931,10 @@ inline size_t CpuCache<Forwarder>::MaxCapacity(size_t size_class) const {
     return 0;
   }
 
+  if (IsColdSizeClass(size_class) && !ColdFeatureActive()) {
+    return 0;
+  }
+
   if (!IsColdSizeClass(size_class) &&
       (size_class % kNumBaseClasses) <= kNumSmall) {
     // Small object sizes are very heavily used and need very deep caches for
@@ -943,33 +943,21 @@ inline size_t CpuCache<Forwarder>::MaxCapacity(size_t size_class) const {
     return kSmallObjectDepth;
   }
 
-  if (ColdFeatureActive()) {
-    // We reduce the number of cached objects for some sizes to fit into the
-    // slab.
-    //
-    // We use fewer number of size classes when using reuse size classes. So,
-    // we may use larger capacity for some sizes.
-    const uint16_t kLargeUninterestingObjectDepth =
-        forwarder_.reuse_size_classes() ? 246 * kWiderSlabMultiplier
-                                        : 123 * kWiderSlabMultiplier;
-    const uint16_t kLargeInterestingObjectDepth =
-        forwarder_.reuse_size_classes() ? 52 * kWiderSlabMultiplier
-                                        : 36 * kWiderSlabMultiplier;
-    absl::Span<const size_t> cold = forwarder_.cold_size_classes();
-    if (absl::c_binary_search(cold, size_class)) {
-      return kLargeInterestingObjectDepth;
-    } else if (!IsColdSizeClass(size_class)) {
-      return kLargeUninterestingObjectDepth;
-    } else {
-      return 0;
-    }
-  }
-
-  if (IsColdSizeClass(size_class)) {
-    return 0;
+  if (!ColdFeatureActive()) {
+    return kLargeObjectDepth;
   }
 
-  return kLargeObjectDepth;
+  // We reduce the number of cached objects for some sizes to fit into the slab.
+  //
+  // We use fewer number of size classes when using reuse size classes. So,
+  // we may use larger capacity for some sizes.
+  const uint16_t kLargeHotObjectDepth = forwarder_.reuse_size_classes()
+                                            ? 246 * kWiderSlabMultiplier
+                                            : 123 * kWiderSlabMultiplier;
+  const uint16_t kColdObjectDepth = forwarder_.reuse_size_classes()
+                                        ? 52 * kWiderSlabMultiplier
+                                        : 36 * kWiderSlabMultiplier;
+  return IsColdSizeClass(size_class) ? kColdObjectDepth : kLargeHotObjectDepth;
 }
 
 // Returns estimated bytes required and the bytes available.

tcmalloc/cpu_cache_test.cc

@@ -276,14 +276,6 @@ class TestStaticForwarder : private Parameters {
     }
   }
 
-  absl::Span<const size_t> cold_size_classes() const {
-    if (size_map_.has_value()) {
-      return size_map_->ColdSizeClasses();
-    } else {
-      return transfer_cache_.cold_size_classes();
-    }
-  }
-
   size_t num_objects_to_move(int size_class) const {
     if (size_map_.has_value()) {
       return size_map_->num_objects_to_move(size_class);

tcmalloc/mock_transfer_cache.h

@@ -271,10 +271,6 @@ class ThreeSizeClassManager : public FakeTransferCacheManager {
     }
   }
 
-  absl::Span<const size_t> cold_size_classes() const {
-    return {cold_size_classes_, 1};
-  }
-
   void InsertRange(int size_class, absl::Span<void*> batch) {
     TC_ASSERT(caches_.contains(size_class));
     caches_[size_class]->InsertRange(size_class, batch);
@@ -292,7 +288,6 @@ class ThreeSizeClassManager : public FakeTransferCacheManager {
   }
 
   absl::flat_hash_map<size_t, std::unique_ptr<TransferCache>> caches_;
-  size_t cold_size_classes_[1] = {kColdSizeClass};
 };
 
 class FakeCpuLayout {

tcmalloc/sizemap.cc

@@ -260,10 +260,11 @@ bool SizeMap::Init(absl::Span<const SizeClassInfo> size_classes) {
               &class_array_[kClassArraySize * i]);
   }
 
-  bool heap_partitioning_active = tc_globals.multiple_non_numa_partitions();
+  const bool heap_partitioning_active =
+      tc_globals.multiple_non_numa_partitions();
+  const bool heap_partitioning_full =
+      Parameters::heap_partitioning_mode() == HeapPartitioningMode::kFull;
   if (kSecurityPartitions > 1 && heap_partitioning_active) {
-    bool heap_partitioning_full =
-        Parameters::heap_partitioning_mode() == HeapPartitioningMode::kFull;
     next_size = 0;
     for (int c = kNumBaseClasses + 1; c < kColdClassesStart; ++c) {
       const int max_size_in_class = class_to_size_[c];
@@ -287,9 +288,6 @@ bool SizeMap::Init(absl::Span<const SizeClassInfo> size_classes) {
   }
 
   if (ColdFeatureActive()) {
-    memset(cold_sizes_, 0, sizeof(cold_sizes_));
-    cold_sizes_count_ = 0;
-
     next_size = 0;
     for (int c = kColdClassesStart + 1; c < kNumClasses; c++) {
       size_t max_size_in_class = class_to_size_[c];
@@ -298,9 +296,6 @@ bool SizeMap::Init(absl::Span<const SizeClassInfo> size_classes) {
         continue;
       }
 
-      cold_sizes_[cold_sizes_count_] = c;
-      ++cold_sizes_count_;
-
       for (int s = next_size; s <= max_size_in_class;
            s += static_cast<size_t>(kAlignment)) {
         class_array_[ClassIndex(s) + kClassArraySize * kColdRegisterStride] = c;
@@ -311,17 +306,16 @@ bool SizeMap::Init(absl::Span<const SizeClassInfo> size_classes) {
       }
     }
     if (kSecurityPartitions > 1) {
-      if (Parameters::heap_partitioning_mode() == HeapPartitioningMode::kFull) {
+      if (heap_partitioning_full) {
         // Point all lookups in Cold New's P1 register to Hot New's P1.
-        std::copy(
-            &class_array_[kClassArraySize], &class_array_[kClassArraySize * 2],
-            &class_array_[kClassArraySize * (2 * kSecurityPartitions + 1)]);
+        std::copy(&class_array_[kClassArraySize],
+                  &class_array_[kClassArraySize * 2],
+                  &class_array_[kClassArraySize * (kColdRegisterStride + 1)]);
       } else {
         // Point all lookups in Cold New's P1 register to Cold New's P0.
-        std::copy(
-            &class_array_[kClassArraySize * (2 * kSecurityPartitions)],
-            &class_array_[kClassArraySize * (2 * kSecurityPartitions + 1)],
-            &class_array_[kClassArraySize * (2 * kSecurityPartitions + 1)]);
+        std::copy(&class_array_[kClassArraySize * kColdRegisterStride],
+                  &class_array_[kClassArraySize * (kColdRegisterStride + 1)],
+                  &class_array_[kClassArraySize * (kColdRegisterStride + 1)]);
       }
     }
   }

tcmalloc/sizemap.h

@@ -176,9 +176,6 @@ class SizeMap {
   // Check that the size classes meet all requirements.
   static bool ValidSizeClasses(absl::Span<const SizeClassInfo> size_classes);
 
-  size_t cold_sizes_[kNumBaseClasses] = {0};
-  size_t cold_sizes_count_ = 0;
-
  public:
   // Returns size classes to use in the current process.
   static const SizeClasses& CurrentClasses();
@@ -345,11 +342,6 @@ class SizeMap {
     return num_objects_to_move_[size_class];
   }
 
-  ABSL_ATTRIBUTE_ALWAYS_INLINE absl::Span<const size_t> ColdSizeClasses()
-      const {
-    return {cold_sizes_, cold_sizes_count_};
-  }
-
   [[nodiscard]] static bool IsValidSizeClass(size_t size, Length num_pages,
                                              size_t num_objects_to_move);
 };

tcmalloc/sizemap_test.cc

@@ -105,19 +105,15 @@ TEST(ColdSizeClassTest, ColdSizeClasses) {
   for (const SizeClasses* sc : kAllSizeClassesConfigs) {
     const auto& classes = sc->classes;
     std::vector<size_t> allowed_alloc_size;
-    std::vector<size_t> expected_cold_size_classes;
     for (int i = 1; i < classes.size(); ++i) {
       allowed_alloc_size.push_back(classes[i].size);
-      expected_cold_size_classes.push_back(i + kColdClassesStart);
     }
 
     SizeMap size_map;
     EXPECT_TRUE(size_map.Init(classes));
     for (const size_t request_size : allowed_alloc_size) {
       VerifyColdSizeClassRelations(size_map, request_size);
     }
-    EXPECT_THAT(size_map.ColdSizeClasses(),
-                ElementsAreArray(expected_cold_size_classes));
   }
 }
 
@@ -201,10 +197,8 @@ TEST(SizeMapTest, PointerPartitionNoCold) {
   for (const SizeClasses* sc : kAllSizeClassesConfigs) {
     const auto& classes = sc->classes;
     std::vector<size_t> allowed_alloc_size;
-    std::vector<size_t> expected_cold_size_classes;
     for (int i = 1; i < classes.size(); ++i) {
       allowed_alloc_size.push_back(classes[i].size);
-      expected_cold_size_classes.push_back(i + kColdClassesStart);
     }
 
     SizeMap size_map;
@@ -231,8 +225,6 @@ TEST(SizeMapTest, PointerPartitionNoCold) {
                 request_size));
       }
     }
-    EXPECT_THAT(size_map.ColdSizeClasses(),
-                ElementsAreArray(expected_cold_size_classes));
   }
 }