diff --git a/tcmalloc/cpu_cache.h b/tcmalloc/cpu_cache.h index e7808d796..224a9a5b6 100644 --- a/tcmalloc/cpu_cache.h +++ b/tcmalloc/cpu_cache.h @@ -161,10 +161,6 @@ class StaticForwarder : private Parameters { return state_.sizemap().class_to_size(size_class); } - absl::Span cold_size_classes() const { - return state_.sizemap().ColdSizeClasses(); - } - size_t num_objects_to_move(int size_class) const { return state_.sizemap().num_objects_to_move(size_class); } @@ -935,6 +931,10 @@ inline size_t CpuCache::MaxCapacity(size_t size_class) const { return 0; } + if (IsColdSizeClass(size_class) && !ColdFeatureActive()) { + return 0; + } + if (!IsColdSizeClass(size_class) && (size_class % kNumBaseClasses) <= kNumSmall) { // Small object sizes are very heavily used and need very deep caches for @@ -943,33 +943,21 @@ inline size_t CpuCache::MaxCapacity(size_t size_class) const { return kSmallObjectDepth; } - if (ColdFeatureActive()) { - // We reduce the number of cached objects for some sizes to fit into the - // slab. - // - // We use fewer number of size classes when using reuse size classes. So, - // we may use larger capacity for some sizes. - const uint16_t kLargeUninterestingObjectDepth = - forwarder_.reuse_size_classes() ? 246 * kWiderSlabMultiplier - : 123 * kWiderSlabMultiplier; - const uint16_t kLargeInterestingObjectDepth = - forwarder_.reuse_size_classes() ? 52 * kWiderSlabMultiplier - : 36 * kWiderSlabMultiplier; - absl::Span cold = forwarder_.cold_size_classes(); - if (absl::c_binary_search(cold, size_class)) { - return kLargeInterestingObjectDepth; - } else if (!IsColdSizeClass(size_class)) { - return kLargeUninterestingObjectDepth; - } else { - return 0; - } - } - - if (IsColdSizeClass(size_class)) { - return 0; + if (!ColdFeatureActive()) { + return kLargeObjectDepth; } - return kLargeObjectDepth; + // We reduce the number of cached objects for some sizes to fit into the slab. + // + // We use fewer number of size classes when using reuse size classes. So, + // we may use larger capacity for some sizes. + const uint16_t kLargeHotObjectDepth = forwarder_.reuse_size_classes() + ? 246 * kWiderSlabMultiplier + : 123 * kWiderSlabMultiplier; + const uint16_t kColdObjectDepth = forwarder_.reuse_size_classes() + ? 52 * kWiderSlabMultiplier + : 36 * kWiderSlabMultiplier; + return IsColdSizeClass(size_class) ? kColdObjectDepth : kLargeHotObjectDepth; } // Returns estimated bytes required and the bytes available. diff --git a/tcmalloc/cpu_cache_test.cc b/tcmalloc/cpu_cache_test.cc index 8310f7b7e..01d56ed36 100644 --- a/tcmalloc/cpu_cache_test.cc +++ b/tcmalloc/cpu_cache_test.cc @@ -276,14 +276,6 @@ class TestStaticForwarder : private Parameters { } } - absl::Span cold_size_classes() const { - if (size_map_.has_value()) { - return size_map_->ColdSizeClasses(); - } else { - return transfer_cache_.cold_size_classes(); - } - } - size_t num_objects_to_move(int size_class) const { if (size_map_.has_value()) { return size_map_->num_objects_to_move(size_class); diff --git a/tcmalloc/mock_transfer_cache.h b/tcmalloc/mock_transfer_cache.h index 84dd02c88..c3409307c 100644 --- a/tcmalloc/mock_transfer_cache.h +++ b/tcmalloc/mock_transfer_cache.h @@ -271,10 +271,6 @@ class ThreeSizeClassManager : public FakeTransferCacheManager { } } - absl::Span cold_size_classes() const { - return {cold_size_classes_, 1}; - } - void InsertRange(int size_class, absl::Span batch) { TC_ASSERT(caches_.contains(size_class)); caches_[size_class]->InsertRange(size_class, batch); @@ -292,7 +288,6 @@ class ThreeSizeClassManager : public FakeTransferCacheManager { } absl::flat_hash_map> caches_; - size_t cold_size_classes_[1] = {kColdSizeClass}; }; class FakeCpuLayout { diff --git a/tcmalloc/sizemap.cc b/tcmalloc/sizemap.cc index b0240ea4c..f234d2176 100644 --- a/tcmalloc/sizemap.cc +++ b/tcmalloc/sizemap.cc @@ -260,10 +260,11 @@ bool SizeMap::Init(absl::Span size_classes) { &class_array_[kClassArraySize * i]); } - bool heap_partitioning_active = tc_globals.multiple_non_numa_partitions(); + const bool heap_partitioning_active = + tc_globals.multiple_non_numa_partitions(); + const bool heap_partitioning_full = + Parameters::heap_partitioning_mode() == HeapPartitioningMode::kFull; if (kSecurityPartitions > 1 && heap_partitioning_active) { - bool heap_partitioning_full = - Parameters::heap_partitioning_mode() == HeapPartitioningMode::kFull; next_size = 0; for (int c = kNumBaseClasses + 1; c < kColdClassesStart; ++c) { const int max_size_in_class = class_to_size_[c]; @@ -287,9 +288,6 @@ bool SizeMap::Init(absl::Span size_classes) { } if (ColdFeatureActive()) { - memset(cold_sizes_, 0, sizeof(cold_sizes_)); - cold_sizes_count_ = 0; - next_size = 0; for (int c = kColdClassesStart + 1; c < kNumClasses; c++) { size_t max_size_in_class = class_to_size_[c]; @@ -298,9 +296,6 @@ bool SizeMap::Init(absl::Span size_classes) { continue; } - cold_sizes_[cold_sizes_count_] = c; - ++cold_sizes_count_; - for (int s = next_size; s <= max_size_in_class; s += static_cast(kAlignment)) { class_array_[ClassIndex(s) + kClassArraySize * kColdRegisterStride] = c; @@ -311,17 +306,16 @@ bool SizeMap::Init(absl::Span size_classes) { } } if (kSecurityPartitions > 1) { - if (Parameters::heap_partitioning_mode() == HeapPartitioningMode::kFull) { + if (heap_partitioning_full) { // Point all lookups in Cold New's P1 register to Hot New's P1. - std::copy( - &class_array_[kClassArraySize], &class_array_[kClassArraySize * 2], - &class_array_[kClassArraySize * (2 * kSecurityPartitions + 1)]); + std::copy(&class_array_[kClassArraySize], + &class_array_[kClassArraySize * 2], + &class_array_[kClassArraySize * (kColdRegisterStride + 1)]); } else { // Point all lookups in Cold New's P1 register to Cold New's P0. - std::copy( - &class_array_[kClassArraySize * (2 * kSecurityPartitions)], - &class_array_[kClassArraySize * (2 * kSecurityPartitions + 1)], - &class_array_[kClassArraySize * (2 * kSecurityPartitions + 1)]); + std::copy(&class_array_[kClassArraySize * kColdRegisterStride], + &class_array_[kClassArraySize * (kColdRegisterStride + 1)], + &class_array_[kClassArraySize * (kColdRegisterStride + 1)]); } } } diff --git a/tcmalloc/sizemap.h b/tcmalloc/sizemap.h index f6f127be0..f3ca74560 100644 --- a/tcmalloc/sizemap.h +++ b/tcmalloc/sizemap.h @@ -176,9 +176,6 @@ class SizeMap { // Check that the size classes meet all requirements. static bool ValidSizeClasses(absl::Span size_classes); - size_t cold_sizes_[kNumBaseClasses] = {0}; - size_t cold_sizes_count_ = 0; - public: // Returns size classes to use in the current process. static const SizeClasses& CurrentClasses(); @@ -345,11 +342,6 @@ class SizeMap { return num_objects_to_move_[size_class]; } - ABSL_ATTRIBUTE_ALWAYS_INLINE absl::Span ColdSizeClasses() - const { - return {cold_sizes_, cold_sizes_count_}; - } - [[nodiscard]] static bool IsValidSizeClass(size_t size, Length num_pages, size_t num_objects_to_move); }; diff --git a/tcmalloc/sizemap_test.cc b/tcmalloc/sizemap_test.cc index f653d34dc..dadb8160d 100644 --- a/tcmalloc/sizemap_test.cc +++ b/tcmalloc/sizemap_test.cc @@ -105,10 +105,8 @@ TEST(ColdSizeClassTest, ColdSizeClasses) { for (const SizeClasses* sc : kAllSizeClassesConfigs) { const auto& classes = sc->classes; std::vector allowed_alloc_size; - std::vector expected_cold_size_classes; for (int i = 1; i < classes.size(); ++i) { allowed_alloc_size.push_back(classes[i].size); - expected_cold_size_classes.push_back(i + kColdClassesStart); } SizeMap size_map; @@ -116,8 +114,6 @@ TEST(ColdSizeClassTest, ColdSizeClasses) { for (const size_t request_size : allowed_alloc_size) { VerifyColdSizeClassRelations(size_map, request_size); } - EXPECT_THAT(size_map.ColdSizeClasses(), - ElementsAreArray(expected_cold_size_classes)); } } @@ -201,10 +197,8 @@ TEST(SizeMapTest, PointerPartitionNoCold) { for (const SizeClasses* sc : kAllSizeClassesConfigs) { const auto& classes = sc->classes; std::vector allowed_alloc_size; - std::vector expected_cold_size_classes; for (int i = 1; i < classes.size(); ++i) { allowed_alloc_size.push_back(classes[i].size); - expected_cold_size_classes.push_back(i + kColdClassesStart); } SizeMap size_map; @@ -231,8 +225,6 @@ TEST(SizeMapTest, PointerPartitionNoCold) { request_size)); } } - EXPECT_THAT(size_map.ColdSizeClasses(), - ElementsAreArray(expected_cold_size_classes)); } }