Skip to content

madvise() away metadata for drained CPUs. #513

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
copybara-service wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

madvise() away metadata for drained CPUs. #513

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions tcmalloc/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ cc_library(
":common_8k_pages",
":malloc_hook",
"//tcmalloc/internal:allocation_guard",
"//tcmalloc/internal:is_aligned_to",
"//tcmalloc/internal:overflow",
"//tcmalloc/internal:page_size",
"@com_google_absl//absl/time",
Expand Down Expand Up @@ -138,6 +139,7 @@ cc_library(
":common_8k_pages",
":malloc_hook",
"//tcmalloc/internal:allocation_guard",
"//tcmalloc/internal:is_aligned_to",
"//tcmalloc/internal:overflow",
"//tcmalloc/internal:page_size",
"@com_google_absl//absl/time",
Expand All @@ -164,6 +166,7 @@ cc_library(
":common_deprecated_perthread",
":malloc_hook",
"//tcmalloc/internal:allocation_guard",
"//tcmalloc/internal:is_aligned_to",
"//tcmalloc/internal:overflow",
"//tcmalloc/internal:page_size",
"@com_google_absl//absl/time",
Expand Down Expand Up @@ -307,6 +310,7 @@ create_tcmalloc_libraries(
"//tcmalloc/internal:exponential_biased",
"//tcmalloc/internal:gwp_asan_state",
"//tcmalloc/internal:hook_list",
"//tcmalloc/internal:is_aligned_to",
"//tcmalloc/internal:linked_list",
"//tcmalloc/internal:logging",
"//tcmalloc/internal:memory_stats",
Expand Down Expand Up @@ -428,6 +432,7 @@ cc_library(
":common_large_pages",
":malloc_hook",
"//tcmalloc/internal:allocation_guard",
"//tcmalloc/internal:is_aligned_to",
"//tcmalloc/internal:overflow",
"//tcmalloc/internal:page_size",
"@com_google_absl//absl/time",
Expand All @@ -453,6 +458,7 @@ cc_library(
":common_256k_pages",
":malloc_hook",
"//tcmalloc/internal:allocation_guard",
"//tcmalloc/internal:is_aligned_to",
"//tcmalloc/internal:overflow",
"//tcmalloc/internal:page_size",
"@com_google_absl//absl/time",
Expand Down Expand Up @@ -481,6 +487,7 @@ cc_library(
":common_256k_pages_numa_aware",
":malloc_hook",
"//tcmalloc/internal:allocation_guard",
"//tcmalloc/internal:is_aligned_to",
"//tcmalloc/internal:overflow",
"//tcmalloc/internal:page_size",
"@com_google_absl//absl/time",
Expand Down Expand Up @@ -510,6 +517,7 @@ cc_library(
":common_small_but_slow",
":malloc_hook",
"//tcmalloc/internal:allocation_guard",
"//tcmalloc/internal:is_aligned_to",
"//tcmalloc/internal:overflow",
"//tcmalloc/internal:page_size",
"@com_google_absl//absl/time",
Expand Down Expand Up @@ -537,6 +545,7 @@ cc_library(
":common_numa_aware",
":malloc_hook",
"//tcmalloc/internal:allocation_guard",
"//tcmalloc/internal:is_aligned_to",
"//tcmalloc/internal:overflow",
"//tcmalloc/internal:page_size",
"@com_google_absl//absl/time",
Expand All @@ -563,6 +572,7 @@ cc_library(
":common_legacy_locking",
":malloc_hook",
"//tcmalloc/internal:allocation_guard",
"//tcmalloc/internal:is_aligned_to",
"//tcmalloc/internal:overflow",
"//tcmalloc/internal:page_size",
"@com_google_absl//absl/time",
Expand Down Expand Up @@ -590,6 +600,7 @@ cc_library(
":malloc_hook",
"//tcmalloc/internal:allocation_guard",
"//tcmalloc/internal:delay_injection",
"//tcmalloc/internal:is_aligned_to",
"//tcmalloc/internal:overflow",
"//tcmalloc/internal:page_size",
"@com_google_absl//absl/time",
Expand Down Expand Up @@ -1435,6 +1446,7 @@ create_tcmalloc_testsuite(
copts = TCMALLOC_DEFAULT_COPTS,
deps = [
":malloc_extension",
"//tcmalloc/internal:is_aligned_to",
"//tcmalloc/internal:page_size",
"//tcmalloc/testing:testutil",
"@com_github_google_benchmark//:benchmark",
Expand Down
52 changes: 52 additions & 0 deletions tcmalloc/cpu_cache.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,10 @@ class StaticForwarder {
return Parameters::per_cpu_caches_dynamic_slab_shrink_threshold();
}

static bool release_drained_slab_metadata() {
return Parameters::release_drained_slab_metadata();
}

bool reuse_size_classes() const {
return state_.size_class_configuration() ==
SizeClassConfiguration::kReuse ||
Expand Down Expand Up @@ -464,6 +468,14 @@ class CpuCache {
// Reports total number of times any CPU has been reclaimed.
uint64_t GetNumReclaims() const;

// Reports number of times the <cpu> has been unpopulated
// (which happens when its metadata gets released, after all per-CPU
// metadata slabs on the same hugepage ave been reclaimed).
uint64_t GetNumUnpopulates(int cpu) const;

// Reports total number of times any CPU has been unpopulated.
uint64_t GetNumUnpopulates() const;

// Reports number of cpus that have touched set to true.
int CountTouchedCpus() const;

Expand Down Expand Up @@ -668,6 +680,9 @@ class CpuCache {
std::atomic<uint64_t> reclaim_used_bytes;
// Tracks number of times this CPU has been reclaimed.
std::atomic<size_t> num_reclaims;
// Tracks number of times this CPU has been unpopulated
// (see GetNumUnpopulates()).
std::atomic<size_t> num_unpopulates;
};

// Determines how we distribute memory in the per-cpu cache to the various
Expand Down Expand Up @@ -1407,6 +1422,8 @@ template <class Forwarder>
inline void CpuCache<Forwarder>::TryReclaimingCaches() {
const int num_cpus = NumCPUs();

bool any_drained = false;

for (int cpu = 0; cpu < num_cpus; ++cpu) {
// Nothing to reclaim if the cpu is not populated.
if (!HasPopulated(cpu)) {
Expand All @@ -1428,6 +1445,7 @@ inline void CpuCache<Forwarder>::TryReclaimingCaches() {
// stayed constant since the last interval.
if (used_bytes != 0 && used_bytes == prev_used_bytes && misses == 0) {
Reclaim(cpu);
any_drained = true;
}

// Takes a snapshot of used bytes in the cache at the end of this interval
Expand All @@ -1438,6 +1456,26 @@ inline void CpuCache<Forwarder>::TryReclaimingCaches() {
resize_[cpu].reclaim_used_bytes.store(used_bytes,
std::memory_order_relaxed);
}

if (any_drained && forwarder_.release_drained_slab_metadata()) {
freelist_.ReleaseSlabMetadataForDrainedCpus(
[this](int cpu) { return HasPopulated(cpu); },
[this](int cpu)
ABSL_NO_THREAD_SAFETY_ANALYSIS { resize_[cpu].lock.lock(); },
[this](int cpu)
ABSL_NO_THREAD_SAFETY_ANALYSIS { resize_[cpu].lock.unlock(); },
[this](int cpu) {
TC_CHECK_EQ(
resize_[cpu].available, resize_[cpu].capacity,
"CPU %u was not actually drained, or available is out of sync",
cpu);
resize_[cpu].populated.store(false, std::memory_order_release);
resize_[cpu].num_unpopulates.fetch_add(1, std::memory_order_relaxed);
},
[this](void* slab_addr, size_t slab_size) {
return MadviseAwaySlabs(slab_addr, slab_size);
});
}
}

template <class Forwarder>
Expand Down Expand Up @@ -2271,6 +2309,20 @@ inline uint64_t CpuCache<Forwarder>::GetNumReclaims() const {
return reclaims;
}

template <class Forwarder>
inline uint64_t CpuCache<Forwarder>::GetNumUnpopulates(int cpu) const {
return resize_[cpu].num_unpopulates.load(std::memory_order_relaxed);
}

template <class Forwarder>
inline uint64_t CpuCache<Forwarder>::GetNumUnpopulates() const {
uint64_t reclaims = 0;
const int num_cpus = NumCPUs();
for (int cpu = 0; cpu < num_cpus; ++cpu)
reclaims += resize_[cpu].num_unpopulates.load(std::memory_order_relaxed);
return reclaims;
}

template <class Forwarder>
inline int CpuCache<Forwarder>::CountTouchedCpus() const {
if (resize_ == nullptr) return 0;
Expand Down
121 changes: 113 additions & 8 deletions tcmalloc/cpu_cache_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,10 @@ class TestStaticForwarder {
return false;
}

bool release_drained_slab_metadata() const {
return release_drained_slab_metadata_;
}

size_t arena_reported_nonresident_bytes_ = 0;
int64_t arena_reported_impending_bytes_ = 0;
size_t shrink_to_usage_limit_calls_ = 0;
Expand All @@ -345,6 +349,7 @@ class TestStaticForwarder {
double dynamic_slab_shrink_threshold_ = -1;
DynamicSlab dynamic_slab_ = DynamicSlab::kNoop;
std::optional<SizeMap> size_map_;
bool release_drained_slab_metadata_ = false;

private:
NumaTopology<kNumaPartitions, kNumBaseClasses> numa_topology_;
Expand Down Expand Up @@ -1267,7 +1272,7 @@ static void ColdCacheOperations(CpuCache& cache, int cpu_id,
// Runs multiple allocate and deallocate operation on the cpu cache to collect
// misses. Once we collect enough misses on this cache, we can shuffle cpu
// caches to steal capacity from colder caches to the hot cache.
static void HotCacheOperations(CpuCache& cache, int cpu_id) {
static void HotCacheOperations(CpuCache& cache, int cpu_id, bool reclaim) {
constexpr size_t kPtrs = 4096;
std::vector<void*> ptrs;
ptrs.resize(kPtrs);
Expand All @@ -1288,10 +1293,13 @@ static void HotCacheOperations(CpuCache& cache, int cpu_id) {
}
}

// We reclaim the cache to reset it so that we record underflows/overflows the
// next time we allocate and deallocate objects. Without reclaim, the cache
// would stay warmed up and it would take more time to drain the colder cache.
cache.Reclaim(cpu_id);
if (reclaim) {
// We reclaim the cache to reset it so that we record underflows/overflows
// the next time we allocate and deallocate objects. Without reclaim, the
// cache would stay warmed up and it would take more time to drain the
// colder cache.
cache.Reclaim(cpu_id);
}
}

class DynamicWideSlabTest : public testing::Test {};
Expand Down Expand Up @@ -1319,7 +1327,7 @@ TEST_F(DynamicWideSlabTest, DynamicSlabThreshold) {
constexpr int kCpuId1 = 1;

// Accumulate overflows and underflows for kCpuId0.
HotCacheOperations(cache, kCpuId0);
HotCacheOperations(cache, kCpuId0, true);
CpuCache::CpuCacheMissStats interval_misses =
cache.GetIntervalCacheMissStats(kCpuId0, MissCount::kSlabResize);
// Make sure that overflows/underflows ratio is greater than the threshold
Expand Down Expand Up @@ -1539,7 +1547,7 @@ TEST(CpuCacheTest, ColdHotCacheShuffleTest) {
CpuCache::kCacheCapacityThreshold * max_cpu_cache_size;
++num_tries) {
ColdCacheOperations(cache, cold_cpu_id, size_class);
HotCacheOperations(cache, hot_cpu_id);
HotCacheOperations(cache, hot_cpu_id, true);
cache.ShuffleCpuCaches();

// Check that the capacity is preserved.
Expand Down Expand Up @@ -1568,7 +1576,7 @@ TEST(CpuCacheTest, ColdHotCacheShuffleTest) {
// change the capacity of either of the caches.
for (int i = 0; i < 100; ++i) {
ColdCacheOperations(cache, cold_cpu_id, size_class);
HotCacheOperations(cache, hot_cpu_id);
HotCacheOperations(cache, hot_cpu_id, true);
cache.ShuffleCpuCaches();

// Check that the capacity is preserved.
Expand Down Expand Up @@ -1611,6 +1619,7 @@ TEST(CpuCacheTest, ReclaimCpuCache) {

// None of the caches should have been reclaimed yet.
EXPECT_EQ(cache.GetNumReclaims(cpu), 0);
EXPECT_EQ(cache.GetNumUnpopulates(cpu), 0);

// Check that caches are empty.
uint64_t used_bytes = cache.UsedBytes(cpu);
Expand Down Expand Up @@ -1709,6 +1718,102 @@ TEST(CpuCacheTest, ReclaimCpuCache) {
cache.Deactivate();
}

TEST(CpuCacheTest, ReclaimCpuCacheAndUnpopulate) {
if (!subtle::percpu::IsFast()) {
return;
}

for (bool enabled : {false, true}) {
SCOPED_TRACE(absl::StrFormat("Feature enabled: %d", enabled));

CpuCache cache;
cache.forwarder().release_drained_slab_metadata_ = enabled;
cache.Activate();

const size_t kSizeClass = 2;

const int num_cpus = NumCPUs();

// Verify that we fill at least three hugepages; one (or more)
// to be unpopulated, one not to be, and one account for misalignment
// before or after.
uint8_t per_cpu_shift = CpuCachePeer::GetSlabShift(cache);
const auto shift = subtle::percpu::ToShiftType(per_cpu_shift);
const size_t slabs_size =
subtle::percpu::GetSlabsAllocSize(shift, num_cpus);
if (slabs_size < 3 * kHugePageSize) {
TC_LOG("Not enough CPUs to run test; skipping.");
return;
}

for (int cpu = 0; cpu < num_cpus; ++cpu) {
SCOPED_TRACE(absl::StrFormat("Failed CPU: %d", cpu));
ColdCacheOperations(cache, cpu, kSizeClass);
EXPECT_TRUE(cache.HasPopulated(cpu));
EXPECT_EQ(cache.GetNumUnpopulates(cpu), 0);
}

// None of the caches are stable, so nothing should be reclaimed
// and nothing should be unpopulated.
cache.TryReclaimingCaches();
EXPECT_EQ(cache.GetNumReclaims(), 0);
EXPECT_EQ(cache.GetNumUnpopulates(), 0);

// Do some work on every other CPUs. This should block all unpopulates,
// as no hugepage will contain all-reclaimed caches. The other ones
// should be reclaimed, though.
int num_idle_cpus = 0;
for (int cpu = 0; cpu < num_cpus; ++cpu) {
if (cpu % 2 == 0) {
HotCacheOperations(cache, cpu, false);
} else {
++num_idle_cpus;
}
}
cache.TryReclaimingCaches();
EXPECT_EQ(cache.GetNumReclaims(), num_idle_cpus);
EXPECT_EQ(cache.GetNumUnpopulates(), 0);

// Now do work on only one CPU, to record some misses on that,
// but let the others stay idle. (We do an extra reclaim first,
// or HotCacheOperations() wouldn't actually cause misses.
// This reclaim gets included in GetNumReclaims() below.)
// We should have unpopulates after another round of reclaim,
// but not everything.
//
// The “arbitrary” CPU must already be touched (so even),
// and we'd like it to be so far in that we know that it
// would actually get unpopulated if untouched.
constexpr int kArbitraryCpu = 80; // Must already be touched.
if (kArbitraryCpu >= num_cpus) {
TC_LOG("Not enough CPUs to run test; skipping.");
return;
}
HotCacheOperations(cache, kArbitraryCpu, false);
cache.TryReclaimingCaches();

EXPECT_EQ(cache.GetNumReclaims(kArbitraryCpu), 0);
EXPECT_EQ(cache.GetNumReclaims(), num_cpus - 1);

if (enabled) {
// The touched CPU cannot be unpopulated, and since it shares hugepage
// with at least one of its neighbors, at least one of those (probably
// both) must remain, too.
EXPECT_EQ(cache.GetNumUnpopulates(kArbitraryCpu), 0);
EXPECT_LT(cache.GetNumUnpopulates(kArbitraryCpu - 1) +
cache.GetNumUnpopulates(kArbitraryCpu + 1),
2);

EXPECT_GT(cache.GetNumUnpopulates(), 0);
EXPECT_LT(cache.GetNumUnpopulates(), num_cpus);
} else {
EXPECT_EQ(cache.GetNumUnpopulates(), 0);
}

cache.Deactivate();
}
}

TEST(CpuCacheTest, SizeClassCapacityTest) {
if (!subtle::percpu::IsFast()) {
return;
Expand Down
4 changes: 4 additions & 0 deletions tcmalloc/huge_page_aware_allocator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,10 @@ class StaticForwarder {
return Parameters::madvise_cold_regions_nohugepage();
}

static bool release_drained_slab_metadata() {
return Parameters::release_drained_slab_metadata();
}

// Arena state.
static Arena& arena();

Expand Down
Loading
Loading