From d8202a0880a2ece87f3b18da1636ad3fe7b1dedb Mon Sep 17 00:00:00 2001 From: nomurak Date: Thu, 15 Oct 2026 15:45:28 -0700 Subject: [PATCH] Write initialized buffers back so read-only tests stay read-only multiload and multichase initialize their buffers with stores (memset of the load buffer, writing the chase pointers), so a read-only test starts with every line dirty. On system level caches where a read hit on a dirty line returns the line dirty and drops the cached copy (Neoverse N4 "Cedar" gMesh SLC, for example), every pass of stream-sum or a pointer chase that fits in the SLC then writes each line back: one SLC re-allocation per hit, and write bandwidth equal to read bandwidth on a read-only test. The measured SLC hit rate drops to 50%. The -F flush does not prevent this: it reads a buffer that was itself filled with memset, only chase threads run it, and a 64 MiB flush is smaller than the last level cache of current server parts. Add clean_invalidate_range() to cpu_util.h. On aarch64 it issues DC CIVAC over the range (line size from CTR_EL0) followed by DSB ISH. DC CIVAC is ARMv8.0 and usable from user space on Linux. On other architectures it is a no-op and HAVE_CLEAN_INVALIDATE_RANGE is 0, so behavior there is unchanged except as noted below. - multiload: clean the load buffer after memset. - multiload and multichase chase setup: on aarch64, a non-zero -F now writes the chase back with clean_invalidate_range() instead of reading the flush buffer, and the flush buffer is not allocated. - multiload: do not allocate the flush buffer in bandwidth-only runs on any architecture. Only chase threads read it, so it only added 64 MiB of dirty lines. Neoverse N4, one thread on cpu 8 (two for chaseload), 20 MiB buffer, chases with -L -H -T 2m, SLC PMU over 3 s of steady state, before -> after: test slc_exec_hit slc_exec_alloc_req latency multiload stream-sum 1353M -> 2697M 1345M -> 1M multiload chase 110M -> 215M 106M -> 1M 31.96 ns both multiload chaseload 1683M -> 2920M 1236M -> 4M multichase chase 111M -> 217M 104M -> 1M 31.96 ns both An idle machine shows about 0.5M allocations over the same window. Hit counts double because once lines are clean the SLC keeps its copy and the core's clean eviction of the line also counts as a hit. Chase latency is unchanged. --- cpu_util.h | 34 ++++++++++++++++++++++++++++++++++ multichase.c | 13 ++++++++++++- multiload.c | 18 +++++++++++++++++- 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/cpu_util.h b/cpu_util.h index ded2a6d..8941b8e 100644 --- a/cpu_util.h +++ b/cpu_util.h @@ -14,6 +14,9 @@ #ifndef PORT_H_INCLUDED #define PORT_H_INCLUDED +#include +#include + // We assume 1024 bytes is good enough alignment to avoid false sharing on all // architectures. #define AVOID_FALSE_SHARING (1024) @@ -47,4 +50,35 @@ static inline void cpu_relax(void) { } while (0) #endif +// clean_invalidate_range() writes back and invalidates [buf, buf + len) to +// the point of coherency, so a buffer that was just initialized is clean in +// every cache level. A read-only test that starts on dirty lines can generate +// write traffic on every pass: on some system level caches a read hit on a +// dirty line hands the line back dirty, and the core later writes it back. +// Only implemented on aarch64, where DC CIVAC is usable from user space on +// Linux. Elsewhere it is a no-op and HAVE_CLEAN_INVALIDATE_RANGE is 0. +#if defined(__aarch64__) +#define HAVE_CLEAN_INVALIDATE_RANGE 1 +static inline void clean_invalidate_range(const void *buf, size_t len) { + // Rounding buf down below would otherwise touch one line when len is 0. + if (len == 0) return; + uint64_t ctr; + asm volatile("mrs %0, ctr_el0" : "=r"(ctr)); + // CTR_EL0.DminLine is log2 of the smallest data cache line in words. + const uintptr_t line = (uintptr_t)4 << ((ctr >> 16) & 0xf); + uintptr_t p = (uintptr_t)buf & ~(line - 1); + const uintptr_t end = (uintptr_t)buf + len; + for (; p < end; p += line) { + asm volatile("dc civac, %0" : : "r"(p) : "memory"); + } + asm volatile("dsb ish" : : : "memory"); +} +#else +#define HAVE_CLEAN_INVALIDATE_RANGE 0 +static inline void clean_invalidate_range(const void *buf, size_t len) { + (void)buf; + (void)len; +} +#endif + #endif diff --git a/multichase.c b/multichase.c index f9c23f8..29c987b 100644 --- a/multichase.c +++ b/multichase.c @@ -496,6 +496,12 @@ static void *thread_start(void *data) { // now flush our caches if (args->x.cache_flush_size) { +#if HAVE_CLEAN_INVALIDATE_RANGE + // Write the chase out to memory instead of reading a flush buffer, so + // the chase starts uncached and clean rather than dirty. + clean_invalidate_range(args->x.genchase_args->arena, + args->x.genchase_args->total_memory); +#else size_t nr_elts = args->x.cache_flush_size / sizeof(size_t); size_t *p = args->x.flush_arena; size_t sum = 0; @@ -505,6 +511,7 @@ static void *thread_start(void *data) { --nr_elts; } args->x.dummy += sum; +#endif } // wait and/or wake up everyone if we're all ready @@ -767,6 +774,9 @@ int main(int argc, char **argv) { "constructing\n" " the chase and before starting the benchmark (use " "with nta)\n" + " on aarch64 any non-zero value instead writes the " + "chase back\n" + " to memory with DC CIVAC\n" " default: %zu\n", DEF_CACHE_FLUSH); fprintf( @@ -850,7 +860,8 @@ int main(int argc, char **argv) { per_thread_t *thread_data = alloc_arena_mmap( default_page_size, false, nr_threads * sizeof(per_thread_t), -1); void *flush_arena = NULL; - if (cache_flush_size) { + // Not read when the chase is cleaned with clean_invalidate_range(). + if (cache_flush_size && !HAVE_CLEAN_INVALIDATE_RANGE) { flush_arena = alloc_arena_mmap(default_page_size, false, cache_flush_size, -1); memset(flush_arena, 1, cache_flush_size); // ensure pages are mapped diff --git a/multiload.c b/multiload.c index c9494ab..8f84c78 100644 --- a/multiload.c +++ b/multiload.c @@ -841,6 +841,12 @@ static void *thread_start(void *data) { } // now flush our caches if (args->x.cache_flush_size) { +#if HAVE_CLEAN_INVALIDATE_RANGE + // Write the chase out to memory instead of reading a flush buffer, so + // the chase starts uncached and clean rather than dirty. + clean_invalidate_range(args->x.genchase_args->arena, + args->x.genchase_args->total_memory); +#else size_t nr_elts = args->x.cache_flush_size / sizeof(size_t); size_t *p = args->x.flush_arena; size_t sum = 0; @@ -850,6 +856,7 @@ static void *thread_start(void *data) { --nr_elts; } args->x.dummy += sum; +#endif } } else { if (verbosity > 2) @@ -861,6 +868,8 @@ static void *thread_start(void *data) { -1) + args->x.load_offset; memset(args->x.load_arena, 1, args->x.load_total_memory); // ensure pages are mapped + // Start clean, so read-only loads do not write dirty lines back. + clean_invalidate_range(args->x.load_arena, args->x.load_total_memory); } if (verbosity > 2) @@ -1179,6 +1188,9 @@ int main(int argc, char **argv) { "constructing\n" " the chase/memload and before starting the benchmark (use " "with nta)\n" + " on aarch64 any non-zero value instead writes the chase " + "back to\n" + " memory with DC CIVAC\n" " default: %zu\n", DEF_CACHE_FLUSH); fprintf(stderr, "-p nnnn[kmg] backing page size to use (default %zu)\n", @@ -1298,7 +1310,11 @@ int main(int argc, char **argv) { default_page_size, false, nr_threads * sizeof(per_thread_t), -1); void *flush_arena = NULL; if (verbosity > 2) printf("allocate cache flush\n"); - if (cache_flush_size) { + // Only chase threads read the flush buffer, and not at all when the chase + // is cleaned with clean_invalidate_range(). Otherwise it only adds 64 MiB + // of dirty lines to the caches. + if (cache_flush_size && !HAVE_CLEAN_INVALIDATE_RANGE && + run_test_type != RUN_BANDWIDTH) { flush_arena = alloc_arena_mmap(default_page_size, false, cache_flush_size, -1); memset(flush_arena, 1, cache_flush_size); // ensure pages are mapped