diff --git a/cpu_util.h b/cpu_util.h index ded2a6d..8941b8e 100644 --- a/cpu_util.h +++ b/cpu_util.h @@ -14,6 +14,9 @@ #ifndef PORT_H_INCLUDED #define PORT_H_INCLUDED +#include +#include + // We assume 1024 bytes is good enough alignment to avoid false sharing on all // architectures. #define AVOID_FALSE_SHARING (1024) @@ -47,4 +50,35 @@ static inline void cpu_relax(void) { } while (0) #endif +// clean_invalidate_range() writes back and invalidates [buf, buf + len) to +// the point of coherency, so a buffer that was just initialized is clean in +// every cache level. A read-only test that starts on dirty lines can generate +// write traffic on every pass: on some system level caches a read hit on a +// dirty line hands the line back dirty, and the core later writes it back. +// Only implemented on aarch64, where DC CIVAC is usable from user space on +// Linux. Elsewhere it is a no-op and HAVE_CLEAN_INVALIDATE_RANGE is 0. +#if defined(__aarch64__) +#define HAVE_CLEAN_INVALIDATE_RANGE 1 +static inline void clean_invalidate_range(const void *buf, size_t len) { + // Rounding buf down below would otherwise touch one line when len is 0. + if (len == 0) return; + uint64_t ctr; + asm volatile("mrs %0, ctr_el0" : "=r"(ctr)); + // CTR_EL0.DminLine is log2 of the smallest data cache line in words. + const uintptr_t line = (uintptr_t)4 << ((ctr >> 16) & 0xf); + uintptr_t p = (uintptr_t)buf & ~(line - 1); + const uintptr_t end = (uintptr_t)buf + len; + for (; p < end; p += line) { + asm volatile("dc civac, %0" : : "r"(p) : "memory"); + } + asm volatile("dsb ish" : : : "memory"); +} +#else +#define HAVE_CLEAN_INVALIDATE_RANGE 0 +static inline void clean_invalidate_range(const void *buf, size_t len) { + (void)buf; + (void)len; +} +#endif + #endif diff --git a/multichase.c b/multichase.c index f9c23f8..29c987b 100644 --- a/multichase.c +++ b/multichase.c @@ -496,6 +496,12 @@ static void *thread_start(void *data) { // now flush our caches if (args->x.cache_flush_size) { +#if HAVE_CLEAN_INVALIDATE_RANGE + // Write the chase out to memory instead of reading a flush buffer, so + // the chase starts uncached and clean rather than dirty. + clean_invalidate_range(args->x.genchase_args->arena, + args->x.genchase_args->total_memory); +#else size_t nr_elts = args->x.cache_flush_size / sizeof(size_t); size_t *p = args->x.flush_arena; size_t sum = 0; @@ -505,6 +511,7 @@ static void *thread_start(void *data) { --nr_elts; } args->x.dummy += sum; +#endif } // wait and/or wake up everyone if we're all ready @@ -767,6 +774,9 @@ int main(int argc, char **argv) { "constructing\n" " the chase and before starting the benchmark (use " "with nta)\n" + " on aarch64 any non-zero value instead writes the " + "chase back\n" + " to memory with DC CIVAC\n" " default: %zu\n", DEF_CACHE_FLUSH); fprintf( @@ -850,7 +860,8 @@ int main(int argc, char **argv) { per_thread_t *thread_data = alloc_arena_mmap( default_page_size, false, nr_threads * sizeof(per_thread_t), -1); void *flush_arena = NULL; - if (cache_flush_size) { + // Not read when the chase is cleaned with clean_invalidate_range(). + if (cache_flush_size && !HAVE_CLEAN_INVALIDATE_RANGE) { flush_arena = alloc_arena_mmap(default_page_size, false, cache_flush_size, -1); memset(flush_arena, 1, cache_flush_size); // ensure pages are mapped diff --git a/multiload.c b/multiload.c index c9494ab..8f84c78 100644 --- a/multiload.c +++ b/multiload.c @@ -841,6 +841,12 @@ static void *thread_start(void *data) { } // now flush our caches if (args->x.cache_flush_size) { +#if HAVE_CLEAN_INVALIDATE_RANGE + // Write the chase out to memory instead of reading a flush buffer, so + // the chase starts uncached and clean rather than dirty. + clean_invalidate_range(args->x.genchase_args->arena, + args->x.genchase_args->total_memory); +#else size_t nr_elts = args->x.cache_flush_size / sizeof(size_t); size_t *p = args->x.flush_arena; size_t sum = 0; @@ -850,6 +856,7 @@ static void *thread_start(void *data) { --nr_elts; } args->x.dummy += sum; +#endif } } else { if (verbosity > 2) @@ -861,6 +868,8 @@ static void *thread_start(void *data) { -1) + args->x.load_offset; memset(args->x.load_arena, 1, args->x.load_total_memory); // ensure pages are mapped + // Start clean, so read-only loads do not write dirty lines back. + clean_invalidate_range(args->x.load_arena, args->x.load_total_memory); } if (verbosity > 2) @@ -1179,6 +1188,9 @@ int main(int argc, char **argv) { "constructing\n" " the chase/memload and before starting the benchmark (use " "with nta)\n" + " on aarch64 any non-zero value instead writes the chase " + "back to\n" + " memory with DC CIVAC\n" " default: %zu\n", DEF_CACHE_FLUSH); fprintf(stderr, "-p nnnn[kmg] backing page size to use (default %zu)\n", @@ -1298,7 +1310,11 @@ int main(int argc, char **argv) { default_page_size, false, nr_threads * sizeof(per_thread_t), -1); void *flush_arena = NULL; if (verbosity > 2) printf("allocate cache flush\n"); - if (cache_flush_size) { + // Only chase threads read the flush buffer, and not at all when the chase + // is cleaned with clean_invalidate_range(). Otherwise it only adds 64 MiB + // of dirty lines to the caches. + if (cache_flush_size && !HAVE_CLEAN_INVALIDATE_RANGE && + run_test_type != RUN_BANDWIDTH) { flush_arena = alloc_arena_mmap(default_page_size, false, cache_flush_size, -1); memset(flush_arena, 1, cache_flush_size); // ensure pages are mapped