Write initialized buffers back so read-only tests stay read-only

cpu_util.h

@@ -14,6 +14,9 @@
 #ifndef PORT_H_INCLUDED
 #define PORT_H_INCLUDED
 
+#include <stddef.h>
+#include <stdint.h>
+
 // We assume 1024 bytes is good enough alignment to avoid false sharing on all
 // architectures.
 #define AVOID_FALSE_SHARING (1024)
@@ -47,4 +50,35 @@ static inline void cpu_relax(void) {
   } while (0)
 #endif
 
+// clean_invalidate_range() writes back and invalidates [buf, buf + len) to
+// the point of coherency, so a buffer that was just initialized is clean in
+// every cache level. A read-only test that starts on dirty lines can generate
+// write traffic on every pass: on some system level caches a read hit on a
+// dirty line hands the line back dirty, and the core later writes it back.
+// Only implemented on aarch64, where DC CIVAC is usable from user space on
+// Linux. Elsewhere it is a no-op and HAVE_CLEAN_INVALIDATE_RANGE is 0.
+#if defined(__aarch64__)
+#define HAVE_CLEAN_INVALIDATE_RANGE 1
+static inline void clean_invalidate_range(const void *buf, size_t len) {
+  // Rounding buf down below would otherwise touch one line when len is 0.
+  if (len == 0) return;
+  uint64_t ctr;
+  asm volatile("mrs %0, ctr_el0" : "=r"(ctr));
+  // CTR_EL0.DminLine is log2 of the smallest data cache line in words.
+  const uintptr_t line = (uintptr_t)4 << ((ctr >> 16) & 0xf);
+  uintptr_t p = (uintptr_t)buf & ~(line - 1);
+  const uintptr_t end = (uintptr_t)buf + len;
+  for (; p < end; p += line) {
+    asm volatile("dc civac, %0" : : "r"(p) : "memory");
+  }
+  asm volatile("dsb ish" : : : "memory");
+}
+#else
+#define HAVE_CLEAN_INVALIDATE_RANGE 0
+static inline void clean_invalidate_range(const void *buf, size_t len) {
+  (void)buf;
+  (void)len;
+}
+#endif
+
 #endif

multichase.c

@@ -496,6 +496,12 @@ static void *thread_start(void *data) {
 
   // now flush our caches
   if (args->x.cache_flush_size) {
+#if HAVE_CLEAN_INVALIDATE_RANGE
+    // Write the chase out to memory instead of reading a flush buffer, so
+    // the chase starts uncached and clean rather than dirty.
+    clean_invalidate_range(args->x.genchase_args->arena,
+                           args->x.genchase_args->total_memory);
+#else
     size_t nr_elts = args->x.cache_flush_size / sizeof(size_t);
     size_t *p = args->x.flush_arena;
     size_t sum = 0;
@@ -505,6 +511,7 @@ static void *thread_start(void *data) {
       --nr_elts;
     }
     args->x.dummy += sum;
+#endif
   }
 
   // wait and/or wake up everyone if we're all ready
@@ -767,6 +774,9 @@ int main(int argc, char **argv) {
             "constructing\n"
             "               the chase and before starting the benchmark (use "
             "with nta)\n"
+            "               on aarch64 any non-zero value instead writes the "
+            "chase back\n"
+            "               to memory with DC CIVAC\n"
             "               default: %zu\n",
             DEF_CACHE_FLUSH);
     fprintf(
@@ -850,7 +860,8 @@ int main(int argc, char **argv) {
   per_thread_t *thread_data = alloc_arena_mmap(
       default_page_size, false, nr_threads * sizeof(per_thread_t), -1);
   void *flush_arena = NULL;
-  if (cache_flush_size) {
+  // Not read when the chase is cleaned with clean_invalidate_range().
+  if (cache_flush_size && !HAVE_CLEAN_INVALIDATE_RANGE) {
     flush_arena = alloc_arena_mmap(default_page_size, false, cache_flush_size,
                                    -1);
     memset(flush_arena, 1, cache_flush_size);  // ensure pages are mapped

multiload.c

@@ -841,6 +841,12 @@ static void *thread_start(void *data) {
     }
     // now flush our caches
     if (args->x.cache_flush_size) {
+#if HAVE_CLEAN_INVALIDATE_RANGE
+      // Write the chase out to memory instead of reading a flush buffer, so
+      // the chase starts uncached and clean rather than dirty.
+      clean_invalidate_range(args->x.genchase_args->arena,
+                             args->x.genchase_args->total_memory);
+#else
       size_t nr_elts = args->x.cache_flush_size / sizeof(size_t);
       size_t *p = args->x.flush_arena;
       size_t sum = 0;
@@ -850,6 +856,7 @@ static void *thread_start(void *data) {
         --nr_elts;
       }
       args->x.dummy += sum;
+#endif
     }
   } else {
     if (verbosity > 2)
@@ -861,6 +868,8 @@ static void *thread_start(void *data) {
                              -1) + args->x.load_offset;
     memset(args->x.load_arena, 1,
            args->x.load_total_memory);  // ensure pages are mapped
+    // Start clean, so read-only loads do not write dirty lines back.
+    clean_invalidate_range(args->x.load_arena, args->x.load_total_memory);
   }
 
   if (verbosity > 2)
@@ -1179,6 +1188,9 @@ int main(int argc, char **argv) {
             "constructing\n"
             "         the chase/memload and before starting the benchmark (use "
             "with nta)\n"
+            "         on aarch64 any non-zero value instead writes the chase "
+            "back to\n"
+            "         memory with DC CIVAC\n"
             "         default: %zu\n",
             DEF_CACHE_FLUSH);
     fprintf(stderr, "-p nnnn[kmg]   backing page size to use (default %zu)\n",
@@ -1298,7 +1310,11 @@ int main(int argc, char **argv) {
       default_page_size, false, nr_threads * sizeof(per_thread_t), -1);
   void *flush_arena = NULL;
   if (verbosity > 2) printf("allocate cache flush\n");
-  if (cache_flush_size) {
+  // Only chase threads read the flush buffer, and not at all when the chase
+  // is cleaned with clean_invalidate_range(). Otherwise it only adds 64 MiB
+  // of dirty lines to the caches.
+  if (cache_flush_size && !HAVE_CLEAN_INVALIDATE_RANGE &&
+      run_test_type != RUN_BANDWIDTH) {
     flush_arena = alloc_arena_mmap(default_page_size, false, cache_flush_size,
                                    -1);
     memset(flush_arena, 1, cache_flush_size);  // ensure pages are mapped