From c427e019950d61cd1ec24287f2675e8a3546c8e7 Mon Sep 17 00:00:00 2001 From: timoheimonen Date: Sat, 29 Aug 2026 21:29:30 +0300 Subject: [PATCH] Simplify 0.63.1 version maintenance Validate bundled standard-memory examples by schema, methodology, completion, and consumed field shape while retaining software version only as provenance. Remove redundant legacy-output tests and make current-state documentation patch-version independent. --- CHANGELOG.md | 6 ++ README.md | 20 +++--- documents/API.md | 32 +++++---- documents/CAPABILITIES.md | 7 +- documents/CORE_TO_CORE_WHITEPAPER.md | 6 +- documents/GPU_BANDWIDTH_WHITEPAPER.md | 4 +- documents/LATENCY_WHITEPAPER.md | 9 +-- documents/LLM_MEMORY_PROFILE_WHITEPAPER.md | 2 +- documents/MANUAL.md | 39 +++++----- documents/PARAMETER_MATRIX.md | 4 +- documents/PROJECT_STRUCTURE.md | 28 ++++---- documents/TECHNICAL_SPECIFICATION.md | 22 +++--- documents/TLB_ANALYSIS_WHITEPAPER.md | 10 +-- script-examples/latency_test_script.sh | 24 ++++--- .../latency_test_script_stride_tlb.sh | 14 ++-- .../plot_M4vsM5_benchmark_comparison.py | 16 +++-- ...plot_benchmark-memory-latency-hierarchy.py | 18 ++--- src/core/config/version.h | 2 +- .../console/messages/program_messages.cpp | 1 - tests/fixtures/README.md | 25 ++++--- tests/test_config.cpp | 15 ++-- tests/test_executable_cli.cpp | 5 -- tests/test_json_schema.cpp | 26 ------- tests/test_messages.cpp | 21 +----- tests/test_pattern_benchmark.cpp | 1 - tests/test_script_examples.py | 71 +++++++++++-------- 26 files changed, 205 insertions(+), 223 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1427470..5664078 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.63.1] - Unreleased + +### Changed + - **CLI help updated**. + - **Test coverage cleaned up**: Removed redundant tests. + ## [0.63.0] - 2026-08-23 ### Added diff --git a/README.md b/README.md index 0796408..f5d260e 100644 --- a/README.md +++ b/README.md @@ -305,7 +305,8 @@ Treat benchmark values as measurements of the configured workload under the obse prefix sharing, sliding-window KV, growing context, and model loading. - The LLM traffic classification version `llm-exact-weight-vs-kv-read-payload-v1` compares exact weight and KV-read bytes only. `near_crossover` means exact equality and is not a measured hardware-bottleneck claim. -- The repository includes two current [Apple M5 CPU-decode working-set samples](results/0.63.0/AppleM5_LLM_working_set_scaling.md) +- The repository includes two + [Apple M5 CPU-decode working-set samples recorded with 0.63.0](results/0.63.0/AppleM5_LLM_working_set_scaling.md) with links to their complete JSON records. They illustrate scaling across 384 MiB and 1,536 MiB data mappings. - TLB-locality controls pointer-chain construction, not hardware TLB residency. Standard locality comparisons combine cache, locality, and translation effects; use `--analyze-tlb` for controlled translation-boundary conclusions. - Core-to-core results are scheduler-influenced acquire/release token-protocol measurements. They do not directly observe @@ -315,9 +316,10 @@ Treat benchmark values as measurements of the configured workload under the obse JSON output records completion and nullable measurement state instead of using zero for unavailable results. Current standard schema 3 requires `configuration.mode: "benchmark"`, a string `configuration.output_file` that preserves the raw output target, plus boolean `results_complete` and `conclusions_valid` fields. The bundled standard-memory examples -track the current producer, require its exact top-level `version` (currently `0.63.0`), sanity-check the current -result locally, and read current schema-3 paths directly. They do not provide compatibility for released standard -schema 2, unversioned historical standard JSON layouts, or any other explicit standard version. +accept compatible producer releases by checking the standard mode, schema 3, the exact +`benchmark-v2-calibrated-seeded-balanced` methodology, completion state, and the shape of the fields they consume. +They retain the top-level `version` as provenance but do not require a particular software release. They do not +translate released standard schema 2, unversioned historical standard JSON layouts, or other methodology identities. Consumers making conclusions should reject incomplete or interrupted runs according to the mode-specific status fields. Every result-producing direct command or CPU sweep using `--output -` reserves stdout for one final JSON document and routes its post-parse human transcript to stderr; file output is atomic. LLM file output checkpoints after each terminal @@ -340,11 +342,11 @@ python3 script-examples/plot_cache_percentiles.py \ The sweep script prefers the repository's local `./memory_benchmark`, then falls back to `memory_benchmark` from `PATH`; set `BENCHMARK_CMD=/path/to/memory_benchmark` to override either choice. Whichever producer is selected must -emit complete current standard schema 3. The sweep helpers return a non-zero status if a planned run fails or does not -produce a complete, parseable result. +emit a complete compatible standard schema-3 result with the expected methodology. The sweep helpers return a non-zero +status if a planned run fails or does not produce a complete, parseable result. -The two standard-result plotters require explicit current inputs; archived 0.53.x standard JSON is retained as -historical evidence and is not a valid current input: +The two standard-result plotters require explicit compatible inputs; archived 0.53.x standard JSON is retained as +historical evidence and is not a compatible input: ```bash python3 script-examples/plot_M4vsM5_benchmark_comparison.py \ @@ -370,7 +372,7 @@ recognizes the current console labels only and is neither JSON-schema nor histor - [LLM Memory Profile Whitepaper](documents/LLM_MEMORY_PROFILE_WHITEPAPER.md): generic schema-v1 vocabulary plus the active CPU and Metal decode/prefill traffic, timing, checksum, and interpretation contracts. - [Apple M5 LLM CPU-decode working-set samples](results/0.63.0/AppleM5_LLM_working_set_scaling.md): two complete - current-version JSON runs and their observed working-set scaling. + 0.63.0 JSON runs and their observed working-set scaling. Runtime behavior and `memory_benchmark -h` are the authoritative sources when documentation differs. diff --git a/documents/API.md b/documents/API.md index d504f22..8676622 100644 --- a/documents/API.md +++ b/documents/API.md @@ -1,8 +1,9 @@ # Machine-Readable Command-Line API -This document defines the supported process-level integration contract for `memory_benchmark` 0.63.0. It describes how -software launches a benchmark, separates machine-readable output from the human transcript, and decides whether a JSON -result is safe to consume. The generated Doxygen pages document C++ internals; they are not this process API. +This document defines the supported process-level integration contract for the current `memory_benchmark` +implementation. It describes how software launches a benchmark, separates machine-readable output from the human +transcript, and decides whether a JSON result is safe to consume. The generated Doxygen pages document C++ internals; +they are not this process API. Runtime behavior and executable integration tests are authoritative if this document and the implementation differ. The documented process runtime baseline is macOS 26 or later on Apple Silicon (ARM64). @@ -199,7 +200,7 @@ predicate. Each `runs[].result` in a sweep retains its nested mode's own schema-version field and completeness contract. Nested standard classification recognizes only current schema 3 with `configuration.mode == "benchmark"` plus typed `results_complete`, `conclusions_valid`, and `configuration.output_file` fields; standard schema 2 and every other -standard version are unsupported. Complete, partial, interrupted, and failed current schema-3 evidence remains +standard schema version are unsupported. Complete, partial, interrupted, and failed current schema-3 evidence remains classifiable and retained rather than being discarded by the complete-result consumer boundary. A non-zero nested execution that initialized a result remains in the envelope: its attempt is failed, but the payload is not replaced by a generic diagnostic. In particular, nested TLB `tlb_analysis.status == "error"` maps to a failed sweep attempt without @@ -557,12 +558,12 @@ persistence and nested file writes are disabled. Schema 3 requires boolean `resu the producer makes `conclusions_valid` true exactly when `results_complete` is true, while consumers must still check the explicit status and both booleans shown in the table. -The bundled standard-memory example scripts are maintained in lockstep with the current producer. Each script performs -only the local version, completion, and field sanity checks needed before reading its current schema-3 metric paths. For -the current producer that includes exact top-level `version == "0.63.0"` in addition to the standard identity and -completeness fields above. The examples are not a versioned compatibility library. Released standard schema 2, -unversioned historical standard JSON layouts, and every other explicit standard version are unsupported inputs and are -not routed through a metric-shape fallback. +The bundled standard-memory example scripts accept compatible producer releases. Before reading their standard +schema-3 metric paths, they require `configuration.mode == "benchmark"`, schema 3, methodology +`benchmark-v2-calibrated-seeded-balanced`, the completion fields above, and the expected types for every consumed +field. They retain a non-empty top-level `version` string as release provenance, but exact software-version equality is +not an acceptance condition. The examples do not translate released standard schema 2, unversioned historical +standard JSON layouts, or other methodology identities through a metric-shape fallback. Graceful interruption or runtime failure after a representable result state has been initialized emits the available partial, interrupted, error, or failed JSON snapshot. The execution status and payload are independent: a non-zero status @@ -610,6 +611,7 @@ memory_benchmark --benchmark --only-bandwidth --buffer-size 512 --count 5 --seed >benchmark.json 2>benchmark.log jq -e '.configuration.mode == "benchmark" and .configuration.benchmark_schema_version == 3 and + .configuration.methodology_version == "benchmark-v2-calibrated-seeded-balanced" and (.configuration.output_file | type) == "string" and .status == "complete" and .results_complete == true and .conclusions_valid == true' benchmark.json @@ -667,7 +669,8 @@ jq -e '.mode == "llm_memory" and .schema_version == 1 and ## Compatibility policy - `version`, the GPU `software_version` field, and LLM `software` identity identify the application release; none is a - result schema version. + result schema version or compatibility selector. Consumers may retain and display this provenance independently of + schema and methodology acceptance. - Current standard schema 3, pattern schema 3, TLB schema 4, core-to-core schema 2, GPU schema 1, and LLM schema 1 remain authoritative at their existing locations. The schema field is intentionally not normalized across these established payloads. @@ -676,9 +679,10 @@ jq -e '.mode == "llm_memory" and .schema_version == 1 and consumers must use the current generic fields, tolerate unknown additive evidence fields, and validate every known field they consume. Future removal/rename/type/meaning changes require schema-version review; software identity alone is not a compatibility substitute. -- Bundled standard-memory examples track the current producer and read current standard schema-3 metric paths directly - after local sanity checks. They provide no compatibility layer for released standard schema 2, unversioned - historical standard JSON layouts, or any other explicit standard version. +- Bundled standard-memory examples accept compatible software releases only when standard mode, schema 3, the exact + methodology identity, completion state, and consumed field shapes match. They retain software-version provenance but + provide no translation layer for released standard schema 2, unversioned historical standard JSON layouts, or other + methodology identities. - Both general and core-to-core sweep envelopes use `configuration.sweep_schema_version == 1`; nested results keep their independent mode schema versions. - Additive optional fields may remain within a schema version only when old consumers can safely ignore them. diff --git a/documents/CAPABILITIES.md b/documents/CAPABILITIES.md index 82562d7..83ecff1 100644 --- a/documents/CAPABILITIES.md +++ b/documents/CAPABILITIES.md @@ -223,9 +223,10 @@ current support matrix and process acceptance procedure are in the [Machine-Read Current standard results use schema 3, which requires `configuration.mode: "benchmark"`, a string `configuration.output_file`, plus boolean `results_complete` and `conclusions_valid`. Bundled standard-memory examples -track the current producer, perform local sanity checks, and read its current schema-3 metric paths directly. They do -not support released standard schema 2, unversioned historical standard JSON layouts, or any other explicit standard -version. +accept compatible producer releases after validating standard mode, schema 3, methodology +`benchmark-v2-calibrated-seeded-balanced`, completion state, and consumed field shapes. Software version is retained +only as provenance. They do not translate released standard schema 2, unversioned historical standard JSON layouts, or +other methodology identities. Sweep output retains completed evidence even when a later run stops. Consumers should check the mode-specific status and completeness indicators before using aggregate conclusions. Exact schemas, checkpoint behavior, and inspection examples are in the [User Manual](MANUAL.md#json-output-format) and [Technical Specification](TECHNICAL_SPECIFICATION.md#18-json-output-contract). diff --git a/documents/CORE_TO_CORE_WHITEPAPER.md b/documents/CORE_TO_CORE_WHITEPAPER.md index 521df6b..4807fdf 100644 --- a/documents/CORE_TO_CORE_WHITEPAPER.md +++ b/documents/CORE_TO_CORE_WHITEPAPER.md @@ -2,7 +2,7 @@ **memory_benchmark — Technical Whitepaper** -*Applies to version 0.63.0; archived examples may use older methodologies* +*Applies to the current repository implementation; archived examples may use older methodologies* --- @@ -13,7 +13,7 @@ protocol between two POSIX threads. The result includes the complete polling and scheduler effects; it is not an isolated measurement of one physical cache-line migration or the coherence fabric. Software version 0.58.0 introduced the change from fixed work in a fixed scenario order to the auditable v2 calibrated/balanced design. Version 0.61.1 introduced distinct 128-byte token and control blocks and the current -methodology identity; version 0.63.0 retains that design as +methodology identity. The current implementation retains that design as `core2core-v3-calibrated-balanced-auditable-128b-isolation`: - every scheduler-hint scenario receives its own excluded pilot after a long calibration warmup; @@ -392,7 +392,7 @@ and one-way arrays are complete for the illustrated scenario. }, "execution_time_sec": 4.2, "timestamp": "YYYY-MM-DDTHH:MM:SSZ", - "version": "0.63.0" + "version": "" } ``` diff --git a/documents/GPU_BANDWIDTH_WHITEPAPER.md b/documents/GPU_BANDWIDTH_WHITEPAPER.md index 6897c4d..0037ab3 100644 --- a/documents/GPU_BANDWIDTH_WHITEPAPER.md +++ b/documents/GPU_BANDWIDTH_WHITEPAPER.md @@ -1,6 +1,6 @@ # Metal GPU Memory Bandwidth Whitepaper -- **Software version:** 0.63.0 +- **Software version:** release provenance; not a compatibility selector - **JSON schema:** 1 - **Methodology:** `gpu-bandwidth-v1-private-runtime-single-cmdbuf-calibrated-balanced` - **Platform:** macOS on Apple Silicon @@ -455,7 +455,7 @@ already-computed measurement state. ## 14. GPU JSON Schema 1 The top-level discriminator is independent of current standard schema 3. Released standard schema 2 is historical and -unsupported by the bundled standard-memory examples, which track the current producer: +unsupported by the bundled standard-memory examples, which accept compatible standard schema-3 producer releases: ```json { diff --git a/documents/LATENCY_WHITEPAPER.md b/documents/LATENCY_WHITEPAPER.md index b13f1e4..797d3ca 100644 --- a/documents/LATENCY_WHITEPAPER.md +++ b/documents/LATENCY_WHITEPAPER.md @@ -2,7 +2,8 @@ ## 1. Purpose -This document explains how `macOS-memory-benchmark` version 0.63.0 measures memory latency on Apple Silicon. +This document explains how the current `macOS-memory-benchmark` implementation measures memory latency on Apple +Silicon. The latency path is designed to measure **load-to-use delay** (pointer chasing), not bulk throughput. It combines: @@ -254,9 +255,9 @@ analysis. `pooled_sample_distribution`, with values and loop-boundary metadata kept separate from continuous loop headlines. - Current standard schema 3 does not serialize the unversioned historical `chain_diagnostics.unique_pages_touched` blocks. Do not use the old `main_memory.latency.chain_diagnostics` or - `cache.*.latency.chain_diagnostics` paths for version 0.63.0 output. Released standard schema 2 and unversioned - historical standard JSON remain archival evidence and are unsupported by the bundled examples, which track the - current producer and read current schema-3 paths directly. + `cache.*.latency.chain_diagnostics` paths for standard schema-3 output. Released standard schema 2 and unversioned + historical standard JSON remain archival evidence and are unsupported by the bundled examples, which accept the + current methodology across compatible software releases and read schema-3 paths directly. When `--latency-tlb-locality-kb` is not explicitly supplied, standard main-memory latency also runs three paired rounds of 16 KiB-locality and global-random chains. The first-measured layout alternates by round, and diff --git a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md index c3e3d7b..eb326f5 100644 --- a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md +++ b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md @@ -22,7 +22,7 @@ The current contract is identified by: | Property | Value | |---|---| -| Software version | `0.63.0` | +| Software version | Release provenance; not a schema or methodology selector | | Mode | `llm_memory` | | Backend | `cpu` or `metal` | | JSON schema | `1` | diff --git a/documents/MANUAL.md b/documents/MANUAL.md index af4a3dd..5987193 100644 --- a/documents/MANUAL.md +++ b/documents/MANUAL.md @@ -967,7 +967,7 @@ middle, and trailing items. `attempted_runs` counts stored entries, while `completed_runs` counts only mode-specific nested results that are genuinely complete. Current standard schema 3 requires nested `configuration.mode: "benchmark"`, `status: "complete"`, `results_complete: true`, and `conclusions_valid: true`, plus a string - `configuration.output_file`. Nested standard schema 2 and every other standard version are unsupported. Pattern + `configuration.output_file`. Nested standard schema 2 and every other standard schema version are unsupported. Pattern requires nested `status: "complete"` with `results_complete: true`; TLB requires nested `tlb_analysis.status: "complete"` with `tlb_analysis.conclusions_valid: true`. Partial, interrupted, and failed nested results never increment it. TLB's @@ -1586,7 +1586,7 @@ each terminal scenario measurement and command terminal, while LLM stdout emits "main_memory": { ... }, "cache": { ... }, "timestamp": "YYYY-MM-DDTHH:MM:SSZ", - "version": "0.63.0" + "version": "" } ``` @@ -1603,10 +1603,11 @@ emits one final snapshot. Schema 3 also requires boolean `conclusions_valid`, wh Bandwidth QoS metadata includes created workers plus per-worker success/failure counts; latency carries the main-thread outcome. These fields describe a best-effort scheduler hint, never hard core pinning. -The bundled standard-memory examples are kept compatible with the current producer. They sanity-check the current -standard result locally, including exact top-level `version: "0.63.0"` for the current producer, and read current -schema-3 metric paths directly; they are not a compatibility library. Released standard schema 2, unversioned -historical standard JSON layouts, and every other explicit standard version are intentionally unsupported inputs. +The bundled standard-memory examples accept compatible producer releases. They require standard mode, schema 3, +methodology `benchmark-v2-calibrated-seeded-balanced`, complete/valid result state, and the expected types for the +fields they consume before reading schema-3 metric paths directly. They retain a non-empty top-level `version` string +as provenance without requiring a particular software release. Released standard schema 2, unversioned historical +standard JSON layouts, and other methodology identities are unsupported inputs and are not translated. ### Pattern benchmark JSON shape @@ -1682,7 +1683,7 @@ historical standard JSON layouts, and every other explicit standard version are } }, "timestamp": "...", - "version": "..." + "version": "" } ``` @@ -1732,8 +1733,8 @@ arrays: ```json { - "software_version": "0.63.0", - "version": "0.63.0", + "software_version": "", + "version": "", "timestamp": "...", "schema_version": 1, "mode": "gpu_bandwidth", @@ -1849,7 +1850,7 @@ the required generic sections. "kv_layout": "contiguous", "methodology_version": "llm-memory-v1-cpu-decode-contiguous", "software": { - "version": "0.63.0", + "version": "", "timestamp": "..." }, "status": "complete", @@ -2139,7 +2140,7 @@ returns only, excludes QoS and pilot outcomes, and does not prove physical place ], "execution_time_sec": 123.4, "timestamp": "YYYY-MM-DDTHH:MM:SSZ", - "version": "0.63.0" + "version": "" } ``` @@ -2526,11 +2527,11 @@ Plotting requires Python 3 and `matplotlib`; the M4/M5 comparison script additio python3 -m pip install matplotlib numpy ``` -The bundled standard-memory scripts are kept in lockstep with the current producer. Each performs only the local -version, completion, and field sanity checks needed by its current schema-3 metric paths. For version 0.63.0, the check -requires top-level `version: "0.63.0"`, standard mode/schema identity, complete/valid result state, and a string output -target before the selected metric path is read. These scripts are examples, not a versioned compatibility layer: -standard schema 2, unversioned historical standard JSON, other modes, and other standard versions are unsupported. The +The bundled standard-memory scripts accept outputs from compatible software releases. Before reading a selected metric +path, each requires a non-empty top-level `version` string as provenance, standard mode, schema 3, methodology +`benchmark-v2-calibrated-seeded-balanced`, complete/valid result state, a string output target, and the expected types +for the fields it consumes. Exact software-version equality is not a compatibility condition. These scripts do not +translate standard schema 2, unversioned historical standard JSON, other modes, or other methodology identities. The separately governed `plot_analyzetlb.py` retains its own TLB-history policy. Standard-memory plotters do not accept GPU schema 1. @@ -2627,8 +2628,8 @@ The repository's historical `results/0.53.7/MacMiniM4_benchmark.json` sample rep - Main memory copy: ~106 GB/s Under heavy concurrent load, expect lower throughput and higher variance than this historical sample. It is an empirical -0.53.7 result, not a guaranteed current-version baseline. Historical pattern files from earlier methodology versions are -not a stability baseline for current pattern schema 3 and should not be compared numerically with +0.53.7 result, not a guaranteed current-methodology baseline. Historical pattern files from earlier methodology +versions are not a stability baseline for current pattern schema 3 and should not be compared numerically with `pattern-v2-phase-calibrated-seeded` results without accounting for the methodology change. --- @@ -2793,7 +2794,7 @@ Make sure you are passing `script-examples/final_output.txt` generated by the la Repository sample result files: -Current 0.63.0 LLM schema-1 examples: +Recorded 0.63.0 LLM schema-1 examples: - [Apple M5 CPU-decode working-set comparison note](../results/0.63.0/AppleM5_LLM_working_set_scaling.md) - [Apple M5, 256 MiB weights and 32,768-token context](../results/0.63.0/apple_m5_cpu_decode_contiguous_weights_256mib_context_32768.json) diff --git a/documents/PARAMETER_MATRIX.md b/documents/PARAMETER_MATRIX.md index 26b99ac..d0d0671 100644 --- a/documents/PARAMETER_MATRIX.md +++ b/documents/PARAMETER_MATRIX.md @@ -1,6 +1,6 @@ # Parameter Compatibility Matrix -Working version `0.63.0` +Applies to the current repository implementation. Runtime platform: macOS 26 or later on Apple Silicon (ARM64). @@ -268,7 +268,7 @@ Additional sweep rules: array but stop further execution and do not increment `completed_runs`. A current standard schema-3 attempt is complete only with nested `configuration.mode: "benchmark"`, `status: "complete"`, `results_complete: true`, `conclusions_valid: true`, and a string `configuration.output_file`. Nested standard schema 2 and every other - standard version are unsupported. A pattern attempt requires nested + standard schema version are unsupported. A pattern attempt requires nested `status: "complete"` and `results_complete: true`; TLB requires nested `tlb_analysis.status: "complete"` and `tlb_analysis.conclusions_valid: true`; core-to-core requires nested `core_to_core_latency.status: "complete"` and `measurements_complete: true`. The authoritative schema-1 sweep acceptance predicate is exactly diff --git a/documents/PROJECT_STRUCTURE.md b/documents/PROJECT_STRUCTURE.md index 2d9b71d..b1093ff 100644 --- a/documents/PROJECT_STRUCTURE.md +++ b/documents/PROJECT_STRUCTURE.md @@ -1,6 +1,5 @@ # Project Structure — macOS-memory-benchmark -**Version:** 0.63.0 **Platform:** ARM64 / AArch64 (Apple Silicon, macOS 26 or later) **License:** GNU General Public License v3.0 or later @@ -173,7 +172,7 @@ Core infrastructure for configuration, memory management, macOS system introspec |---|---| | `config.h` | `BenchmarkConfig` structure for standard, pattern, standalone TLB, and their common sweep settings; core-to-core, GPU, and LLM use separate config types | | `constants.h` | Named constants for CPU/GPU/LLM memory limits, calibration, grid/dispatch/payload guardrails, buffer sizing, and latency access counts | -| `version.h` | `SOFTVERSION` macro (semantic version string, currently `"0.63.0"`) | +| `version.h` | `SOFTVERSION` macro containing the semantic software-release provenance string | | `mode_selector.h` / `.cpp` | Pure primary-mode scan and conflict detection before mode-specific parsing; routes standard, pattern, TLB, core-to-core, GPU, and LLM deterministically while skipping one opaque output value | | `argument_parser.cpp` | Parses standard, pattern, and standalone TLB options into `BenchmarkConfig`; core-to-core, GPU, and LLM are pre-routed to dedicated parsers | | `config_validator.cpp` | Validates the parsed configuration; emits errors for out-of-range or conflicting settings | @@ -397,7 +396,7 @@ installed. All `.cpp` files are picked up automatically by the Makefile. Tests n | `test_hash_utils.cpp` | `HashUtilsTest` | CommonCrypto SHA-256 standard vectors and source-provenance helper behavior | | `test_analysis.cpp` | `AnalysisTest` | Injected TLB coordination, counters/status, boundary detection, validation, and paired analysis | | `test_json_schema.cpp` | `JsonSchemaTest` | Current standard schema-3 output structure, completion fields, and other mode schema contracts | -| `test_script_examples.py` | Python script-example entry test | Current schema-3 JSON entry paths, local sanity checks, and metric extraction for the bundled standard-memory examples, including the optional jq-backed shell path when jq is installed | +| `test_script_examples.py` | Python script-example entry test | Compatible standard schema-3/methodology inputs, provenance-version handling, completion and field-shape checks, and metric extraction for the bundled standard-memory examples, including the optional jq-backed shell path when jq is installed | | `test_json_output_session.cpp` | `JsonOutputTargetTest`, `JsonOutputSessionTest` | Exact sentinel/path classification, lazy checkpoint dispatch, atomic-file parity, stdout routing/restoration, and stream failure containment | | `test_json_utils.cpp` | `JsonUtilsTest`, `JsonFileWriterTest` | JSON parse/statistics and atomic writer success/failure contracts | | `test_output_printer.cpp` | `OutputPrinterTest`, `OutputPrinterCustomCacheUnitsTest` | Status-aware partial output, mode/cache composition, and custom-cache size-unit boundaries | @@ -450,17 +449,17 @@ Four shared helper headers support deterministic setup and output capture across ## 4. results/ — Benchmark result data Historical JSON, CSV, and text output from benchmark runs on specific hardware, organized by software-version -subdirectory. The files are retained as examples and historical schema evidence; they are not current 0.63.0 -methodology baselines unless explicitly identified as such. Historical standard JSON is not an input to current -bundled standard-result consumers; separately governed TLB tooling retains its documented legacy-input policy. +subdirectory. The files are retained as examples and historical schema evidence; they are not baselines for the current +methodology unless explicitly identified as such. Historical standard JSON is not an input to current bundled +standard-result consumers; separately governed TLB tooling retains its documented legacy-input policy. ``` results/ 0.63.0/ AppleM5_LLM_working_set_scaling.md — Apple M5 CPU decode working-set comparison note M4MacMiniVSM5MacbookAir_LLM.md — Informal LLM CPU decode comparison - apple_m5_cpu_decode_contiguous_weights_256mib_context_32768.json — Current LLM schema-1 sample - apple_m5_cpu_decode_contiguous_weights_1024mib_context_131072.json — Current LLM schema-1 sample + apple_m5_cpu_decode_contiguous_weights_256mib_context_32768.json — Recorded LLM schema-1 sample + apple_m5_cpu_decode_contiguous_weights_1024mib_context_131072.json — Recorded LLM schema-1 sample 0.53.7/ MacMiniM4_analyzetlb.json — TLB analysis run, Mac Mini M4 MacMiniM4_benchmark.json — Standard benchmark run, Mac Mini M4 @@ -475,7 +474,7 @@ results/ *.json / *.csv / *.txt — Pre-versioned historical results ``` -The current-version result notes include an +The 0.63.0 result notes include an [Apple M5 CPU decode working-set comparison](../results/0.63.0/AppleM5_LLM_working_set_scaling.md) with links to its complete schema-1 JSON records, and an [informal M4 Mac mini versus M5 MacBook Air LLM CPU decode comparison](../results/0.63.0/M4MacMiniVSM5MacbookAir_LLM.md). @@ -509,11 +508,12 @@ Tracked PNG chart archive generated from benchmark result data. Several files ar ## 6. script-examples/ — Run and plotting helpers Example shell workflows and Python/Matplotlib plotters for tracked benchmark outputs. The four standard-memory examples -are maintained in lockstep with the current producer rather than as a compatibility library. Each of those JSON-reading -entry points performs a small local release-version/schema-3/completion/field sanity check and then reads the current -metric paths it needs. Version 0.63.0 therefore requires exact top-level `version: "0.63.0"`. Standard schema 2, -unversioned historical layouts, alternative modes, and other explicit standard versions are unsupported by those four -examples. Separately governed mode-specific tools retain only the history policy documented in their own row below. +accept compatible producer releases rather than requiring an exact software version. Each JSON-reading entry point +checks for a non-empty provenance version, standard mode, schema 3, methodology +`benchmark-v2-calibrated-seeded-balanced`, completion state, and the shapes of the fields it consumes before reading +those metric paths. Standard schema 2, unversioned historical layouts, alternative modes, and other methodology +identities are not translated by those four examples. Separately governed mode-specific tools retain only the history +policy documented in their own row below. | File | Purpose | |---|---| diff --git a/documents/TECHNICAL_SPECIFICATION.md b/documents/TECHNICAL_SPECIFICATION.md index 393b399..1505b4a 100644 --- a/documents/TECHNICAL_SPECIFICATION.md +++ b/documents/TECHNICAL_SPECIFICATION.md @@ -2,7 +2,7 @@ ## 1. Scope and Status -This document specifies the current implementation in this repository (version `0.63.0`) for `memory_benchmark` on macOS Apple Silicon. +This document specifies the current `memory_benchmark` implementation in this repository for macOS Apple Silicon. It is intentionally implementation-driven and reflects real behavior in code paths under `main.cpp`, `src/core`, `src/benchmark`, `src/pattern_benchmark`, `src/gpu_bandwidth`, `src/llm_memory`, `src/output`, and `src/asm`. @@ -691,8 +691,8 @@ reference cohort: the completed 0.61.0 automatic and fixed-work populations esta for their exact hardware, OS, compiler, kernel, and methodology identity. The frozen pre-remediation validation identity uses `gpu-linear-word-mod32-tg-reduce-v2`, the frozen 8192-threadgroup cap, canonical MSL SHA-256 `b9a242d2b959c9c11f6f130a52afd66f111d6761be2193beec1f051baa094296`, and the exact executable identity retained -with the local validation record. The current canonical source SHA-256 retained unchanged in 0.63.0 (introduced in -0.61.2) is +with the local validation record. The canonical source SHA-256 introduced in 0.61.2 and retained by the current +implementation is `21def2d75d3545dba31aa4897ea57ec2fd0e4481cd86ce21725338ab0f322ac5` after removing three unread shared-parameter fields; runtime Metal integration revalidates compilation and correctness, while the performance population remains tied to the frozen pre-remediation identity. Automatic read/write/copy @@ -1143,7 +1143,7 @@ measurement schema. [API.md](API.md) is the process-integration contract and sup logical cadence but defers serialization until the terminal envelope. `completed_runs` requires current standard schema 3 to have nested `configuration.mode: "benchmark"`, `status: "complete"`, `results_complete: true`, `conclusions_valid: true`, and a string `configuration.output_file`. Nested standard schema 2 and every other - standard version are unsupported. Pattern requires nested + standard schema version are unsupported. Pattern requires nested `status: "complete"` and `results_complete: true` for pattern; `tlb_analysis.status: "complete"` and `tlb_analysis.conclusions_valid: true` for TLB; or `core_to_core_latency.status: "complete"` and `measurements_complete: true` for core-to-core. Partial, interrupted, @@ -1162,10 +1162,11 @@ classifying the failure. Command completeness for current standard schema 3 requires `configuration.mode == "benchmark"`, `status == "complete" && results_complete == true && conclusions_valid == true`, with string -`configuration.output_file`. Bundled standard-memory examples track this current schema-3 producer, perform local -sanity checks (including exact top-level `version == "0.63.0"` for the current producer), and read the metric paths they -need directly. They do not support released standard schema 2, unversioned historical standard JSON layouts, or any -other explicit standard version. Pattern requires +`configuration.output_file`. Bundled standard-memory examples accept compatible producer releases when +`configuration.methodology_version == "benchmark-v2-calibrated-seeded-balanced"`, the command is complete, and each +consumed field has the expected shape. They retain a non-empty top-level `version` string as provenance without +requiring exact software-version equality. They do not translate released standard schema 2, unversioned historical +standard JSON layouts, or other methodology identities. Pattern requires `status == "complete" && results_complete == true`. TLB requires `tlb_analysis.status == "complete" && tlb_analysis.conclusions_valid == true`; core-to-core requires `core_to_core_latency.status == "complete" && core_to_core_latency.measurements_complete == true`. Metric consumers must @@ -1196,8 +1197,9 @@ In addition to standard fields (buffer size, iterations, loop count, thread coun - Calibration targets/windows and phase/operation schedule policies. Schema 3 makes `configuration.output_file` and top-level `conclusions_valid` mandatory without changing the standard -methodology version. Bundled standard-memory examples identify the current producer explicitly rather than inferring -standard identity from metric layout; schema 2 and unversioned historical standard JSON are unsupported inputs. +methodology version. Bundled standard-memory examples identify compatible input from the standard mode, schema, +methodology, completion fields, and consumed field shapes rather than inferring identity from metric layout or requiring +an exact software version; schema 2 and unversioned historical standard JSON are unsupported inputs. ### 18.2 Main-memory latency keys diff --git a/documents/TLB_ANALYSIS_WHITEPAPER.md b/documents/TLB_ANALYSIS_WHITEPAPER.md index 6bcec63..da6cc3b 100644 --- a/documents/TLB_ANALYSIS_WHITEPAPER.md +++ b/documents/TLB_ANALYSIS_WHITEPAPER.md @@ -2,7 +2,8 @@ ## 1. Purpose -This document specifies how `macOS-memory-benchmark` implements standalone TLB analysis mode (`--analyze-tlb`) in version `0.63.0`. +This document specifies how the current `macOS-memory-benchmark` implementation provides standalone TLB analysis mode +(`--analyze-tlb`). The goal is to provide a reproducible, implementation-accurate description of: @@ -518,8 +519,9 @@ it is not an additional completeness condition. ## 10. Current Schema Worked Example (Deterministic Exporter Fixture) -The regression test `JsonSchemaTest.TlbAnalysisExporterIncludesModeAndCoreCounts` generates a current-version payload with -the production serializer. Its deliberately synthetic inputs make this a contract example, not an Apple hardware claim: +The regression test `JsonSchemaTest.TlbAnalysisExporterIncludesModeAndCoreCounts` generates a current-schema payload +with the production serializer. Its deliberately synthetic inputs make this a contract example, not an Apple hardware +claim: ```json { @@ -564,7 +566,7 @@ the production serializer. Its deliberately synthetic inputs make this a contrac "active_cache_line_footprint_bytes": 2097152 } }, - "version": "0.63.0" + "version": "" } ``` diff --git a/script-examples/latency_test_script.sh b/script-examples/latency_test_script.sh index a0a5f32..3562213 100755 --- a/script-examples/latency_test_script.sh +++ b/script-examples/latency_test_script.sh @@ -13,8 +13,8 @@ mkdir -p "${TMP_DIR}" # Prefer the binary built in the repository; allow an explicit override or an # installed memory_benchmark from PATH when the local binary is unavailable. -# Whichever producer is selected must emit complete current standard schema 3; -# extraction below intentionally rejects older standard JSON contracts. +# Whichever producer is selected must emit the supported standard schema 3 and +# methodology; extraction below intentionally rejects incompatible contracts. DEFAULT_BENCHMARK="${SCRIPT_DIR}/../memory_benchmark" if [ -x "${DEFAULT_BENCHMARK}" ]; then BENCHMARK_CMD="${BENCHMARK_CMD:-${DEFAULT_BENCHMARK}}" @@ -139,7 +139,7 @@ final_output="${SCRIPT_DIR}/final_output.txt" # Clear/create the final output file > "${final_output}" -# Function to extract current standard schema-3 pooled sample statistics. +# Function to extract supported standard schema-3 pooled sample statistics. extract_with_jq() { local json_file=$1 local cache_size=$2 @@ -148,14 +148,16 @@ extract_with_jq() { if ! jq ' if type != "object" or (.configuration | type) != "object" - or .version != "0.63.0" + or (.version | type) != "string" + or (.version | length) == 0 or .configuration.mode != "benchmark" or .configuration.benchmark_schema_version != 3 + or .configuration.methodology_version != "benchmark-v2-calibrated-seeded-balanced" or .status != "complete" or .results_complete != true or .conclusions_valid != true or (.configuration.output_file | type) != "string" - then error("not a complete current standard schema-3 benchmark result") + then error("not a complete supported standard schema-3 benchmark result") else .cache.custom.latency.headline_ns.pooled_sample_distribution.statistics end | if type == "object" @@ -177,7 +179,7 @@ extract_with_jq() { rm -f "${extracted_file}" } -# Function to extract current standard schema-3 pooled sample statistics. +# Function to extract supported standard schema-3 pooled sample statistics. extract_with_python() { local json_file=$1 local cache_size=$2 @@ -188,26 +190,28 @@ import json import sys -def require_current_standard_result(data): +def require_supported_standard_result(data): configuration = data.get("configuration") if isinstance(data, dict) else None if not ( isinstance(configuration, dict) - and data.get("version") == "0.63.0" + and isinstance(data.get("version"), str) + and bool(data["version"]) and configuration.get("mode") == "benchmark" and type(configuration.get("benchmark_schema_version")) is int and configuration["benchmark_schema_version"] == 3 + and configuration.get("methodology_version") == "benchmark-v2-calibrated-seeded-balanced" and data.get("status") == "complete" and data.get("results_complete") is True and data.get("conclusions_valid") is True and isinstance(configuration.get("output_file"), str) ): - raise RuntimeError("not a complete current standard schema-3 benchmark result") + raise RuntimeError("not a complete supported standard schema-3 benchmark result") try: with open(sys.argv[1], 'r') as f: data = json.load(f) - require_current_standard_result(data) + require_supported_standard_result(data) latency = data['cache']['custom']['latency'] stats = latency['headline_ns']['pooled_sample_distribution']['statistics'] required_statistics = ('average', 'median', 'p90', 'p95', 'p99', 'min', 'max', 'stddev') diff --git a/script-examples/latency_test_script_stride_tlb.sh b/script-examples/latency_test_script_stride_tlb.sh index 747e5c8..c57f0b8 100755 --- a/script-examples/latency_test_script_stride_tlb.sh +++ b/script-examples/latency_test_script_stride_tlb.sh @@ -14,8 +14,8 @@ JSON_DIR="${RUN_DIR}/json" SUMMARY_CSV="${RUN_DIR}/latency_summary.csv" DEFAULT_BENCHMARK="${SCRIPT_DIR}/../memory_benchmark" -# Whichever BENCHMARK_CMD/PATH producer is selected must emit complete current -# standard schema 3; summary extraction intentionally rejects older contracts. +# Whichever BENCHMARK_CMD/PATH producer is selected must emit the supported +# standard schema 3 and methodology; extraction rejects incompatible contracts. if [ -x "${DEFAULT_BENCHMARK}" ]; then BENCHMARK_CMD="${BENCHMARK_CMD:-${DEFAULT_BENCHMARK}}" else @@ -125,20 +125,22 @@ import sys from pathlib import Path -def require_current_standard_result(data): +def require_supported_standard_result(data): configuration = data.get("configuration") if isinstance(data, dict) else None if not ( isinstance(configuration, dict) - and data.get("version") == "0.63.0" + and isinstance(data.get("version"), str) + and bool(data["version"]) and configuration.get("mode") == "benchmark" and type(configuration.get("benchmark_schema_version")) is int and configuration["benchmark_schema_version"] == 3 + and configuration.get("methodology_version") == "benchmark-v2-calibrated-seeded-balanced" and data.get("status") == "complete" and data.get("results_complete") is True and data.get("conclusions_valid") is True and isinstance(configuration.get("output_file"), str) ): - raise RuntimeError("not a complete current standard schema-3 benchmark result") + raise RuntimeError("not a complete supported standard schema-3 benchmark result") return configuration @@ -151,7 +153,7 @@ errors = [] for path in sorted(json_dir.glob("*.json")): try: data = json.loads(path.read_text(encoding="utf-8")) - cfg = require_current_standard_result(data) + cfg = require_supported_standard_result(data) headline = data["cache"]["custom"]["latency"]["headline_ns"] stats = headline["pooled_sample_distribution"]["statistics"] measurements = headline.get("measurements", []) or [] diff --git a/script-examples/plot_M4vsM5_benchmark_comparison.py b/script-examples/plot_M4vsM5_benchmark_comparison.py index 6b986df..5600cba 100644 --- a/script-examples/plot_M4vsM5_benchmark_comparison.py +++ b/script-examples/plot_M4vsM5_benchmark_comparison.py @@ -39,11 +39,11 @@ def parse_args(): parser.add_argument( "-4", "--m4-file", required=True, - help="Path to a current standard schema-3 M4 benchmark JSON") + help="Path to a supported standard schema-3 M4 benchmark JSON") parser.add_argument( "-5", "--m5-file", required=True, - help="Path to a current standard schema-3 M5 benchmark JSON") + help="Path to a supported standard schema-3 M5 benchmark JSON") parser.add_argument( "--metric", default="average", @@ -76,22 +76,24 @@ def resolve_input_path(raw_path: str) -> Path: raise RuntimeError(f"Input JSON file not found: {raw_path}") -def require_current_standard_result(data: object, source: str) -> dict: - """Require the complete standard schema emitted by the current producer.""" +def require_supported_standard_result(data: object, source: str) -> dict: + """Require the supported standard schema and methodology contract.""" configuration = data.get("configuration") if isinstance(data, dict) else None if not ( isinstance(configuration, dict) - and data.get("version") == "0.63.0" + and isinstance(data.get("version"), str) + and bool(data["version"]) and configuration.get("mode") == "benchmark" and type(configuration.get("benchmark_schema_version")) is int and configuration["benchmark_schema_version"] == 3 + and configuration.get("methodology_version") == "benchmark-v2-calibrated-seeded-balanced" and data.get("status") == "complete" and data.get("results_complete") is True and data.get("conclusions_valid") is True and isinstance(configuration.get("output_file"), str) ): raise RuntimeError( - f"{source} is not a complete current standard schema-3 benchmark result") + f"{source} is not a complete supported standard schema-3 benchmark result") return configuration @@ -112,7 +114,7 @@ def load_data(path: Path, metric: str) -> dict: with path.open("r", encoding="utf-8") as fh: data = json.load(fh) - config = require_current_standard_result(data, str(path)) + config = require_supported_standard_result(data, str(path)) cpu_name = str(config.get("cpu_name", "Unknown CPU")) version = data["version"] diff --git a/script-examples/plot_benchmark-memory-latency-hierarchy.py b/script-examples/plot_benchmark-memory-latency-hierarchy.py index 508cedc..35e391c 100644 --- a/script-examples/plot_benchmark-memory-latency-hierarchy.py +++ b/script-examples/plot_benchmark-memory-latency-hierarchy.py @@ -30,7 +30,7 @@ def parse_args(): "-f", "--file", required=True, - help="Path to a current standard schema-3 benchmark JSON or console-text statistics file", + help="Path to a supported standard schema-3 benchmark JSON or console-text statistics file", ) parser.add_argument( "--metric", @@ -75,22 +75,24 @@ def resolve_input_path(raw_path: str) -> Path: raise RuntimeError(f"Input file not found: {raw_path}") -def require_current_standard_result(data: object, source: str) -> dict: - """Require the complete standard schema emitted by the current producer.""" +def require_supported_standard_result(data: object, source: str) -> dict: + """Require the supported standard schema and methodology contract.""" configuration = data.get("configuration") if isinstance(data, dict) else None if not ( isinstance(configuration, dict) - and data.get("version") == "0.63.0" + and isinstance(data.get("version"), str) + and bool(data["version"]) and configuration.get("mode") == "benchmark" and type(configuration.get("benchmark_schema_version")) is int and configuration["benchmark_schema_version"] == 3 + and configuration.get("methodology_version") == "benchmark-v2-calibrated-seeded-balanced" and data.get("status") == "complete" and data.get("results_complete") is True and data.get("conclusions_valid") is True and isinstance(configuration.get("output_file"), str) ): raise RuntimeError( - f"{source} is not a complete current standard schema-3 benchmark result") + f"{source} is not a complete supported standard schema-3 benchmark result") return configuration @@ -98,7 +100,7 @@ def pick_stat(metric_block: dict, metric: str, label: str) -> float: try: return float(metric_block["statistics"][metric]) except (KeyError, TypeError, ValueError) as exc: - raise RuntimeError(f"Missing current '{metric}' statistic for '{label}'.") from exc + raise RuntimeError(f"Missing supported '{metric}' statistic for '{label}'.") from exc def infer_cpu_name_from_path(path: Path) -> str: @@ -199,7 +201,7 @@ def load_latency_data(path: Path, metric: str): except json.JSONDecodeError: return parse_text_statistics(path, metric) - config = require_current_standard_result(data, str(path)) + config = require_supported_standard_result(data, str(path)) cpu_name = str(config.get("cpu_name", "Unknown CPU")) version = data["version"] @@ -213,7 +215,7 @@ def load_latency_data(path: Path, metric: str): except (KeyError, TypeError) as exc: raise RuntimeError( "JSON is missing required fields for memory hierarchy plot. " - "Expected current standard schema-3 headline/locality fields." + "Expected supported standard schema-3 headline/locality fields." ) from exc categories = [ diff --git a/src/core/config/version.h b/src/core/config/version.h index a47e813..b3a1585 100644 --- a/src/core/config/version.h +++ b/src/core/config/version.h @@ -29,6 +29,6 @@ * @def SOFTVERSION * @brief Software version number (semantic versioning format as string) */ -#define SOFTVERSION "0.63.0" +#define SOFTVERSION "0.63.1" #endif // VERSION_H diff --git a/src/output/console/messages/program_messages.cpp b/src/output/console/messages/program_messages.cpp index 8a6c371..babca4c 100644 --- a/src/output/console/messages/program_messages.cpp +++ b/src/output/console/messages/program_messages.cpp @@ -141,7 +141,6 @@ std::string usage_header(const std::string& version) { std::string usage_options(const std::string& prog_name) { std::ostringstream oss; oss << "Usage: " << prog_name << " [options]\n" - << "Long options require --; single dash is only for one-character aliases.\n" << "Platform: macOS 26 or later on Apple Silicon (ARM64).\n" << "Options:\n" << " -B, --benchmark Run calibrated, seeded, balanced standard bandwidth/latency benchmark.\n" diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index ad28156..5a277c4 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -1,21 +1,20 @@ -# Current script-example fixtures +# Script-example fixtures -## Current schema-3 producer captures +## Supported schema-3 fixtures -`standard-schema-v3-complete-current.json` and `standard-schema-v3-custom-complete-current.json` are exact captures from -the repository's real 0.63.0 standard producer in the Phase 0 working tree based on source HEAD -`0fbd050312c29c40859ffd89c7fb61bcee5d3b9e`. The commands minimize volatile populations at capture time to one loop and -one latency sample while preserving the producer's configuration, work metadata, completion fields, metric blocks, and -raw output target: +`standard-schema-v3-complete-current.json` and `standard-schema-v3-custom-complete-current.json` are captured complete +results for the bundled standard-memory script examples. They cover the supported schema and methodology, +configuration, work metadata, completion fields, metric blocks, and raw output targets consumed by those examples. + +To refresh the fixtures, build `memory_benchmark` and run: ```bash ./memory_benchmark --benchmark --iterations 1 --buffer-size 1 --count 1 --latency-samples 1 --output /tmp/standard-schema-v3-complete-current.capture.json ./memory_benchmark --benchmark --only-latency --buffer-size 0 --cache-size 16 --count 1 --latency-samples 1 --latency-tlb-locality-kb 16 --latency-stride-bytes 256 --output /tmp/standard-schema-v3-custom-complete-current.capture.json ``` -The captures have SHA-256 digests -`81a35dbf70c60c03d72039e9bd0c6b00d71d38f41180b6c0e78e1b293c5019f6` and -`06afb3b7b35c642b039446fef6acaa4e2ffd4fb4acf11aa4042fd567ace07a94`, respectively. The full run supplies the -bandwidth, L1/L2 headline latency, and automatic-locality paths used by both plotters. The custom-cache run supplies the -pooled sample distribution and work metadata used by the shell examples. The fixtures intentionally cover only the -current producer shape and are refreshed together with the examples when that shape changes. +Replace the corresponding fixture files with the captured outputs. The full run supplies the bandwidth, L1/L2 headline +latency, and automatic-locality paths used by both plotters. The custom-cache run supplies the pooled sample distribution +and work metadata used by the shell examples. Refresh the fixtures, examples, and entry-path assertions together when +the consumed schema, methodology, or result shape changes. A `SOFTVERSION`-only change does not require a fixture +refresh. Run `make test-script-examples` after updating them. diff --git a/tests/test_config.cpp b/tests/test_config.cpp index 55ab129..aa5462b 100644 --- a/tests/test_config.cpp +++ b/tests/test_config.cpp @@ -956,17 +956,10 @@ TEST(ConfigTest, ParseMissingValue) { EXPECT_EQ(result, EXIT_FAILURE); } -TEST(ConfigTest, RejectsUnknownAndLegacySingleDashLongOptions) { - const std::vector> cases = { - {"program", "-unknown", "value"}, - {"program", "-buffersize", "1024"}, - }; - - for (const std::vector& arguments : cases) { - SCOPED_TRACE(arguments[1]); - BenchmarkConfig config; - EXPECT_EQ(parse_capturing_stderr(arguments, config).result, EXIT_FAILURE); - } +TEST(ConfigTest, RejectsUnknownSingleDashLongOption) { + BenchmarkConfig config; + EXPECT_EQ(parse_capturing_stderr({"program", "-unknown", "value"}, config).result, + EXIT_FAILURE); } // Test parsing help flag diff --git a/tests/test_executable_cli.cpp b/tests/test_executable_cli.cpp index a25e675..ffe98f7 100644 --- a/tests/test_executable_cli.cpp +++ b/tests/test_executable_cli.cpp @@ -1111,8 +1111,6 @@ TEST(ExecutableCliIntegrationTest, EXPECT_TRUE(measurement["synthetic_memory_work_units_per_second"] .is_number()); EXPECT_TRUE(measurement["effective_model_payload_gb_s"].is_number()); - EXPECT_FALSE(measurement.contains("synthetic_step_latency_seconds")); - EXPECT_FALSE(measurement.contains("effective_payload_gb_s")); } expect_complete_llm_checkpoint_lifecycle(json); @@ -2378,9 +2376,6 @@ TEST(ExecutableCliIntegrationTest, PatternModeRunsPatternOrchestrationIntegratio EXPECT_NE(result.output.find("Running Pattern Benchmarks"), std::string::npos); EXPECT_NE(result.output.find("Sequential Forward:"), std::string::npos); EXPECT_EQ(result.output.find('\r'), std::string::npos); - EXPECT_EQ(result.output.find("Pattern Efficiency Analysis:"), std::string::npos); - EXPECT_EQ(result.output.find("Prefetcher effectiveness"), std::string::npos); - EXPECT_EQ(result.output.find("TLB pressure"), std::string::npos); EXPECT_NE(result.output.find("2 MiB stride"), std::string::npos); } diff --git a/tests/test_json_schema.cpp b/tests/test_json_schema.cpp index da507a8..8abc40b 100644 --- a/tests/test_json_schema.cpp +++ b/tests/test_json_schema.cpp @@ -238,7 +238,6 @@ TEST(JsonSchemaTest, BenchmarkSchemaV3IncludesCompletionAndNullableMeasurements) EXPECT_TRUE(measurements["main_write_bandwidth"]["value"].is_null()); EXPECT_EQ(output["main_memory"]["bandwidth"]["read_gb_s"]["value"], 12.5); EXPECT_TRUE(output["main_memory"]["bandwidth"]["write_gb_s"]["value"].is_null()); - EXPECT_FALSE(output.dump().find("page_walk_penalty_ns") != std::string::npos); } TEST(JsonSchemaTest, BenchmarkSchemaV3SerializesExactCompletionContract) { @@ -751,8 +750,6 @@ TEST(JsonSchemaTest, TlbAnalysisExporterIncludesModeAndCoreCounts) { "chain_layout")); EXPECT_TRUE(output_json[JsonKeys::CONFIGURATION]["main_thread_qos"]["requested"]); EXPECT_TRUE(output_json[JsonKeys::CONFIGURATION]["main_thread_qos"]["applied"]); - EXPECT_FALSE(output_json[JsonKeys::CONFIGURATION].contains( - "schema_compatibility")); EXPECT_FALSE(output_json[JsonKeys::CONFIGURATION].contains( JsonKeys::LATENCY_SAMPLE_COUNT)); EXPECT_FALSE(output_json[JsonKeys::CONFIGURATION].contains( @@ -838,10 +835,6 @@ TEST(JsonSchemaTest, TlbAnalysisExporterIncludesModeAndCoreCounts) { EXPECT_EQ(output_json["tlb_analysis"]["measurement_counter_scope"] ["total"], "all serialized measurement passes"); - EXPECT_FALSE(output_json["tlb_analysis"].contains( - "completed_measurement_pairs")); - EXPECT_FALSE(output_json["tlb_analysis"].contains( - "completed_raw_measurements")); EXPECT_TRUE(output_json["tlb_analysis"]["conclusions_valid"]); EXPECT_EQ(output_json["tlb_analysis"]["measurement_records"].size(), 7u); EXPECT_EQ(output_json["tlb_analysis"]["measurement_records"][0]["pass"], "base"); @@ -854,8 +847,6 @@ TEST(JsonSchemaTest, TlbAnalysisExporterIncludesModeAndCoreCounts) { EXPECT_EQ(output_json["tlb_analysis"]["measurement_records"][0] ["paired_control"]["packed"]["seed"], "18446744073709551615"); - EXPECT_FALSE(output_json["tlb_analysis"]["measurement_records"][0] - .contains("latency_ns")); EXPECT_EQ(output_json["tlb_analysis"]["measurement_records"][2]["pass"], "validation"); EXPECT_TRUE(output_json["tlb_analysis"]["measurement_records"][0] @@ -870,10 +861,6 @@ TEST(JsonSchemaTest, TlbAnalysisExporterIncludesModeAndCoreCounts) { EXPECT_TRUE(output_json["tlb_analysis"]["pass_summaries"][0]["converged"]); EXPECT_EQ(output_json["tlb_analysis"]["sweep"][0]["requested_pages"], 1); EXPECT_EQ(output_json["tlb_analysis"]["sweep"][0]["actual_pages"], 1); - EXPECT_FALSE(output_json["tlb_analysis"]["sweep"][0].contains( - "pointer_count")); - EXPECT_FALSE(output_json["tlb_analysis"]["sweep"][0].contains( - "actual_node_count")); EXPECT_EQ(output_json["tlb_analysis"]["sweep"][0]["pointer_nodes"], 1); EXPECT_EQ(output_json["tlb_analysis"]["sweep"][0] ["spread_pointers_per_page_max"], @@ -887,10 +874,6 @@ TEST(JsonSchemaTest, TlbAnalysisExporterIncludesModeAndCoreCounts) { Constants::CACHE_LINE_SIZE_BYTES); EXPECT_TRUE(output_json["tlb_analysis"]["sweep"][0] ["short_cycle_diagnostic"]); - EXPECT_FALSE(output_json["tlb_analysis"]["sweep"][0].contains( - "p50_latency_ns")); - EXPECT_FALSE(output_json["tlb_analysis"]["sweep"][0].contains( - "loop_latencies_ns")); EXPECT_DOUBLE_EQ(output_json["tlb_analysis"]["sweep"][0] ["translation_delta_p50_ns"], 5.0); @@ -902,13 +885,8 @@ TEST(JsonSchemaTest, TlbAnalysisExporterIncludesModeAndCoreCounts) { output_json["tlb_analysis"]["sweep"][0]["spread_chain"]; EXPECT_EQ(spread_chain["pointer_nodes"], 1); EXPECT_EQ(spread_chain["pointers_per_page_max"], 1); - EXPECT_FALSE(spread_chain.contains("node_count")); - EXPECT_FALSE(spread_chain.contains("max_nodes_per_page")); ASSERT_TRUE(output_json["tlb_analysis"].contains( "large_locality_paired_comparison")); - EXPECT_FALSE(output_json["tlb_analysis"].contains( - "large_locality_latency_delta")); - EXPECT_FALSE(output_json["tlb_analysis"].contains("page_walk_penalty")); const nlohmann::json paired_large = output_json["tlb_analysis"]["large_locality_paired_comparison"]; EXPECT_TRUE(paired_large["available"]); @@ -924,7 +902,6 @@ TEST(JsonSchemaTest, TlbAnalysisExporterIncludesModeAndCoreCounts) { EXPECT_EQ(paired_large["active_cache_line_footprint_bytes"], 2 * Constants::BYTES_PER_MB); EXPECT_EQ(paired_large["pointer_nodes"], 32768); - EXPECT_FALSE(paired_large.contains("node_count")); EXPECT_EQ(paired_large["measurements"].size(), 3u); EXPECT_NE(paired_large["interpretation"].get().find( "not DRAM latency"), @@ -1044,9 +1021,6 @@ TEST(JsonSchemaTest, TlbAnalysisBuilderCoversStatusesAndExporterHandlesUnavailab .get() .find("suppressed"), std::string::npos); - EXPECT_FALSE(output_json["tlb_analysis"].contains("page_walk_penalty")); - EXPECT_FALSE(output_json["tlb_analysis"].contains( - "large_locality_latency_delta")); EXPECT_FALSE(paired_large["available"]); EXPECT_FALSE(paired_large.contains("translation_delta_p50_ns")); EXPECT_NE(paired_large["reason"].get().find("incomplete"), diff --git a/tests/test_messages.cpp b/tests/test_messages.cpp index d402574..9d1f120 100644 --- a/tests/test_messages.cpp +++ b/tests/test_messages.cpp @@ -38,17 +38,6 @@ void expect_exact_messages(const std::vector& cases) { } } -void expect_capability_based_llm_metal_status(const std::string& usage) { - constexpr const char* forbidden_phrases[] = { - "Apple7/M1", "M4 evidence", "M5", "validation remains pending", - "baseline smoke", "cross-family", "device matrix", "experimental", - "preview"}; - for (const char* phrase : forbidden_phrases) { - SCOPED_TRACE(phrase); - EXPECT_EQ(usage.find(phrase), std::string::npos); - } -} - } // namespace TEST(MessagesFormattingTest, LinearHelpersHaveExactOutput) { @@ -503,9 +492,6 @@ TEST(MessagesTest, LlmMemoryCliMessagesHaveExactOutput) { "(4096 bytes); the result may be cache-dominant"}, }; expect_exact_messages(cases); - const std::string usage = - Messages::llm_memory_usage_options("memory_benchmark"); - expect_capability_based_llm_metal_status(usage); } TEST(MessagesTest, GeneralHelpAdvertisesTheLlmBoundaryExactlyOnce) { @@ -526,7 +512,6 @@ TEST(MessagesTest, GeneralHelpAdvertisesTheLlmBoundaryExactlyOnce) { std::string::npos); EXPECT_NE(usage.find("and never falls back to CPU"), std::string::npos); - expect_capability_based_llm_metal_status(usage); EXPECT_NE(usage.find("--weight-size-mb"), std::string::npos); EXPECT_NE(usage.find("--query-heads"), std::string::npos); EXPECT_NE(usage.find("--context-tokens"), std::string::npos); @@ -550,8 +535,6 @@ TEST(MessagesTest, GeneralHelpAdvertisesTheLlmBoundaryExactlyOnce) { std::string::npos); EXPECT_NE(usage.find("checkpoints each terminal scenario"), std::string::npos); - EXPECT_EQ(usage.find("execution is unavailable"), std::string::npos); - EXPECT_EQ(usage.find("future LLM"), std::string::npos); } TEST(MessagesErrorTest, GpuMessagesHaveExactMethodologyOutput) { @@ -862,11 +845,9 @@ TEST(MessagesFormattingTest, ReportTlbResourceSummaryRetainsFailures) { EXPECT_NE(not_requested.find("QoS not requested"), std::string::npos); } -TEST(MessagesFormattingTest, ReportTlbWorkEstimateIsConcise) { +TEST(MessagesFormattingTest, ReportTlbWorkEstimateAndCompletionHaveExpectedContent) { const std::string work = Messages::report_tlb_work_estimate("base", 15, 10, 20, 3.75, 7.5); EXPECT_NE(work.find("Work Estimate [base]"), std::string::npos); - EXPECT_EQ(work.find("pointer accesses"), std::string::npos); - EXPECT_EQ(work.find("peak"), std::string::npos); EXPECT_NE(work.find("3.75-7.50 s"), std::string::npos); const std::string completion = Messages::report_tlb_pass_completion("base", 12, "CI target reached"); diff --git a/tests/test_pattern_benchmark.cpp b/tests/test_pattern_benchmark.cpp index 1162499..6aacfa8 100644 --- a/tests/test_pattern_benchmark.cpp +++ b/tests/test_pattern_benchmark.cpp @@ -1080,7 +1080,6 @@ TEST(PatternBenchmarkTest, ConsoleRendersUnavailableMeasurementsAsStatusNotZero) const std::string output = testing::internal::GetCapturedStdout(); EXPECT_NE(output.find("N/A [skipped: test skip]"), std::string::npos); EXPECT_EQ(output.find("0.000 GB/s"), std::string::npos); - EXPECT_EQ(output.find("Pattern Efficiency Analysis"), std::string::npos); EXPECT_NE(output.find("2 MiB stride"), std::string::npos); } diff --git a/tests/test_script_examples.py b/tests/test_script_examples.py index e49c874..bb61650 100644 --- a/tests/test_script_examples.py +++ b/tests/test_script_examples.py @@ -2,7 +2,7 @@ # Copyright 2026 Timo Heimonen # SPDX-License-Identifier: GPL-3.0-or-later -"""Entry-path tests for the bundled current-version script examples.""" +"""Entry-path tests for the bundled standard-memory script examples.""" import csv import importlib.util @@ -51,7 +51,7 @@ def load_plotter(path: Path, module_name: str): class PlotterEntryPathTest(unittest.TestCase): - def test_comparison_loader_accepts_current_full_result(self): + def test_comparison_loader_accepts_supported_full_result(self): with tempfile.TemporaryDirectory(prefix="script-example-plotter-") as temporary: _, script = copy_script(Path(temporary), "plot_M4vsM5_benchmark_comparison.py") plotter = load_plotter(script, "current_comparison_plotter") @@ -63,7 +63,7 @@ def test_comparison_loader_accepts_current_full_result(self): self.assertAlmostEqual(result["l1_lat"], 1.1562554254729616) self.assertAlmostEqual(result["global_random"], 5.622018794239533) - def test_hierarchy_loader_accepts_current_full_result(self): + def test_hierarchy_loader_accepts_supported_full_result(self): with tempfile.TemporaryDirectory(prefix="script-example-plotter-") as temporary: _, script = copy_script(Path(temporary), "plot_benchmark-memory-latency-hierarchy.py") plotter = load_plotter(script, "current_hierarchy_plotter") @@ -76,7 +76,7 @@ def test_hierarchy_loader_accepts_current_full_result(self): self.assertAlmostEqual(locality_delta, 0.6151600647670126) self.assertEqual(version, "0.63.0") - def test_hierarchy_loader_accepts_current_console_statistics(self): + def test_hierarchy_loader_accepts_supported_console_statistics(self): statistics = """\ L1 Cache: Average: 1.25 @@ -103,35 +103,48 @@ def test_hierarchy_loader_accepts_current_console_statistics(self): self.assertEqual(locality_delta, 25.25) self.assertIsNone(version) - def test_plotter_loaders_reject_noncurrent_results(self): - current = json.loads(CURRENT_FULL_FIXTURE.read_text(encoding="utf-8")) - wrong_schema = json.loads(json.dumps(current)) + def test_plotter_loaders_enforce_supported_contract(self): + supported = json.loads(CURRENT_FULL_FIXTURE.read_text(encoding="utf-8")) + other_version = json.loads(json.dumps(supported)) + other_version["version"] = "0.64.0" + wrong_schema = json.loads(json.dumps(supported)) wrong_schema["configuration"]["benchmark_schema_version"] = 2 - wrong_version = json.loads(json.dumps(current)) - wrong_version["version"] = "0.61.0" - - with tempfile.TemporaryDirectory(prefix="script-example-wrong-schema-") as temporary: + wrong_methodology = json.loads(json.dumps(supported)) + wrong_methodology["configuration"]["methodology_version"] = "benchmark-v1" + wrong_shape = json.loads(json.dumps(supported)) + wrong_shape["configuration"]["output_file"] = None + missing_provenance = json.loads(json.dumps(supported)) + missing_provenance["version"] = "" + + with tempfile.TemporaryDirectory(prefix="script-example-contract-") as temporary: root = Path(temporary) - inputs = { + accepted_path = root / "other-version.json" + accepted_path.write_text(json.dumps(other_version), encoding="utf-8") + incompatible_inputs = { "schema 2": wrong_schema, - "version 0.61.0": wrong_version, + "methodology benchmark-v1": wrong_methodology, + "non-string output_file": wrong_shape, + "empty provenance version": missing_provenance, } loaders = ( - ("plot_M4vsM5_benchmark_comparison.py", "wrong_schema_comparison", "load_data"), + ("plot_M4vsM5_benchmark_comparison.py", "contract_comparison", "load_data"), ( "plot_benchmark-memory-latency-hierarchy.py", - "wrong_schema_hierarchy", + "contract_hierarchy", "load_latency_data", ), ) for filename, module_name, loader_name in loaders: _, script = copy_script(root / module_name, filename) plotter = load_plotter(script, module_name) - for name, data in inputs.items(): + accepted = getattr(plotter, loader_name)(accepted_path, "average") + accepted_version = accepted["version"] if isinstance(accepted, dict) else accepted[-1] + self.assertEqual(accepted_version, "0.64.0") + for name, data in incompatible_inputs.items(): input_path = root / f"{module_name}-{name.replace(' ', '-')}.json" input_path.write_text(json.dumps(data), encoding="utf-8") with self.subTest(plotter=filename, input=name): - with self.assertRaisesRegex(RuntimeError, "complete current standard schema-3"): + with self.assertRaisesRegex(RuntimeError, "complete supported standard schema-3"): getattr(plotter, loader_name)(input_path, "average") @@ -210,7 +223,7 @@ def assert_success(self, script_name: str, completed: subprocess.CompletedProces ) @unittest.skipUnless(shutil.which("jq"), "jq is optional and is not installed") - def test_latency_workflow_accepts_current_result_with_jq(self): + def test_latency_workflow_accepts_supported_result_with_jq(self): with tempfile.TemporaryDirectory(prefix="script-example-latency-jq-") as temporary: script_dir, completed = self.run_workflow( Path(temporary), "latency_test_script.sh", CURRENT_CUSTOM_FIXTURE @@ -222,7 +235,7 @@ def test_latency_workflow_accepts_current_result_with_jq(self): self.assertEqual(final_output.count("TLB Locality:"), 120) self.assertIn('"average": 0.6893617567937856', final_output) - def test_latency_workflow_accepts_current_result_with_python(self): + def test_latency_workflow_accepts_supported_result_with_python(self): with tempfile.TemporaryDirectory(prefix="script-example-latency-python-") as temporary: script_dir, completed = self.run_workflow( Path(temporary), @@ -237,7 +250,7 @@ def test_latency_workflow_accepts_current_result_with_python(self): self.assertEqual(final_output.count("TLB Locality:"), 120) self.assertIn('"average": 0.6893617567937856', final_output) - def test_stride_tlb_workflow_accepts_current_result(self): + def test_stride_tlb_workflow_accepts_supported_result(self): with tempfile.TemporaryDirectory(prefix="script-example-stride-tlb-") as temporary: script_dir, completed = self.run_workflow( Path(temporary), "latency_test_script_stride_tlb.sh", CURRENT_CUSTOM_FIXTURE @@ -255,19 +268,19 @@ def test_stride_tlb_workflow_accepts_current_result(self): self.assertEqual(rows[0]["average"], "0.6893617567937856") self.assertEqual(rows[0]["chain_node_count"], "64") - def test_shell_workflows_do_not_extract_metrics_from_noncurrent_results(self): - current = json.loads(CURRENT_CUSTOM_FIXTURE.read_text(encoding="utf-8")) - wrong_schema_data = json.loads(json.dumps(current)) + def test_shell_workflows_reject_incompatible_contracts(self): + supported = json.loads(CURRENT_CUSTOM_FIXTURE.read_text(encoding="utf-8")) + wrong_schema_data = json.loads(json.dumps(supported)) wrong_schema_data["configuration"]["benchmark_schema_version"] = 2 - wrong_version_data = json.loads(json.dumps(current)) - wrong_version_data["version"] = "0.61.0" + wrong_methodology_data = json.loads(json.dumps(supported)) + wrong_methodology_data["configuration"]["methodology_version"] = "benchmark-v1" - with tempfile.TemporaryDirectory(prefix="script-example-shell-wrong-schema-") as temporary: + with tempfile.TemporaryDirectory(prefix="script-example-shell-contract-") as temporary: root = Path(temporary) wrong_schema = root / "wrong-schema.json" wrong_schema.write_text(json.dumps(wrong_schema_data), encoding="utf-8") - wrong_version = root / "wrong-version.json" - wrong_version.write_text(json.dumps(wrong_version_data), encoding="utf-8") + wrong_methodology = root / "wrong-methodology.json" + wrong_methodology.write_text(json.dumps(wrong_methodology_data), encoding="utf-8") latency_dir, latency = self.run_workflow( root / "latency", @@ -278,7 +291,7 @@ def test_shell_workflows_do_not_extract_metrics_from_noncurrent_results(self): final_output = (latency_dir / "final_output.txt").read_text(encoding="utf-8") stride_dir, stride = self.run_workflow( - root / "stride", "latency_test_script_stride_tlb.sh", wrong_version + root / "stride", "latency_test_script_stride_tlb.sh", wrong_methodology ) summaries = { path.resolve()