From 6ef6af08d74726ebe5a37fac3ff675bbf04cb0bc Mon Sep 17 00:00:00 2001 From: timoheimonen Date: Sat, 5 Sep 2026 12:19:35 +0300 Subject: [PATCH 01/16] Validate CPU decode append state after timed execution --- documents/API.md | 21 ++++ documents/LLM_MEMORY_PROFILE_WHITEPAPER.md | 9 ++ documents/MANUAL.md | 9 ++ src/llm_memory/llm_cpu_backend.cpp | 13 +- src/llm_memory/llm_executor.cpp | 51 +++++++- src/llm_memory/llm_executor.h | 4 + src/llm_memory/llm_json.cpp | 8 ++ src/llm_memory/llm_runner.cpp | 4 + tests/test_llm_memory_executor.cpp | 134 +++++++++++++++++++++ tests/test_llm_memory_json.cpp | 72 ++++++++++- tests/test_llm_memory_output.cpp | 6 +- tests/test_llm_memory_runner.cpp | 69 +++++++++++ 12 files changed, 391 insertions(+), 9 deletions(-) diff --git a/documents/API.md b/documents/API.md index 8676622..989e790 100644 --- a/documents/API.md +++ b/documents/API.md @@ -334,6 +334,27 @@ equal-multiplicity table permutations are therefore distinguishable. These four-byte lookups and physical suffix padding are reported evidence, not effective model payload. + +CPU final-state validation evidence: + +CPU contiguous-decode checks every byte of both K and V append records for every layer and batch +against the cold affine oracle at final task-local work-unit ordinal `T - 1`, after the last worker +stops the timer and all workers join. Failure is `decode-post-validation-failed`, retained as an +invalid attempt with null rates and excluded from aggregate populations. The timed payload and +checksum algorithm are unchanged. A matching runtime checksum does not replace this final-state check. + +In LLM schema 1, CPU `measurements[].execution.post_validation_evaluated` retains its existing +meaning: the phase/layout final-state validation pipeline was reached, including a no-op path when +no write check applies. `post_validation_valid` is that combined pipeline result. Additive CPU fields +`kv_write_validation_applicable`, `kv_write_validation_evaluated`, and `kv_write_validation_valid` +distinguish a KV-writing scenario from `weights_only`. When applicability is false, evaluated and +valid serialize as null. When applicable but unevaluated, evaluated is false and valid is null; +acceptance requires evaluated and valid to be true. All three are null without available CPU evidence. +For paged and prefill profiles, the valid field reports the existing combined phase/layout final-state +check, including any applicable padding check; it is not an independent append-only verdict. Prefill +retains its documented sampling limits. `weights_only` does not claim KV-write validation; the existing +CPU paged unexpected-KV-write and padding protections still run in the combined pipeline. + For prefill, let `P` be prompt length, `Q` query-tile length, and `K = L*2*R`. With `C = ceil(P/Q)`, tile ends `e_j = min((j+1)*Q, P)`, and `S(P,Q) = sum(e_j)`, one `prefill_operation` has: diff --git a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md index eb326f5..17816f5 100644 --- a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md +++ b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md @@ -692,6 +692,15 @@ The independent oracle does not call the assembly helper. Excluded post-validati first/middle/last byte samples, including samples at canonical word and owner boundaries, against final operation ordinal `T-1`; it does not reread every prompt record. +CPU contiguous-decode checks every byte of both K/V append records for all layers and batches +against the final task-local ordinal `T-1`, after timer stop and all worker joins. A mismatch yields +`decode-post-validation-failed`: the invalid attempt is retained with null rates and excluded from +aggregates. The timed workload and checksum algorithm are unchanged. KV-write evidence applies only +to KV-bearing scenarios; paged/prefill evidence retains the combined final-state check and its existing +padding/sampling limits. CPU paged weights-only unexpected-write/padding protection remains active. +See the [CPU final-state evidence contract](API.md#result-schemas-and-completion) for field and null semantics. + + Only exact checksum agreement, successful phase/layout-specific post-validation (decode current-token append and paged padding canaries, or prefill final-ordinal representative/boundary samples), complete worker lifecycle, successful kernel status, and a finite positive elapsed time permit `measured` status. A mismatch is terminal invalid evidence and diff --git a/documents/MANUAL.md b/documents/MANUAL.md index 5987193..ff1b0aa 100644 --- a/documents/MANUAL.md +++ b/documents/MANUAL.md @@ -270,6 +270,15 @@ depends on pool, physical ID, and physical offset. Post-task validation checks current-token writes and padding canaries without adding to elapsed time. +CPU contiguous-decode checks every byte of both K/V append records for all layers and batches +against the final task-local ordinal `T-1`, after timer stop and all worker joins. A mismatch yields +`decode-post-validation-failed`: the invalid attempt is retained with null rates and excluded from +aggregates. The timed workload and checksum algorithm are unchanged. KV-write evidence applies only +to KV-bearing scenarios; paged/prefill evidence retains the combined final-state check and its existing +padding/sampling limits. CPU paged weights-only unexpected-write/padding protection remains active. +See the [CPU final-state evidence contract](API.md#result-schemas-and-completion) for field and null semantics. + + For prefill, prompt length `P` and query-tile length `Q` are explicit. Define `C = ceil(P/Q)`, tile ends `e_j = min((j+1)*Q, P)`, and `S(P,Q) = sum(e_j)`. One full-prompt work unit has: diff --git a/src/llm_memory/llm_cpu_backend.cpp b/src/llm_memory/llm_cpu_backend.cpp index 9de5f87..57f113c 100644 --- a/src/llm_memory/llm_cpu_backend.cpp +++ b/src/llm_memory/llm_cpu_backend.cpp @@ -183,9 +183,13 @@ LlmTaskExecutionResult adapt_llm_cpu_executor_result(const LlmMemoryWorkPlan& mo const LlmCpuExecutionPlan* cpu_plan = get_llm_cpu_execution_plan(model_plan); const size_t effective_workers = cpu_plan == nullptr ? 0 : cpu_plan->effective_workers; const bool lifecycle_complete = cpu_lifecycle_complete(executor_result, effective_workers); + const bool write_required = scenario_plan.scenario != LlmScenario::WeightsOnly; + const bool write_evidence_complete = + executor_result.kv_write_validation_applicable == write_required && + (!write_required || executor_result.kv_write_validation_evaluated); const bool checksum_evidence_complete = lifecycle_complete && executor_result.checksum_evaluated && - executor_result.post_validation_evaluated; + executor_result.post_validation_evaluated && write_evidence_complete; result.timing.evaluated = executor_result.timer_started && executor_result.timer_stopped; result.timing.elapsed_seconds = executor_result.elapsed_seconds; result.timing.valid = result.timing.evaluated && std::isfinite(executor_result.elapsed_seconds) && @@ -193,7 +197,8 @@ LlmTaskExecutionResult adapt_llm_cpu_executor_result(const LlmMemoryWorkPlan& mo result.validation.evaluated = checksum_evidence_complete; result.validation.valid = checksum_evidence_complete && executor_result.checksum_valid && - executor_result.post_validation_valid; + executor_result.post_validation_valid && + (!write_required || executor_result.kv_write_validation_valid); const std::string_view original_reason = executor_result.reason_code; const bool accepted = executor_result.valid && original_reason == LlmExecutorReason::VALID && lifecycle_complete && @@ -207,13 +212,15 @@ LlmTaskExecutionResult adapt_llm_cpu_executor_result(const LlmMemoryWorkPlan& mo result.completion.completed_layout_metadata_read_bytes = scenario_plan.layout_metadata_read_bytes; result.completion.completed_task_accounted_bytes = scenario_plan.task_accounted_bytes; } else if (original_reason == LlmExecutorReason::CHECKSUM_MISMATCH || + original_reason == LlmExecutorReason::DECODE_POST_VALIDATION_FAILED || original_reason == LlmExecutorReason::PAGED_POST_VALIDATION_FAILED || original_reason == LlmExecutorReason::PREFILL_POST_VALIDATION_FAILED || (lifecycle_complete && result.validation.evaluated && !result.validation.valid)) { result.status = LlmTaskExecutionStatus::Invalid; - result.reason_code = original_reason == + result.reason_code = original_reason == LlmExecutorReason::DECODE_POST_VALIDATION_FAILED || + original_reason == LlmExecutorReason::PAGED_POST_VALIDATION_FAILED || original_reason == LlmExecutorReason::PREFILL_POST_VALIDATION_FAILED diff --git a/src/llm_memory/llm_executor.cpp b/src/llm_memory/llm_executor.cpp index ea78e3d..7e5af8c 100644 --- a/src/llm_memory/llm_executor.cpp +++ b/src/llm_memory/llm_executor.cpp @@ -3013,6 +3013,41 @@ LlmExpectedChecksumResult calculate_paged_expected_checksums( return result; } +/** + * Validate every contiguous decode append byte against the cold affine oracle. + * Called only after descriptor admission, timer stop, and worker joins; mapping + * ownership remains live and no worker can modify the checked records. Admission + * proves nonzero visible-token/record counts and checked layer/batch/sequence + * products, with each complete append record inside both K and V mappings. + */ +bool validate_decode_post_execution(const LlmMemoryWorkPlan& model_plan, + const LlmScenarioWorkPlan& scenario_plan, + const LlmExecutionResources& resources) noexcept { + const auto& geometry = model_plan.geometry; + if (!geometry.decode.has_value() || scenario_plan.work_units == 0 || !resources.buffers.complete()) { + return false; + } + const auto* k = static_cast(resources.buffers.k.get()); + const auto* v = static_cast(resources.buffers.v.get()); + const size_t record_bytes = geometry.k_or_v_record_bytes_per_layer; + for (size_t layer = 0; layer < geometry.layer_count; ++layer) { + for (size_t batch = 0; batch < geometry.batch_size; ++batch) { + const size_t row = layer * geometry.batch_size + batch; + const size_t offset = row * geometry.k_or_v_sequence_visible_bytes + + (geometry.decode->visible_context_tokens - 1) * record_bytes; + for (size_t byte = 0; byte < record_bytes; ++byte) { + if (k[offset + byte] != append_byte(scenario_plan.scenario_seed, scenario_plan.work_units - 1, + layer, batch, byte, LlmChecksumComponent::K) || + v[offset + byte] != append_byte(scenario_plan.scenario_seed, scenario_plan.work_units - 1, + layer, batch, byte, LlmChecksumComponent::V)) { + return false; + } + } + } + } + return true; +} + bool validate_paged_post_execution( const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan& scenario_plan, @@ -4067,6 +4102,8 @@ LlmExecutorResult execute_llm_scenario(const LlmMemoryWorkPlan& model_plan, cons const LlmExecutionResources& resources, HighResTimer& timer, LlmKernelAdapter kernel, const LlmExecutorTestControl* test_control) noexcept { LlmExecutorResult result; + result.kv_write_validation_applicable = + (llm_scenario_flags(scenario_plan.scenario) & kLlmScenarioFlagKv) != 0; const LlmCpuExecutionPlan* const cpu_plan = get_llm_cpu_execution_plan(model_plan); result.requested_workers = @@ -4300,9 +4337,13 @@ LlmExecutorResult execute_llm_scenario(const LlmMemoryWorkPlan& model_plan, cons : model_plan.phase == LlmPhase::Prefill ? validate_prefill_post_execution(model_plan, scenario_plan, resources) - : model_plan.kv_layout != LlmKvLayout::Paged || - validate_paged_post_execution(model_plan, scenario_plan, - resources); + : model_plan.kv_layout == LlmKvLayout::Paged + ? validate_paged_post_execution(model_plan, scenario_plan, resources) + : !result.kv_write_validation_applicable || + validate_decode_post_execution(model_plan, scenario_plan, resources); + result.kv_write_validation_evaluated = result.kv_write_validation_applicable; + result.kv_write_validation_valid = + result.kv_write_validation_evaluated && result.post_validation_valid; if (!result.checksum_valid) { result.reason_code = LlmExecutorReason::CHECKSUM_MISMATCH; return result; @@ -4311,7 +4352,9 @@ LlmExecutorResult execute_llm_scenario(const LlmMemoryWorkPlan& model_plan, cons result.reason_code = model_plan.phase == LlmPhase::Prefill ? LlmExecutorReason::PREFILL_POST_VALIDATION_FAILED - : LlmExecutorReason::PAGED_POST_VALIDATION_FAILED; + : model_plan.kv_layout == LlmKvLayout::Paged + ? LlmExecutorReason::PAGED_POST_VALIDATION_FAILED + : LlmExecutorReason::DECODE_POST_VALIDATION_FAILED; return result; } diff --git a/src/llm_memory/llm_executor.h b/src/llm_memory/llm_executor.h index 7f6c6fd..2dbfc4d 100644 --- a/src/llm_memory/llm_executor.h +++ b/src/llm_memory/llm_executor.h @@ -45,6 +45,7 @@ inline constexpr const char* OUTPUT_NOT_EMPTY = "output-not-empty"; inline constexpr const char* WEIGHT_MAPPING_FAILED = "weight-mapping-failed"; inline constexpr const char* K_MAPPING_FAILED = "k-mapping-failed"; inline constexpr const char* V_MAPPING_FAILED = "v-mapping-failed"; +inline constexpr const char* DECODE_POST_VALIDATION_FAILED = "decode-post-validation-failed"; inline constexpr const char* PAGED_POST_VALIDATION_FAILED = "paged-post-validation-failed"; inline constexpr const char* PREFILL_POST_VALIDATION_FAILED = @@ -338,6 +339,9 @@ struct LlmExecutorResult { bool checksum_valid = false; bool post_validation_evaluated = false; bool post_validation_valid = false; + bool kv_write_validation_applicable = false; ///< KV-writing scenario; independent of pipeline completion. + bool kv_write_validation_evaluated = false; ///< Phase/layout write check ran after stop and join. + bool kv_write_validation_valid = false; ///< Combined phase/layout final-state result, including paged padding. std::vector expected_checksums; std::vector actual_checksums; LlmRunChecksum expected_run_checksum{0, 0}; diff --git a/src/llm_memory/llm_json.cpp b/src/llm_memory/llm_json.cpp index 481c75d..d90a4e3 100644 --- a/src/llm_memory/llm_json.cpp +++ b/src/llm_memory/llm_json.cpp @@ -1620,6 +1620,14 @@ OrderedJson measurement_execution_json(const LlmMeasurementState& measurement) { cpu_available ? OrderedJson(cpu->post_validation_evaluated) : OrderedJson(nullptr); output["post_validation_valid"] = cpu_available && cpu->post_validation_evaluated ? OrderedJson(cpu->post_validation_valid) : OrderedJson(nullptr); + output["kv_write_validation_applicable"] = + cpu_available ? OrderedJson(cpu->kv_write_validation_applicable) : OrderedJson(nullptr); + output["kv_write_validation_evaluated"] = + cpu_available && cpu->kv_write_validation_applicable + ? OrderedJson(cpu->kv_write_validation_evaluated) : OrderedJson(nullptr); + output["kv_write_validation_valid"] = + cpu_available && cpu->kv_write_validation_applicable && cpu->kv_write_validation_evaluated + ? OrderedJson(cpu->kv_write_validation_valid) : OrderedJson(nullptr); if (metal != nullptr) { output["metal"] = metal_task_evidence_json(*metal); } diff --git a/src/llm_memory/llm_runner.cpp b/src/llm_memory/llm_runner.cpp index 94dc5b4..989e45c 100644 --- a/src/llm_memory/llm_runner.cpp +++ b/src/llm_memory/llm_runner.cpp @@ -62,6 +62,7 @@ constexpr std::string_view kLlmExecutorReasons[] = { LlmExecutorReason::WEIGHT_MAPPING_FAILED, LlmExecutorReason::K_MAPPING_FAILED, LlmExecutorReason::V_MAPPING_FAILED, + LlmExecutorReason::DECODE_POST_VALIDATION_FAILED, LlmExecutorReason::PAGED_POST_VALIDATION_FAILED, LlmExecutorReason::PREFILL_POST_VALIDATION_FAILED, LlmExecutorReason::DESCRIPTOR_ALLOCATION_FAILED, @@ -1441,6 +1442,9 @@ void retain_task_evidence(LlmTaskExecutionResult& retained, destination.post_validation_evaluated = input.post_validation_evaluated; destination.post_validation_valid = input.post_validation_valid; + destination.kv_write_validation_applicable = input.kv_write_validation_applicable; + destination.kv_write_validation_evaluated = input.kv_write_validation_evaluated; + destination.kv_write_validation_valid = input.kv_write_validation_valid; destination.expected_run_checksum = input.expected_run_checksum; destination.actual_run_checksum = input.actual_run_checksum; const LlmCpuExecutionPlan* cpu_plan = diff --git a/tests/test_llm_memory_executor.cpp b/tests/test_llm_memory_executor.cpp index 2ef9a49..c7ba7ad 100644 --- a/tests/test_llm_memory_executor.cpp +++ b/tests/test_llm_memory_executor.cpp @@ -480,6 +480,27 @@ bool fake_kernel(void* opaque, const LlmKernelInvocation& invocation) { if (static_cast(invocation.worker_index) == context.fail_worker) { return false; } + // Materialize the admitted decode append independently; this fake never calls ASM. + if (invocation.phase == LlmPhase::Decode && invocation.kv_layout == LlmKvLayout::Contiguous && + (invocation.scenario_flags & kLlmScenarioFlagKv) != 0) { + for (size_t layer = 0; layer < invocation.layer_count; ++layer) { + const auto& descriptor = invocation.layers[layer]; + for (size_t local = 0; local < descriptor.sequence_count; ++local) { + const auto& sequence = invocation.sequences[descriptor.first_sequence_index + local]; + for (bool is_k : {true, false}) { + auto* bytes = is_k ? sequence.k_append_ptr : sequence.v_append_ptr; + const size_t count = is_k ? sequence.k_append_bytes : sequence.v_append_bytes; + for (size_t byte = 0; byte < count; ++byte) { + const size_t canonical = sequence.append_record_byte_offset + byte; + const uint64_t word = scalar_append_word(invocation.scenario_seed, invocation.work_unit_count - 1, + descriptor.layer_index, sequence.batch_sequence_index, canonical / 8, + is_k ? LlmChecksumComponent::K : LlmChecksumComponent::V); + bytes[byte] = static_cast(word >> (8 * (canonical % 8))); + } + } + } + } + } *invocation.output = (*context.expected)[invocation.worker_index]; if (static_cast(invocation.worker_index) == context.corrupt_worker) { ++invocation.output->k.state_b; @@ -487,6 +508,36 @@ bool fake_kernel(void* opaque, const LlmKernelInvocation& invocation) { return true; } +struct DecodeCorruptionContext { + uint8_t* target = nullptr; + std::atomic changed{false}; +}; + +bool decode_corrupting_kernel(void* opaque, const LlmKernelInvocation& invocation) { + auto& context = *static_cast(opaque); + if (!production_llm_kernel_adapter().invoke(nullptr, invocation)) return false; + if (context.target == nullptr) return true; + for (size_t layer = 0; layer < invocation.layer_count; ++layer) { + const auto& descriptor = invocation.layers[layer]; + for (size_t local = 0; local < descriptor.sequence_count; ++local) { + const auto& sequence = invocation.sequences[descriptor.first_sequence_index + local]; + for (size_t byte = 0; byte < sequence.k_append_bytes; ++byte) { + if (sequence.k_append_ptr + byte == context.target) { + *context.target ^= 1; + context.changed.store(true); + } + } + for (size_t byte = 0; byte < sequence.v_append_bytes; ++byte) { + if (sequence.v_append_ptr + byte == context.target) { + *context.target ^= 1; + context.changed.store(true); + } + } + } + } + return true; +} + bool throwing_kernel(void*, const LlmKernelInvocation&) { throw std::runtime_error("injected kernel exception"); } enum class PagedCorruption { @@ -2228,6 +2279,9 @@ TEST_F(LlmMemoryExecutorTest, CpuAdapterOwnsCpuLifecycleAcceptanceAndPreservesTa execution.checksum_valid = true; execution.post_validation_evaluated = true; execution.post_validation_valid = true; + execution.kv_write_validation_applicable = true; + execution.kv_write_validation_evaluated = true; + execution.kv_write_validation_valid = true; execution.expected_checksums.resize(cpu_plan.effective_workers); execution.actual_checksums = execution.expected_checksums; execution.expected_run_checksum = {11, 22}; @@ -2264,6 +2318,19 @@ TEST_F(LlmMemoryExecutorTest, CpuAdapterOwnsCpuLifecycleAcceptanceAndPreservesTa EXPECT_EQ(retained->expected_checksums.size(), cpu_plan.effective_workers); EXPECT_EQ(retained->actual_checksums.size(), cpu_plan.effective_workers); + LlmExecutorResult missing_write = successful_cpu_evidence(); + missing_write.kv_write_validation_evaluated = false; + result = adapt_llm_cpu_executor_result(plan, scenario, context, std::move(missing_write)); + EXPECT_EQ(result.status, LlmTaskExecutionStatus::Failed); + EXPECT_FALSE(result.validation.evaluated); + EXPECT_EQ(result.completion.completed_work_units, 0u); + + LlmExecutorResult not_applicable = successful_cpu_evidence(); + not_applicable.kv_write_validation_applicable = false; + result = adapt_llm_cpu_executor_result(plan, scenario, context, std::move(not_applicable)); + EXPECT_EQ(result.status, LlmTaskExecutionStatus::Failed); + EXPECT_FALSE(result.validation.evaluated); + LlmExecutorResult malformed = successful_cpu_evidence(); malformed.actual_checksums.pop_back(); result = adapt_llm_cpu_executor_result(plan, scenario, context, @@ -2294,3 +2361,70 @@ TEST_F(LlmMemoryExecutorTest, CpuAdapterOwnsCpuLifecycleAcceptanceAndPreservesTa EXPECT_TRUE(result.timing.evaluated); EXPECT_FALSE(result.timing.valid); } + +TEST_F(LlmMemoryExecutorTest, ContiguousDecodeAppendCorruptionIntegration) { + for (size_t record_bytes : {1u, 7u, 8u, 9u, 33u}) { + const auto plan = build_executor_ready_plan({257, 2, 1, 1, record_bytes, 1, 5, 3}, 3); + ASSERT_TRUE(plan.valid) << plan.reason_code; + LlmExecutionResources resources; + ASSERT_TRUE(prepare_llm_execution_resources(plan, resources).valid); + for (auto scenario : {LlmScenario::KvOnly, LlmScenario::Mixed}) { + for (size_t work : {1u, 3u}) { + const auto task = build_llm_scenario_work_plan(plan, scenario, work, true); + ASSERT_TRUE(task.valid); + for (size_t layer = 0; layer < 2; ++layer) { + for (size_t batch = 0; batch < 3; ++batch) { + for (bool is_k : {true, false}) { + for (size_t byte : {size_t(0), record_bytes - 1}) { + SCOPED_TRACE(::testing::Message() << record_bytes << "/" << work << "/" << layer + << "/" << batch << "/" << is_k << "/" << byte); + auto* base = static_cast(is_k ? resources.buffers.k.get() : resources.buffers.v.get()); + DecodeCorruptionContext context; + context.target = base + (layer * 3 + batch) * plan.geometry.k_or_v_sequence_visible_bytes + + 4 * record_bytes + byte; + auto timer = HighResTimer::create(); + ASSERT_TRUE(timer.has_value()); + const auto result = execute_llm_scenario(plan, task, resources, *timer, + {decode_corrupting_kernel, &context}); + EXPECT_TRUE(context.changed.load()); + EXPECT_TRUE(result.checksum_valid); + EXPECT_TRUE(result.timer_stopped); + EXPECT_EQ(result.completed_workers, result.requested_workers); + EXPECT_FALSE(result.valid); + EXPECT_TRUE(result.post_validation_evaluated); + EXPECT_FALSE(result.post_validation_valid); + EXPECT_TRUE(result.kv_write_validation_applicable); + EXPECT_TRUE(result.kv_write_validation_evaluated); + EXPECT_FALSE(result.kv_write_validation_valid); + EXPECT_EQ(result.reason_code, LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); + } + } + } + } + } + } + } +} + +TEST_F(LlmMemoryExecutorTest, CpuProfilesWriteValidationControlsIntegration) { + for (const auto& geometry : {LlmGeometryRequest{257, 2, 1, 1, 33, 1, 5, 3}, + paged_geometry(5, 4), prefill_geometry(), paged_prefill_geometry()}) { + const auto plan = build_executor_ready_plan(geometry, 3); + ASSERT_TRUE(plan.valid) << plan.reason_code; + LlmExecutionResources resources; + ASSERT_TRUE(prepare_llm_execution_resources(plan, resources).valid); + for (auto scenario : {LlmScenario::WeightsOnly, LlmScenario::KvOnly, LlmScenario::Mixed}) { + for (size_t work : {1u, 3u}) { + const auto task = build_llm_scenario_work_plan(plan, scenario, work, true); + auto timer = HighResTimer::create(); + ASSERT_TRUE(timer.has_value()); + const auto result = execute_llm_scenario(plan, task, resources, *timer); + EXPECT_TRUE(result.valid) << result.reason_code; + const bool writes = scenario != LlmScenario::WeightsOnly; + EXPECT_EQ(result.kv_write_validation_applicable, writes); + EXPECT_EQ(result.kv_write_validation_evaluated, writes); + EXPECT_EQ(result.kv_write_validation_valid, writes); + } + } + } +} diff --git a/tests/test_llm_memory_json.cpp b/tests/test_llm_memory_json.cpp index d9f2e32..4fcf59d 100644 --- a/tests/test_llm_memory_json.cpp +++ b/tests/test_llm_memory_json.cpp @@ -677,6 +677,9 @@ class FakeLlmBackend final : public LlmBackend { const LlmRunnerTaskContext& context) override { LlmExecutorResult execution = executor_ ? executor_(model_plan, scenario_plan, context) : successful_execution(model_plan); + execution.kv_write_validation_applicable = scenario_plan.scenario != LlmScenario::WeightsOnly; + execution.kv_write_validation_evaluated = execution.kv_write_validation_applicable && execution.post_validation_evaluated; + execution.kv_write_validation_valid = execution.kv_write_validation_evaluated && execution.post_validation_valid; return adapt_llm_cpu_executor_result(model_plan, scenario_plan, context, std::move(execution)); } @@ -809,6 +812,9 @@ TEST(LlmMemoryJsonTest, MetalDocumentPublishesSegmentationBackendTaskAndCompactE EXPECT_TRUE(document["measurements"][0]["qos_failed_workers"].is_null()); const OrderedJson& execution = document["measurements"][0]["execution"]; EXPECT_TRUE(execution["requested_workers"].is_null()); + EXPECT_TRUE(execution["kv_write_validation_applicable"].is_null()); + EXPECT_TRUE(execution["kv_write_validation_evaluated"].is_null()); + EXPECT_TRUE(execution["kv_write_validation_valid"].is_null()); EXPECT_TRUE(execution["qos_successful_workers"].is_null()); EXPECT_DOUBLE_EQ(execution["elapsed_seconds"].get(), 0.0025); const OrderedJson& task = execution["metal"]; @@ -1774,7 +1780,8 @@ TEST(LlmMemoryJsonTest, CompleteDocumentHasExactTopLevelIdentityAndAuditableNest document["measurements"][0]["execution"], {"status", "reason_code", "valid", "elapsed_seconds", "requested_workers", "created_workers", "completed_workers", "qos_successful_workers", "qos_failed_workers", "worker_startup_failed", "kernel_succeeded", "timer_started", - "timer_stopped", "post_validation_evaluated", "post_validation_valid"}); + "timer_stopped", "post_validation_evaluated", "post_validation_valid", + "kv_write_validation_applicable", "kv_write_validation_evaluated", "kv_write_validation_valid"}); expect_exact_keys(document["measurements"][0]["checksum"], {"status", "reason_code", "initialization_pattern_version", "write_pattern_version", "checksum_pattern_version", "checksum_valid", "expected_worker_checksums", @@ -2647,6 +2654,9 @@ TEST(LlmMemoryJsonTest, InterruptedRunnerSerializesUnavailableMetricsExecutionQo EXPECT_TRUE(serialized["execution"]["valid"].is_null()); EXPECT_TRUE(serialized["execution"]["post_validation_evaluated"].is_null()); EXPECT_TRUE(serialized["execution"]["post_validation_valid"].is_null()); + EXPECT_TRUE(serialized["execution"]["kv_write_validation_applicable"].is_null()); + EXPECT_TRUE(serialized["execution"]["kv_write_validation_evaluated"].is_null()); + EXPECT_TRUE(serialized["execution"]["kv_write_validation_valid"].is_null()); EXPECT_EQ(serialized["checksum"]["status"], "not_evaluated"); EXPECT_EQ(serialized["checksum"]["reason_code"], LlmRunnerReason::INTERRUPTION_BEFORE_TASK); EXPECT_TRUE(serialized["checksum"]["checksum_valid"].is_null()); @@ -2689,6 +2699,9 @@ TEST(LlmMemoryJsonTest, ExecutorExceptionUsesRunnerReasonAndNullUnavailableExecu EXPECT_TRUE(measurement["execution"]["kernel_succeeded"].is_null()); EXPECT_TRUE(measurement["execution"]["post_validation_evaluated"].is_null()); EXPECT_TRUE(measurement["execution"]["post_validation_valid"].is_null()); + EXPECT_TRUE(measurement["execution"]["kv_write_validation_applicable"].is_null()); + EXPECT_TRUE(measurement["execution"]["kv_write_validation_evaluated"].is_null()); + EXPECT_TRUE(measurement["execution"]["kv_write_validation_valid"].is_null()); EXPECT_EQ(measurement["checksum"]["status"], "not_evaluated"); EXPECT_EQ(measurement["checksum"]["reason_code"], LlmRunnerReason::RUNNER_EXCEPTION); EXPECT_TRUE(measurement["checksum"]["checksum_valid"].is_null()); @@ -3034,3 +3047,60 @@ TEST(LlmMemoryJsonTest, LoopRecordsExposeOnlyRealizedPrefixAndAllMeasurementInde EXPECT_EQ(loop["measurement_indexes"].size(), kLlmScenarioCount); EXPECT_EQ(loop["measurement_indexes"], (OrderedJson::array({0, 1, 2}))); } + +TEST(LlmMemoryJsonTest, DecodeWriteValidationFailureRetainsInvalidAttemptAndPopulation) { + const auto config = explicit_config(1); + const auto plan = admitted_plan(config); + ASSERT_TRUE(plan.valid); + FakeLlmBackend backend([](const LlmMemoryWorkPlan& model, const LlmScenarioWorkPlan& scenario, + const LlmRunnerTaskContext& context) { + auto execution = successful_execution(model); + if (context.kind == LlmRunnerTaskKind::Measurement && scenario.scenario == LlmScenario::KvOnly) { + execution.valid = false; + execution.reason_code = LlmExecutorReason::DECODE_POST_VALIDATION_FAILED; + execution.post_validation_valid = false; + } + return execution; + }); + LlmMemoryResult result; + ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_FAILURE); + const auto document = build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); + const auto& weights = document["measurements"][0]; + EXPECT_FALSE(weights["execution"]["kv_write_validation_applicable"]); + EXPECT_TRUE(weights["execution"]["kv_write_validation_evaluated"].is_null()); + EXPECT_TRUE(weights["execution"]["kv_write_validation_valid"].is_null()); + const auto& measurement = document["measurements"][1]; + EXPECT_EQ(measurement["status"], "invalid"); + EXPECT_EQ(measurement["reason_code"], LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); + EXPECT_TRUE(measurement["execution"]["kv_write_validation_applicable"]); + EXPECT_TRUE(measurement["execution"]["kv_write_validation_evaluated"]); + EXPECT_FALSE(measurement["execution"]["kv_write_validation_valid"]); + EXPECT_TRUE(measurement["checksum"]["checksum_valid"]); + EXPECT_TRUE(measurement["synthetic_memory_work_units_per_second"].is_null()); + EXPECT_TRUE(measurement["effective_model_payload_gb_s"].is_null()); + EXPECT_EQ(document["aggregates"]["scenarios"]["kv_only"]["effective_model_payload_gb_s"]["sample_count"], 0u); +} + +TEST(LlmMemoryJsonTest, RequiredCpuWriteValidationNotRunIsFailedWithNullVerdict) { + const auto config = explicit_config(1); + const auto plan = admitted_plan(config); + ASSERT_TRUE(plan.valid); + FakeLlmBackend backend([](const LlmMemoryWorkPlan& model, const LlmScenarioWorkPlan& scenario, + const LlmRunnerTaskContext& context) { + auto execution = successful_execution(model); + if (context.kind == LlmRunnerTaskKind::Measurement && scenario.scenario == LlmScenario::KvOnly) { + execution.post_validation_evaluated = false; + } + return execution; + }); + LlmMemoryResult result; + ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_FAILURE); + const auto document = build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); + const auto& measurement = document["measurements"][1]; + EXPECT_EQ(measurement["status"], "failed"); + EXPECT_TRUE(measurement["execution"]["kv_write_validation_applicable"]); + EXPECT_FALSE(measurement["execution"]["kv_write_validation_evaluated"]); + EXPECT_TRUE(measurement["execution"]["kv_write_validation_valid"].is_null()); + EXPECT_TRUE(measurement["synthetic_memory_work_units_per_second"].is_null()); + EXPECT_TRUE(measurement["effective_model_payload_gb_s"].is_null()); +} diff --git a/tests/test_llm_memory_output.cpp b/tests/test_llm_memory_output.cpp index 5b3ce82..451e913 100644 --- a/tests/test_llm_memory_output.cpp +++ b/tests/test_llm_memory_output.cpp @@ -424,7 +424,11 @@ class FakeLlmBackend final : public LlmBackend { LlmTaskExecutionResult execute_task(const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan& scenario_plan, const LlmRunnerTaskContext& context) override { - return adapt_llm_cpu_executor_result(model_plan, scenario_plan, context, successful_fake_execution(model_plan)); + auto execution = successful_fake_execution(model_plan); + execution.kv_write_validation_applicable = scenario_plan.scenario != LlmScenario::WeightsOnly; + execution.kv_write_validation_evaluated = execution.kv_write_validation_applicable; + execution.kv_write_validation_valid = execution.kv_write_validation_applicable; + return adapt_llm_cpu_executor_result(model_plan, scenario_plan, context, std::move(execution)); } const LlmBackendEvidence& evidence() const noexcept override { return evidence_; } diff --git a/tests/test_llm_memory_runner.cpp b/tests/test_llm_memory_runner.cpp index 77ddd90..ac6981c 100644 --- a/tests/test_llm_memory_runner.cpp +++ b/tests/test_llm_memory_runner.cpp @@ -3085,3 +3085,72 @@ TEST(LlmMemoryRunnerTest, StableTaskAndCheckpointTokensCoverUnknownValues) { } } // namespace + +TEST(LlmMemoryRunnerTest, + DecodePostValidationFailureCanonicalizesExactlyThroughRunnerFailure) { + std::string owned_reason = + LlmExecutorReason::DECODE_POST_VALIDATION_FAILED; + const std::string_view canonical = + canonicalize_llm_result_reason_code(owned_reason); + std::fill(owned_reason.begin(), owned_reason.end(), 'x'); + EXPECT_EQ(canonical, LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); + EXPECT_NE(canonical, LlmRunnerReason::RUNNER_UNKNOWN_EXCEPTION); + EXPECT_EQ( + canonical.data(), + canonicalize_llm_result_reason_code( + LlmExecutorReason::DECODE_POST_VALIDATION_FAILED) + .data()); + + LlmMemoryConfig config = explicit_config(1); + const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); + ASSERT_TRUE(plan.valid) << plan.reason_code; + FakeLlmBackend executor; + executor.mutate = [](const LlmMemoryWorkPlan&, + const LlmScenarioWorkPlan&, + const LlmRunnerTaskContext& context, size_t, + LlmTaskExecutionResult& execution) { + if (context.kind != LlmRunnerTaskKind::Measurement || context.scenario != LlmScenario::KvOnly) { + return; + } + execution.status = LlmTaskExecutionStatus::Invalid; + execution.reason_code = + std::string(LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); + execution.validation.evaluated = true; + execution.validation.valid = false; + LlmCpuTaskEvidence evidence; + evidence.executor.valid = false; + evidence.executor.reason_code = + std::string(LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); + evidence.executor.post_validation_evaluated = true; + evidence.executor.post_validation_valid = false; + evidence.executor.kv_write_validation_applicable = true; + evidence.executor.kv_write_validation_evaluated = true; + evidence.executor.kv_write_validation_valid = false; + execution.backend_evidence = std::move(evidence); + }; + LlmMemoryResult result; + + EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result), + EXIT_FAILURE); + EXPECT_EQ(result.status, LlmRunStatus::Failed); + EXPECT_EQ(result.reason_code, + LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); + EXPECT_NE(result.reason_code, LlmRunnerReason::RUNNER_UNKNOWN_EXCEPTION); + ASSERT_FALSE(result.measurements.empty()); + const LlmMeasurementState& measurement = result.measurements[1]; + EXPECT_EQ(measurement.status, LlmMeasurementStatus::Invalid); + EXPECT_EQ(measurement.reason_code, + LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); + EXPECT_EQ(measurement.execution.reason_code, + LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); + const auto* retained = + std::get_if(&measurement.execution.backend_evidence); + ASSERT_NE(retained, nullptr); + EXPECT_EQ(retained->executor.reason_code, + LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); + EXPECT_TRUE(retained->executor.post_validation_evaluated); + EXPECT_FALSE(retained->executor.post_validation_valid); + EXPECT_TRUE(retained->executor.kv_write_validation_applicable); + EXPECT_TRUE(retained->executor.kv_write_validation_evaluated); + EXPECT_FALSE(retained->executor.kv_write_validation_valid); +} From dd9a5fcc70586019f94ba52c3643782eb96e1dec Mon Sep 17 00:00:00 2001 From: timoheimonen Date: Sat, 5 Sep 2026 12:25:47 +0300 Subject: [PATCH 02/16] Cover bounded CPU checksum collisions and work mutations --- tests/test_llm_memory_kernels.cpp | 108 ++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/tests/test_llm_memory_kernels.cpp b/tests/test_llm_memory_kernels.cpp index e0d1243..c61f3ce 100644 --- a/tests/test_llm_memory_kernels.cpp +++ b/tests/test_llm_memory_kernels.cpp @@ -745,6 +745,37 @@ void expect_worker_equal(const LlmWorkerChecksum& actual, const LlmWorkerChecksu expect_component_equal(actual.v, expected.v); } +// Four direct ABI calls keep these fault examples scoped to the shared weight +// span. They do not exercise KV writes, lookup mixing, or backend acceptance. +LlmWorkerChecksum run_weights_only_fault_probe(size_t profile, const uint8_t* bytes, + size_t byte_count, size_t work_units) { + LlmWorkerChecksum actual{}; + constexpr uint64_t seed = 424242; + switch (profile) { + case 0: { + LlmLayerDescriptor layer{bytes, byte_count, 0, 0, 0, 0}; + llm_decode_memory_asm(&layer, nullptr, 1, work_units, kLlmScenarioFlagWeight, seed, &actual); + break; + } + case 1: { + LlmPagedLayerDescriptor layer{bytes, byte_count, 0, 0, 0, 0}; + llm_decode_memory_paged_asm(&layer, nullptr, 1, work_units, kLlmScenarioFlagWeight, seed, &actual); + break; + } + case 2: { + LlmPrefillLayerDescriptor layer{bytes, byte_count, 0, 0, 0, 0}; + llm_prefill_memory_asm(&layer, nullptr, 1, work_units, kLlmScenarioFlagWeight, seed, &actual); + break; + } + case 3: { + LlmPagedPrefillLayerDescriptor layer{bytes, byte_count, 0, 0, 0, 0}; + llm_prefill_memory_paged_asm(&layer, nullptr, 1, work_units, kLlmScenarioFlagWeight, seed, &actual); + break; + } + } + return actual; +} + LlmMemoryWorkPlan build_real_executor_plan(const LlmGeometryRequest& geometry, size_t workers) { LlmMemoryWorkPlanRequest request; request.geometry = geometry; @@ -904,6 +935,83 @@ struct ManualPagedDescriptorFixture { } // namespace +TEST(LlmMemoryKernelIntegrationTest, AllCpuWeightSpansExposeBoundedParityCollisions) { + const std::array original = {11, 23, 37, 43, 59, 67, 79, 83}; + auto swapped = original; + std::rotate(swapped.begin(), swapped.begin() + 4, swapped.end()); + auto cancelling = original; + ++cancelling[0]; + --cancelling[2]; + auto one_bit = original; + one_bit[0] ^= 1; + ASSERT_NE(original, swapped); + ASSERT_NE(original, cancelling); + ASSERT_NE(original, one_bit); + + for (size_t profile = 0; profile < 4; ++profile) { + SCOPED_TRACE(profile); + for (size_t work_units : {1U, 2U}) { + SCOPED_TRACE(work_units); + LlmWorkerChecksum expected = profile < 2 + ? oracle_initial_worker() + : LlmWorkerChecksum{oracle_initial(LlmChecksumComponent::Weight), {}, {}}; + for (size_t unit = 0; unit < work_units; ++unit) { + oracle_absorb(expected.weight, reinterpret_cast(original.data()), sizeof(original)); + } + for (const auto* contents : std::array*, 3>{&original, &swapped, &cancelling}) { + const auto actual = run_weights_only_fault_probe( + profile, reinterpret_cast(contents->data()), sizeof(original), work_units); + expect_worker_equal(actual, expected); + } + const auto changed = run_weights_only_fault_probe( + profile, reinterpret_cast(one_bit.data()), sizeof(original), work_units); + EXPECT_NE(changed.weight.state_a, expected.weight.state_a); + EXPECT_EQ(changed.weight.exact_bytes_read, expected.weight.exact_bytes_read); + EXPECT_EQ(changed.weight.span_count, expected.weight.span_count); + } + } +} + +TEST(LlmMemoryKernelIntegrationTest, AllCpuWeightSpansDetectBoundedReadAndWorkUnitMutations) { + // The final 32 bytes duplicate the first 32. The frozen expected workload is + // exactly the first 64 bytes, twice. Mutations retain that original oracle; + // these selected descriptor faults do not model every possible kernel bug. + const std::array contents = {11, 23, 37, 43, 59, 67, 79, 83, 11, 23, 37, 43}; + constexpr size_t planned_bytes = 64; + constexpr size_t planned_work_units = 2; + const auto* bytes = reinterpret_cast(contents.data()); + struct Mutation { + const char* name; + size_t span_bytes; + size_t work_units; + }; + const std::array mutations = {{{"skip-32-byte-range", 32, 2}, + {"duplicate-32-byte-range", 96, 2}, + {"missing-work-unit", 64, 1}, + {"extra-work-unit", 64, 3}}}; + for (size_t profile = 0; profile < 4; ++profile) { + SCOPED_TRACE(profile); + LlmWorkerChecksum expected = profile < 2 + ? oracle_initial_worker() + : LlmWorkerChecksum{oracle_initial(LlmChecksumComponent::Weight), {}, {}}; + for (size_t unit = 0; unit < planned_work_units; ++unit) { + oracle_absorb(expected.weight, bytes, planned_bytes); + } + expect_worker_equal(run_weights_only_fault_probe(profile, bytes, planned_bytes, planned_work_units), expected); + for (const auto& mutation : mutations) { + SCOPED_TRACE(mutation.name); + const auto actual = run_weights_only_fault_probe(profile, bytes, mutation.span_bytes, mutation.work_units); + EXPECT_NE(actual.weight.exact_bytes_read, expected.weight.exact_bytes_read); + EXPECT_EQ(actual.weight.exact_bytes_read, mutation.span_bytes * mutation.work_units); + EXPECT_EQ(actual.weight.span_count, mutation.work_units); + EXPECT_TRUE(actual.weight.state_a != expected.weight.state_a || actual.weight.state_b != expected.weight.state_b); + // No KV descriptors or resources were used by any of these probes. + expect_component_equal(actual.k, expected.k); + expect_component_equal(actual.v, expected.v); + } + } +} + TEST(LlmMemoryKernelIntegrationTest, SafeZeroNullAndInvalidTopLevelBoundariesReturnInitialStates) { // A null output is the sole case where no initialized result can be stored. llm_decode_memory_asm(nullptr, nullptr, 0, 0, 0, 0, nullptr); From d411d2ee336a58181943503532f7ccc69988d881 Mon Sep 17 00:00:00 2001 From: timoheimonen Date: Sat, 5 Sep 2026 12:36:33 +0300 Subject: [PATCH 03/16] Test Metal checksum helper collisions on the GPU --- Makefile | 8 ++- tests/test_llm_metal_checksum.mm | 108 +++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 tests/test_llm_metal_checksum.mm diff --git a/Makefile b/Makefile index 4912674..a07870c 100644 --- a/Makefile +++ b/Makefile @@ -80,7 +80,8 @@ $(TARGET): $(OBJ_FILES) # Test directory and files TEST_DIR = tests TEST_SRCS := $(sort $(wildcard $(TEST_DIR)/*.cpp)) -TEST_OBJS := $(TEST_SRCS:.cpp=.o) +TEST_OBJCXX_SRCS := $(sort $(wildcard $(TEST_DIR)/*.mm)) +TEST_OBJS := $(TEST_SRCS:.cpp=.o) $(TEST_OBJCXX_SRCS:.mm=.o) # Dependency files generated by DEPFLAGS. Test dependencies include tests/*.h # helpers through the compiler's actual include graph rather than a manually @@ -105,6 +106,11 @@ $(TEST_DIR)/%.o: $(TEST_DIR)/%.cpp @echo "Compiling test $< -> $@..." $(CXX) $(TEST_CXXFLAGS) $(DEPFLAGS) -c $< -o $@ +# Objective-C++ tests retain test flags and ARC, including generated dependencies. +$(TEST_DIR)/%.o: $(TEST_DIR)/%.mm + @echo "Compiling Objective-C++ test $< -> $@..." + $(CXX) $(TEST_CXXFLAGS) -fobjc-arc $(DEPFLAGS) -c $< -o $@ + # Objective-C++ production boundaries share the normal C++ build settings. %.o: %.mm @echo "Compiling Objective-C++ $< -> $@..." diff --git a/tests/test_llm_metal_checksum.mm b/tests/test_llm_metal_checksum.mm new file mode 100644 index 0000000..0ee366d --- /dev/null +++ b/tests/test_llm_metal_checksum.mm @@ -0,0 +1,108 @@ +// Copyright 2026 Timo Heimonen +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +#include + +#import +#import + +#include +#include +#include + +#include "llm_memory/llm_metal_kernels_source.h" + +namespace { + +// This extra entrypoint exercises the unchanged shared helper, not a workload +// entrypoint or full-backend acceptance. The canonical embedded source is kept +// intact and receives the same decode-contiguous compile selector as production. +constexpr char kHelperProbe[] = R"MSL( +kernel void llm_checksum_helper_probe(device const uint* input [[buffer(0)]], + device uint2* output [[buffer(1)]], + uint id [[thread_position_in_grid]]) { + uint2 checksum = uint2(0u); + for (uint i = 0; i < 4u; ++i) { + mix_checksum_word(checksum, input[id * 4u + i], ulong(i), 12345u + i * 17u); + } + output[id] = checksum; +} +)MSL"; + +} // namespace + +TEST(LlmMetalChecksumHelperIntegrationTest, SharedAffineLanesExposeBoundedContentCollisions) { + @autoreleasepool { + id device = MTLCreateSystemDefaultDevice(); + if (device == nil) { + GTEST_SKIP() << "metal-device-unavailable"; + } + std::string source(LlmMetalKernelContract::kSource); + source += kHelperProbe; + NSString* source_string = [[NSString alloc] initWithBytes:source.data() + length:source.size() + encoding:NSUTF8StringEncoding]; + ASSERT_NE(source_string, nil); + MTLCompileOptions* options = [[MTLCompileOptions alloc] init]; + options.languageVersion = MTLLanguageVersion2_3; + options.preprocessorMacros = @{@"LLM_METAL_DECODE_CONTIGUOUS" : @1, + @"LLM_METAL_DECODE_PAGED" : @0, + @"LLM_METAL_PREFILL_CONTIGUOUS" : @0, + @"LLM_METAL_PREFILL_PAGED" : @0}; + NSError* error = nil; + id library = [device newLibraryWithSource:source_string options:options error:&error]; + ASSERT_NE(library, nil) << (error == nil ? "no compiler diagnostic" : error.description.UTF8String); + id function = [library newFunctionWithName:@"llm_checksum_helper_probe"]; + ASSERT_NE(function, nil); + id pipeline = [device newComputePipelineStateWithFunction:function error:&error]; + ASSERT_NE(pipeline, nil) << (error == nil ? "no pipeline diagnostic" : error.description.UTF8String); + ASSERT_GE(pipeline.maxTotalThreadsPerThreadgroup, 4U); + + const std::array input = {11, 23, 37, 43, // original + 43, 37, 23, 11, // content permutation, same visits + 12, 22, 37, 43, // cancelling +1/-1 + 10, 23, 37, 43}; // one changed bit, detectable control + id input_buffer = [device newBufferWithBytes:input.data() + length:sizeof(input) + options:MTLResourceStorageModeShared]; + id output_buffer = [device newBufferWithLength:8 * sizeof(uint32_t) + options:MTLResourceStorageModeShared]; + ASSERT_NE(input_buffer, nil); + ASSERT_NE(output_buffer, nil); + id queue = [device newCommandQueue]; + ASSERT_NE(queue, nil); + id command = [queue commandBuffer]; + ASSERT_NE(command, nil); + id encoder = [command computeCommandEncoder]; + ASSERT_NE(encoder, nil); + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:input_buffer offset:0 atIndex:0]; + [encoder setBuffer:output_buffer offset:0 atIndex:1]; + [encoder dispatchThreads:MTLSizeMake(4, 1, 1) threadsPerThreadgroup:MTLSizeMake(4, 1, 1)]; + [encoder endEncoding]; + [command commit]; + [command waitUntilCompleted]; + ASSERT_EQ(command.status, MTLCommandBufferStatusCompleted) + << (command.error == nil ? "no command diagnostic" : command.error.description.UTF8String); + const auto* actual = static_cast(output_buffer.contents); + ASSERT_NE(actual, nullptr); + + // Independent hand-derived mod-2^32 goldens: sum(value)=114 (control 113), + // sum(index)=6, sum(domain)=4*12345+17*6=49482. + // a=sum(value)+49482; + // b=sum(value)*0x9e3779b1+6*0x85ebca77+49482*0x7feb352d mod 2^32. + // Both affine lanes therefore share the same content-sum collision. + constexpr std::array expected = { + 49596, 3847175070U, 49596, 3847175070U, + 49596, 3847175070U, 49595, 1192739309U}; + for (size_t word = 0; word < expected.size(); ++word) { + EXPECT_EQ(actual[word], expected[word]) << "result word " << word; + } + EXPECT_NE(actual[0], actual[6]); + EXPECT_NE(actual[1], actual[7]); + } +} From f5bbfa19caf5a05e0c1bd5fd2e192fdc2b116877 Mon Sep 17 00:00:00 2001 From: timoheimonen Date: Sat, 5 Sep 2026 12:46:49 +0300 Subject: [PATCH 04/16] Document profile-specific checksum fault boundaries --- documents/API.md | 32 +++++-- documents/LLM_MEMORY_PROFILE_WHITEPAPER.md | 106 ++++++++++++++++++--- documents/MANUAL.md | 26 +++-- 3 files changed, 135 insertions(+), 29 deletions(-) diff --git a/documents/API.md b/documents/API.md index 989e790..012b70a 100644 --- a/documents/API.md +++ b/documents/API.md @@ -238,6 +238,18 @@ worker lifecycle, QoS, elapsed-time, and checksum fields are null. For a `not_ru `qos_successful_workers` and `qos_failed_workers` are also null. These are absence-of-evidence states, not zero-worker or successful-checksum observations. +Checksum validity describes agreement with the versioned accumulator, not an exhaustive proof of all contents, +addresses, or visit multiplicities. A post-validation failure can retain `checksum_valid: true`; the attempt remains +invalid, its rates are null, and it does not enter aggregates. Missing checksum evidence remains null rather than false +or successful. CPU paged decode weights-only checks append slots and padding; CPU paged prefill weights-only checks padding only. +Metal weights-only +reports KV-write unevaluated and padding inapplicable, so no successful KV-write observation may be inferred from its +combined lifecycle result. Prefill validation remains sampled on both backends. Programmed byte/lookup counts and +plan-derived completion do not measure physical DRAM traffic. Source hashes, ABI goldens, and generated semantic +lists qualify a specific implementation rather than adding runtime observations to the JSON. See the +[profile fault matrix and exact collision limits](LLM_MEMORY_PROFILE_WHITEPAPER.md#checksum-fault-model-and-evidence-limits). +These limits do not change schema 1, the checksum identities, or the completion/acceptance predicate below. + The required generic top-level field set is: ```text @@ -330,8 +342,8 @@ The Metal paged kernel assigns each layer/batch/logical-block owner to exactly o grid-stride schedule. A named lane loads each `device const volatile uint` table entry, publishes the physical ID in threadgroup memory, and executes a `mem_threadgroup` barrier before address construction. The timed checksum non-separably binds logical table index, physical ID, append/K-read/V-read visit kind, and work-unit ordinal; -equal-multiplicity table permutations are -therefore distinguishable. These four-byte lookups and physical suffix padding are reported evidence, not effective +the maintained real-device tests distinguish selected equal-multiplicity non-tail two-entry swaps, not all +possible table permutations. These four-byte lookups and physical suffix padding are reported evidence, not effective model payload. @@ -352,8 +364,9 @@ valid serialize as null. When applicable but unevaluated, evaluated is false and acceptance requires evaluated and valid to be true. All three are null without available CPU evidence. For paged and prefill profiles, the valid field reports the existing combined phase/layout final-state check, including any applicable padding check; it is not an independent append-only verdict. Prefill -retains its documented sampling limits. `weights_only` does not claim KV-write validation; the existing -CPU paged unexpected-KV-write and padding protections still run in the combined pipeline. +retains its documented sampling limits. `weights_only` does not claim KV-write validation. Its combined CPU paged +pipeline checks append slots and padding for decode, but only structure and padding for prefill; prefill weights-only +does not check unexpected writes to valid prompt contents. For prefill, let `P` be prompt length, `Q` query-tile length, and `K = L*2*R`. With `C = ceil(P/Q)`, tile ends `e_j = min((j+1)*Q, P)`, and `S(P,Q) = sum(e_j)`, one `prefill_operation` has: @@ -369,8 +382,8 @@ mixed_payload = W + B * (P + S(P,Q)) * K Each operation writes owner-local prompt tokens in ascending order, K then V for each token. Those writes precede that owner's reads; no global worker barrier is implied. Each tile reads its complete owned K prefix before its complete -owned V prefix. The operation ordinal is bound into write/checksum evidence, and the timed checksum covers every -tile-read visit. For a KV-bearing CPU task, excluded post-validation checks each owner's deterministic +owned V prefix. The operation ordinal is bound into write/checksum evidence, and the intended checksum accumulation includes each +tile-read visit; equality alone does not establish every executed visit. For a KV-bearing CPU task, excluded post-validation checks each owner's deterministic first/middle/last canonical-word samples, including bytes clipped to owner boundaries, against the final operation ordinal `T-1`. For a KV-bearing Metal prefill task, it checks representative and boundary byte locations per layer/batch sequence in both K and V against that ordinal; paged Metal additionally validates applicable terminal @@ -516,8 +529,8 @@ the weight-vector grid-stride schedule and reports zero metadata work. The same published for every Metal `weights_only` task. Contiguous KV-bearing grids retain the grid geometry but publish null cost unit/minimum/maximum/imbalance fields and an empty cost vector. The cyclic KV schedule is deliberately not described as weighted-balanced. -Partial prefix visits stop at the exact tile or prompt end, and complete status requires full-prompt write validation -plus applicable suffix-padding validation. +Partial prefix visits stop at the exact tile or prompt end. KV-bearing prefill tasks require final-operation +representative/boundary sample validation of the prompt writes, plus applicable suffix-padding validation. Paged `weights_only` performs no block-table access and reports zero layout-metadata work. For decode `kv_only` and `mixed`, each layer/batch pair performs one paired K/V append lookup, `N` K-scan lookups, and `N` V-scan lookups per @@ -551,7 +564,8 @@ Metal prefill with contiguous KV uses the same logical prefill traffic contract `P` K/V records, and each lane scans only its own written slices of the complete K prefix followed by the complete V prefix at each tile end. No grid-wide barrier is implied. All `T` operations execute inside the single timed workload dispatch. The commutative checksum binds scenario, layer, batch, byte domain, and work-unit ordinal; the -source-hash-bound entrypoint audit separately locks the write-before-tiled-read loop nesting. +source-hash-bound entrypoint audit separately checks the build’s write-before-tiled-read loop nesting, rather than +recording a runtime GPU trace. The traffic classification version is `llm-exact-weight-vs-kv-read-payload-v1`: it compares exact active-weight bytes with exact KV-read bytes only. `near_crossover` means equality, not a tolerance band and not an observed hardware diff --git a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md index 17816f5..921fee8 100644 --- a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md +++ b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md @@ -62,7 +62,7 @@ Schema-v1 vocabulary includes `cpu|metal`, `decode|prefill`, `contiguous|paged`, for both phases on both backends. The four Metal profiles never receive hidden fallback. The prefill implementation resolves checked tile/prefix/payload formulas, versioned atomic CPU ownership evidence, -owner-local semantic traces, and operation-ordinal checksum oracles. CPU contiguous prefill uses token-range ownership; +generated owner-local semantic event lists, and operation-ordinal checksum oracles. CPU contiguous prefill uses token-range ownership; paged prefill uses block-exclusive weighted ownership plus a read-only uint32 table, deterministic permutation, full physical K/V pools, a dedicated descriptor ABI, and a separate ARM64 executor. Metal prefill uses scenario-specialized MSL pipelines, one in-kernel weight pass per operation when weights are active, full-prompt K/V @@ -529,9 +529,8 @@ exactly one threadgroup under a cyclic grid-stride schedule. At every visit one K/V writes. Each block owner issues its separate K and V scan lookups, giving exactly `L * B * (2 * N + 1)` lookup evidence for each KV-active work unit independent of threadgroup count. Segment/block/address selection depends on the loaded physical ID. The timed checksum non-separably mixes logical table index, physical ID, append/K-read/V-read kind, -and work-unit -ordinal, so swapping equal-multiplicity non-tail IDs is detectable. Excluded post-validation checks both append bytes -and terminal-block padding canaries. +and work-unit ordinal; the selected equal-multiplicity non-tail ID swaps in the mutation tests are detectable. +For KV-bearing Metal tasks, excluded post-validation checks both append bytes and terminal-block padding canaries. For contiguous prefill KV work, every operation/layer/batch owner first writes all of its token ranges in increasing logical order. It then visits query tiles in increasing order; for each tile it scans every owned range intersecting the @@ -545,7 +544,8 @@ bytes, not one pass per prompt token or query tile. In KV-bearing entrypoints ea across every prompt K/V record before reading those slices, then scans each tile's complete K prefix before its complete V prefix. No grid-wide barrier is required because a lane reads only the bytes it wrote. The dispatch loops over all `T` operations and uses exact vector prefixes, scalar tails, and segment-boundary splits. The commutative dual-mod32 -checksum proves contents and multiplicity; a separate audit ties this loop nesting to the exact MSL source hash. +checksum supplies a bounded consistency witness, with the collision limits described below. A separate source-bound +audit checks this loop nesting for the recorded MSL build; it is not a trace collected from GPU execution. CPU paged prefill uses the same logical order with weighted whole-block assignments. For each operation/layer/batch owner it first loads each owned logical block's physical ID and populates all valid prompt K/V bytes in that block. It @@ -559,7 +559,8 @@ threadgroups with a deterministic cyclic grid stride rather than CPU weighted re each volatile uint32 table load and publishes the physical ID before the owner threadgroup addresses the block. Prompt population contributes `N` paired lookups and the tiled scans contribute `M` K-prefix plus `M` V-prefix lookups per layer/batch pair, so each KV-active operation has exactly `L * B * (N + 2 * M)` loads independent of grid size. The -source-hash-bound event-trace audit freezes full-prompt population before tile-ordered K-then-V reads. Grid evidence +source-hash-bound semantic event enumeration checks full-prompt population before tile-ordered K-then-V reads +in the described implementation, rather than recording those events from GPU execution. Grid evidence reports exact per-threadgroup accounted-byte vectors and their minimum, maximum, and imbalance without claiming weighted balance. Every Metal `weights_only` grid reports the same exact weight-vector grid-stride cost evidence; contiguous KV-bearing grids leave threadgroup-cost evidence unavailable. Excluded validation checks final-ordinal @@ -615,7 +616,7 @@ The task-local step restarts at zero for every warmup, calibration, and measurem uses the canonical record byte offset; every partial fragment writes only the corresponding little-endian bytes without widening its bounds. -Every read contributes to an observable `llm-read-checksum-v1` state. For each worker, weight, K, and V components each +The intended read path accumulates each read into an observable `llm-read-checksum-v1` state. For each worker, weight, K, and V components each contain two 64-bit states plus exact bytes read and span count. Cold-path code independently derives the expected values from frozen descriptors, initialization references, append formula, scenario, and step count. After the timer stops, it compares every worker/component tuple and folds expected and actual results in stable worker/weight/K/V order. @@ -672,8 +673,9 @@ This checksum is workload-liveness and bounds evidence, not a cryptographic inte Paged execution uses the separate `llm-paged-read-checksum-v1` contract. Its timed accumulator binds each semantic visit to the logical table index, loaded physical ID, visit kind (paired write/append, K scan, or V scan), and work-unit ordinal in one -non-separable mix. Summing physical IDs alone is prohibited because every permutation has the same ID sum. A test that -swaps two non-tail blocks with equal read multiplicity must therefore produce a mismatch. An independent bounded scalar +non-separable mix. Summing physical IDs alone is prohibited because every permutation has the same ID sum. The maintained tests +show a mismatch for selected two-entry non-tail swaps with equal read multiplicity; this is not a guarantee for all +possible table mutations or modulo-arithmetic collisions. An independent bounded scalar oracle computes the cold-path expected value without calling the assembly helper or rereading a multi-GiB pool. Post-validation checks the logical decode current-token K/V append or prefill final-ordinal samples at their resolved physical locations and every last-block padding canary. @@ -682,9 +684,9 @@ CPU prefill uses its versioned full-prompt affine64 write pattern. Metal contigu `llm-metal-prefill-contiguous-full-prompt-affine32-v1`, while Metal paged prefill uses `llm-metal-prefill-paged-full-prompt-affine32-v1` with checksum `llm-metal-paged-prefill-dual-mod32-lookup-address-mix-v1`. Every K/V word binds scenario seed, operation ordinal, layer, -batch, logical token, record-word index, and K/V domain. Timed checksum agreement proves the expected contents -and visit multiplicity across all T operations, including every tile-read visit; it does not by itself prove event -order. CPU source/disassembly audit and the Metal entrypoint/source-hash audit establish the locked order: one +batch, logical token, record-word index, and K/V domain. Timed checksum agreement checks the expected accumulator +across T operations, including planned tile-read contributions. It does not establish every executed load, content value, visit multiplicity, or event order. +CPU source/disassembly and Metal entrypoint/source-hash audits qualify the implementation of the specified order: one weight pass per operation when weights are active, complete prompt population for KV-bearing scenarios, then for every increasing tile the complete K prefix followed by the complete V prefix. In paged prefill, population and prefix fragments additionally bind the table index and loaded physical ID and stop at exact partial-block boundaries. @@ -697,7 +699,8 @@ against the final task-local ordinal `T-1`, after timer stop and all worker join `decode-post-validation-failed`: the invalid attempt is retained with null rates and excluded from aggregates. The timed workload and checksum algorithm are unchanged. KV-write evidence applies only to KV-bearing scenarios; paged/prefill evidence retains the combined final-state check and its existing -padding/sampling limits. CPU paged weights-only unexpected-write/padding protection remains active. +padding/sampling limits. CPU paged decode weights-only checks its append slots and padding; CPU paged prefill weights-only checks padding, +not valid prompt contents. See the [CPU final-state evidence contract](API.md#result-schemas-and-completion) for field and null semantics. @@ -716,6 +719,81 @@ the paged profile also checks applicable suffix-padding canaries. It does not re checksum, K/V-write, or applicable padding mismatch invalidates the current task and prevents the next task; there is no retry and no numeric comparison with the CPU checksum algorithm. +## Checksum fault model and evidence limits + +Three kinds of evidence have different owners: the immutable work plan describes intended geometry, payload, and +visits; a task records runtime checksum, timing, lifecycle, and post-validation observations; kernel qualification +checks a particular build using independent oracles, ABI/layout goldens, source/disassembly, and real-device tests. +A source hash or generated semantic event list is a build/semantic change guard, not a GPU execution trace. Exact-byte +and lookup counters are programmed observations or plan-derived completion quantities, not physical DRAM counters. +Checksum equality is necessary for acceptance, but is not an exhaustive content or address proof. + +The following matrix covers CPU/Metal × decode/prefill × contiguous/paged. **O** means an observed, specifically +bounded test case; **E** means detection is conditional on the checksum/validator and has no corresponding injected +kernel-fault proof here; **S** identifies a known blind spot. **—** is inapplicable. Multiple marks distinguish a +selected detected mutation from other undetected mutations in the same broad class. They are not coverage percentages. + +| Fault | CPU decode contiguous | CPU decode paged | CPU prefill contiguous | CPU prefill paged | Metal decode contiguous | Metal decode paged | Metal prefill contiguous | Metal prefill paged | +|---|---|---|---|---|---|---|---|---| +| Skipped/duplicated read or wrong work-unit count | O W; E other reads | O W; E KV | O W; E KV | O W; E KV | E | E | E | E | +| Wrong address or equal-valued substitution | S equal value | O T; S other substitutions | S equal value | O T; S other substitutions | S equal value | O T; S other substitutions | S equal value | O T; S other substitutions | +| CPU within-span 32-byte swap or same-parity +1/−1 | S W | S W | S W | S W | — | — | — | — | +| Metal content permutation or cancellation with the same visits | — | — | — | — | S H | S H | S H | S H | +| Wrong paged table ID / lookup | — | O T; E skipped lookup | — | O T; E skipped lookup | — | O T; E skipped lookup | — | O T; E skipped lookup | +| Wrong K/V append or prefill final state | O A | O A | O P; S unsampled post-checksum writes | O P; S unsampled post-checksum writes | E A | E A | E P; S unsampled post-checksum writes | E P; S unsampled post-checksum writes | +| Padding or other extra write | E outside checked append | O C; E elsewhere | E outside samples | O C; E elsewhere | E outside checked append | O C; E elsewhere | E outside samples | O C; E elsewhere | + +- **W — shared CPU weight span only.** In all four real ASM entrypoints, the 64-byte word vector + `11,23,37,43,59,67,79,83` retains its checksum after swapping its two 32-byte halves or adding one to word zero and + subtracting one from word two, at T=1 and T=2. Both mutations preserve the even/odd word sums. A one-bit change is + detected. Separate invocations against the original independent 64-byte × T=2 oracle detect a 32-byte range removal, + a duplicated 32-byte range, and T=1/T=3. These are selected descriptor/work-count mutations, not tests of every + possible skipped machine load. They do not establish a whole KV/backend acceptance collision. CPU prefill KV uses + logical-word parity sums with its own visit/lookup rules; the weight-span test does not replace that separate oracle. +- **H — actual Metal shared helper, not full-backend acceptance.** The test adds a small test-only entrypoint to the + unchanged embedded MSL and calls `mix_checksum_word` on four words. For fixed visits, modulo 2^32: + `a=sum(value)+sum(domain)` and + `b=0x9e3779b1*sum(value)+0x85ebca77*sum(word_index)+0x7feb352d*sum(domain)`. + Thus both lanes depend on the same content sum; they are not an independent 64-bit content proof. The original + `11,23,37,43`, its reversal, and `12,22,37,43` each produce `(49596,3847175070)` for domains `12345+17*i`. + The one-bit control `10,23,37,43` produces `(49595,1192739309)`. This shared-helper result applies to its use by + all four profiles with the same visit/domain contributions; additional lookup, write, and lifecycle checks can + still reject a complete task. Content terms and separately added address terms do not establish arbitrary + content-to-address binding. +- **T — selected real table mutations.** CPU decode/prefill tests swap two table IDs while preserving ID multiplicity + and using identical initial physical block contents. The actual ASM K/V checksums differ from the independent + correct-table oracle even when byte/span counts match. Metal decode/prefill tests swap equal-read-multiplicity + nonterminal IDs after generating the canonical expected summary and before uploading the actual GPU table. + The real workload dispatch returns a checksum mismatch. This does not prove detection of every table permutation, + lookup omission, or modulo collision. +- **A/P — append versus sampled prefill state.** CPU contiguous decode checks all bytes of both final append records, + across layers/batches after timer stop and worker joins. Its K/V first/last-byte corruptions retain a valid timed + checksum but fail post-validation. CPU paged decode also has real append-corruption coverage. CPU prefill tests + corrupt checked canonical-word/boundary samples after the real kernel; Metal prefill checks representative/boundary + locations per layer/batch sequence. Neither prefill validator scans every prompt record or every intermediate + operation. A mutation confined to unsampled prompt bytes after the timed checksum has been collected is a known blind spot + of the cold sampler on both backends; it cannot change that already collected checksum. Metal's forced `kv_write` result test is an acceptance hook, not an actual shader write-corruption test; + hence E rather than O for that fault in the matrix. +- **C — bounded canaries.** CPU paged decode/prefill tests alter real final-state/padding memory after the kernel. + Metal paged tests use a real blit write into K padding before post-validation. CPU paged decode weights-only checks its append slots and padding. CPU paged prefill weights-only checks + structure and padding only, not unexpected writes to valid prompt contents. Metal weights-only does not + evaluate KV-write or padding validation. Tail/guard-page, layout, and ABI tests additionally qualify the intended + implementation; none is an all-memory post-execution scan. + +The maintained examples are in [CPU kernel tests](../tests/test_llm_memory_kernels.cpp), notably +`AllCpuWeightSpansExposeBoundedParityCollisions`, `AllCpuWeightSpansDetectBoundedReadAndWorkUnitMutations`, +`PagedWrongSameMultiplicityBlockTableDoesNotMatchOracle`, and +`PagedPrefillWrongSameMultiplicityBlockTableDoesNotMatchOracle`; in +[CPU executor tests](../tests/test_llm_memory_executor.cpp) for append/prefill/padding corruptions; and in +[Metal helper tests](../tests/test_llm_metal_checksum.mm) for +`SharedAffineLanesExposeBoundedContentCollisions`. The +[Metal backend tests](../tests/test_llm_metal_backend.cpp) contain +`DecodePagedPermutationAndPaddingHooksAreDetectedIntegration` and +`PrefillPagedPermutationAndPaddingHooksAreDetectedIntegration`. Their real table/blit mutations differ from +`force_timed_checksum_mismatch` (host readback altered after execution) and `force_kv_write_validation_mismatch` +(forced validation boolean), which test result handling. Pure independent checksum-oracle goldens test arithmetic; +they are not GPU execution evidence. Algorithms and their existing identities remain unchanged by this fault model. + ## Timing boundary and backend lifecycle A CPU scenario task follows this boundary: @@ -1096,7 +1174,7 @@ Correctness gates cover: exact payload and vector tails, final-ordinal K/V-write validation, and source-hash-bound full-write → tile-K → tile-V loop-order audit; - Metal paged-prefill `N + 2*M` lookup accounting, cyclic owner-ordinal scheduling, partial terminal visits, - per-threadgroup accounted-cost evidence, multi-segment table/K/V addressing, final-ordinal full-prompt write and + per-threadgroup accounted-cost evidence, multi-segment table/K/V addressing, final-ordinal representative/boundary prompt-write samples and padding-canary validation, wrong-table detection, and source-hash-bound loop-order audit; - synchronized worker timing, startup cancellation, QoS evidence, timer/error containment, and AAPCS64 preservation; - scenario-specific calibration, frozen plans, cyclic balance, aggregate population, interruption, and checkpoint diff --git a/documents/MANUAL.md b/documents/MANUAL.md index ff1b0aa..a5b8e23 100644 --- a/documents/MANUAL.md +++ b/documents/MANUAL.md @@ -275,10 +275,21 @@ against the final task-local ordinal `T-1`, after timer stop and all worker join `decode-post-validation-failed`: the invalid attempt is retained with null rates and excluded from aggregates. The timed workload and checksum algorithm are unchanged. KV-write evidence applies only to KV-bearing scenarios; paged/prefill evidence retains the combined final-state check and its existing -padding/sampling limits. CPU paged weights-only unexpected-write/padding protection remains active. +padding/sampling limits. CPU paged decode weights-only checks its append slots and padding; CPU paged prefill weights-only checks padding, +not valid prompt contents. See the [CPU final-state evidence contract](API.md#result-schemas-and-completion) for field and null semantics. +Treat a matching LLM checksum as a bounded consistency check. All four CPU weight paths have demonstrated +within-span parity collisions, and the shared Metal helper has two affine lanes that depend on the same content sum. +The Metal helper test is not a full-backend acceptance test. Separate final-state checks can reject a task whose +checksum still matches; such a task keeps null rates and stays outside aggregates. Prefill checks selected final-state +locations, not every prompt byte. CPU paged decode weights-only checks append slots and padding, while paged prefill weights-only checks padding only. Metal +weights-only does not perform those KV checks. Source hashes and generated event lists describe a build, not a GPU +runtime trace. Consult the [profile fault matrix](LLM_MEMORY_PROFILE_WHITEPAPER.md#checksum-fault-model-and-evidence-limits) +for detected mutations, conditional coverage, and blind spots; use the [API contract](API.md#result-schemas-and-completion) +for status/null/acceptance rules. + For prefill, prompt length `P` and query-tile length `Q` are explicit. Define `C = ceil(P/Q)`, tile ends `e_j = min((j+1)*Q, P)`, and `S(P,Q) = sum(e_j)`. One full-prompt work unit has: @@ -291,7 +302,8 @@ Prefill payload is `W`, `B*(P+S(P,Q))*K`, or `W+B*(P+S(P,Q))*K` for weights-only its prompt tokens in ascending order, K then V for each token, before that owner's reads; no global worker barrier is implied. Each increasing tile then reads the complete owned K prefix followed by the complete owned V prefix. `Q=P` scans one full prefix; `Q=1` gives `S=triangular(P)`. Reported causal-pair and logical attention/FMA counts are audit -metadata, not executed compute. The timed checksum covers every tile-read visit. For a KV-bearing CPU task, excluded +metadata, not executed compute. The intended checksum accumulation includes each tile-read visit; equality alone +does not establish every executed visit. For a KV-bearing CPU task, excluded post-validation checks each owner's deterministic first/middle/last canonical-word samples, including bytes clipped to owner boundaries, against the final operation ordinal `T-1`. For a KV-bearing Metal prefill task, it checks representative/boundary byte locations per layer/batch sequence in both K and V against that ordinal; paged Metal @@ -705,8 +717,8 @@ middle, and trailing items. without lookup duplication. Grid evidence reports exact per-threadgroup accounted-byte minimum, maximum, imbalance, and the underlying vector with `cost_unit=actual-threadgroup-cost` without claiming weighted balance. Every Metal `weights_only` task uses a weight-vector grid stride and reports the same exact cost unit. Contiguous KV-bearing grids - publish no threadgroup-cost evidence. Partial prefix visits stop at the exact tile or prompt end, and complete status - requires full-prompt write validation plus applicable padding-canary validity. In contiguous + publish no threadgroup-cost evidence. Partial prefix visits stop at the exact tile or prompt end. KV-bearing tasks + require final-operation representative/boundary prompt-write sample validation plus applicable padding-canary validity. In contiguous prefill, each lane writes its disjoint slices across all prompt K/V records before reading those slices in the tiled prefixes; it scans the complete K prefix before the V prefix at every tile end. A weight-bearing scenario performs one weight pass per `prefill_operation`. No @@ -2765,8 +2777,10 @@ remain unsuitable for balanced comparative conclusions. High CV, cache-dominant power state do not turn a measurement into zero; they remain explicit quality warnings. For paged results, also inspect the permutation identity/hash, physical K/V lengths, layout padding, table protection, -lookup/metadata counts, phase-specific K/V-write post-validation, and padding-canary result. Decode validates the -current-token write; prefill validates the full-prompt write against the final operation ordinal. A different block +lookup/metadata counts, phase-specific K/V-write post-validation, and padding-canary result. For KV-bearing tasks, +decode validates the current-token write; prefill checks representative canonical-word/boundary samples against the +final operation ordinal, rather than all prompt contents. CPU paged decode weights-only checks append slots and padding, +CPU paged prefill weights-only checks padding only, and Metal weights-only performs neither KV-write nor padding checks. A different block size or permutation is a different comparison cohort even when logical model geometry is unchanged. If an otherwise identical command fails memory admission only after a non-empty `--output` target is added, include the From 151d36331b952ec3766ce689d9ca71848d15ae58 Mon Sep 17 00:00:00 2001 From: timoheimonen Date: Sat, 5 Sep 2026 12:53:58 +0300 Subject: [PATCH 05/16] Classify LLM runtime-dependent tests as integrations --- tests/test_llm_memory_config.cpp | 2 +- tests/test_llm_memory_executor.cpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_llm_memory_config.cpp b/tests/test_llm_memory_config.cpp index 92d47b8..2fae6a2 100644 --- a/tests/test_llm_memory_config.cpp +++ b/tests/test_llm_memory_config.cpp @@ -1538,7 +1538,7 @@ TEST(LlmMemoryConfigTest, ConvertsWeightMiBWithCheckedArithmetic) { } TEST(LlmMemoryConfigTest, - TimerSetupFailureKeepsStdoutEmptyAndPreservesLegacyDiagnosticBoundary) { + TimerSetupFailureKeepsStdoutEmptyAndPreservesLegacyDiagnosticBoundaryIntegration) { std::vector arguments = valid_llm_arguments(); arguments.insert(arguments.end(), {"--iterations", "1", "--count", "1", "--output", "-"}); diff --git a/tests/test_llm_memory_executor.cpp b/tests/test_llm_memory_executor.cpp index c7ba7ad..fee3ba8 100644 --- a/tests/test_llm_memory_executor.cpp +++ b/tests/test_llm_memory_executor.cpp @@ -1814,7 +1814,7 @@ TEST_F(LlmMemoryExecutorTest, } TEST_F(LlmMemoryExecutorTest, - PagedProductionDispatchPassesAllScenariosAndExactByteTails) { + PagedProductionDispatchPassesAllScenariosAndExactByteTailsIntegration) { for (size_t tail_bytes : {31u, 32u, 33u}) { SCOPED_TRACE(::testing::Message() << "tail bytes " << tail_bytes); const LlmMemoryWorkPlan plan = build_executor_ready_plan( @@ -1884,7 +1884,7 @@ TEST_F(LlmMemoryExecutorTest, } TEST_F(LlmMemoryExecutorTest, - PagedAppendAndPaddingCorruptionFailExcludedPostValidation) { + PagedAppendAndPaddingCorruptionFailExcludedPostValidationIntegration) { for (PagedCorruption corruption : {PagedCorruption::Append, PagedCorruption::Padding}) { SCOPED_TRACE(corruption == PagedCorruption::Append ? "append" @@ -1920,7 +1920,7 @@ TEST_F(LlmMemoryExecutorTest, } TEST_F(LlmMemoryExecutorTest, - PagedWeightsOnlyRejectsUnexpectedValidKvWrite) { + PagedWeightsOnlyRejectsUnexpectedValidKvWriteIntegration) { const LlmMemoryWorkPlan plan = build_executor_ready_plan(paged_geometry(5, 4), 1); ASSERT_TRUE(plan.valid) << plan.reason_code; From c4a9ec4451baa7553f462b54eedc763b4aad2a55 Mon Sep 17 00:00:00 2001 From: timoheimonen Date: Sat, 5 Sep 2026 16:22:07 +0300 Subject: [PATCH 06/16] Retain independent LLM cold validation observations --- src/llm_memory/llm_backend.h | 1 + src/llm_memory/llm_executor.cpp | 90 ++++++++++++++++++--------- src/llm_memory/llm_executor.h | 2 + src/llm_memory/llm_metal_backend.h | 9 +++ src/llm_memory/llm_metal_backend.mm | 43 ++++++++++++- src/llm_memory/llm_runner.cpp | 2 + src/llm_memory/llm_runner.h | 1 + src/llm_memory/llm_validation.h | 66 ++++++++++++++++++++ tests/test_llm_memory_executor.cpp | 94 +++++++++++++++++++++++++++++ tests/test_llm_memory_runner.cpp | 18 ++++++ tests/test_llm_metal_backend.cpp | 57 +++++++++++++++++ 11 files changed, 353 insertions(+), 30 deletions(-) create mode 100644 src/llm_memory/llm_validation.h diff --git a/src/llm_memory/llm_backend.h b/src/llm_memory/llm_backend.h index 032b514..9f96b00 100644 --- a/src/llm_memory/llm_backend.h +++ b/src/llm_memory/llm_backend.h @@ -224,6 +224,7 @@ struct LlmMetalDualMod32Checksum { /** Complete per-task Metal timing, dispatch, checksum, and validation evidence. */ struct LlmMetalTaskEvidence { + LlmColdChecks cold_checks; bool timed_pipeline_available = false; std::string pipeline_label; size_t pipeline_thread_execution_width = 0; diff --git a/src/llm_memory/llm_executor.cpp b/src/llm_memory/llm_executor.cpp index 7e5af8c..7093337 100644 --- a/src/llm_memory/llm_executor.cpp +++ b/src/llm_memory/llm_executor.cpp @@ -3022,11 +3022,13 @@ LlmExpectedChecksumResult calculate_paged_expected_checksums( */ bool validate_decode_post_execution(const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan& scenario_plan, - const LlmExecutionResources& resources) noexcept { + const LlmExecutionResources& resources, LlmColdChecks& checks) noexcept { + constexpr auto reason = LlmExecutorReason::DECODE_POST_VALIDATION_FAILED; const auto& geometry = model_plan.geometry; if (!geometry.decode.has_value() || scenario_plan.work_units == 0 || !resources.buffers.complete()) { - return false; + return resolve_llm_cold_check(checks, 0, false, reason); } + resolve_llm_cold_check(checks, 0, true); const auto* k = static_cast(resources.buffers.k.get()); const auto* v = static_cast(resources.buffers.v.get()); const size_t record_bytes = geometry.k_or_v_record_bytes_per_layer; @@ -3040,23 +3042,25 @@ bool validate_decode_post_execution(const LlmMemoryWorkPlan& model_plan, layer, batch, byte, LlmChecksumComponent::K) || v[offset + byte] != append_byte(scenario_plan.scenario_seed, scenario_plan.work_units - 1, layer, batch, byte, LlmChecksumComponent::V)) { - return false; + return resolve_llm_cold_check(checks, 1, false, reason); } } } } + for (size_t slot = 0; slot < checks.size(); ++slot) resolve_llm_cold_check(checks, slot, true); return true; } bool validate_paged_post_execution( const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan& scenario_plan, - const LlmExecutionResources& resources) noexcept { + const LlmExecutionResources& resources, LlmColdChecks& checks) noexcept { + constexpr auto reason = LlmExecutorReason::PAGED_POST_VALIDATION_FAILED; const LlmCpuExecutionPlan* const cpu_plan = get_llm_cpu_execution_plan(model_plan); if (cpu_plan == nullptr || !cpu_plan->paged.has_value() || resources.block_table == nullptr || !resources.buffers.complete()) { - return false; + return resolve_llm_cold_check(checks, 0, false, reason); } const LlmKvLayoutPlan& layout = cpu_plan->paged->layout; const uint8_t* const k = @@ -3073,8 +3077,10 @@ bool validate_paged_post_execution( (layout.blocks_per_sequence - 1); const uint32_t physical_id = resources.block_table[table_index]; if (physical_id >= layout.physical_blocks_per_layer) { - return false; + return resolve_llm_cold_check(checks, 0, false, reason); } + const bool final_row = layer + 1 == layout.layer_count && batch + 1 == layout.batch_size; + if (final_row) resolve_llm_cold_check(checks, 0, true); const size_t global_block = layer * layout.physical_blocks_per_layer + physical_id; const size_t block_offset = global_block * layout.block_bytes; @@ -3101,9 +3107,10 @@ bool validate_paged_post_execution( model_plan.v_buffer_seed, layer, physical_id, block_byte_offset, 1)); if (k[byte_offset] != expected_k || v[byte_offset] != expected_v) { - return false; + return resolve_llm_cold_check(checks, 1, false, reason); } } + if (final_row) resolve_llm_cold_check(checks, 1, true); for (size_t padding_byte = layout.last_block_valid_bytes; padding_byte < layout.block_bytes; ++padding_byte) { const uint8_t expected_k = static_cast( @@ -3114,18 +3121,20 @@ bool validate_paged_post_execution( physical_id, padding_byte, 1)); if (k[block_offset + padding_byte] != expected_k || v[block_offset + padding_byte] != expected_v) { - return false; + return resolve_llm_cold_check(checks, 2, false, reason); } } } } + for (size_t slot = 0; slot < checks.size(); ++slot) resolve_llm_cold_check(checks, slot, true); return true; } bool validate_paged_prefill_post_execution( const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan& scenario_plan, - const LlmExecutionResources& resources) noexcept { + const LlmExecutionResources& resources, LlmColdChecks& checks) noexcept { + constexpr auto reason = LlmExecutorReason::PREFILL_POST_VALIDATION_FAILED; const LlmCpuExecutionPlan* const cpu_plan = get_llm_cpu_execution_plan(model_plan); if (model_plan.phase != LlmPhase::Prefill || @@ -3134,7 +3143,7 @@ bool validate_paged_prefill_post_execution( !cpu_plan->prefill.has_value() || !cpu_plan->paged.has_value() || resources.block_table == nullptr || !resources.buffers.complete() || scenario_plan.work_units == 0) { - return false; + return resolve_llm_cold_check(checks, 0, false, reason); } const LlmKvLayoutPlan& layout = cpu_plan->paged->layout; const uint8_t* const k = @@ -3153,11 +3162,11 @@ bool validate_paged_prefill_post_execution( logical_block < layout.blocks_per_sequence; ++logical_block) { const size_t table_index = table_row + logical_block; if (table_index >= resources.block_table_entries) { - return false; + return resolve_llm_cold_check(checks, 0, false, reason); } const uint32_t physical_id = resources.block_table[table_index]; if (physical_id >= layout.physical_blocks_per_layer) { - return false; + return resolve_llm_cold_check(checks, 0, false, reason); } size_t global_block = 0; size_t physical_offset = 0; @@ -3171,27 +3180,31 @@ bool validate_paged_prefill_post_execution( global_block, layout.block_bytes, physical_offset) || !NumericUtils::checked_multiply( logical_block, layout.block_bytes, logical_offset)) { - return false; + return resolve_llm_cold_check(checks, 0, false, reason); } const size_t valid_bytes = logical_block + 1 == layout.blocks_per_sequence ? layout.last_block_valid_bytes : layout.block_bytes; if (valid_bytes == 0) { - return false; + return resolve_llm_cold_check(checks, 0, false, reason); } + const bool final_block = layer + 1 == layout.layer_count && + batch + 1 == layout.batch_size && logical_block + 1 == layout.blocks_per_sequence; + if (final_block && !wrote_kv) resolve_llm_cold_check(checks, 0, true); if (wrote_kv) { size_t logical_end = 0; if (!NumericUtils::checked_add( logical_offset, valid_bytes, logical_end)) { - return false; + return resolve_llm_cold_check(checks, 0, false, reason); } const size_t midpoint = logical_offset + valid_bytes / 2; const std::array representative_words = { logical_offset / sizeof(uint64_t), midpoint / sizeof(uint64_t), (logical_end - 1) / sizeof(uint64_t)}; - for (size_t logical_word : representative_words) { + for (size_t sample = 0; sample < representative_words.size(); ++sample) { + const size_t logical_word = representative_words[sample]; const size_t word_start = logical_word * sizeof(uint64_t); const size_t sample_start = std::max(logical_offset, word_start); size_t sample_end = 0; @@ -3199,8 +3212,10 @@ bool validate_paged_prefill_post_execution( sample_start, sizeof(uint64_t) - sample_start % sizeof(uint64_t), sample_end)) { - return false; + return resolve_llm_cold_check(checks, 0, false, reason); } + if (final_block && sample + 1 == representative_words.size()) + resolve_llm_cold_check(checks, 0, true); sample_end = std::min(logical_end, sample_end); for (size_t logical_byte = sample_start; logical_byte < sample_end; ++logical_byte) { @@ -3214,11 +3229,12 @@ bool validate_paged_prefill_post_execution( LlmPrefillKvDomain::V, logical_byte); if (k[mapping_offset] != expected_k || v[mapping_offset] != expected_v) { - return false; + return resolve_llm_cold_check(checks, 1, false, reason); } } } } + if (final_block) resolve_llm_cold_check(checks, 1, true); for (size_t padding_byte = valid_bytes; padding_byte < layout.block_bytes; ++padding_byte) { const uint8_t expected_k = static_cast( @@ -3231,19 +3247,21 @@ bool validate_paged_prefill_post_execution( padding_byte, 1)); if (k[physical_offset + padding_byte] != expected_k || v[physical_offset + padding_byte] != expected_v) { - return false; + return resolve_llm_cold_check(checks, 2, false, reason); } } } } } + for (size_t slot = 0; slot < checks.size(); ++slot) resolve_llm_cold_check(checks, slot, true); return true; } bool validate_prefill_post_execution( const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan& scenario_plan, - const LlmExecutionResources& resources) noexcept { + const LlmExecutionResources& resources, LlmColdChecks& checks) noexcept { + constexpr auto reason = LlmExecutorReason::PREFILL_POST_VALIDATION_FAILED; const LlmCpuExecutionPlan* const cpu_plan = get_llm_cpu_execution_plan(model_plan); if (model_plan.phase != LlmPhase::Prefill || @@ -3251,10 +3269,11 @@ bool validate_prefill_post_execution( !model_plan.geometry.prefill.has_value() || cpu_plan == nullptr || !cpu_plan->prefill.has_value() || !resources.buffers.complete() || scenario_plan.work_units == 0) { - return false; + return resolve_llm_cold_check(checks, 0, false, reason); } if ((llm_scenario_flags(scenario_plan.scenario) & kLlmScenarioFlagKv) == 0) { + resolve_llm_cold_check(checks, 0, true); return true; } const uint8_t* const k = @@ -3264,7 +3283,7 @@ bool validate_prefill_post_execution( const size_t final_operation = scenario_plan.work_units - 1; const size_t scenario_index = static_cast(scenario_plan.scenario); if (scenario_index >= kLlmScenarioCount) { - return false; + return resolve_llm_cold_check(checks, 0, false, reason); } const size_t per_scenario = cpu_plan->prefill->sequence_descriptors_per_scenario_per_worker; @@ -3280,7 +3299,7 @@ bool validate_prefill_post_execution( !NumericUtils::checked_multiply( row, model_plan.geometry.k_or_v_sequence_visible_bytes, row_offset)) { - return false; + return resolve_llm_cold_check(checks, 0, false, reason); } for (size_t worker = 0; worker < cpu_plan->effective_workers; ++worker) { @@ -3307,8 +3326,11 @@ bool validate_prefill_post_execution( owned_bytes == 0 || end_byte > model_plan.geometry.k_or_v_sequence_visible_bytes) { - return false; + return resolve_llm_cold_check(checks, 0, false, reason); } + if (layer + 1 == model_plan.geometry.layer_count && + batch + 1 == model_plan.geometry.batch_size && worker + 1 == cpu_plan->effective_workers) + resolve_llm_cold_check(checks, 0, true); const size_t midpoint = first_byte + owned_bytes / 2; const std::array representative_words = { first_byte / sizeof(uint64_t), @@ -3330,13 +3352,14 @@ bool validate_prefill_post_execution( scenario_plan.scenario_seed, final_operation, layer, batch, LlmPrefillKvDomain::V, logical_byte)) { - return false; + return resolve_llm_cold_check(checks, 1, false, reason); } } } } } } + for (size_t slot = 0; slot < checks.size(); ++slot) resolve_llm_cold_check(checks, slot, true); return true; } @@ -4106,6 +4129,15 @@ LlmExecutorResult execute_llm_scenario(const LlmMemoryWorkPlan& model_plan, cons (llm_scenario_flags(scenario_plan.scenario) & kLlmScenarioFlagKv) != 0; const LlmCpuExecutionPlan* const cpu_plan = get_llm_cpu_execution_plan(model_plan); + const bool paged_checks = model_plan.kv_layout == LlmKvLayout::Paged; + const bool prefill_checks = model_plan.phase == LlmPhase::Prefill; + const bool padding_checks = paged_checks && cpu_plan != nullptr && cpu_plan->paged.has_value() && + cpu_plan->paged->layout.last_block_valid_bytes < cpu_plan->paged->layout.block_bytes; + result.cold_checks = make_llm_cold_checks( + paged_checks || prefill_checks || result.kv_write_validation_applicable, + prefill_checks ? LlmColdCheckKind::KvPrefillFinalSamples : + result.kv_write_validation_applicable ? LlmColdCheckKind::KvAppendFinal : LlmColdCheckKind::KvAppendUnchanged, + result.kv_write_validation_applicable || (paged_checks && !prefill_checks), padding_checks); result.requested_workers = cpu_plan == nullptr ? 0 : cpu_plan->effective_workers; try { @@ -4333,14 +4365,14 @@ LlmExecutorResult execute_llm_scenario(const LlmMemoryWorkPlan& model_plan, cons model_plan.phase == LlmPhase::Prefill && model_plan.kv_layout == LlmKvLayout::Paged ? validate_paged_prefill_post_execution( - model_plan, scenario_plan, resources) + model_plan, scenario_plan, resources, result.cold_checks) : model_plan.phase == LlmPhase::Prefill ? validate_prefill_post_execution(model_plan, scenario_plan, - resources) + resources, result.cold_checks) : model_plan.kv_layout == LlmKvLayout::Paged - ? validate_paged_post_execution(model_plan, scenario_plan, resources) + ? validate_paged_post_execution(model_plan, scenario_plan, resources, result.cold_checks) : !result.kv_write_validation_applicable || - validate_decode_post_execution(model_plan, scenario_plan, resources); + validate_decode_post_execution(model_plan, scenario_plan, resources, result.cold_checks); result.kv_write_validation_evaluated = result.kv_write_validation_applicable; result.kv_write_validation_valid = result.kv_write_validation_evaluated && result.post_validation_valid; diff --git a/src/llm_memory/llm_executor.h b/src/llm_memory/llm_executor.h index 2dbfc4d..24dc79b 100644 --- a/src/llm_memory/llm_executor.h +++ b/src/llm_memory/llm_executor.h @@ -21,6 +21,7 @@ #ifndef LLM_EXECUTOR_H #define LLM_EXECUTOR_H +#include "llm_memory/llm_validation.h" #include #include #include @@ -339,6 +340,7 @@ struct LlmExecutorResult { bool checksum_valid = false; bool post_validation_evaluated = false; bool post_validation_valid = false; + LlmColdChecks cold_checks; ///< Independent observations from the existing cold walk. bool kv_write_validation_applicable = false; ///< KV-writing scenario; independent of pipeline completion. bool kv_write_validation_evaluated = false; ///< Phase/layout write check ran after stop and join. bool kv_write_validation_valid = false; ///< Combined phase/layout final-state result, including paged padding. diff --git a/src/llm_memory/llm_metal_backend.h b/src/llm_memory/llm_metal_backend.h index 6e15796..bc62caf 100644 --- a/src/llm_memory/llm_metal_backend.h +++ b/src/llm_memory/llm_metal_backend.h @@ -42,6 +42,15 @@ #include "llm_memory/llm_backend.h" +/** Interpret only completed cold-command shader bits. Incomplete commands leave + * applicable sets unresolved. Invalid parameters prevent clear content bits from + * proving completion; observed mismatches still resolve false. Pure, reentrant, + * allocation-free; flags must come from the current command, not a test injection. + */ +LlmColdChecks interpret_llm_metal_cold_checks(LlmPhase phase, bool writes_kv, + bool padding, bool completed, + uint32_t flags) noexcept; + /** Canonical CPU mirror of the MSL 2.3 foundation parameter block. */ struct alignas(8) LlmMetalFoundationParams { uint64_t byte_count = 0; diff --git a/src/llm_memory/llm_metal_backend.mm b/src/llm_memory/llm_metal_backend.mm index 4c4cef1..54fc3f0 100644 --- a/src/llm_memory/llm_metal_backend.mm +++ b/src/llm_memory/llm_metal_backend.mm @@ -50,6 +50,28 @@ #include #include +LlmColdChecks interpret_llm_metal_cold_checks(LlmPhase phase, bool writes_kv, + bool padding, bool completed, + uint32_t flags) noexcept { + LlmColdChecks checks = make_llm_cold_checks(writes_kv, + phase == LlmPhase::Prefill ? LlmColdCheckKind::KvPrefillFinalSamples : LlmColdCheckKind::KvAppendFinal, + writes_kv, writes_kv && padding); + if (!completed || !writes_kv) return checks; + const bool structure_valid = (flags & LlmMetalKernelContract::kValidationInvalidParametersBit) == 0; + resolve_llm_cold_check(checks, 0, structure_valid, + structure_valid ? "valid" : "post-validation-invalid-parameters"); + const uint32_t bits[] = {LlmMetalKernelContract::kKvWriteValidationMismatchBit, + LlmMetalKernelContract::kPaddingCanaryMismatchBit}; + const char* reasons[] = {LlmBackendReason::KV_WRITE_VALIDATION_MISMATCH, + LlmBackendReason::PADDING_CANARY_MISMATCH}; + for (size_t index = 0; index < 2; ++index) { + const bool mismatch = (flags & bits[index]) != 0; + if (mismatch || structure_valid) + resolve_llm_cold_check(checks, index + 1, !mismatch, mismatch ? reasons[index] : "valid"); + } + return checks; +} + namespace { constexpr size_t kFoundationPipelineCount = 6; @@ -3601,6 +3623,17 @@ LlmBackendLifecycleResult prepare_resources(const LlmMemoryWorkPlan& model_plan) } } + /** Initialize unresolved applicability without allocation, including exception exits. */ + LlmColdChecks pending_cold_checks(const LlmMemoryWorkPlan& model_plan, + const LlmScenarioWorkPlan& scenario_plan) const noexcept { + const bool padding = model_plan.kv_layout == LlmKvLayout::Paged && + resolved_execution_plan_.resources.paged_layout.has_value() && + resolved_execution_plan_.resources.paged_layout->last_block_valid_bytes < + resolved_execution_plan_.resources.paged_layout->block_bytes; + return interpret_llm_metal_cold_checks(model_plan.phase, + scenario_plan.scenario != LlmScenario::WeightsOnly, padding, false, 0); + } + LlmTaskExecutionResult execute_task(const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan& scenario_plan, const LlmRunnerTaskContext& context) noexcept override { @autoreleasepool { @@ -3612,6 +3645,7 @@ LlmTaskExecutionResult execute_task(const LlmMemoryWorkPlan& model_plan, const L LlmBackendReason::TIMED_COMMAND_BUFFER_ERROR}; result.identity = metal_task_identity(model_plan, scenario_plan, context); LlmMetalTaskEvidence metal; + metal.cold_checks = pending_cold_checks(model_plan, scenario_plan); metal.error = internal_error(exception.what()); result.backend_evidence = std::move(metal); return result; @@ -3619,7 +3653,9 @@ LlmTaskExecutionResult execute_task(const LlmMemoryWorkPlan& model_plan, const L LlmTaskExecutionResult result{LlmTaskExecutionStatus::Failed, LlmBackendReason::TIMED_COMMAND_BUFFER_ERROR}; result.identity = metal_task_identity(model_plan, scenario_plan, context); - result.backend_evidence = LlmMetalTaskEvidence{}; + LlmMetalTaskEvidence metal; + metal.cold_checks = pending_cold_checks(model_plan, scenario_plan); + result.backend_evidence = std::move(metal); return result; } } @catch (NSException* exception) { @@ -3627,6 +3663,7 @@ LlmTaskExecutionResult execute_task(const LlmMemoryWorkPlan& model_plan, const L LlmBackendReason::TIMED_COMMAND_BUFFER_ERROR}; result.identity = metal_task_identity(model_plan, scenario_plan, context); LlmMetalTaskEvidence metal; + metal.cold_checks = pending_cold_checks(model_plan, scenario_plan); metal.error = internal_error(ns_string(exception.reason)); result.backend_evidence = std::move(metal); return result; @@ -6064,6 +6101,9 @@ bool run_task_post_validation(id argument, id status, resolved_execution_plan_.resources.paged_layout.has_value() && resolved_execution_plan_.resources.paged_layout->last_block_valid_bytes < resolved_execution_plan_.resources.paged_layout->block_bytes; + // The acceptance-injection hook is deliberately excluded from observed bits. + task.cold_checks = interpret_llm_metal_cold_checks(active_phase_, validate_kv_writes, + padding_applicable, true, flags); task.padding_canary_applicable = padding_applicable; task.padding_canary_evaluated = padding_applicable; task.padding_canary_valid = @@ -6083,6 +6123,7 @@ LlmTaskExecutionResult execute_task_impl(const LlmMemoryWorkPlan& model_plan, result.identity = metal_task_identity(model_plan, scenario_plan, context); result.completion.planned_work_units = scenario_plan.work_units; LlmMetalTaskEvidence task; + task.cold_checks = pending_cold_checks(model_plan, scenario_plan); task.checksum_algorithm_version = model_plan.component_identities.checksum_pattern_version; const bool supported_profile = diff --git a/src/llm_memory/llm_runner.cpp b/src/llm_memory/llm_runner.cpp index 989e45c..2c15dba 100644 --- a/src/llm_memory/llm_runner.cpp +++ b/src/llm_memory/llm_runner.cpp @@ -415,6 +415,7 @@ LlmTaskExecutionEvidence compact_execution( cpu->expected_checksums.size() == cpu->requested_workers && cpu->actual_checksums.size() == cpu->requested_workers; evidence.cpu_evidence_available = true; + evidence.cpu_cold_checks = cpu->cold_checks; evidence.requested_workers = cpu->requested_workers; evidence.created_workers = cpu->created_workers; evidence.completed_workers = cpu->completed_workers; @@ -1442,6 +1443,7 @@ void retain_task_evidence(LlmTaskExecutionResult& retained, destination.post_validation_evaluated = input.post_validation_evaluated; destination.post_validation_valid = input.post_validation_valid; + destination.cold_checks = input.cold_checks; destination.kv_write_validation_applicable = input.kv_write_validation_applicable; destination.kv_write_validation_evaluated = input.kv_write_validation_evaluated; destination.kv_write_validation_valid = input.kv_write_validation_valid; diff --git a/src/llm_memory/llm_runner.h b/src/llm_memory/llm_runner.h index 31d8810..d8953c4 100644 --- a/src/llm_memory/llm_runner.h +++ b/src/llm_memory/llm_runner.h @@ -64,6 +64,7 @@ enum class LlmCheckpointKind : uint8_t { /** Compact excluded-task evidence without retained per-worker vectors. */ struct LlmTaskExecutionEvidence { + LlmColdChecks cpu_cold_checks; ///< Compact CPU copy; Metal owns its own array. bool available = false; ///< Complete generic task evidence was retained. bool valid = false; std::string_view reason_code = LlmRunnerReason::NOT_STARTED; diff --git a/src/llm_memory/llm_validation.h b/src/llm_memory/llm_validation.h new file mode 100644 index 0000000..d2fb6cc --- /dev/null +++ b/src/llm_memory/llm_validation.h @@ -0,0 +1,66 @@ +// Copyright 2026 Timo Heimonen +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +/** @file llm_validation.h + * @brief Fixed, allocation-free observations from existing cold validation walks. + */ +#ifndef LLM_VALIDATION_H +#define LLM_VALIDATION_H + +#include +#include +#include + +/** Bounded coverage kinds; these do not certify untimed or timed access traces. */ +enum class LlmColdCheckKind { + PostValidationStructure, + KvAppendFinal, + KvPrefillFinalSamples, + KvAppendUnchanged, + KvPaddingCanary +}; + +/** A verdict is meaningful only when applicable and evaluated. Reasons reference + * static storage. Partial walks without a mismatch remain unresolved. */ +struct LlmColdCheckResult { + LlmColdCheckKind kind = LlmColdCheckKind::PostValidationStructure; + bool applicable = false; + bool evaluated = false; + bool valid = false; + std::string_view reason_code = "not-evaluated"; +}; + +using LlmColdChecks = std::array; + +/** Construct structure/write/padding slots before any execution can fail. */ +inline LlmColdChecks make_llm_cold_checks(bool structure, LlmColdCheckKind write_kind, + bool write, bool padding) noexcept { + return {{{LlmColdCheckKind::PostValidationStructure, structure}, + {write_kind, write}, {LlmColdCheckKind::KvPaddingCanary, padding}}}; +} + +/** Resolve one observed set, leaving other sets unchanged. Requires slot < 3 and + * a reason with static storage lifetime. No allocation or throw. */ +inline bool resolve_llm_cold_check(LlmColdChecks& checks, size_t slot, bool valid, + std::string_view reason = "valid") noexcept { + if (checks[slot].applicable) { + checks[slot].evaluated = true; + checks[slot].valid = valid; + checks[slot].reason_code = reason; + } + return valid; +} + +#endif diff --git a/tests/test_llm_memory_executor.cpp b/tests/test_llm_memory_executor.cpp index fee3ba8..a65cc5a 100644 --- a/tests/test_llm_memory_executor.cpp +++ b/tests/test_llm_memory_executor.cpp @@ -2393,6 +2393,11 @@ TEST_F(LlmMemoryExecutorTest, ContiguousDecodeAppendCorruptionIntegration) { EXPECT_FALSE(result.valid); EXPECT_TRUE(result.post_validation_evaluated); EXPECT_FALSE(result.post_validation_valid); + EXPECT_TRUE(result.cold_checks[0].evaluated); + EXPECT_TRUE(result.cold_checks[0].valid); + EXPECT_TRUE(result.cold_checks[1].evaluated); + EXPECT_FALSE(result.cold_checks[1].valid); + EXPECT_FALSE(result.cold_checks[2].applicable); EXPECT_TRUE(result.kv_write_validation_applicable); EXPECT_TRUE(result.kv_write_validation_evaluated); EXPECT_FALSE(result.kv_write_validation_valid); @@ -2420,6 +2425,12 @@ TEST_F(LlmMemoryExecutorTest, CpuProfilesWriteValidationControlsIntegration) { ASSERT_TRUE(timer.has_value()); const auto result = execute_llm_scenario(plan, task, resources, *timer); EXPECT_TRUE(result.valid) << result.reason_code; + for (const auto& check : result.cold_checks) { + EXPECT_EQ(check.evaluated, check.applicable); + EXPECT_EQ(check.valid, check.applicable); + } + EXPECT_EQ(result.cold_checks[1].applicable, scenario != LlmScenario::WeightsOnly || + (plan.kv_layout == LlmKvLayout::Paged && plan.phase == LlmPhase::Decode)); const bool writes = scenario != LlmScenario::WeightsOnly; EXPECT_EQ(result.kv_write_validation_applicable, writes); EXPECT_EQ(result.kv_write_validation_evaluated, writes); @@ -2428,3 +2439,86 @@ TEST_F(LlmMemoryExecutorTest, CpuProfilesWriteValidationControlsIntegration) { } } } + +// Single-worker real-kernel wrapper targets canonical rows after the kernel completes. +TEST_F(LlmMemoryExecutorTest, PagedColdCheckCompletionIntegration) { + struct Mutation { uint8_t* byte; }; + const auto mutate = +[](void* opaque, const LlmKernelInvocation& invocation) { + const auto kernel = production_llm_kernel_adapter(); + if (!kernel.invoke(kernel.context, invocation)) return false; + *static_cast(opaque)->byte ^= 0x80; + return true; + }; + for (bool prefill : {false, true}) { + for (bool final_row : {false, true}) { + for (bool padding : {false, true}) { + for (auto scenario : {LlmScenario::KvOnly, LlmScenario::WeightsOnly}) { + if (prefill && !padding && scenario == LlmScenario::WeightsOnly) continue; + // Prior case resources have left scope; recycle the bounded fake mapping arena. + state = {}; + const auto plan = build_executor_ready_plan(prefill ? paged_prefill_geometry() : paged_geometry(5, 4), 1); + ASSERT_TRUE(plan.valid); + LlmExecutionResources resources; + ASSERT_TRUE(prepare_llm_execution_resources(plan, resources).valid); + const auto& layout = get_llm_cpu_execution_plan(plan)->paged->layout; + const size_t layer = final_row ? layout.layer_count - 1 : 0; + const size_t batch = final_row ? layout.batch_size - 1 : 0; + const size_t logical_block = padding || !prefill ? layout.blocks_per_sequence - 1 : 0; + const size_t physical = resources.block_table[batch * layout.blocks_per_sequence + logical_block]; + const size_t offset = (layer * layout.physical_blocks_per_layer + physical) * layout.block_bytes + + (padding ? layout.last_block_valid_bytes : prefill ? 0 : layout.decode_append_offset_in_last_block); + Mutation mutation{static_cast(resources.buffers.k.get()) + offset}; + const auto task = build_llm_scenario_work_plan(plan, scenario, 1, true); + auto timer = HighResTimer::create(); + ASSERT_TRUE(timer.has_value()); + const auto result = execute_llm_scenario(plan, task, resources, *timer, {mutate, &mutation}); + EXPECT_FALSE(result.valid); + EXPECT_TRUE(result.checksum_valid); + const size_t failed = padding ? 2 : 1; + EXPECT_TRUE(result.cold_checks[failed].evaluated); + EXPECT_FALSE(result.cold_checks[failed].valid); + if (padding) { + EXPECT_EQ(result.cold_checks[0].evaluated, final_row); + EXPECT_EQ(result.cold_checks[1].evaluated, final_row && (!prefill || scenario != LlmScenario::WeightsOnly)); + EXPECT_EQ(result.cold_checks[0].valid, final_row); + EXPECT_EQ(result.cold_checks[1].valid, final_row && (!prefill || scenario != LlmScenario::WeightsOnly)); + } else { + EXPECT_FALSE(result.cold_checks[2].evaluated); + EXPECT_EQ(result.cold_checks[0].evaluated, !prefill && final_row); + } + } + } + } + } +} + +TEST_F(LlmMemoryExecutorTest, ColdApplicabilitySurvivesResourceFailure) { + ScopedExecutorTimer timer_calls; + size_t kernel_calls = 0; + const auto count_kernel = +[](void* context, const LlmKernelInvocation&) { + ++*static_cast(context); + return false; + }; + for (const auto& geometry : {LlmGeometryRequest{257, 2, 1, 1, 33, 1, 5, 3}, + paged_geometry(5, 4), paged_geometry(4, 4), + prefill_geometry(), paged_prefill_geometry()}) { + const auto plan = build_executor_ready_plan(geometry, 1); + ASSERT_TRUE(plan.valid); + for (auto scenario : {LlmScenario::WeightsOnly, LlmScenario::KvOnly}) { + const auto task = build_llm_scenario_work_plan(plan, scenario, 1, true); + LlmExecutionResources absent; + auto timer = HighResTimer::create(); + ASSERT_TRUE(timer.has_value()); + const auto result = execute_llm_scenario(plan, task, absent, *timer, {count_kernel, &kernel_calls}); + EXPECT_EQ(kernel_calls, 0u); + EXPECT_EQ(result.reason_code, LlmExecutorReason::INVALID_RESOURCES); + const bool paged = plan.kv_layout == LlmKvLayout::Paged; + const bool prefill = plan.phase == LlmPhase::Prefill; + EXPECT_EQ(result.cold_checks[0].applicable, paged || prefill || scenario != LlmScenario::WeightsOnly); + EXPECT_EQ(result.cold_checks[1].applicable, scenario != LlmScenario::WeightsOnly || (paged && !prefill)); + EXPECT_EQ(result.cold_checks[2].applicable, + paged && plan.geometry.last_block_valid_bytes < plan.geometry.kv_block_bytes); + for (const auto& check : result.cold_checks) EXPECT_FALSE(check.evaluated); + } + } +} diff --git a/tests/test_llm_memory_runner.cpp b/tests/test_llm_memory_runner.cpp index ac6981c..b37c601 100644 --- a/tests/test_llm_memory_runner.cpp +++ b/tests/test_llm_memory_runner.cpp @@ -2675,6 +2675,10 @@ TEST(LlmMemoryRunnerTest, CalibrationFailureRetainsAttemptAndFinalizesMeasuremen size_t, LlmTaskExecutionResult& execution) { if (context.purpose == "pilot") { fail_execution(execution); + LlmCpuTaskEvidence cold; + cold.executor.cold_checks = make_llm_cold_checks(true, LlmColdCheckKind::KvAppendFinal, true, true); + resolve_llm_cold_check(cold.executor.cold_checks, 2, false, LlmExecutorReason::PAGED_POST_VALIDATION_FAILED); + execution.backend_evidence = std::move(cold); } }; std::vector checkpoints; @@ -2687,6 +2691,13 @@ TEST(LlmMemoryRunnerTest, CalibrationFailureRetainsAttemptAndFinalizesMeasuremen EXPECT_FALSE(result.calibration_attempts[0][1].valid); EXPECT_EQ(result.calibration_attempts[0][1].reason_code, LlmBackendReason::RESOURCES_NOT_PREPARED); + const auto& checks = result.calibration_attempts[0][1].execution.cpu_cold_checks; + EXPECT_TRUE(checks[0].applicable); + EXPECT_FALSE(checks[0].evaluated); + EXPECT_FALSE(checks[1].evaluated); + EXPECT_TRUE(checks[2].evaluated); + EXPECT_FALSE(checks[2].valid); + EXPECT_EQ(checks[2].reason_code, LlmExecutorReason::PAGED_POST_VALIDATION_FAILED); EXPECT_EQ(result.counters.attempted_measurements, 0u); EXPECT_EQ(result.counters.terminal_measurements, 6u); for (const LlmMeasurementState& measurement : result.measurements) { @@ -3126,6 +3137,9 @@ TEST(LlmMemoryRunnerTest, evidence.executor.kv_write_validation_applicable = true; evidence.executor.kv_write_validation_evaluated = true; evidence.executor.kv_write_validation_valid = false; + evidence.executor.cold_checks = make_llm_cold_checks(true, LlmColdCheckKind::KvAppendFinal, true, false); + resolve_llm_cold_check(evidence.executor.cold_checks, 0, true); + resolve_llm_cold_check(evidence.executor.cold_checks, 1, false, LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); execution.backend_evidence = std::move(evidence); }; LlmMemoryResult result; @@ -3153,4 +3167,8 @@ TEST(LlmMemoryRunnerTest, EXPECT_TRUE(retained->executor.kv_write_validation_applicable); EXPECT_TRUE(retained->executor.kv_write_validation_evaluated); EXPECT_FALSE(retained->executor.kv_write_validation_valid); + EXPECT_TRUE(retained->executor.cold_checks[0].valid); + EXPECT_TRUE(retained->executor.cold_checks[1].evaluated); + EXPECT_FALSE(retained->executor.cold_checks[1].valid); + EXPECT_EQ(retained->executor.cold_checks[1].reason_code, LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); } diff --git a/tests/test_llm_metal_backend.cpp b/tests/test_llm_metal_backend.cpp index 1a1688c..36de318 100644 --- a/tests/test_llm_metal_backend.cpp +++ b/tests/test_llm_metal_backend.cpp @@ -4370,6 +4370,10 @@ TEST(LlmMetalBackendFailureInjectionIntegrationTest, EXPECT_TRUE(task->padding_canary_evaluated); EXPECT_EQ(task->padding_canary_valid, !cases[index].padding_mismatch); EXPECT_EQ(task->post_validation_valid, !cases[index].padding_mismatch); + EXPECT_TRUE(task->cold_checks[0].valid); + EXPECT_TRUE(task->cold_checks[1].valid); + EXPECT_TRUE(task->cold_checks[2].evaluated); + EXPECT_EQ(task->cold_checks[2].valid, !cases[index].padding_mismatch); } } @@ -4456,6 +4460,10 @@ TEST(LlmMetalBackendFailureInjectionIntegrationTest, EXPECT_TRUE(task->padding_canary_evaluated); EXPECT_EQ(task->padding_canary_valid, !cases[index].padding_mismatch); EXPECT_EQ(task->post_validation_valid, !cases[index].padding_mismatch); + EXPECT_TRUE(task->cold_checks[0].valid); + EXPECT_TRUE(task->cold_checks[1].valid); + EXPECT_TRUE(task->cold_checks[2].evaluated); + EXPECT_EQ(task->cold_checks[2].valid, !cases[index].padding_mismatch); } } @@ -4538,6 +4546,11 @@ TEST(LlmMetalBackendFailureInjectionIntegrationTest, const LlmMetalTaskEvidence* task = get_llm_metal_task_evidence(result); ASSERT_NE(task, nullptr); EXPECT_TRUE(task->timed_pipeline_available); + EXPECT_TRUE(task->cold_checks[0].applicable); + EXPECT_TRUE(task->cold_checks[1].applicable); + EXPECT_EQ(task->cold_checks[0].evaluated, test_case.index != 2 && test_case.index != 4); + // Host acceptance injection is not an observed shader-bit mismatch. + EXPECT_EQ(task->cold_checks[1].valid, test_case.index != 2 && test_case.index != 4); EXPECT_EQ(task->timing_evaluated, test_case.index != 4); EXPECT_EQ(task->timing_valid, test_case.index != 0 && test_case.index != 4); @@ -4711,3 +4724,47 @@ TEST(LlmMetalBackendFailureInjectionIntegrationTest, } } // namespace + +TEST(LlmMetalBackendTest, ColdReadbackIndependentVerdicts) { + for (bool completed : {false, true}) { + for (unsigned combination = 0; combination < 8; ++combination) { + const uint32_t flags = ((combination & 1) ? LlmMetalKernelContract::kValidationInvalidParametersBit : 0) | + ((combination & 2) ? LlmMetalKernelContract::kKvWriteValidationMismatchBit : 0) | + ((combination & 4) ? LlmMetalKernelContract::kPaddingCanaryMismatchBit : 0); + const auto checks = interpret_llm_metal_cold_checks(LlmPhase::Prefill, true, true, completed, flags); + EXPECT_EQ(checks[0].evaluated, completed); + EXPECT_EQ(checks[0].valid, completed && !(combination & 1)); + for (size_t slot = 1; slot < 3; ++slot) { + const bool mismatch = (combination & (1U << slot)) != 0; + EXPECT_EQ(checks[slot].evaluated, completed && (mismatch || !(combination & 1))); + EXPECT_EQ(checks[slot].valid, completed && !mismatch && !(combination & 1)); + } + const auto none = interpret_llm_metal_cold_checks(LlmPhase::Decode, false, true, completed, flags); + for (const auto& check : none) EXPECT_FALSE(check.applicable); + const auto contiguous = interpret_llm_metal_cold_checks(LlmPhase::Decode, true, false, completed, flags); + EXPECT_FALSE(contiguous[2].applicable); + EXPECT_EQ(contiguous[1].kind, LlmColdCheckKind::KvAppendFinal); + EXPECT_EQ(checks[1].kind, LlmColdCheckKind::KvPrefillFinalSamples); + } + } +} + +TEST(LlmMetalBackendTest, PendingColdChecksPreserveAllApplicabilityCombinations) { + for (auto phase : {LlmPhase::Decode, LlmPhase::Prefill}) { + for (bool writes : {false, true}) { + for (bool padding : {false, true}) { + const auto checks = interpret_llm_metal_cold_checks(phase, writes, padding, false, 0); + EXPECT_EQ(checks[0].applicable, writes); + EXPECT_EQ(checks[1].applicable, writes); + EXPECT_EQ(checks[2].applicable, writes && padding); + EXPECT_EQ(checks[1].kind, phase == LlmPhase::Decode ? LlmColdCheckKind::KvAppendFinal + : LlmColdCheckKind::KvPrefillFinalSamples); + for (const auto& check : checks) { + EXPECT_FALSE(check.evaluated); + EXPECT_FALSE(check.valid); + EXPECT_EQ(check.reason_code, "not-evaluated"); + } + } + } + } +} From 84853205edf36a1013c6ee4fff2b3968bffc781e Mon Sep 17 00:00:00 2001 From: timoheimonen Date: Sat, 5 Sep 2026 19:08:47 +0300 Subject: [PATCH 07/16] Normalize LLM schema 2 results and bound snapshots --- README.md | 14 +- documents/API.md | 329 +++-- documents/LLM_MEMORY_PROFILE_WHITEPAPER.md | 351 +++-- documents/MANUAL.md | 484 +++---- src/core/config/constants.h | 17 +- src/llm_memory/llm_backend.h | 28 +- src/llm_memory/llm_cpu_backend.cpp | 7 + src/llm_memory/llm_executor.h | 15 +- src/llm_memory/llm_json.cpp | 416 +++--- src/llm_memory/llm_memory.cpp | 19 +- src/llm_memory/llm_output.cpp | 9 +- src/llm_memory/llm_prefill.cpp | 49 +- src/llm_memory/llm_prefill.h | 19 +- src/llm_memory/llm_runner.cpp | 732 ++++++----- src/llm_memory/llm_runner.h | 148 ++- src/llm_memory/llm_work_plan.cpp | 43 +- src/llm_memory/llm_work_plan.h | 9 +- .../console/messages/llm_memory_messages.cpp | 25 +- src/output/console/messages/messages_api.h | 8 +- .../console/messages/program_messages.cpp | 13 +- .../json/json_output/json_output_session.cpp | 11 +- .../json/json_output/json_output_session.h | 10 + tests/test_executable_cli.cpp | 212 +-- tests/test_json_output_session.cpp | 37 + tests/test_llm_memory_config.cpp | 30 +- tests/test_llm_memory_contract.cpp | 522 +------- tests/test_llm_memory_json.cpp | 1168 +++++++++-------- tests/test_llm_memory_output.cpp | 28 +- tests/test_llm_memory_runner.cpp | 607 ++++++--- tests/test_llm_memory_work_plan.cpp | 152 ++- tests/test_messages.cpp | 67 +- 31 files changed, 2926 insertions(+), 2653 deletions(-) diff --git a/README.md b/README.md index f5d260e..770a2e0 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ checkpoints are required; see the [Machine-Readable CLI API](documents/API.md) s | `--analyze-core2core` | Calibrated two-thread acquire/release token-protocol round-trip latency under best-effort macOS scheduler hints. | | `--gpu-bandwidth` | Standalone Metal GPU read/write/copy effective compute-payload bandwidth. | | `--llm-memory` | Standalone synthetic LLM memory profile: CPU or Metal decode/prefill with contiguous or paged KV. | -| `--sweep ` | Cartesian parameter sweep for supported CPU, pattern, TLB, and core-to-core modes; requires `--output`. GPU schema 1 and LLM schema 1 do not support sweeps. | +| `--sweep ` | Cartesian parameter sweep for supported CPU, pattern, TLB, and core-to-core modes; requires `--output`. GPU schema 1 and LLM schema 2 do not support sweeps. | Primary modes are intentionally separate and accept different option sets. Use `memory_benchmark -h` or the [User Manual](documents/MANUAL.md) for defaults, valid combinations, and the complete option reference. @@ -243,7 +243,7 @@ memory_benchmark --gpu-bandwidth --buffer-size 512 --count 3 --seed 42 --output >gpu_bandwidth.json 2>gpu_bandwidth.log ``` -Reproducible fixed-work LLM memory profile with atomic scenario and command-terminal file checkpoints: +Reproducible fixed-work LLM memory profile with bounded completed-loop and command-terminal atomic file snapshots: ```bash caffeinate -i -d memory_benchmark --llm-memory --weight-size-mb 4096 --layers 32 \ @@ -265,7 +265,7 @@ For prefill, replace the decode context with explicit prompt/tile geometry: Add `--kv-layout paged --kv-block-tokens 16` to combine that prefill geometry with deterministic paged KV. -The same schema 1 payload can be captured once from final-only stdout: +The same LLM schema 2 payload can be captured once from final-only stdout: ```bash memory_benchmark --llm-memory --weight-size-mb 64 --layers 4 \ @@ -322,9 +322,11 @@ They retain the top-level `version` as provenance but do not require a particula translate released standard schema 2, unversioned historical standard JSON layouts, or other methodology identities. Consumers making conclusions should reject incomplete or interrupted runs according to the mode-specific status fields. Every result-producing direct command or CPU sweep using `--output -` reserves stdout for one final JSON document and -routes its post-parse human transcript to stderr; file output is atomic. LLM file output checkpoints after each terminal -scenario measurement and at command terminal, while its stdout checkpoints remain logical lazy transitions followed by -one final document. Exact process acceptance rules are in the +routes its post-parse human transcript to stderr; file output is atomic. LLM files receive a progress snapshot every +`K=max(1,ceil(count/8))` completed loops plus a terminal snapshot: at most eight progress writes, with up to `3K` +completed attempts potentially missing after abrupt termination. Stdout emits one final document. A correct complete +one-loop LLM run can have `run_accepted: true` while position balance is incomplete and quality is insufficient for a +comparison. Inspect sample count, observed CV, duration and environment separately. Exact process acceptance rules are in the [Machine-Readable CLI API](documents/API.md), with schema and checkpoint details in the [User Manual](documents/MANUAL.md), [Technical Specification](documents/TECHNICAL_SPECIFICATION.md), and mode whitepapers. diff --git a/documents/API.md b/documents/API.md index 012b70a..bad8f9b 100644 --- a/documents/API.md +++ b/documents/API.md @@ -25,7 +25,7 @@ transport. GPU schema 1 remains a direct-only mode and does not support sweeps. | Direct `--gpu-bandwidth` | Yes | Yes | | Direct `--llm-memory` | Yes | Yes | -GPU schema 1 and LLM schema 1 are direct-only modes and do not support sweeps. +GPU schema 1 and LLM schema 2 are direct-only modes and do not support sweeps. ## Invocation and stream contract @@ -48,7 +48,7 @@ A direct GPU command uses the same stream transport with its existing top-level memory_benchmark --gpu-bandwidth --buffer-size 512 --count 3 --seed 42 --output - ``` -A direct LLM command likewise emits its top-level schema 1 payload. CPU decode and prefill each support contiguous or +A direct LLM command likewise emits its top-level schema 2 payload. CPU decode and prefill each support contiguous or paged KV: ```bash @@ -133,8 +133,8 @@ Some runtime setup failures also occur before a schema-valid payload exists. The TLB setup, memory-budget, or allocation failures before analysis-state initialization, a GPU backend-factory failure, and LLM logical preflight or JSON-output peak-estimation failure before runner-result initialization therefore produce stderr plus a non-zero status and leave stdout empty. After the LLM runner has initialized a representable result, a -selected Metal backend that fails its runtime capability check emits one terminal schema-1 `unsupported` document; -runtime compiler, pipeline, resource, or task failures emit a terminal schema-1 `failed` run with failed or invalid +selected Metal backend that fails its runtime capability check emits one terminal schema-2 `unsupported` document; +runtime compiler, pipeline, resource, or task failures emit a terminal schema-2 `failed` run with failed or invalid measurement evidence as applicable. All return non-zero, and none is retried on CPU. Once a mode has initialized a representable result, graceful interruption or a normal runtime failure emits the available partial, interrupted, error, failed, or unsupported payload. Core-to-core measurement failures, TLB measurement errors, GPU @@ -158,21 +158,38 @@ Real file targets retain their existing persistence behavior: - parameter sweeps atomically checkpoint their combined envelope after each attempted run and also checkpoint a terminal zero-attempt envelope when the run plan is empty or interruption is observed before a run; - GPU mode retains its mode-specific terminal-measurement and failure checkpoints; -- LLM mode atomically checkpoints after every terminal scenario measurement and once at command terminal; +- LLM mode atomically snapshots bounded completed-loop progress and command terminal as specified below; - a temporary `.tmp` file is replaced atomically, and a failed replacement preserves the preceding destination when possible. -Stdout is final-only. Intermediate standard, sweep, GPU, and LLM checkpoint requests are successful lazy no-ops: their +Stdout is final-only. Intermediate standard, sweep and GPU checkpoint requests are successful lazy no-ops: their payload builders are not invoked, while all logical state changes, stop observations, counters, cleanup, and final result -construction still occur. In particular, GPU and LLM perform the same checkpoint-boundary stop reads as file output. +construction still occur. LLM skips intermediate snapshot preparation on stdout but preserves task-boundary stop observations. The command serializes one terminal snapshot after orchestration finishes. Stdout is not JSON Lines and never contains a sequence of checkpoint documents. -File-output ownership is mode-aware. Standard and sweep file producers retain their existing checkpoints, GPU retains -its terminal-measurement, failure, and post-release replacement cadence, and LLM owns its scenario-terminal plus -command-terminal cadence. A failed LLM checkpoint is terminal and is not retried at a final file-write boundary; the -last successfully replaced destination remains the recoverable evidence when possible. The stdout command boundary -alone emits the retained final document. +For LLM, let `N=planned_loops` and `K=max(1,ceil(N/8))`, calculated without addition overflow. File targets +write progress after every Kth fully completed loop, at most eight times, and one command-terminal snapshot on +success, graceful interruption or representable failure. The current unmerged cadence gives at most 4/7/9 normal +snapshots for N=3/12/48. One late command exception after successful terminal persistence may write one corrective +failure snapshot; a failed checkpoint is terminal and never retried or followed by a disguised final write. + +An abrupt exit can lose up to `3K` truly completed measurement attempts after the last successful snapshot, including +while its replacement is in progress. Unstarted tail placeholders do not count toward this bound. Before the first +successful snapshot there may be no file. Graceful SIGINT retains the finished prefix and an interrupted/null tail +when terminal persistence succeeds; SIGKILL or a crash does not promise terminal output. Atomic rename does not imply +power-loss durability. Stop checks remain at each existing task/terminal boundary even when a snapshot is skipped. +Stdout builds one final DOM; disabled output builds none. + +`checkpoint_lifecycle` publishes `checkpoint_failed`, `prior_file_writer_attempts`, `prior_successful_file_writes`, +`observation_point: "before-current-snapshot-preparation"`, `current_request` (`progress`, `terminal`, or +`late-command-error-correction`), `current_persistence_success: null`, `snapshot_interval_loops`, +`checkpoint_policy: "bounded-loop-snapshots"`, `file_checkpoint_failure_is_terminal_and_not_retried: true`, and +`stdout_intermediate_checkpoints_are_lazy: true`. The prior counts observe writer entry and successful return before +this snapshot, not the current write's eventual success. Builder failure is not writer entry; stdout/disabled no-ops +are not file writes. For a successful N=3 file command the terminal document normally reports three prior successes, +not four self-inclusive successes. No extra write is made just to count itself. A retained prior snapshot cannot attest +to a future write/cleanup failure; final collection must retain actual process outcome and terminal context. Use a real file target when crash-resilient intermediate checkpoints are required. @@ -188,7 +205,7 @@ contract version 1 first appears in software version `0.62.0`. | TLB | `configuration.schema_version == 4` | `tlb_analysis.status == "complete" && tlb_analysis.conclusions_valid == true` | | Core-to-core | `configuration.schema_version == 2` | `core_to_core_latency.status == "complete" && core_to_core_latency.measurements_complete == true` | | GPU | `schema_version == 1` | `status == "complete" && results_complete == true && conclusions_valid == true` | -| LLM memory profile | `mode == "llm_memory" && schema_version == 1` | Requested backend/phase/layout match, methodology is the exact derived identity, `status == "complete"`, `results_complete == true`, `conclusions_valid == true`, and every planned measurement is `measured` | +| LLM memory profile | `mode == "llm_memory" && schema_version == 2` | Requested backend/phase/layout match, methodology and run-policy match the exact identities, `status == "complete"`, `results_complete == true`, `run_accepted == true`, and every planned measurement is `measured` | | General CPU sweep | `configuration.sweep_schema_version == 1` | `status == "complete" && conclusions_valid == true` | | Core-to-core sweep | `configuration.sweep_schema_version == 1` | `status == "complete" && conclusions_valid == true` | @@ -214,16 +231,16 @@ conclusions additionally require `affinity_hint_comparison_interpretable == true additionally requires `operation_order_balance_complete == true`; consumers of an operation also require a measured, non-null value and its applicable validation/quality fields. -LLM schema 1 is an unpublished generic contract. It has top-level `mode: "llm_memory"`, `schema_version: 1`, and exact +LLM schema 2 replaces the unpublished schema-1 contract without compatibility aliases or a fallback reader. It has top-level `mode: "llm_memory"`, `schema_version: 2`, and exact `backend`, `phase`, `kv_layout`, and `methodology_version` selectors. Methodology is derived as -`llm-memory-v1---`. This revision activates `cpu`/`decode`/`contiguous`, +`llm-memory-v2---`. This revision activates `cpu`/`decode`/`contiguous`, `cpu`/`decode`/`paged`, `cpu`/`prefill`/`contiguous`, and `cpu`/`prefill`/`paged`, with exact methodologies -`llm-memory-v1-cpu-decode-contiguous`, `llm-memory-v1-cpu-decode-paged`, -`llm-memory-v1-cpu-prefill-contiguous`, and `llm-memory-v1-cpu-prefill-paged`. It also activates +`llm-memory-v2-cpu-decode-contiguous`, `llm-memory-v2-cpu-decode-paged`, +`llm-memory-v2-cpu-prefill-contiguous`, and `llm-memory-v2-cpu-prefill-paged`. It also activates `metal`/`decode`/`contiguous`, `metal`/`decode`/`paged`, `metal`/`prefill`/`contiguous`, and -`metal`/`prefill`/`paged`, with methodologies `llm-memory-v1-metal-decode-contiguous`, -`llm-memory-v1-metal-decode-paged`, `llm-memory-v1-metal-prefill-contiguous`, and -`llm-memory-v1-metal-prefill-paged`. No selection is silently replaced by another backend, phase, or layout. +`metal`/`prefill`/`paged`, with methodologies `llm-memory-v2-metal-decode-contiguous`, +`llm-memory-v2-metal-decode-paged`, `llm-memory-v2-metal-prefill-contiguous`, and +`llm-memory-v2-metal-prefill-paged`. No selection is silently replaced by another backend, phase, or layout. Run statuses are `not_started`, `complete`, `partial`, `interrupted`, `unsupported`, and `failed`; measurement statuses are `not_run`, `measured`, `interrupted`, `invalid`, and `failed`. `unsupported` is a terminal, non-acceptable result, @@ -243,12 +260,12 @@ addresses, or visit multiplicities. A post-validation failure can retain `checks invalid, its rates are null, and it does not enter aggregates. Missing checksum evidence remains null rather than false or successful. CPU paged decode weights-only checks append slots and padding; CPU paged prefill weights-only checks padding only. Metal weights-only -reports KV-write unevaluated and padding inapplicable, so no successful KV-write observation may be inferred from its +reports KV-write and padding inapplicable, so no successful KV-write observation may be inferred from its combined lifecycle result. Prefill validation remains sampled on both backends. Programmed byte/lookup counts and plan-derived completion do not measure physical DRAM traffic. Source hashes, ABI goldens, and generated semantic lists qualify a specific implementation rather than adding runtime observations to the JSON. See the [profile fault matrix and exact collision limits](LLM_MEMORY_PROFILE_WHITEPAPER.md#checksum-fault-model-and-evidence-limits). -These limits do not change schema 1, the checksum identities, or the completion/acceptance predicate below. +Independent named checks retain these existing detection limits and checksum identities. The required generic top-level field set is: @@ -256,7 +273,9 @@ The required generic top-level field set is: schema_version, mode, backend, phase, kv_layout, methodology_version, software, configuration, resolved_plan, backend_evidence, memory_budget, calibration, measurements, aggregates, status, reason_code, -results_complete, conclusions_valid, interpretation +results_complete, run_accepted, interpretation, diagnostic, interruption_requested, +scenario_order_balance_complete, seeds, counters, checkpoint_lifecycle, loop_records, +environment, quality_warnings, build_manifest ``` `configuration` preserves exact `argv`, the raw output target, requested/resolved inputs, and a `resolved_sources` @@ -275,8 +294,8 @@ requested/available/effective worker contract. - Exactly one of `geometry.decode` and `geometry.prefill` is an object. Phase-specific fields are never overloaded. Decode has integer `visible_context_tokens`. Prefill has integer - `prompt_tokens`/`attention_query_tile_tokens` and decimal-string tile, prefix-visit, causal-pair, attention-pair, and - FMA-term counts. Decode-only crossover numerator, denominator, and context, current visible context, weight/KV-read + `prompt_tokens`/`attention_query_tile_tokens` and decimal-string tile and prefix-visit counts. Theoretical + causal-pair, attention-pair and FMA-term values instead belong to `model_context.prefill`. Decode-only crossover numerator, denominator, and context, current visible context, weight/KV-read ratio, and `current_context_classification` are null for prefill. `classification_version` and `classification_is_payload_only` remain populated because they describe the schema field's semantics. - `layout.kv_layout` is the selected `contiguous` or `paged` token. Contiguous results use null for paged-only fields @@ -355,18 +374,22 @@ stops the timer and all workers join. Failure is `decode-post-validation-failed` invalid attempt with null rates and excluded from aggregate populations. The timed payload and checksum algorithm are unchanged. A matching runtime checksum does not replace this final-state check. -In LLM schema 1, CPU `measurements[].execution.post_validation_evaluated` retains its existing -meaning: the phase/layout final-state validation pipeline was reached, including a no-op path when -no write check applies. `post_validation_valid` is that combined pipeline result. Additive CPU fields -`kv_write_validation_applicable`, `kv_write_validation_evaluated`, and `kv_write_validation_valid` -distinguish a KV-writing scenario from `weights_only`. When applicability is false, evaluated and -valid serialize as null. When applicable but unevaluated, evaluated is false and valid is null; -acceptance requires evaluated and valid to be true. All three are null without available CPU evidence. -For paged and prefill profiles, the valid field reports the existing combined phase/layout final-state -check, including any applicable padding check; it is not an independent append-only verdict. Prefill -retains its documented sampling limits. `weights_only` does not claim KV-write validation. Its combined CPU paged -pipeline checks append slots and padding for decode, but only structure and padding for prefill; prefill weights-only -does not check unexpected writes to valid prompt contents. +Both measurements and excluded calibration attempts publish cold checks at `execution.validation.checks[]`. +Each entry contains exactly `kind`, `applicable`, `evaluated`, `valid`, and `reason_code`. The three slots are structure, +the scenario's write/unchanged check, and padding. The five kind tokens are `post-validation-structure`, +`kv-append-final`, `kv-prefill-final-samples`, `kv-append-unchanged`, and `kv-padding-canary`. +Structure describes existing checked prerequisites, not an exhaustive plan proof. + +- Inapplicable: `applicable=false`, `evaluated=null`, `valid=null`, reason `not-applicable`. +- Applicable but unresolved: `evaluated=false`, `valid=null`; missing observed evidence cannot pass. +- Evaluated: `evaluated=true`, `valid` is a boolean. An observed mismatch stays false even if another check is unresolved. + +KV-bearing decode checks final append state; prefill checks final-ordinal samples, retaining its sampling limits. +CPU paged decode weights-only checks unchanged append slots and applicable padding; CPU paged prefill weights-only +checks structure and applicable padding, not unexpected changes to valid prompt contents. Metal weights-only checks +neither writes nor padding. Padding applies only where terminal suffix padding exists. CPU contiguous decode +weights-only has no applicable structure/write/padding check. CPU prefill retains its structural prerequisite check. +The separate checksum verdict may remain true when a cold check fails. For prefill, let `P` be prompt length, `Q` query-tile length, and `K = L*2*R`. With `C = ceil(P/Q)`, tile ends `e_j = min((j+1)*Q, P)`, and `S(P,Q) = sum(e_j)`, one `prefill_operation` has: @@ -424,10 +447,9 @@ publishes whole-uint32 table segmentation. `backend_evidence.metal.resources` re validation, the same permutation identity, and combined K/V layout padding. Each task grid reports `paged_semantic_lookups` and `serial_range_visits_per_lane`; task validation reports the phase-neutral K/V-write result and applicable padding-canary evaluation. A complete KV-active task requires the expected lookup count, valid checksum, -valid K/V-write validation and, when terminal padding exists, an evaluated-valid padding canary. Under -`measurements[].execution.metal.validation`, the generic fields are `kv_write_evaluated` and `kv_write_valid`; they apply -to decode current-token writes and prefill full-prompt writes without changing meaning by phase. Human-readable Metal -task output renders the same contract as `kv_write=valid|invalid|not-evaluated|not-applicable`. +valid K/V-write validation and, when terminal padding exists, an evaluated-valid padding canary. The runtime checks are published at +`measurements[].execution.validation.checks` and the corresponding calibration execution path. Human-readable Metal +task output summarizes the write result as `kv_write=valid|invalid|not-evaluated|not-applicable`. The canonical embedded MSL revision and its exact SHA-256 are computed from the selected runtime source and reported rather than duplicated here. Every profile publishes its selected scenario-pipeline and parameter-layout-probe @@ -466,43 +488,50 @@ pair, and ends with the frozen-plan warmup. Successful explicit preparation has records only its frozen-plan warmup. A failed or interrupted preparation retains only the attempts reached before its terminal boundary. -Every measurement has stable generic work accounting: - -```text -work_unit_kind, planned_work_units, completed_work_units, -weight_read_bytes_per_work_unit, kv_read_bytes_per_work_unit, -kv_write_bytes_per_work_unit, kv_write_kind, -effective_model_payload_bytes_per_work_unit, -layout_metadata_lookup_count_per_work_unit, -layout_metadata_read_bytes_per_work_unit, accounted_bytes_per_work_unit, -planned_effective_model_payload_bytes, completed_effective_model_payload_bytes, -planned_layout_metadata_lookup_count, completed_layout_metadata_lookup_count, -planned_layout_metadata_read_bytes, completed_layout_metadata_read_bytes, -planned_task_accounted_bytes, completed_task_accounted_bytes, -synthetic_work_unit_latency_seconds, -synthetic_memory_work_units_per_second, -effective_model_payload_gb_s -``` - -Decode uses `work_unit_kind: "decode_step"` and `kv_write_kind: "current_token_append"` for KV-bearing scenarios. -Prefill uses `work_unit_kind: "prefill_operation"` and `kv_write_kind: "full_prompt_population"`. -`weights_only` uses `kv_write_kind: "none"` in both phases. `planned_work_units` and `completed_work_units` are -integers. -All listed byte, lookup, metadata, and accounted quantities are canonical decimal strings, including applicable zero. -The three derived metrics are finite numbers only for a successfully measured record and otherwise null. The effective -model numerator contains versioned logical W/K/V reads and writes; timed layout metadata is reported separately and -included in `accounted_bytes_per_work_unit`, not in `effective_model_payload_gb_s`. -Measurement checksum evidence uses the phase-neutral keys `write_pattern_version` and `checksum_pattern_version`; -schema 1 does not publish decode-specific `append_pattern_version` or `read_checksum_version` aliases. Metal uses -profile-specific dual-mod32 checksums with separate W/K/V lanes. Its measurement execution evidence records the -selected pipeline, threadgroup/grid geometry, raw `GPUStartTime` and `GPUEndTime`, authoritative GPU elapsed time, -the host submit/wait -envelope, command/encoder/dispatch counts and statuses, timed checksum comparison, and excluded phase-neutral K/V-write -validation. Decode validates its current-token write. Prefill checks representative/boundary byte locations per -layer/batch sequence in both K and V from the full-prompt population against the final work-unit ordinal `T - 1`; -paged prefill also requires applicable terminal-padding canaries to remain valid. -Queue delay is null unless it is measured in the same clock domain. CPU and Metal checksum values are backend-specific -and are never compared numerically. +Canonical work and runtime observations have separate owners: + +- `resolved_plan.plan_identity`, `geometry`, `layout`, `resources`, `component_identities` and `model_work_plan` + describe the model once. The root methodology selector is not repeated in nested objects. +- `resolved_plan.scenario_plans[]` contains unique scenario/T/explicit plans, including calibration shapes. Plans sort + by scenario (`weights_only`, `kv_only`, `mixed`), increasing `work_units`, then `explicit_iterations` + (`false` before `true`). Each has `model_ref: "resolved_plan"`, exact `plan_identity`, scenario seed, work kind, + work policy, limits, per-work-unit and total payload/lookup/accounted quantities. +- `resolved_plan.frozen_plan_refs` has the three scenario keys, with a zero-based plan-array index or null when unresolved. + Measurements and `calibration.attempts.[]` use `plan_ref` into that same document. Unknown/out-of-range or + wrong-scenario references are rejected. Internal insertion handles stay stable, but public indices can change between + snapshots as calibration plans arrive; never join references across documents. +- `measurements[].measurement_id` is its zero-based position in the scheduled measurement array. A measurement retains + scenario, loop/order, attempted/status/reason, authoritative duration, completed work and mutable runtime evidence. + `completed_work_units` is an integer. `completed_effective_model_payload_bytes`, `completed_layout_metadata_lookup_count`, + `completed_layout_metadata_read_bytes` and `completed_task_accounted_bytes` are decimal strings. CPU accepted completion + has `completion_derivation: "accepted-plan-derived"`; this is not a hardware work counter. +- Work kind (`decode_step` or `prefill_operation`), `work_units`, `kv_write_kind` + (`none`, `current_token_append`, `full_prompt_population`), per-work-unit quantities and planned totals belong to the + referenced plan. Its total names are `effective_model_payload_bytes`, `layout_metadata_lookup_count`, + `layout_metadata_read_bytes` and `task_accounted_bytes`, without a `planned_` prefix. Measurement workers, QoS, + calibration attempt indexes, working-set description and mixed payload fractions remain derived metadata. + +Each canonical plan has an `expected_checksum` object with exactly `status`, `reason_code`, +`expected_worker_checksums`, and `expected_run_checksum`. Status `available` means the cold expectation was +reconstructed once for that plan; it is not an observed runtime pass. Status `unavailable` makes both expected fields +null and retains the reason. CPU available worker arrays have increasing `worker_index` and exactly effective-workers +entries; each has `weight`, `k`, `v` objects with decimal-string `state_a_uint64_decimal`, `state_b_uint64_decimal`, +`exact_bytes_read`, and `span_count_uint64_decimal`. Zero-work domains retain actual algorithm initial values. +CPU run checksum has the two state fields. Metal expected workers are null; its run object has `weight`, `k`, `v`, +each with `a_uint32_decimal` and `b_uint32_decimal`. CPU and Metal checksums are never compared numerically. + +Measurement `checksum` contains exactly `status`, `reason_code`, `checksum_valid`, `actual_worker_checksums`, and +`actual_run_checksum`. Unevaluated checksum has null validity and both actual fields null; evaluated invalid retains +actuals and false validity. Calibration keeps the compact actual-run-only counterpart at +`calibration.attempts.[].execution.checksum`, with no actual-worker vector. All expectations resolve through +that record's plan reference. Algorithm identity comes from canonical `component_identities.checksum_pattern_version`. + +The one accepted `elapsed_seconds` and completed work/payload derive `synthetic_work_unit_latency_seconds`, +`synthetic_memory_work_units_per_second`, and `effective_model_payload_gb_s`. Invalid/excluded measurements have null +rates and accepted elapsed; any observed time is retained as `execution.timing.diagnostic_elapsed_seconds`. Metal retains +raw GPU start/end, host submit/wait envelope, nullable same-clock queue delay, pipeline/grid and command/encoder/dispatch +observations. Queue delay is null unless measured in the same clock domain. Effective payload includes logical W/K/V +bytes; timed table metadata stays outside that numerator. Each Metal task uses one reset command buffer, one timed command buffer with one explicitly serial compute encoder and one workload dispatch, and one excluded post-validation command buffer. The task's `T` work units loop inside the @@ -571,14 +600,27 @@ The traffic classification version is `llm-exact-weight-vs-kv-read-payload-v1`: with exact KV-read bytes only. `near_crossover` means equality, not a tolerance band and not an observed hardware bottleneck. -`results_complete` requires all planned scenario measurements to be terminal and measured. `conclusions_valid` also -requires valid geometry/checksums, no checkpoint failure, and a complete cyclic position balance. A complete one-loop -run is therefore inspectable with `results_complete: true` but has `conclusions_valid: false`. Consumers of comparative -LLM conclusions must apply the exact selector/methodology predicate, require every planned measurement to be -`measured`, and then check the selected scenario metric's non-null value plus relevant checksum, quality, and environment -evidence. Comparisons also require identical phase geometry and, for paged results, identical `kv_block_tokens`, -permutation identity, physical-resource geometry, and component identities. Contiguous and paged measurements are not -members of the same comparison cohort merely because their logical model geometry matches. +`results_complete` means all planned measurements are terminal and measured. `run_accepted` additionally requires +complete run status, required accepted timing/checksum/cold-check and backend completion/lifecycle evidence, and no +known checkpoint or command failure. It is independent of position balance, sample count, CV, duration and environment. +A correct count-one run can have both booleans true and `scenario_order_balance_complete: false`. A late command error +can leave the measured population complete while run status is failed and `run_accepted` is false. Consumers retain +process exit status as well as the snapshot; an older preserved file cannot report a later failure. + +Each `aggregates.scenarios..accepted_measurement_ids` defines one population shared by all three metrics. +Excluded calibration never enters it. Derive each rate before applying the shared linear percentile interpolation; +for even n, median(work/duration) generally differs from work/median(duration). Exact median/MAD/percentiles are +prepared only at snapshots or terminal, with reusable workspaces; adding or printing a raw measurement does not sort +all earlier samples. Statistics retain average, min/max, median, P90/P95/P99, sample stddev, CV percent and MAD. +Empty populations have null statistics/headlines; n=1 has stddev=0 and positive-mean CV=0, while classification remains +`insufficient-samples`. `observed_cv_classification` is `insufficient-samples` for n<3, `undefined` for undefined CV, +`above-threshold` for payload CV strictly greater than `cv_warning_threshold_pct` (5), otherwise `below-threshold`. +Equality is below-threshold. These describe observed samples, not reproducibility guarantees; percentiles are not +confidence levels. Default LLM console output shows n, median, min/max, CV and MAD, without tail percentiles. + +A comparison may impose stricter quality criteria, but those are separate policy and do not delete or retry noisy +measurements. Match phase/model geometry and, for paged results, block size, permutation, physical-resource geometry +and component identities. Contiguous and paged results are distinct cohorts even with identical logical geometry. `quality_warnings` merges and deduplicates runner tokens `weights_only-high-cv`, `kv_only-high-cv`, `mixed-high-cv`, and `scenario-order-not-balanced` with final-report tokens `environment-not-nominal`, @@ -605,6 +647,56 @@ partial, interrupted, error, or failed JSON snapshot. The execution status and p must not cause a caller to discard evidence without parsing it. Exit status zero is used for human help, complete execution, and established graceful-interruption paths; it alone does not prove that JSON conclusions are complete. +## LLM schema-1 to schema-2 field map + +This table describes the intentional cutover, not an automatic reader. Unlisted supported geometry, resource, +seed, runtime and interpretation fields retain their meaning. Schema/version checks must precede field consumption. + +| Schema 1 path or behavior | Schema 2 owner or behavior | +|---|---| +| `schema_version: 1`, `llm-memory-v1-*` selectors | `schema_version: 2`, the eight exact v2 selectors above | +| `conclusions_valid` | `run_accepted`; correctness is independent of balance/quality | +| `resolved_plan.methodology_version`, `resolved_plan.model_work_plan.methodology_version`, `resolved_plan.methodology.methodology_version` | root `methodology_version` | +| methodology component-version copies | `resolved_plan.component_identities` only; `run_policy_version` is `llm-run-policy-bounded-loop-snapshots-v1` | +| `resolved_plan.model_work_plan.plan_identity` | `resolved_plan.plan_identity` | +| `resolved_plan.model_work_plan.component_identity` | `resolved_plan.component_identities.identity` | +| `backend_evidence.cpu.resources.model_plan_identity` | `model_ref: "resolved_plan"` when resolved, otherwise null | +| `resolved_plan.frozen_scenario_work_plans` wrapper and `scenarios[]` | `resolved_plan.scenario_plans[]` plus `resolved_plan.frozen_plan_refs` | +| frozen wrapper/scenario `model_plan_identity` | canonical scenario `model_ref: "resolved_plan"`; wrapper removed | +| `measurements[].frozen_plan_index`, `frozen_work_plan_identity`, execution model/scenario identity copies | measurement `plan_ref`; canonical plan carries its exact identity and `model_ref` | +| `measurements[].scenario_seed_uint64_decimal`, `explicit_iterations`, `work_policy`, `work_unit_kind`, `kv_write_kind`, `*_bytes_per_work_unit`, `planned_*` | referenced canonical plan, with `work_units` and total names without `planned_` | +| implicit measurement array position | explicit `measurement_id`, still the zero-based schedule index | +| `measurements[].checksum.expected_*`, `execution.metal.checksum` | referenced plan's `expected_checksum`; runtime actuals remain in measurement `checksum` | +| `calibration.attempts.[].work_plan_identity`, planned payload and execution expected run checksum | `calibration.attempts.[].plan_ref`; compact runtime actual stays under `execution.checksum` | +| runtime checksum algorithm/write-version copies | canonical component checksum/write versions | +| `execution.post_validation_evaluated/valid`, CPU `kv_write_validation_*`, Metal `execution.metal.validation` | measurement/calibration `execution.validation.checks[]` independent observations | +| duplicate accepted elapsed values and retained metric sample vectors | authoritative measurement elapsed/work/payload; derived metrics and shared `accepted_measurement_ids` | +| `aggregates.scenarios..stability_quality` | `observed_cv_classification` plus explicit threshold and sample counts | +| `checkpoint_lifecycle.logical_checkpoint_attempts`, `successful_logical_checkpoints`, terminal checkpoint flags | prior actual writer attempts/successes at the named observation point; current persistence unresolved | +| `geometry.prefill` causal-pair, logical-attention-pair and FMA context | nullable `resolved_plan.model_context.prefill` scalars with individual `*_reason_code` | +| no build reservation | always-present `build_manifest` object; unavailable fields remain null | + +`model_context.prefill` is null for decode. For prefill its exact children are +`causal_token_pairs_per_sequence`, `logical_attention_pairs`, `logical_attention_fma_terms` and each one's +`_reason_code`. Available values are decimal strings with reason `valid`; theoretical arithmetic overflow gives +null with `arithmetic-overflow`. Available values retain numeric exact-identity encoding; unavailable identity fields +use `unavailable:arithmetic-overflow`. Actual byte/prefix/lookup/allocation overflows still reject the workload. +`h_q` (`--query-heads`) controls theoretical attention context; query tile `Q` (`--attention-query-tile-tokens`) +controls executed prefix-read geometry. No Transformer or FMA computation is performed. + +`build_manifest` has `manifest_version: 1`, `status: "unavailable"`, +`reason_code: "build-provenance-not-provided"`, and null `binary_sha256`, `git_commit`, `git_dirty`, `compiler`, +`compile_flags`, `link_flags`, `target_arch`, `sdk`, and `min_os`. This revision reserves the contract and does not +supply build attestation. Optional `execution.timing.cpu_raw` is not emitted here; absence cannot be promoted to +raw-tick verification. Software identity and MSL-source provenance alone are not independent build/runtime proof. + +All byte, lookup, seed and checksum values follow canonical unsigned decimal-string encoding (`0` or +`[1-9][0-9]*`); Metal lanes retain uint32 bounds. Indexes, work units and validated small inputs remain bounded JSON +integers below 2^53, never booleans or coerced strings. Null separates unavailable/inapplicable from known zero. +Canonical plan/expected vector capacities, transient expected reconstruction, accepted-ID maps, exact-stat workspaces, +DOM and stdout serialized strings are charged together at their simultaneous peak; bounded snapshot count does not +reduce the single-snapshot peak allowance. + ## Consumer acceptance procedure A caller accepts a benchmark conclusion only after all of the following checks succeed: @@ -618,7 +710,9 @@ A caller accepts a benchmark conclusion only after all of the following checks s 5. Check the supported mode and schema-version field. 6. Require the expected successful process status before accepting conclusions. 7. Apply the mode-specific command-completeness predicate above. -8. Apply the selected metric's status, non-null, and quality predicates. +8. Apply the selected metric's mode-specific status, non-null, and quality predicates described above. + For LLM, balance, observed CV, duration quality, and environment are separate comparison-quality context; + they do not add correctness predicates to `run_accepted`. Do not wait for process exit before reading sequential pipe captures. If either stdout or stderr fills its pipe buffer, the child can block before exit and the parent's wait can deadlock. @@ -678,42 +772,61 @@ jq -e '.schema_version == 1 and .mode == "gpu_bandwidth" and .status == "complete" and .results_complete == true and .conclusions_valid == true' gpu.json -jq -e '.mode == "llm_memory" and .schema_version == 1 and - .backend == "cpu" and .phase == "decode" and - .kv_layout == "paged" and - .methodology_version == "llm-memory-v1-cpu-decode-paged" and +jq -e '. as $root | .mode == "llm_memory" and .schema_version == 2 and + .backend == "cpu" and .phase == "decode" and .kv_layout == "paged" and + .methodology_version == "llm-memory-v2-cpu-decode-paged" and .resolved_plan.layout.kv_block_tokens == 16 and (.resolved_plan.layout.permutation_sha256 | type == "string" and test("^[0-9a-f]{64}$")) and - .status == "complete" and .results_complete == true and - .conclusions_valid == true and - ([.measurements[] | select(.status != "measured")] | length) == 0' llm_memory.json - -jq -e '.mode == "llm_memory" and .schema_version == 1 and - .backend == "cpu" and .phase == "prefill" and - .kv_layout == "contiguous" and - .methodology_version == "llm-memory-v1-cpu-prefill-contiguous" and + .resolved_plan.component_identities.run_policy_version == + "llm-run-policy-bounded-loop-snapshots-v1" and + .status == "complete" and .results_complete == true and .run_accepted == true and + (.resolved_plan.scenario_plans | type) == "array" and + (.measurements | type) == "array" and + .counters.planned_measurements > 0 and + (.measurements | length) == .counters.planned_measurements and + all(.measurements[]; + .status == "measured" and (.plan_ref | type) == "number" and + .plan_ref >= 0 and .plan_ref == (.plan_ref | floor) and + .plan_ref < ($root.resolved_plan.scenario_plans | length) and + $root.resolved_plan.scenario_plans[.plan_ref].scenario == .scenario)' llm_memory.json + +jq -e '. as $root | .mode == "llm_memory" and .schema_version == 2 and + .backend == "cpu" and .phase == "prefill" and .kv_layout == "contiguous" and + .methodology_version == "llm-memory-v2-cpu-prefill-contiguous" and .resolved_plan.geometry.decode == null and .resolved_plan.geometry.prefill.prompt_tokens == 512 and .resolved_plan.geometry.prefill.attention_query_tile_tokens == 64 and - .status == "complete" and .results_complete == true and - .conclusions_valid == true and - ([.measurements[] | select(.status != "measured")] | length) == 0' llm_prefill.json + .resolved_plan.component_identities.run_policy_version == + "llm-run-policy-bounded-loop-snapshots-v1" and + .status == "complete" and .results_complete == true and .run_accepted == true and + (.resolved_plan.scenario_plans | type) == "array" and + (.measurements | type) == "array" and + .counters.planned_measurements > 0 and + (.measurements | length) == .counters.planned_measurements and + all(.measurements[]; + .status == "measured" and (.plan_ref | type) == "number" and + .plan_ref >= 0 and .plan_ref == (.plan_ref | floor) and + .plan_ref < ($root.resolved_plan.scenario_plans | length) and + $root.resolved_plan.scenario_plans[.plan_ref].scenario == .scenario)' llm_prefill.json ``` +The two LLM examples consume the producer's acceptance and validate same-document plan joins; they do not independently +re-execute checksum/timing or attest to the build. Supply your requested geometry and cohort checks in addition to the +shown block-size or prompt/tile requirements. A noisy or one-loop correct result passes; comparison-quality policy is +separate. Require the successful producer process outcome before applying `jq -e`. + ## Compatibility policy - `version`, the GPU `software_version` field, and LLM `software` identity identify the application release; none is a result schema version or compatibility selector. Consumers may retain and display this provenance independently of schema and methodology acceptance. -- Current standard schema 3, pattern schema 3, TLB schema 4, core-to-core schema 2, GPU schema 1, and LLM schema 1 remain +- Current standard schema 3, pattern schema 3, TLB schema 4, core-to-core schema 2, GPU schema 1, and LLM schema 2 remain authoritative at their existing locations. The schema field is intentionally not normalized across these established payloads. -- The prior LLM schema-1 producer shape was unpublished. This revision deliberately replaces its CPU/step-specific - vocabulary with the generic v1 contract without a compatibility reader, field alias, or schema bump. LLM schema-1 - consumers must use the current generic fields, tolerate unknown additive evidence fields, and validate every known - field they consume. Future removal/rename/type/meaning changes require schema-version review; software identity alone - is not a compatibility substitute. +- LLM schema 2 deliberately replaces the unpublished schema-1 shape and changes methodology selectors. The concrete + field map describes relocation and changed acceptance; no automatic v1 reader, alias or migration layer is provided. + Future removal/rename/type/meaning changes require schema-version review. - Bundled standard-memory examples accept compatible software releases only when standard mode, schema 3, the exact methodology identity, completion state, and consumed field shapes match. They retain software-version provenance but provide no translation layer for released standard schema 2, unversioned historical standard JSON layouts, or other diff --git a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md index 921fee8..ab1c553 100644 --- a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md +++ b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md @@ -25,10 +25,10 @@ The current contract is identified by: | Software version | Release provenance; not a schema or methodology selector | | Mode | `llm_memory` | | Backend | `cpu` or `metal` | -| JSON schema | `1` | +| JSON schema | `2` | | Phase selector | `decode` or `prefill` | | Work unit | `decode_step` or `prefill_operation` | -| Methodology | `llm-memory-v1---` | +| Methodology | `llm-memory-v2---` | | Model/scenario plan identity prefix | `llm-memory-work-plan-v1` | | Component identity prefix | `llm-memory-components-v1` | | Logical profile version | phase-specific decode or prefill profile identity | @@ -43,6 +43,7 @@ The current contract is identified by: | Write pattern | backend/phase-specific append or full-prompt affine identity | | Checksum pattern | Backend/phase/layout-specific checksum identity; Metal uses profile-specific dual-mod32 variants | | MSL revision/source SHA-256 | null / null for CPU; exact selected runtime source identity for Metal | +| Run policy | `llm-run-policy-bounded-loop-snapshots-v1` | | Traffic classification | `llm-exact-weight-vs-kv-read-payload-v1` | Component identities use a fixed-order, length-prefixed canonical serialization beginning with @@ -131,11 +132,11 @@ falls back to another profile. Output targets follow the shared process contract: - omitted or empty output means console only; -- exact `--output -` reserves stdout for one final schema 1 document and routes the post-parse human transcript to +- exact `--output -` reserves stdout for one final schema 2 document and routes the post-parse human transcript to stderr; - every other non-empty raw token is a file, including `./-` and flag-shaped names; -- file output uses atomic `.tmp` replacement after each terminal scenario measurement and at command terminal; -- stdout performs the same logical checkpoint and stop transitions without intermediate serialization. +- file output uses atomic `.tmp` replacement every `K=max(1,ceil(count/8))` completed loops plus command terminal; +- stdout preserves task-boundary stop observations but performs no intermediate snapshot preparation or serialization. Parser/logical-preflight or JSON-output peak-estimation failure before runner-result initialization leaves stdout empty. A Metal runtime capability failure after output-session creation emits terminal `unsupported` JSON when enabled and @@ -249,7 +250,11 @@ mixed = W + B * (P + S(P,Q)) * K The weight pass occurs once per full prompt, not once per token or tile. Audit metadata separately records `causal_token_pairs_per_sequence = triangular(P)`, `logical_attention_pairs = L*B*h_q*triangular(P)`, and -`logical_attention_fma_terms = logical_attention_pairs*d_h`. These values are not payload and no FMA is executed. +`logical_attention_fma_terms = logical_attention_pairs*d_h`. These values are not payload and no FMA is executed. Schema 2 places them in `resolved_plan.model_context.prefill`, +with an individual `*_reason_code`: available decimal string/`valid`, or null/`arithmetic-overflow`. Theoretical overflow +does not reject otherwise representable byte work; byte/prefix/lookup/allocation overflow still rejects it. Available +identity values retain their numeric encoding; unavailable values use `unavailable:arithmetic-overflow`. Query heads +`h_q` affect theoretical context, whereas query tile `Q` changes actual prefix-read traffic. ### Paged physical geometry, table, and lookup traffic @@ -386,7 +391,7 @@ The exact logical weight/KV-read crossover is: traffic_crossover_context_tokens = W / (B * K) ``` -Schema 1 preserves the numerator and denominator as exact decimal strings plus a floating estimate. For the vector +Schema 2 preserves the numerator and denominator as exact decimal strings plus a floating estimate. For the vector above, the ratio is `4294967296 / 131072`, exactly 32768 visible tokens. The current-context classification is versioned as `llm-exact-weight-vs-kv-read-payload-v1`: @@ -421,8 +426,8 @@ A non-empty file or stdout JSON target also reserves a conservative peak for one transport text before final memory admission. The estimate covers fixed schema storage, captured input strings, every planned measurement record, and both expected and actual worker-checksum trees. The input-string term also includes the frozen model-plan, methodology, component/layout identities, and applicable prefill aggregate, scenario, execution, -and scope identities. Frozen identities are charged once. Scenario-plan identities scale with the maximum retained -calibration attempts and planned measurement loops. Each variable-length addition is checked, and preliminary and finalized-plan +and scope identities. Frozen identities are charged once. Canonical scenario-plan identities scale with the bounded maximum retained calibration shapes plus frozen plans, +independently of measured loop count. Actual measurement records/checksums still scale with planned loops. Each variable-length addition is checked, and preliminary and finalized-plan estimates use the same canonical identity-size formula. Omitted or empty output adds no serialization reserve; normal console-only execution does not serialize a schema document. @@ -579,7 +584,7 @@ Before every paged decode task, the executor restores only the mutable current-t initialization pattern. This allocation-free reset is outside the timed interval; it does not rewrite history blocks or suffix-padding canaries. Paged prefill instead rewrites every owned prompt byte during each timed operation. The planner uses shared SplitMix64 derivation with frozen domains to derive separate weight/K/V buffer seeds, -permutation seed, and scenario seeds from one base seed; schema 1 stores exact seeds as decimal strings. +permutation seed, and scenario seeds from one base seed; schema 2 stores exact seeds as decimal strings. The initialization pattern treats each mapping as a zero-based stream of little-endian 64-bit words. For mapping word index `i` and that mapping's domain-separated buffer seed: @@ -698,8 +703,8 @@ CPU contiguous-decode checks every byte of both K/V append records for all layer against the final task-local ordinal `T-1`, after timer stop and all worker joins. A mismatch yields `decode-post-validation-failed`: the invalid attempt is retained with null rates and excluded from aggregates. The timed workload and checksum algorithm are unchanged. KV-write evidence applies only -to KV-bearing scenarios; paged/prefill evidence retains the combined final-state check and its existing -padding/sampling limits. CPU paged decode weights-only checks its append slots and padding; CPU paged prefill weights-only checks padding, +to KV-bearing scenarios; schema 2 retains independent write, structure and applicable padding verdicts with the +existing sampling limits. No new memory scan or stronger fault-coverage claim follows from naming these checks. CPU paged decode weights-only checks its append slots and padding; CPU paged prefill weights-only checks padding, not valid prompt contents. See the [CPU final-state evidence contract](API.md#result-schemas-and-completion) for field and null semantics. @@ -881,15 +886,22 @@ Measured duration-quality values are `within-target-window`, `above-target-singl `not-run`. Every measured non-window value produces the corresponding `-duration-` warning. The base order is `weights_only`, `kv_only`, `mixed`; loop `i` rotates it by `i mod 3`. A complete block of three loops -gives every scenario one first, middle, and last position. Count need not be divisible by three, but comparative -conclusions require exact position balance. - -Only checksum-valid `measured` records enter aggregates. Each scenario stores work-unit latency, -`synthetic_memory_work_units_per_second`, and `effective_model_payload_gb_s` values. One value is its own headline; -multiple values use median P50. Statistics include average, median, P90/P95/P99, sample standard deviation, CV, MAD, -minimum, and maximum. Fewer than three measurements is `insufficient-samples`; the effective-model-payload-GB/s CV -above 5% selects `noisy` and the -scenario's diagnostic high-CV warning. Values are not removed, winsorized, or retried because of performance. +gives every scenario one first, middle, and last position. Count need not be divisible by three. Balance is reported independently; a comparison policy may require it, but +producer correctness acceptance does not. + +Only `measured` records with accepted required timing, checksum and cold-check evidence enter aggregates. +One retained authoritative duration/work/payload sample feeds all three metrics through one +`accepted_measurement_ids` population. Each rate is derived before statistics: median(work/duration) need not equal +work/median(duration). Exact statistics use the existing shared linear interpolation and sample standard deviation +helper only at snapshots/terminal, with reusable extraction/sort/MAD workspaces. Raw sample console output does not +sort earlier samples. One sample is its own headline; otherwise the headline is median P50. + +JSON retains average, median, P90/P95/P99, sample stddev, CV, MAD, min/max. Empty populations have null statistics; +n=1 has stddev=0 and positive-mean CV=0. `observed_cv_classification` is `insufficient-samples` for n<3, `undefined` +for undefined CV, `above-threshold` for payload CV strictly greater than 5%, otherwise `below-threshold`. Equality is +below-threshold; `cv_warning_threshold_pct` states the bound. These describe observed samples, not confidence levels +or reproducibility guarantees. Default console uses n/median/min/max/CV/MAD. Quality does not remove, winsorize or retry +samples and does not change correctness acceptance. Mixed payload fractions are exact byte fractions for weight read, KV read, and KV append write. The single mixed elapsed time is not used to publish separate independent weight and KV bandwidths. @@ -926,194 +938,133 @@ measurement remains measured even if the signal arrived during it. Once stop is remaining slots become interrupted/null. A real backend-task, timer, checksum, or checkpoint failure remains authoritative over a simultaneous interruption. -The runner offers one logical checkpoint after every terminal scenario measurement and a distinct command-terminal -checkpoint. File targets serialize each with atomic replacement. Stdout targets preserve the same state transitions and -stop observations but build no intermediate payload and emit exactly one final document. A failed file checkpoint ends -the run, marks checkpoint failure, and is not retried at command terminal. - -`results_complete` means every planned scenario measurement is measured. `scenario_order_balance_complete` requires -the complete realized position matrix to be balanced. `conclusions_valid` requires complete status/results, balanced -order, and no checkpoint failure. A count-one command can therefore finish every planned task and retain valid numeric -measurements while correctly reporting `conclusions_valid: false`. - -## JSON schema 1 - -The old schema-1 producer shape was unpublished. The generic vocabulary below replaces its CPU/decode/step-specific -field names without compatibility aliases, a fallback reader, or a schema-version increment. - -The required top-level schema keys are: - -```text -schema_version, mode, backend, phase, kv_layout, methodology_version, -software, configuration, resolved_plan, backend_evidence, memory_budget, -calibration, measurements, aggregates, status, reason_code, -results_complete, conclusions_valid, interpretation -``` +Files receive a snapshot after every Kth fully completed loop, where `K=max(1,ceil(N/8))` for N planned loops. +The overflow-safe calculation is `max(1,N/8 + (N%8 != 0))`. At most eight progress snapshots and one terminal snapshot +are written (normal maxima 4/7/9 for N=3/12/48). A successful terminal may be followed by one corrective failure +snapshot for a late command exception. Failed persistence is terminal and never retried at the final-write boundary. -Top-level `backend`, `phase`, and `kv_layout` are canonical selectors. `methodology_version` is derived exactly as -`llm-memory-v1---`. `configuration` preserves exact argv plus `resolved_sources`; a default is -recorded as `default`, not as fabricated argv. Additional diagnostic, interruption, checkpoint, loop-order, checksum, -environment, warning, and traffic-classification evidence may be present. +At abrupt termination, at most `3K` truly completed attempts may be missing after the last successful snapshot, +including while its replacement is in progress. Unstarted tail placeholders do not count. Before the first snapshot +there may be no file; SIGKILL/crash cannot promise terminal output. Graceful SIGINT retains the completed prefix and +interrupted tail if terminal persistence succeeds. Atomic rename does not imply power-loss durability. Task-boundary +stop observations remain active even when snapshots are skipped. Stdout builds one terminal DOM, disabled output none. -`resolved_plan` has four required ownership groups: +`checkpoint_lifecycle` distinguishes actual writer entry/return from snapshot construction and logical task transitions. +`prior_file_writer_attempts` and `prior_successful_file_writes` have observation point +`before-current-snapshot-preparation`. `current_request` is `progress`, `terminal` or `late-command-error-correction`; +`current_persistence_success` is null, never predicted. Builder failure is not writer entry; stdout/disabled no-ops +are not persistent successes. No extra snapshot is written to count itself. An older preserved file cannot encode +future write/cleanup failure, so final collection retains actual process outcome and terminal context. -```text -resolved_plan.geometry -resolved_plan.layout -resolved_plan.resources -resolved_plan.component_identities -``` +`results_complete` describes the fully measured planned population. `run_accepted` requires complete run status, +accepted required timing/checksum/cold-check/backend lifecycle and completion evidence, and no known command or +checkpoint error. Balance, count, CV, duration and environment are independent quality context. Count one can be +correct and accepted while position balance is false and sample classification is insufficient. A late command error +may preserve `results_complete: true` while status is failed and `run_accepted: false`. -`geometry.decode` and `geometry.prefill` are object-or-null. Active decode populates integer -`decode.visible_context_tokens` and uses null prefill; active prefill does the reverse and never reuses context fields. -Its object has integer `prompt_tokens` and -`attention_query_tile_tokens` plus decimal-string `tile_count`, -`attention_prefix_token_visits_per_sequence`, `causal_token_pairs_per_sequence`, `logical_attention_pairs`, and -`logical_attention_fma_terms`. Decode-only crossover numerator/denominator/context, weight/KV-read ratio, and -context-classification fields are null for prefill. -`layout.kv_layout` is a string. Paged populates integer `kv_block_tokens`; decimal-string -`blocks_per_sequence`, `physical_blocks_per_layer`, `last_block_tokens`, `last_block_valid_bytes`, -`block_table_entries`, and `block_table_bytes`; plus `permutation_domain_uint64_hex`, -`permutation_seed_uint64_decimal`, `permutation_algorithm_version`, and `permutation_sha256`. These paged-only fields -are null for contiguous. -`resources` stores canonical decimal-string `weight_logical_bytes`, `k_logical_bytes`, `v_logical_bytes`, -`k_physical_length_bytes`, `v_physical_length_bytes`, `k_layout_padding_bytes`, and `v_layout_padding_bytes`; -`block_table_bytes` is decimal-string-or-null. - -`component_identities` contains, in canonical fixed order: +## JSON schema 2 -```text -logical_profile_version -kv_layout_version -permutation_version -backend_executor_version -resource_abi_version -schedule_version -timer_policy_version -buffer_pattern_version -write_pattern_version -checksum_pattern_version -msl_revision -msl_source_sha256 -``` +Schema 2 deliberately replaces the unpublished schema-1 shape and adopts v2 methodology selectors. There is no +compatibility alias or fallback reader. The normative [API field map](API.md#llm-schema-1-to-schema-2-field-map) +lists each relocation and changed predicate. -Always-applicable values are strings. `permutation_version` is null for contiguous; MSL fields are null for CPU. The -serialized identity begins `llm-memory-components-v1` and appends every field in that order as `|key=:` -or `|key=null`. - -`backend_evidence` contains both `cpu` and `metal` object-or-null branches. Exactly the selected backend is populated; -CPU profiles have a CPU object and `metal: null`. Within the CPU object, `prefill` is null for decode and -populated for either prefill layout. The prefill evidence fixes `cost_unit: "worker-cost"`, execution and scope identities, -descriptors per scenario/worker, decimal-string worker cost vectors, and scenario-specific decimal-string minimum, -maximum, and max-minus-min imbalance per work unit. Its `paged` sibling is populated for either paged phase, so paged -prefill has both objects. Metal profiles use `cpu: null` and populate `metal` with worker/QoS applicability false; -lifecycle status/reasons; device/family/unified/Tier-2/compiler/MSL/source/pipeline evidence; argument-buffer layout -probe; actual resource options, lengths, optional allocated sizes, and memory totals. Paged Metal evidence additionally -publishes whole-entry table segments, upload/readback validation, materialized permutation identity, combined K/V -padding, exact grid lookup count, owner/threadgroup geometry, per-threadgroup `actual-threadgroup-cost` accounted-byte -vector and minimum/maximum/imbalance, and K/V-write/padding-canary validity. Every Metal `weights_only` task publishes -the same cost unit for its weight-vector grid-stride schedule; contiguous KV-bearing grids publish null cost summary -fields and an empty vector. Runtime unsupported/failure diagnostics remain -bounded and separate from stable reason codes. -`memory_budget` separates allocation-time evidence into required -canonical decimal-string `resource_rounding_bytes`, `transient_peak_bytes`, `known_owned_peak_bytes`, and -`admitted_budget_bytes`. `calibration` owns excluded work-resolution attempts. `aggregates` contains measured-only -scenario values. - -Every measurement exposes this stable backend-neutral accounting vocabulary: +The exact active methodology selectors are: -```text -work_unit_kind -planned_work_units -completed_work_units -weight_read_bytes_per_work_unit -kv_read_bytes_per_work_unit -kv_write_bytes_per_work_unit -kv_write_kind -effective_model_payload_bytes_per_work_unit -layout_metadata_lookup_count_per_work_unit -layout_metadata_read_bytes_per_work_unit -accounted_bytes_per_work_unit -planned_effective_model_payload_bytes -completed_effective_model_payload_bytes -planned_layout_metadata_lookup_count -completed_layout_metadata_lookup_count -planned_layout_metadata_read_bytes -completed_layout_metadata_read_bytes -planned_task_accounted_bytes -completed_task_accounted_bytes -synthetic_work_unit_latency_seconds -synthetic_memory_work_units_per_second -effective_model_payload_gb_s -``` +- `llm-memory-v2-cpu-decode-contiguous`, `llm-memory-v2-cpu-decode-paged`; +- `llm-memory-v2-cpu-prefill-contiguous`, `llm-memory-v2-cpu-prefill-paged`; +- `llm-memory-v2-metal-decode-contiguous`, `llm-memory-v2-metal-decode-paged`; +- `llm-memory-v2-metal-prefill-contiguous`, `llm-memory-v2-metal-prefill-paged`. -Measurement checksum evidence uses generic `write_pattern_version` and `checksum_pattern_version` keys; it does not -reuse decode-specific append terminology for prefill. Metal measurement evidence additionally publishes pipeline/grid, -raw GPU start/end and authoritative elapsed, host envelope and nullable same-clock queue delay, -command-buffer/encoder/dispatch counts, dual-mod32 W/K/V expected/actual values, and excluded phase-neutral -`kv_write_evaluated`/`kv_write_valid` state. The console renders the same contract as `kv_write=`. - -Decode uses `decode_step` and KV-bearing `current_token_append`; prefill uses `prefill_operation` and KV-bearing -`full_prompt_population`. `weights_only` uses `kv_write_kind: "none"` in both phases. Planned/completed work units are -integer numbers. All listed byte, lookup, metadata, and accounted fields are canonical decimal strings even when the -applicable value is zero. Derived elapsed/rate/statistic -values are finite JSON numbers only for successful measured evidence and otherwise null. - -The field type never changes by backend, phase, layout, scenario, or magnitude. Schema/control indexes and validated -small inputs such as worker count and visible-context tokens are JSON integers bounded to the exact IEEE-754 integer -range. Potentially large bytes, capacities, block/table/lookup counts, token visits, causal pairs, FMA terms, seeds, and -checksums are canonical decimal strings. A non-applicable object or scalar is null; an applicable count of zero is -number `0` or decimal string `"0"` according to the field's fixed type. The string `"not_applicable"` is never used. - -The complete document additionally retains: - -- exact raw argv/output plus requested/default configuration; -- methodology/component identities, fixed policies, geometry, MHA/GQA/MQA metadata, and exact traffic/crossover inputs; -- requested, page-rounded committed, transient, known-owned, allowed, and available memory evidence; -- full mapping/init/descriptor or Metal segment/argument-buffer evidence and backend-applicable worker fields; -- base, buffer-domain, and scenario-domain seeds; -- immutable model and per-scenario work-plan identities, limits, exact work units, model payload, metadata, and accounted - totals; -- excluded warmup/pilot/trial/correction attempts; -- planned/attempted/completed loop, measurement, work-unit, payload, and checkpoint counters; -- planned/realized cyclic order, every status-bearing measurement, checksum evidence, and measured-only aggregates; -- CPU/OS/cache/page/QoS plus start/end thermal, Low Power Mode, and physical-memory snapshots; -- quality-warning tokens and the non-inference/non-DRAM interpretation boundary. - -The authoritative consumer acceptance predicate is exactly: +The required top-level fields are: ```text -mode == "llm_memory" -schema_version == 1 -backend == requested_backend -phase == requested_phase -kv_layout == requested_kv_layout -methodology_version == - "llm-memory-v1-" + backend + "-" + phase + "-" + kv_layout -status == "complete" -results_complete == true -conclusions_valid == true -every planned measurement has status == "measured" +schema_version, mode, backend, phase, kv_layout, methodology_version, +software, configuration, resolved_plan, backend_evidence, memory_budget, +calibration, measurements, aggregates, status, reason_code, +results_complete, run_accepted, interpretation, diagnostic, interruption_requested, +scenario_order_balance_complete, seeds, counters, checkpoint_lifecycle, loop_records, +environment, quality_warnings, build_manifest ``` -For active commands the requested values are CPU or Metal decode/prefill with contiguous or paged KV. Paged acceptance -and comparison must additionally match `G`, `N`, tail geometry, logical/physical/padding/table resources, -permutation -version/domain/resolved seed/hash, lookup/accounted bytes, worker schedule, descriptor/executor, timer, and checksum -identities. `unsupported`, `partial`, `interrupted`, `invalid`, and `failed` evidence is never accepted as performance. -Metal comparison additionally requires matching device capability, MSL revision/source hash, pipeline/grid identity, -exact segment geometry, and timer/checksum contracts. CPU and Metal samples are never pooled and their checksums are -not numerically comparable. -After the predicate, a consumer must still require the selected scenario metric's non-null value plus checksum and -quality conditions relevant to its conclusion. Process exit success alone is insufficient because graceful -interruption is an established success-return path. - -The `interpretation` object always preserves the generic boundary: the workload is synthetic and memory-only; -effective GB/s is exact logical W/K/V payload divided by authoritative elapsed time, not measured physical DRAM -traffic; timed paged-table metadata is excluded from that numerator; prefill profiles do not perform Transformer compute -or predict TTFT; private Metal storage on unified memory is not separate VRAM; cache/SLC/DRAM residency is unmeasured; -and results with differing backend, phase, layout, phase geometry, paged geometry, methodology, or component identity -must not be pooled as one performance distribution. +`configuration` preserves exact argv/output target and resolved/default input sources. Omitted iterations means +automatic calibration; file output still needs a target. Omitted/empty output is disabled, with no automatic filename. + +`resolved_plan` owns `plan_identity`, logical `geometry`, `model_context`, `layout`, `resources`, +`component_identities`, `methodology`, `model_work_plan`, `scenario_plans`, and `frozen_plan_refs`. +Exactly one geometry phase object is populated. Decode uses integer visible-context tokens; prefill has integer P/Q +and decimal-string tile/prefix visits. Decode-only crossover and classification values are null for prefill. +`model_context.prefill` contains nullable theoretical quantities as described above; it is null for decode. + +Paged layout retains integer block size, decimal block/tail/table geometry and materialized permutation domain, +seed, version and hash. Before Metal table preparation, admitted geometry survives with null runtime permutation. +Resources retain exact logical/physical K/V, suffix padding, table and Metal segment/argument-buffer geometry. +Component identity retains fixed-order length-prefixed logical/layout/permutation/backend/ABI/schedule/timer/ +buffer/write/checksum/MSL fields under `llm-memory-components-v1`; CPU MSL and contiguous permutation fields are null. +The separately published `run_policy_version` is `llm-run-policy-bounded-loop-snapshots-v1`. + +Canonical `scenario_plans[]` stores unique scenario/T/explicit content once, including excluded calibration shapes, +ordered by weights/KV/mixed, increasing T, then false/true explicit policy. Each retains exact identity, scenario seed, +`model_ref: "resolved_plan"`, work kind, work units, limits and all planned byte/lookup quantities. `frozen_plan_refs` +has the three scenario keys and integer index or unresolved null. Measurement and calibration `plan_ref` refer to the +same document's array; public indexes may change between snapshots while internal insertion handles remain stable. +Unknown/out-of-range and wrong-scenario references reject publication. Canonical content is immutable after registration. + +A measurement retains `measurement_id` (its scheduled array index), scenario/loop/order, status/reason, authoritative +accepted `elapsed_seconds`, completed work/payload/metadata/accounted bytes and actual runtime evidence. +Work kind, `work_units`, per-work-unit bytes and planned totals are read through `plan_ref`. Canonical total names are +`effective_model_payload_bytes`, `layout_metadata_lookup_count`, `layout_metadata_read_bytes`, `task_accounted_bytes`; +measurement totals keep `completed_`. CPU `completion_derivation: "accepted-plan-derived"` is not a hardware counter. +Derived workers/QoS, calibration indexes, working-set and mixed fractions remain available. Invalid attempts retain +observed diagnostic time and actual checksum evidence while accepted elapsed/rates are null. + +Expected checksum lives once at `scenario_plans[].expected_checksum`, with exactly status/reason, +`expected_worker_checksums`, `expected_run_checksum`. Available means the canonical cold expectation was reconstructed, +not that execution passed; unavailable makes both expected fields null. CPU worker expectations are ordered by worker +and include W/K/V component state, exact bytes and span count; CPU run checksum has state A/B. Metal expected workers +are null; its run object contains W/K/V dual-mod32 pairs. Measurement checksum contains status/reason/validity and +actual worker/run witnesses. Calibration retains compact actual-run-only evidence at `execution.checksum`, with +expectations resolved via its plan reference; no actual-worker array is invented. Algorithm versions have one owner in +canonical components, and CPU/Metal numerical witnesses are not cross-backend comparisons. + +Named checks under measurement/calibration `execution.validation.checks[]` preserve the independently completed +structure, applicable phase/scenario write/unchanged and padding observations. Their kind tokens are +`post-validation-structure`, `kv-append-final`, `kv-prefill-final-samples`, `kv-append-unchanged`, `kv-padding-canary`. +CPU paged decode weights-only checks unchanged append and applicable padding; CPU paged prefill weights-only checks +structure/padding, not valid prompt content. Metal weights-only has no write/padding check. Inapplicable evaluated/valid are null; +applicable unresolved valid is null; observed mismatch remains false. Checksum agreement may coexist with failed cold +validation and cannot override it. The [fault matrix](#checksum-fault-model-and-evidence-limits) retains its scope: +new names add no collision resistance, unsampled-byte coverage or runtime trace. + +`backend_evidence` retains tagged CPU/Metal branches, lifecycle, capability, resources and bounded error diagnostics. +CPU prefill preserves execution/scope identities and exact worker-cost vectors; paged prefill also has paged evidence. +Metal has null workers and preserves MSL/layout-probe/pipeline, actual resource options/lengths, grid/owner costs, +GPU raw start/end, host envelope, command/encoder/dispatch observations and nullable same-clock queue delay. +`memory_budget` separates immutable geometry from rounded/committed/transient/known-owned/admitted runtime evidence. +Canonical plan/expected vector capacities, calibration, accepted-ID maps, exact-stat scratch and DOM/serialized-string +simultaneous peak are admitted together; paged Metal canonical expectations also reserve table/validation/hash scratch. +The available-memory sample is excluded from immutable identities. Snapshot count reduction does not reduce peak size. + +Indexes, bounded small inputs and work units are JSON integers below 2^53, never booleans. Bytes, lookups, visits, +seeds and checksums are canonical unsigned decimal strings (`0` or `[1-9][0-9]*`), with the relevant uint64/uint32 bounds. +Known zero is not null. Unavailable/inapplicable values retain null plus applicability/status/reason semantics. + +`build_manifest` reserves `manifest_version: 1`, `status: "unavailable"`, +`reason_code: "build-provenance-not-provided"`, and null `binary_sha256`, `git_commit`, `git_dirty`, `compiler`, +`compile_flags`, `link_flags`, `target_arch`, `sdk`, `min_os`. Optional CPU raw-tick evidence is not emitted in this +revision. Neither this reservation nor software/MSL version identity is independent build attestation. + +Consumer acceptance requires successful process outcome, exact mode/schema2/backend/phase/layout/v2 methodology, +`run_policy_version: "llm-run-policy-bounded-loop-snapshots-v1"`, complete status, `results_complete: true`, +`run_accepted: true`, every planned measurement measured, valid same-document references, and a non-null selected +metric. This consumes producer acceptance; it is not an independent re-execution of the checksum, timing or build proof. +A comparison policy separately checks matched geometry, physical layout/permutation, seeds, exact work, backend/device, +component/MSL/pipeline/timer identity and environment. Position balance and CV can inform that policy but never redefine +correctness or justify performance-based sample filtering. + +The `interpretation` object preserves the memory-only boundary: effective GB/s is logical W/K/V divided by authoritative +elapsed time, not physical DRAM; timed table metadata is excluded; prefill does no Transformer math and predicts no TTFT; +private Metal memory is not separate VRAM; cache/SLC/DRAM residency is unmeasured. Distinct backend/phase/layout/model/ +component cohorts must not be pooled as one distribution. ## Console contract and quality warnings @@ -1122,8 +1073,8 @@ weight/KV-read/KV-write bytes, and up to one measured headline per scenario. Dec prefill prints P/Q/C, prefix visits, causal pairs, and logical attention/FMA audit counts. It uses phase-specific labels such as `ms/decode step` or `ms/prefill operation`; JSON remains backend-neutral with `synthetic_work_unit_latency_seconds`, `synthetic_memory_work_units_per_second`, and -`effective_model_payload_gb_s`. Metal task output uses `kv_write=valid|invalid|not-evaluated|not-applicable`, matching -the generic JSON validation fields rather than a decode-only append label. The report never uses bare `tokens/s` and +`effective_model_payload_gb_s`. Metal task output uses `kv_write=valid|invalid|not-evaluated|not-applicable`, summarizing +the phase-specific named JSON checks. The report never uses bare `tokens/s` and states that effective model payload is not a physical DRAM counter. A scenario without a headline does not receive a fabricated numeric console value; its status, reason, and null observations remain in JSON. @@ -1195,7 +1146,7 @@ Any change to traffic formulas, context semantics, buffer sizing/layout, tempora output-serialization peak admission, timing boundary, checksum observability, calibration/frozen-plan rules, interruption/checkpoint lifecycle, or meaning of a reported field requires methodology and schema compatibility review. Removing or renaming a field, changing its type, or changing its meaning requires a schema-version bump. Additive -evidence may remain schema 1 only when existing consumers can safely ignore it. +evidence may remain schema 2 only when existing consumers can safely ignore it. Runtime paged allocation/free lists, prefix sharing, sliding windows, growing context, chunked prefill, Metal execution outside the active profiles, ANE execution, model presets, quantization metadata, diff --git a/documents/MANUAL.md b/documents/MANUAL.md index a5b8e23..3400fda 100644 --- a/documents/MANUAL.md +++ b/documents/MANUAL.md @@ -196,7 +196,10 @@ boundaries, parallelism, cache behavior, resource modes, and validation overhead ### Synthetic LLM memory payload -LLM-memory schema 1 uses generic backend/phase/layout/work-unit vocabulary. This revision activates all eight +LLM-memory schema 2 uses generic backend/phase/layout/work-unit vocabulary. Theoretical attention context +uses query-head count `h_q`; actual prefix-read geometry uses tile `Q`. They are different inputs. Theoretical +causal/attention/FMA overflow becomes null with `arithmetic-overflow` in `resolved_plan.model_context.prefill`; +actual byte/lookup/size overflow still rejects the workload. No FMA computation is performed. This revision activates all eight backend/phase/layout profiles: CPU and Metal each support decode and prefill with contiguous or paged KV. Prefill uses `work_unit_kind: "prefill_operation"`; decode uses `"decode_step"`. Metal never falls back to CPU. @@ -274,8 +277,8 @@ CPU contiguous-decode checks every byte of both K/V append records for all layer against the final task-local ordinal `T-1`, after timer stop and all worker joins. A mismatch yields `decode-post-validation-failed`: the invalid attempt is retained with null rates and excluded from aggregates. The timed workload and checksum algorithm are unchanged. KV-write evidence applies only -to KV-bearing scenarios; paged/prefill evidence retains the combined final-state check and its existing -padding/sampling limits. CPU paged decode weights-only checks its append slots and padding; CPU paged prefill weights-only checks padding, +to KV-bearing scenarios; schema 2 retains independent write, structure and applicable padding verdicts with the +existing sampling limits. No new memory scan or stronger fault-coverage claim follows from naming these checks. CPU paged decode weights-only checks its append slots and padding; CPU paged prefill weights-only checks padding, not valid prompt contents. See the [CPU final-state evidence contract](API.md#result-schemas-and-completion) for field and null semantics. @@ -632,10 +635,10 @@ middle, and trailing items. #### `--llm-memory` - Selects the standalone CPU/Metal synthetic memory mode. Active methodologies are - `llm-memory-v1-cpu-decode-contiguous`, `llm-memory-v1-cpu-decode-paged`, - `llm-memory-v1-cpu-prefill-contiguous`, `llm-memory-v1-cpu-prefill-paged`, - `llm-memory-v1-metal-decode-contiguous`, `llm-memory-v1-metal-decode-paged`, - `llm-memory-v1-metal-prefill-contiguous`, and `llm-memory-v1-metal-prefill-paged`. It never enters the general + `llm-memory-v2-cpu-decode-contiguous`, `llm-memory-v2-cpu-decode-paged`, + `llm-memory-v2-cpu-prefill-contiguous`, `llm-memory-v2-cpu-prefill-paged`, + `llm-memory-v2-metal-decode-contiguous`, `llm-memory-v2-metal-decode-paged`, + `llm-memory-v2-metal-prefill-contiguous`, and `llm-memory-v2-metal-prefill-paged`. It never enters the general `BenchmarkConfig` parser or CPU sweep runner - The four active Metal methodologies are governed by runtime capability checks - Requires each common model option `--weight-size-mb `, `--layers `, `--query-heads `, @@ -690,10 +693,11 @@ middle, and trailing items. Cyclic measured order gives all scenarios each position once when count is three - Uses task-boundary completion-wins interruption. A started scenario is not polled in its hot kernel; a completed and checksum-valid current task stays measured, while no next task starts after the stop is observed -- Output values retain the shared raw-target syntax: empty disables JSON, exact `-` emits one final schema 1 document, +- Output values retain the shared raw-target syntax: empty disables JSON, exact `-` emits one final LLM schema 2 document, and every other non-empty value is a file target, including `./-` and flag-shaped names. File output atomically - checkpoints each terminal scenario measurement and command terminal; stdout performs the same logical transitions - without intermediate serialization + snapshots every `K=max(1,ceil(count/8))` completed loops plus command terminal, with at most eight progress + snapshots and an abrupt-loss bound of `3K` completed attempts. Stdout retains task stop observations without + intermediate snapshot preparation - CPU prefill uses layout-specific ARM64 executors and reports scenario-specific cost-balanced owner identities plus minimum/maximum/imbalance worker-cost evidence. Paged prefill additionally reports deterministic block-exclusive ownership, table/permutation identity, exact lookup metadata, and padding validation @@ -924,8 +928,8 @@ middle, and trailing items. - For every result-producing direct mode and the CPU modes' supported sweeps, an exact raw value of `-` selects machine-readable stdout. The sentinel is classified before path normalization - A supported stdout-target command emits exactly one two-space-indented UTF-8 JSON document followed by one newline - after orchestration reaches its terminal state. Intermediate standard, sweep, GPU, and LLM checkpoint requests are - lazy no-ops and do not emit documents + after orchestration reaches its terminal state. Intermediate standard, sweep and GPU checkpoint requests are + lazy no-ops; LLM skips intermediate snapshot preparation. Neither emits intermediate documents - Post-parse human output is routed to stderr while the stdout target is active. Parse, preflight, and other pre-result failures leave stdout empty; `--help` remains a human-facing stdout command when that mode accepts the combination - For result-producing modes, every other non-empty value is a file target, including `./-` and flag-shaped names such @@ -934,12 +938,13 @@ middle, and trailing items. syntax in `configuration.output_file` - Standard and sweep file targets retain atomic intermediate checkpoints. Pattern, TLB, and core-to-core direct file targets each receive one final atomic write. GPU file output retains its terminal-measurement/failure checkpoints and - post-release replacement. LLM file output checkpoints after every terminal scenario measurement and once at command - terminal. A failed LLM checkpoint is terminal and is not retried as a final file write. Use a real file when - crash-resilient intermediate checkpoints are required -- With GPU or LLM `--output -`, every logical checkpoint transition and post-checkpoint stop observation still runs, but its + post-release replacement. LLM files snapshot every `K=max(1,ceil(count/8))` completed loops plus command terminal. + At most eight progress writes occur; abrupt loss is bounded by `3K` completed attempts, including during replacement. + No file is promised before the first successful snapshot. Failed LLM persistence is terminal without final-write retry +- With GPU `--output -`, every logical checkpoint transition and post-checkpoint stop observation still runs, but its lazy payload builder is not invoked and no intermediate document is serialized. The command boundary writes one - terminal schema 1 document. `configuration.output_file` remains the raw string `"-"`, and captured `argv` is not + terminal GPU schema 1 document. LLM retains task stop observations and emits one terminal schema 2 document, with + no intermediate DOM. `configuration.output_file` remains the raw string `"-"`, and captured `argv` is not rewritten - GPU syntax/config errors, including a buffer below 64 MB, fail before result JSON is created. A backend-factory failure also precedes result initialization. Initialized capability, compilation, allocation, or work-plan failures remain @@ -968,7 +973,7 @@ middle, and trailing items. - `--benchmark --only-latency` supports `buffer-size`, `cache-size`, and latency chain/locality/stride keys - `--analyze-tlb` supports `latency-stride-bytes`, `latency-chain-mode`, and `tlb-density` - `--analyze-core2core` supports `count` and `latency-samples` -- `--gpu-bandwidth` and `--llm-memory` do not support `--sweep` or `--sweep-max-runs` in their schema-1 modes +- `--gpu-bandwidth` and `--llm-memory` do not support `--sweep` or `--sweep-max-runs` (GPU schema 1, LLM schema 2) #### `--sweep-max-runs ` @@ -1086,7 +1091,7 @@ memory_benchmark --gpu-bandwidth --buffer-size 512 --iterations 24 --count 9 --s memory_benchmark --llm-memory --weight-size-mb 4096 --layers 32 --query-heads 32 --kv-heads 8 \ --head-dim 128 --context-tokens 8192 --count 3 --seed 123456789 --output llm_memory.json -# Standalone LLM automation: reproducible fixed work and one final schema 1 document on stdout +# Standalone LLM automation: reproducible fixed work and one final schema 2 document on stdout memory_benchmark --llm-memory --weight-size-mb 64 --layers 4 --query-heads 8 --kv-heads 2 \ --head-dim 64 --context-tokens 512 --iterations 1 --count 3 --seed 42 --output - \ >llm_memory_stdout.json 2>llm_memory.log @@ -1177,7 +1182,7 @@ memory_benchmark --gpu-bandwidth --sweep buffer-size=64,128 --output gpu_sweep.j memory_benchmark --llm-memory --weight-size-mb 64 --layers 4 --query-heads 8 --kv-heads 2 \ --head-dim 64 --context-tokens 512 --non-cacheable -# invalid: LLM schema 1 has no sweep support +# invalid: LLM schema 2 has no sweep support memory_benchmark --llm-memory --weight-size-mb 64 --layers 4 --query-heads 8 --kv-heads 2 \ --head-dim 64 --context-tokens 512 --sweep context-tokens=512,1024 --output llm_sweep.json @@ -1554,8 +1559,8 @@ invalid measurement is not printed as zero and remains status-bearing/null in JS effective model-payload GB/s; JSON uses the backend-neutral fields `synthetic_work_unit_latency_seconds`, `synthetic_memory_work_units_per_second`, and `effective_model_payload_gb_s`, without calling any value tokens/s; - warnings for incomplete position balance, non-nominal environment, QoS failure, cache-dominant working sets, high - effective-model-payload CV, or duration outside the intended window. JSON retains the complete repeatability statistics; - the console prints the CV value only in a high-CV warning; + effective-model-payload CV, or duration outside the intended window. Console statistics include n, median, min/max, + CV and MAD; JSON additionally retains descriptive P90/P95/P99, without treating them as confidence levels; - an interpretation reminder that the payload is logical and memory-only, not physical DRAM traffic or model inference. Paged suffix padding is capacity and validation evidence, not payload. A padding-canary or write/checksum mismatch makes @@ -1579,8 +1584,8 @@ report. Every result-producing direct mode and the CPU modes' supported sweeps can serialize their payload either to a real file or, with the exact raw target `--output -`, once to stdout. The stdout transport does not wrap the result or change its -measurement schema; sweep stdout is one final envelope rather than a checkpoint stream. LLM file targets checkpoint -each terminal scenario measurement and command terminal, while LLM stdout emits one final schema 1 document. See +measurement schema; sweep stdout is one final envelope rather than a checkpoint stream. LLM files snapshot bounded completed-loop progress +plus command terminal, while LLM stdout emits one final schema 2 document. See [API.md](API.md) for stream handling, process-status checks, current schema acceptance, and the transport support matrix. ### Standard benchmark JSON shape (schema 3) @@ -1851,194 +1856,211 @@ computed measurement status. ### LLM memory-profile JSON shape -LLM file and stdout output use the same separate top-level generic schema 1. It is not a standard benchmark payload and -must be classified by `mode` and `schema_version`, then by the exact backend/phase/layout/methodology identity. The old -unpublished CPU/step-specific schema-v1 shape has no compatibility aliases or fallback reader; schema 1 is intentionally -re-frozen in this generic form without a version bump. +LLM file and stdout output use the same separate top-level schema 2. Select by `mode`, `schema_version`, and exact +backend/phase/layout/methodology. This replaces unpublished schema 1 with an explicit schema and methodology bump; +there is no compatibility alias or fallback reader. The [API field map](API.md#llm-schema-1-to-schema-2-field-map) +is the normative old-to-new mapping. -This abbreviated structural selection shows CPU/decode/contiguous. Either paged phase populates the paged-only fields; -either prefill layout instead populates `geometry.prefill` and uses null decode/crossover/weight-to-KV-read ratio -fields. The paged-prefill methodology is `llm-memory-v1-cpu-prefill-paged`. A real document contains complete -calibration, measurement, checksum, loop, checkpoint, environment, warning, and interpretation evidence in addition to -the required generic sections. +This structural projection comes from a complete one-loop CPU paged-decode run (1 MiB weights, one layer/head/worker, +head dimension 8, one-byte KV elements, context 32, block size 16, seed 42, one work unit). It keeps all three plan +reference targets but displays only measurement ID 1 and selected nested fields. Geometry, plan identities, expected +worker vectors, other measurements/calibration and full statistics are omitted. It is **not a complete acceptable +artifact** and must not be passed to a result-acceptance predicate. The displayed checksum states are observed fixture +data; they do not independently prove the implementation. ```json { - "schema_version": 1, + "schema_version": 2, "mode": "llm_memory", "backend": "cpu", "phase": "decode", - "kv_layout": "contiguous", - "methodology_version": "llm-memory-v1-cpu-decode-contiguous", - "software": { - "version": "", - "timestamp": "..." - }, + "kv_layout": "paged", + "methodology_version": "llm-memory-v2-cpu-decode-paged", "status": "complete", - "reason_code": "complete", "results_complete": true, - "conclusions_valid": true, - "configuration": { - "argv": ["memory_benchmark", "--llm-memory", "...", "--output", "-"], - "resolved_sources": { - "backend": "default", - "phase": "default", - "kv_layout": "default" - } - }, + "run_accepted": true, + "scenario_order_balance_complete": false, "resolved_plan": { - "geometry": { - "decode": {"visible_context_tokens": 512}, - "prefill": null - }, - "layout": { - "kv_layout": "contiguous", - "kv_block_tokens": null, - "blocks_per_sequence": null, - "physical_blocks_per_layer": null, - "last_block_tokens": null, - "last_block_valid_bytes": null, - "block_table_entries": null, - "block_table_bytes": null, - "permutation_domain_uint64_hex": null, - "permutation_seed_uint64_decimal": null, - "permutation_algorithm_version": null, - "permutation_sha256": null - }, - "resources": { - "weight_logical_bytes": "67108864", - "k_logical_bytes": "524288", - "v_logical_bytes": "524288", - "k_physical_length_bytes": "524288", - "v_physical_length_bytes": "524288", - "k_layout_padding_bytes": "0", - "v_layout_padding_bytes": "0", - "block_table_bytes": null - }, "component_identities": { - "logical_profile_version": "decode_steady_fixed_context", - "kv_layout_version": "contiguous_layer_batch_token_head_dimension", - "permutation_version": null, - "backend_executor_version": "llm-cpu-executor-v1-arm64-decode-contiguous", - "resource_abi_version": "llm-memory-descriptor-abi-v1", - "schedule_version": "worker-local-layer-order-no-per-layer-global-barrier", - "timer_policy_version": "synchronized-start-to-last-worker-completion-per-scenario-task", - "buffer_pattern_version": "llm-buffer-pattern-v1", - "write_pattern_version": "llm-kv-append-affine64-v1", - "checksum_pattern_version": "llm-read-checksum-v1", - "msl_revision": null, - "msl_source_sha256": null + "run_policy_version": "llm-run-policy-bounded-loop-snapshots-v1" + }, + "scenario_plans": [ + { + "scenario": "weights_only", + "work_units": 1, + "model_ref": "resolved_plan" + }, + { + "scenario": "kv_only", + "work_units": 1, + "model_ref": "resolved_plan", + "expected_checksum": { + "status": "available", + "reason_code": "valid", + "expected_run_checksum": { + "state_a_uint64_decimal": "13746298198528704984", + "state_b_uint64_decimal": "13236122334346434487" + } + } + }, + { + "scenario": "mixed", + "work_units": 1, + "model_ref": "resolved_plan" + } + ], + "frozen_plan_refs": { + "weights_only": 0, + "kv_only": 1, + "mixed": 2 + } + }, + "measurements": [ + { + "measurement_id": 1, + "plan_ref": 1, + "scenario": "kv_only", + "status": "measured", + "completed_work_units": 1, + "completed_effective_model_payload_bytes": "528", + "checksum": { + "status": "valid", + "checksum_valid": true, + "actual_run_checksum": { + "state_a_uint64_decimal": "13746298198528704984", + "state_b_uint64_decimal": "13236122334346434487" + } + }, + "execution": { + "validation": { + "checks": [ + { + "kind": "post-validation-structure", + "applicable": true, + "evaluated": true, + "valid": true, + "reason_code": "valid" + }, + { + "kind": "kv-append-final", + "applicable": true, + "evaluated": true, + "valid": true, + "reason_code": "valid" + }, + { + "kind": "kv-padding-canary", + "applicable": false, + "evaluated": null, + "valid": null, + "reason_code": "not-applicable" + } + ] + } + } + } + ], + "aggregates": { + "scenarios": { + "kv_only": { + "accepted_measurement_ids": [ + 1 + ], + "observed_cv_classification": "insufficient-samples", + "cv_warning_threshold_pct": 5.0 + } } }, - "backend_evidence": { - "cpu": {}, - "metal": null + "checkpoint_lifecycle": { + "checkpoint_failed": false, + "observation_point": "before-current-snapshot-preparation", + "prior_file_writer_attempts": 0, + "prior_successful_file_writes": 0, + "current_request": "terminal", + "current_persistence_success": null, + "snapshot_interval_loops": 1, + "checkpoint_policy": "bounded-loop-snapshots", + "file_checkpoint_failure_is_terminal_and_not_retried": true, + "stdout_intermediate_checkpoints_are_lazy": true }, - "memory_budget": {}, - "calibration": {}, - "measurements": [], - "aggregates": {}, - "interpretation": {} + "build_manifest": { + "manifest_version": 1, + "status": "unavailable", + "reason_code": "build-provenance-not-provided", + "binary_sha256": null, + "git_commit": null, + "git_dirty": null, + "compiler": null, + "compile_flags": null, + "link_flags": null, + "target_arch": null, + "sdk": null, + "min_os": null + } } ``` -For a paged result, `configuration.kv_layout` is `"paged"`, `configuration.kv_block_tokens` is the explicit integer -`G`, and `resolved_sources.kv_layout` records `"explicit"`. The `resolved_plan.layout` block populates `G`, -`blocks_per_sequence = ceil(A/G)`, `physical_blocks_per_layer = B*ceil(A/G)`, terminal-token/byte counts, uint32 table -entry/byte counts, permutation domain, resolved seed, algorithm version, and lowercase 64-hex SHA-256. The resource -block reports full physical K/V lengths and their difference from logical K/V as layout padding. These are fixed-schema -integers or canonical decimal strings according to the type rules below; they are not human-formatted sizes. - -Metal paged requests that terminate before table preparation retain admitted block geometry and table sizes, but use -JSON null for the runtime permutation and combined layout identity. A complete paged result has materialized table -evidence and therefore populates those fields. - -Schema 1 rules: - -- Canonical selectors are `backend: cpu|metal`, `phase: decode|prefill`, `kv_layout: contiguous|paged`, - `work_unit_kind: decode_step|prefill_operation`, and - `kv_write_kind: none|current_token_append|full_prompt_population`. All eight backend/phase/layout profiles are active; - CPU and Metal each support decode and prefill with contiguous or paged KV. -- Methodology is always `llm-memory-v1---`. Active exact identities are - `llm-memory-v1-cpu-decode-contiguous`, `llm-memory-v1-cpu-decode-paged`, - `llm-memory-v1-cpu-prefill-contiguous`, `llm-memory-v1-cpu-prefill-paged`, - `llm-memory-v1-metal-decode-contiguous`, `llm-memory-v1-metal-decode-paged`, - `llm-memory-v1-metal-prefill-contiguous`, and `llm-memory-v1-metal-prefill-paged`. -- `backend_evidence.cpu.prefill` is null for decode. Either prefill layout populates it with `cost_unit: - "worker-cost"`, one execution identity, descriptors per scenario/worker, and scenario-specific partition/scope - identities, decimal-string worker costs, minimum, maximum, and max-minus-min imbalance per work unit. Paged prefill - also populates `backend_evidence.cpu.paged`. -- Run statuses are `not_started`, `complete`, `partial`, `interrupted`, `unsupported`, and `failed`. Measurement statuses are - `not_run`, `measured`, `interrupted`, `invalid`, and `failed`. Multiword status tokens use underscores; multiword - reason-code and duration-quality tokens use hyphens. `unsupported` is terminal, invalid for performance acceptance, - and never authorizes a backend fallback. -- `resolved_plan.geometry.decode` and `.prefill`, the layout-specific scalars, and `backend_evidence.cpu`/`.metal` use - JSON null when not applicable. Applicable but absent scenario traffic is numeric zero or decimal-string `"0"` - according to the field's fixed type; it is never the string `"not_applicable"`. -- Metal worker and worker-QoS fields are null and the corresponding applicability booleans are false. CPU retains - numeric requested/available/effective worker evidence. -- Metal task validation uses the generic `kv_write_evaluated` and `kv_write_valid` keys under - `measurements[].execution.metal.validation`. Decode and prefill do not publish phase-specific aliases for that contract. -- Schema/control indexes, validated small configuration values, and planned/completed work units are integer numbers. - Potentially large byte, capacity, block/table/lookup, token-visit, causal-pair, and FMA-term quantities are canonical - decimal strings even when their value is zero. UInt64 seeds are canonical decimal strings. Unavailable elapsed, - rate, ratio, and statistics values are null. -- When a backend call throws before returning evidence for a measurement or excluded task, `execution.status` is - `unavailable`, the reason remains the runner-exception token, and missing lifecycle/QoS/checksum fields are null. A - `not_run` measurement's top-level successful/failed QoS-worker counts are likewise null rather than zero. -- `memory_budget` reports canonical decimal-string `resource_rounding_bytes`, `transient_peak_bytes`, - `known_owned_peak_bytes`, and `admitted_budget_bytes` separately. Immutable logical/physical resource geometry stays - under `resolved_plan.resources`. Paged admission covers full physical K/V blocks, the table, page rounding, and the - table-validation transient before materialization. The available-memory sample and its derived admitted budget are - runtime evidence, not resource/execution/model/frozen-plan identity inputs; identical fixed-seed work keeps its - identity when only that sample changes. A non-empty JSON target also reserves checked storage for all - variable-length component/layout identities and the prefill execution, scenario, and scope identities; its preflight - and finalized-plan estimates cover the same identity set. -- `measurements[]` preserves scenario/order/status/reason, frozen-plan identity, executor/checksum evidence, and these - fixed generic work-accounting fields: - - ```text - work_unit_kind, planned_work_units, completed_work_units, - weight_read_bytes_per_work_unit, kv_read_bytes_per_work_unit, - kv_write_bytes_per_work_unit, kv_write_kind, - effective_model_payload_bytes_per_work_unit, - layout_metadata_lookup_count_per_work_unit, - layout_metadata_read_bytes_per_work_unit, accounted_bytes_per_work_unit, - planned_effective_model_payload_bytes, completed_effective_model_payload_bytes, - planned_layout_metadata_lookup_count, completed_layout_metadata_lookup_count, - planned_layout_metadata_read_bytes, completed_layout_metadata_read_bytes, - planned_task_accounted_bytes, completed_task_accounted_bytes, - synthetic_work_unit_latency_seconds, - synthetic_memory_work_units_per_second, - effective_model_payload_gb_s - ``` - - Measurement checksum evidence uses phase-neutral `write_pattern_version` and `checksum_pattern_version` keys. The - unpublished schema has no decode-specific append/read aliases. - - Contiguous records and paged `weights_only` report metadata lookup/read additions as decimal-string zero; non-weights - decode scenarios use `kv_write_kind: "current_token_append"`; prefill KV-bearing scenarios use - `"full_prompt_population"`. Paged KV-bearing decode reports `L * B * (2 * N + 1)` table lookups. Paged prefill - reports `L * B * (N + 2 * M)`, where `M = sum(ceil(e_j/G))`; both use four bytes per lookup. These bytes contribute to - `accounted_bytes_per_work_unit`, not to effective-model-payload GB/s. Derived rates are non-null only for a successful - measured record. -- `aggregates` contains measured-only work-unit latency, work-unit rate, and effective model-payload GB/s values, - headlines, statistics, and stability quality. -- `quality_warnings` combines runner high-CV/order tokens with evidence-backed environment, QoS, cache-dominance, and - per-scenario duration-quality tokens; warning presence never causes measurement filtering or retry. -- Traffic classification tokens are `weight_payload_dominant`, `near_crossover`, and - `kv_read_payload_dominant`. `near_crossover` means exact equality of weight and KV-read payload, not a tolerance band. -- The authoritative acceptance predicate requires `mode == "llm_memory"`, `schema_version == 1`, backend/phase/layout - equal to the requested profile, `methodology_version` equal to the exact derived identity, - `status == "complete"`, `results_complete == true`, `conclusions_valid == true`, and every planned measurement to be - `measured`. Count one can be complete but has invalid comparative conclusions because its cyclic scenario positions - are not balanced. Paged comparisons additionally require identical `kv_block_tokens`, physical geometry, - permutation identity, and component identities. - -LLM file output checkpoints after every terminal scenario measurement and at command terminal. Exact `--output -` -retains those logical transitions without building intermediate payloads and emits one final document. A started task -uses completion-wins interruption; remaining slots become interrupted/null. A real backend-task, timer, checksum, or -checkpoint failure wins over interruption, and a failed file checkpoint is not retried. +Schema 2 ownership and interpretation: + +- All eight exact selectors are `llm-memory-v2-{cpu|metal}-{decode|prefill}-{contiguous|paged}`, with the canonical + `run_policy_version` shown above. These braces describe the eight combinations, not a prefix-matching acceptance rule. +- `resolved_plan` owns model geometry, resources, layout/permutation and component identities once. + `scenario_plans[]` owns each unique scenario/T/explicit shape, its seed, work kind, per-work-unit byte/lookup values, + planned totals and cold expected checksum once. Its order is scenario, increasing T, then false/true explicit policy. + Frozen refs, measurements and calibration attempts reference that same document's array. Public indexes can change + between snapshots; unknown, out-of-range or wrong-scenario references are rejected. Null is allowed for unresolved plans. +- Measurement `measurement_id` is the schedule-array index. `elapsed_seconds`, completed work/payload, actual checksums, + timing/lifecycle observations and mutable quality remain measurement-owned. Work kind and planned totals come from + `plan_ref`; total plan names have no `planned_` prefix. CPU `completion_derivation: "accepted-plan-derived"` is not a + hardware work counter. Worker/QoS/calibration-index/working-set and mixed-fraction metadata remain available. +- Canonical `expected_checksum` has status/reason, `expected_worker_checksums`, and `expected_run_checksum`. + CPU worker expectations have weight/K/V domains and exactly the effective worker count. Metal worker expectations + are null and its run checksum contains W/K/V dual-mod32 lanes. Available expectation is not observed validation. + Runtime actuals stay under measurement `checksum`; compact calibration retains only its existing actual-run witness. +- `execution.validation.checks[]` describes structure, phase/scenario write or unchanged-append, and padding separately. + The five kinds are `post-validation-structure`, `kv-append-final`, `kv-prefill-final-samples`, `kv-append-unchanged`, + `kv-padding-canary`. Inapplicable evaluated/valid are null; applicable unresolved valid is null; an observed mismatch + remains false. A required unevaluated check prevents acceptance. Prefill remains sampled, and checksum agreement does + not replace final-state checks. CPU paged decode weights-only checks unchanged append; CPU paged prefill weights-only + checks structure/padding; Metal weights-only has no write/padding check. Padding applies only to a physical suffix. +- Run statuses are `not_started`, `complete`, `partial`, `interrupted`, `unsupported`, `failed`; measurement statuses + are `not_run`, `measured`, `interrupted`, `invalid`, `failed`. Unsupported never authorizes backend fallback. + Unavailable execution evidence has unavailable status and null worker/timing/checksum observations. +- Indexes, bounded configuration values and work units are integer JSON numbers; potentially large bytes, lookup, + seed and checksum quantities are canonical unsigned decimal strings. Known zero is not null. Metal workers are null. + Prefill uses `geometry.prefill`, decode uses `geometry.decode`; the other is null. Paged resources distinguish logical + K/V from full physical blocks, suffix padding and table bytes; unsupported-before-preparation retains admitted geometry + with null runtime permutation. CPU paged prefill has both backend `prefill` and `paged` evidence objects. +- Theoretical causal/attention/FMA values live in `resolved_plan.model_context.prefill` with individual `*_reason_code`: + decimal value/`valid` or null/`arithmetic-overflow`. `h_q` is query-head count; tile `Q` determines executed prefix reads. + No FMA is executed. Actual byte/lookup/size overflows still reject work. Decode model-context prefill is null. +- Memory admission covers full resources, rounded allocation, canonical identity/expected capacity, actual retained + checksums, calibration, accepted-ID maps, exact-stat workspaces and the simultaneous DOM/serialized-string peak. + Metal paged expected reconstruction also reserves its temporary table/validation/hash workspace. Available memory is + runtime evidence, not part of immutable work identity. Reducing snapshot frequency does not remove peak admission. +- `run_accepted` requires complete status, every planned measurement measured, accepted required runtime evidence and + no known command/checkpoint error. `results_complete` only describes that measured population. Correct count one can + be accepted while balance is false. Quality, CV, duration and environment remain separate comparison criteria. +- One `accepted_measurement_ids` population feeds all three metrics. Derive each rate before statistics; exact median, + MAD and linearly interpolated P90/P95/P99 are prepared at snapshots/terminal. Empty populations are null; n=1 stddev + and positive-mean CV are zero. Classification is `insufficient-samples` for n<3, `undefined` for undefined CV, + `above-threshold` for payload CV strictly over 5%, otherwise `below-threshold`; equality is below-threshold. + These descriptive percentiles are not confidence levels. Default console shows n/median/min/max/CV/MAD. +- `build_manifest` is currently an unavailable reservation with the exact keys shown above. Optional CPU raw ticks are + not emitted; software/version or MSL provenance alone is not build attestation or independent verification. + +For N planned loops, files snapshot every `K=max(1,ceil(N/8))` completed loops, at most eight progress snapshots plus +terminal: maximum 4/7/9 normal snapshots for N=3/12/48. Abrupt termination can lose up to `3K` truly completed attempts, +including during replacement; unstarted placeholders do not count. No file is promised before the first successful +snapshot. Graceful SIGINT retains the completed prefix and interrupted tail if terminal writing succeeds; SIGKILL +cannot promise terminal output. Atomic replacement is not power-loss durability. + +Prior writer counts are observed before the current snapshot preparation; its own persistence remains null, with +`current_request` distinguishing progress, terminal and `late-command-error-correction`. Builder failure is not a +writer attempt, and stdout/disabled no-ops are not file successes. A failed checkpoint is terminal with no final-write +retry. One later command exception after successful terminal persistence may write one corrective failure snapshot. +A previous file cannot attest to a later failure: retain process outcome and terminal context. Stdout builds one final +DOM; omitted/empty output disables JSON and builds none. Automatic calibration does not invent a default filename. ### Core-to-core JSON shape @@ -2514,28 +2536,43 @@ jq -e 'select(.mode == "gpu_bandwidth" and .schema_version == 1 and # Inspect GPU exact payload, pass count, timing, and validation status jq '.measurements[] | {operation, status, value_gb_s, passes: .work_plan.passes, exact_payload_bytes: .work_plan.exact_payload_bytes, gpu_elapsed_seconds: .timed.gpu_elapsed_seconds, validation_status: .validation.validation_status}' gpu_bandwidth.json -# Reject incomplete, identity-mismatched, or position-unbalanced active-profile LLM schema 1 output -jq -e 'select(.mode == "llm_memory" and .schema_version == 1 and - .backend == "cpu" and .phase == "decode" and - .kv_layout == "paged" and - .methodology_version == "llm-memory-v1-cpu-decode-paged" and - .resolved_plan.layout.kv_block_tokens == 16 and - (.resolved_plan.layout.permutation_sha256 | - type == "string" and test("^[0-9a-f]{64}$")) and - .status == "complete" and .results_complete == true and - .conclusions_valid == true and - ([.measurements[] | select(.status != "measured")] | length) == 0)' llm_memory.json - -# Inspect scenario status, generic work accounting, work-unit latency, and effective model-payload GB/s -jq '.measurements[] | {scenario, status, reason_code, - work_unit_kind, - planned_work_units, - planned_task_accounted_bytes, - layout_metadata_lookup_count_per_work_unit, - layout_metadata_read_bytes_per_work_unit, - synthetic_work_unit_latency_seconds, - effective_model_payload_gb_s, - checksum_valid: .checksum.checksum_valid}' llm_memory.json +# Consume producer acceptance for CPU paged-decode schema 2; quality is separate. +jq -e '. as $root | .mode == "llm_memory" and .schema_version == 2 and + .backend == "cpu" and .phase == "decode" and .kv_layout == "paged" and + .methodology_version == "llm-memory-v2-cpu-decode-paged" and + .resolved_plan.layout.kv_block_tokens == 16 and + (.resolved_plan.layout.permutation_sha256 | + type == "string" and test("^[0-9a-f]{64}$")) and + .resolved_plan.component_identities.run_policy_version == + "llm-run-policy-bounded-loop-snapshots-v1" and + .status == "complete" and .results_complete == true and .run_accepted == true and + (.resolved_plan.scenario_plans | type) == "array" and + (.measurements | type) == "array" and + .counters.planned_measurements > 0 and + (.measurements | length) == .counters.planned_measurements and + all(.measurements[]; + .status == "measured" and (.plan_ref | type) == "number" and + .plan_ref >= 0 and .plan_ref == (.plan_ref | floor) and + .plan_ref < ($root.resolved_plan.scenario_plans | length) and + $root.resolved_plan.scenario_plans[.plan_ref].scenario == .scenario)' llm_memory.json + +# Inspect the accepted document, joining each reference against the same root. +jq '. as $root | .measurements[] as $m | + $m.plan_ref as $ref | + if ($ref | type) != "number" then error("missing numeric plan_ref") + elif $ref < 0 or $ref != ($ref | floor) or + $ref >= ($root.resolved_plan.scenario_plans | length) then error("invalid plan_ref") + elif $root.resolved_plan.scenario_plans[$ref].scenario != $m.scenario then error("scenario mismatch") + else $root.resolved_plan.scenario_plans[$ref] as $plan | + {scenario: $m.scenario, status: $m.status, reason_code: $m.reason_code, + work_unit_kind: $plan.work_unit_kind, planned_work_units: $plan.work_units, + planned_task_accounted_bytes: $plan.task_accounted_bytes, + layout_metadata_lookup_count_per_work_unit: $plan.layout_metadata_lookup_count_per_work_unit, + layout_metadata_read_bytes_per_work_unit: $plan.layout_metadata_read_bytes_per_work_unit, + synthetic_work_unit_latency_seconds: $m.synthetic_work_unit_latency_seconds, + effective_model_payload_gb_s: $m.effective_model_payload_gb_s, + checksum_valid: $m.checksum.checksum_valid} + end' llm_memory.json ``` --- @@ -2676,7 +2713,8 @@ versions are not a stability baseline for current pattern schema 3 and should no model and phase geometry, batch, requested/effective CPU workers, fixed or automatic policy, frozen work-plan identity, seed, hardware, software, and environment. Paged comparisons also require identical block size, physical geometry, table/permutation identity, and padding. Prefer a count divisible by three and require every planned - measurement to be measured plus `conclusions_valid: true` for comparative conclusions. + measurement to be measured plus `run_accepted: true`. Apply balance/CV/environment requirements as a separate + comparison-quality policy; correct one-loop runs can pass producer acceptance. - Size contiguous LLM experiments from `W + K_mapping + V_mapping`; size paged experiments from `W + K_physical + V_physical + block_table`, then include the recorded transient, auxiliary, and page-rounded peak before launch. A non-empty JSON target also reserves the recorded DOM/serialization peak in orchestration auxiliary bytes. Start with @@ -2769,11 +2807,11 @@ policy. ### LLM result is incomplete, invalid, or not comparable -Check top-level backend/phase/layout/methodology, status/reason, `results_complete`, `conclusions_valid`, scenario-order +Check top-level backend/phase/layout/methodology, status/reason, `results_complete`, `run_accepted`, scenario-order balance, counters, every measurement's status/reason, checksum evidence, duration quality, QoS, and environment warnings. A started task may -finish after an interrupt, but remaining slots are interrupted/null. Count one may complete all three scenarios yet -remain unsuitable for balanced comparative conclusions. High CV, cache-dominant working sets, or non-nominal thermal/ +finish after an interrupt, but remaining slots are interrupted/null. Count one can be correct and accepted while +remaining unsuitable for a comparison policy requiring position balance. High CV, cache-dominant working sets, or non-nominal thermal/ power state do not turn a measurement into zero; they remain explicit quality warnings. For paged results, also inspect the permutation identity/hash, physical K/V lengths, layout padding, table protection, @@ -2785,7 +2823,7 @@ size or permutation is a different comparison cohort even when logical model geo If an otherwise identical command fails memory admission only after a non-empty `--output` target is added, include the conservative JSON DOM/serialization peak in the diagnosis. It scales with planned measurement records, retained -calibration attempts, effective worker checksum trees, and their copied scenario-plan identities. For prefill it also +calibration attempts, effective worker checksum trees and once-per-canonical-plan identities/expectations. For prefill it also scales with the scenario-specific ownership-scope identity set, including layer/batch scopes; reduce the model/count/worker demand or free memory rather than treating the reserve as measured payload. @@ -2811,7 +2849,7 @@ Make sure you are passing `script-examples/final_output.txt` generated by the la - [GPU_BANDWIDTH_WHITEPAPER.md](GPU_BANDWIDTH_WHITEPAPER.md) - Metal GPU memory-bandwidth methodology, validation, schema 1, capability boundaries, and maintenance policy - [LLM_MEMORY_PROFILE_WHITEPAPER.md](LLM_MEMORY_PROFILE_WHITEPAPER.md) - synthetic CPU and Metal - decode/prefill methodology for both layouts, execution, schema 1, validation, and interpretation limits + decode/prefill methodology for both layouts, execution, schema 2, validation, and interpretation limits - [TECHNICAL_SPECIFICATION.md](TECHNICAL_SPECIFICATION.md) - architecture and implementation details - [CHANGELOG.md](../CHANGELOG.md) - release history diff --git a/src/core/config/constants.h b/src/core/config/constants.h index 40eb425..a311264 100644 --- a/src/core/config/constants.h +++ b/src/core/config/constants.h @@ -175,19 +175,19 @@ namespace Constants { constexpr size_t LLM_MAX_ACCOUNTED_BYTES_PER_TASK = 64ULL * 1024ULL * BYTES_PER_MB; constexpr double LLM_STREAMING_CV_WARNING_PCT = 5.0; - constexpr int LLM_JSON_SCHEMA_VERSION = 1; + constexpr int LLM_JSON_SCHEMA_VERSION = 2; constexpr size_t LLM_JSON_MAX_SAFE_INTEGER = (1ULL << 53) - 1; constexpr const char* LLM_JSON_MODE_NAME = "llm_memory"; constexpr size_t LLM_WEIGHT_PASSES_PER_WORK_UNIT = 1; constexpr size_t LLM_KV_REPLAY_FACTOR = 1; constexpr const char* LLM_CPU_DECODE_CONTIGUOUS_METHODOLOGY_VERSION = - "llm-memory-v1-cpu-decode-contiguous"; + "llm-memory-v2-cpu-decode-contiguous"; constexpr const char* LLM_CPU_DECODE_PAGED_METHODOLOGY_VERSION = - "llm-memory-v1-cpu-decode-paged"; + "llm-memory-v2-cpu-decode-paged"; constexpr const char* LLM_CPU_PREFILL_CONTIGUOUS_METHODOLOGY_VERSION = - "llm-memory-v1-cpu-prefill-contiguous"; + "llm-memory-v2-cpu-prefill-contiguous"; constexpr const char* LLM_CPU_PREFILL_PAGED_METHODOLOGY_VERSION = - "llm-memory-v1-cpu-prefill-paged"; + "llm-memory-v2-cpu-prefill-paged"; constexpr const char* LLM_WORK_PLAN_IDENTITY_VERSION = "llm-memory-work-plan-v1"; constexpr const char* LLM_COMPONENT_IDENTITY_VERSION = @@ -276,6 +276,13 @@ namespace Constants { constexpr const char* LLM_PAGED_READ_CHECKSUM_VERSION = "llm-paged-read-checksum-v1"; constexpr size_t LLM_KV_BLOCK_TABLE_HASH_CHUNK_ENTRIES = 1024; + // Canonical Metal oracle reconstruction owns a temporary table/bitset (charged + // separately) and fixed scratch: the layout hasher's 4 KiB serialization + // buffer, at most 4 KiB for fixed uint64/hex permutation strings and their + // construction copies, and 8 KiB for SHA state, table/summary objects and + // fixed call-frame headroom. This conservative envelope is shared by both + // runner estimates; it is not a claim of additional persistent resources. + constexpr size_t LLM_CANONICAL_PAGED_EXPECTED_FIXED_SCRATCH_BYTES = 16 * 1024; constexpr const char* LLM_METAL_SEGMENT_LAYOUT_VERSION = "whole-element-segments-v1"; diff --git a/src/llm_memory/llm_backend.h b/src/llm_memory/llm_backend.h index 9f96b00..31c74c5 100644 --- a/src/llm_memory/llm_backend.h +++ b/src/llm_memory/llm_backend.h @@ -182,8 +182,7 @@ struct LlmAuthoritativeTiming { }; /** Exact planned and completed logical work reported by one task. */ -struct LlmTaskCompletion { - size_t planned_work_units = 0; +struct LlmCompletedWork { size_t completed_work_units = 0; size_t completed_effective_model_payload_bytes = 0; size_t completed_layout_metadata_lookup_count = 0; @@ -191,6 +190,11 @@ struct LlmTaskCompletion { size_t completed_task_accounted_bytes = 0; }; +/** Transient completion assertion checked against the canonical plan. */ +struct LlmTaskCompletion : LlmCompletedWork { + size_t planned_work_units = 0; +}; + /** Backend-independent post-validation outcome. */ struct LlmTaskValidation { bool evaluated = false; @@ -223,7 +227,7 @@ struct LlmMetalDualMod32Checksum { }; /** Complete per-task Metal timing, dispatch, checksum, and validation evidence. */ -struct LlmMetalTaskEvidence { +struct LlmMetalRuntimeEvidence { LlmColdChecks cold_checks; bool timed_pipeline_available = false; std::string pipeline_label; @@ -235,7 +239,6 @@ struct LlmMetalTaskEvidence { bool timing_valid = false; double gpu_start_seconds = 0.0; double gpu_end_seconds = 0.0; - double gpu_elapsed_seconds = 0.0; bool host_timing_evaluated = false; double host_submit_to_completion_seconds = 0.0; double host_wait_seconds = 0.0; @@ -249,10 +252,8 @@ struct LlmMetalTaskEvidence { std::string reset_command_status = "not-run"; std::string timed_command_status = "not-run"; std::string post_validation_command_status = "not-run"; - std::string checksum_algorithm_version = "llm-metal-dual-mod32-v1"; bool checksum_evaluated = false; bool checksum_valid = false; - LlmMetalDualMod32Checksum expected_checksum; LlmMetalDualMod32Checksum actual_checksum; bool kv_write_validation_evaluated = false; bool kv_write_validation_valid = false; @@ -264,6 +265,13 @@ struct LlmMetalTaskEvidence { LlmMetalErrorDiagnostic error; }; +/** Transient backend result; canonical retention owns its expectation separately. */ +struct LlmMetalTaskEvidence : LlmMetalRuntimeEvidence { + double gpu_elapsed_seconds = 0.0; + std::string checksum_algorithm_version = "llm-metal-dual-mod32-v1"; + LlmMetalDualMod32Checksum expected_checksum; +}; + using LlmTaggedTaskEvidence = std::variant; /** Generic result consumed synchronously by the logical runner. */ @@ -409,6 +417,14 @@ class LlmBackend { virtual LlmBackendLifecycleResult initialize(const LlmMemoryConfig& config) = 0; virtual LlmBackendLifecycleResult resolve_execution_plan(const LlmMemoryWorkPlan& model_plan) = 0; virtual LlmBackendLifecycleResult prepare_resources(const LlmMemoryWorkPlan& model_plan) = 0; + /** Reconstruct CPU expectation for one new canonical plan using prepared, + * matching resources. Read-only and synchronous; returned vectors transfer + * ownership to the caller. Default is explicitly unavailable. No reset or + * execution is performed, and the caller invokes this only once per plan. */ + virtual LlmExpectedChecksumResult expected_cpu_checksum(const LlmMemoryWorkPlan&, + const LlmScenarioWorkPlan&) const noexcept { + return LlmExpectedChecksumResult{}; + } virtual LlmTaskExecutionResult execute_task(const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan& scenario_plan, const LlmRunnerTaskContext& context) = 0; diff --git a/src/llm_memory/llm_cpu_backend.cpp b/src/llm_memory/llm_cpu_backend.cpp index 57f113c..43371fd 100644 --- a/src/llm_memory/llm_cpu_backend.cpp +++ b/src/llm_memory/llm_cpu_backend.cpp @@ -128,6 +128,13 @@ class LlmCpuBackend final : public LlmBackend { return evidence_.preparation; } + LlmExpectedChecksumResult expected_cpu_checksum(const LlmMemoryWorkPlan& model_plan, + const LlmScenarioWorkPlan& scenario_plan) const noexcept override { + if (!resources_prepared_ || !plan_resolved_ || model_plan.backend != LlmMemoryBackend::Cpu || + resolved_plan_identity_ != model_plan.plan_identity) return LlmExpectedChecksumResult{}; + return calculate_llm_expected_checksums(model_plan, scenario_plan, resources_); + } + LlmTaskExecutionResult execute_task(const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan& scenario_plan, const LlmRunnerTaskContext& context) override { if (!resources_prepared_ || !timer_.has_value() || resolved_plan_identity_ != model_plan.plan_identity) { diff --git a/src/llm_memory/llm_executor.h b/src/llm_memory/llm_executor.h index 24dc79b..04c7c06 100644 --- a/src/llm_memory/llm_executor.h +++ b/src/llm_memory/llm_executor.h @@ -322,11 +322,10 @@ struct LlmExpectedChecksumResult { LlmRunChecksum run_checksum{0, 0}; }; -/** One synchronized task result; checksum vectors are worker-index ordered. */ -struct LlmExecutorResult { +/** Actual CPU runtime observations; no canonical checksum expectations. */ +struct LlmCpuRuntimeEvidence { bool valid = false; std::string reason_code = LlmExecutorReason::INVALID_RESOURCES; - double elapsed_seconds = 0.0; size_t requested_workers = 0; size_t created_workers = 0; size_t completed_workers = 0; @@ -344,12 +343,18 @@ struct LlmExecutorResult { bool kv_write_validation_applicable = false; ///< KV-writing scenario; independent of pipeline completion. bool kv_write_validation_evaluated = false; ///< Phase/layout write check ran after stop and join. bool kv_write_validation_valid = false; ///< Combined phase/layout final-state result, including paged padding. - std::vector expected_checksums; std::vector actual_checksums; - LlmRunChecksum expected_run_checksum{0, 0}; LlmRunChecksum actual_run_checksum{0, 0}; }; +/** Transient synchronized executor result. Expected data is consumed by validation + * and never retained once per measurement. */ +struct LlmExecutorResult : LlmCpuRuntimeEvidence { + double elapsed_seconds = 0.0; + std::vector expected_checksums; + LlmRunChecksum expected_run_checksum{0, 0}; +}; + /** Dedicated ARM64 implementation; @p output must be non-null. */ extern "C" void llm_decode_memory_asm(const LlmLayerDescriptor* layers, const LlmKvSequenceDescriptor* sequences, uint64_t layer_count, uint64_t work_unit_count, uint64_t scenario_flags, diff --git a/src/llm_memory/llm_json.cpp b/src/llm_memory/llm_json.cpp index d90a4e3..28b3609 100644 --- a/src/llm_memory/llm_json.cpp +++ b/src/llm_memory/llm_json.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -507,14 +508,12 @@ OrderedJson methodology_json(const LlmMemoryWorkPlan& plan) { : "worker-join-and-json-serialization"); OrderedJson output; - output["methodology_version"] = plan.methodology_version; output["backend"] = llm_memory_backend_to_string(plan.backend); output["phase"] = llm_phase_to_string(plan.phase); output["kv_layout"] = llm_kv_layout_to_string(plan.kv_layout); output["work_unit_kind"] = llm_work_unit_kind_to_string(plan.work_unit_kind); output["weight_passes_per_work_unit"] = plan.weight_passes_per_work_unit; output["kv_replay_factor"] = plan.kv_replay_factor; - output["schedule_version"] = plan.component_identities.schedule_version; output["warmup_policy"] = "same-shape-excluded-steady-state-warm-memory"; output["context_policy"] = plan.phase == LlmPhase::Decode ? "fixed-visible-context-including-current-token-slot" : "full-prompt-population-with-tiled-causal-prefix-scans"; @@ -542,11 +541,8 @@ OrderedJson methodology_json(const LlmMemoryWorkPlan& plan) { output["maximum_accounted_bytes_per_task"] = decimal_string(Constants::LLM_MAX_ACCOUNTED_BYTES_PER_TASK); output["repeatability_cv_warning_threshold_pct"] = Constants::LLM_STREAMING_CV_WARNING_PCT; output["calibration_excluded_from_results"] = true; + output["snapshot_policy"] = "bounded-loop-snapshots"; output["timed_region_exclusions"] = std::move(exclusions); - output["resource_abi_version"] = plan.component_identities.resource_abi_version; - output["buffer_pattern_version"] = plan.component_identities.buffer_pattern_version; - output["write_pattern_version"] = plan.component_identities.write_pattern_version; - output["checksum_pattern_version"] = plan.component_identities.checksum_pattern_version; return output; } @@ -566,9 +562,7 @@ OrderedJson geometry_json(const LlmGeometry& geometry) { {"tile_count", decimal_string(geometry.prefill->tile_count)}, {"attention_prefix_token_visits_per_sequence", decimal_string(geometry.prefill->attention_prefix_token_visits_per_sequence)}, - {"causal_token_pairs_per_sequence", decimal_string(geometry.prefill->causal_token_pairs_per_sequence)}, - {"logical_attention_pairs", decimal_string(geometry.prefill->logical_attention_pairs)}, - {"logical_attention_fma_terms", decimal_string(geometry.prefill->logical_attention_fma_terms)}}; + }; } OrderedJson output; @@ -900,18 +894,13 @@ OrderedJson statistics_json(const DescriptiveStatistics& statistics, bool availa } OrderedJson metric_aggregate_json(const LlmMetricAggregate& aggregate, const char* units) { - OrderedJson values = OrderedJson::array(); - for (double value : aggregate.values) { - values.push_back(finite_or_null(value)); - } + const size_t count = aggregate.statistics.sample_count; OrderedJson output; output["units"] = units; - output["sample_count"] = aggregate.values.size(); - output["headline_semantics"] = - aggregate.values.empty() ? "unavailable" : (aggregate.values.size() == 1 ? "single_measurement" : "median_p50"); + output["sample_count"] = count; + output["headline_semantics"] = count == 0 ? "unavailable" : count == 1 ? "single_measurement" : "median_p50"; output["headline"] = optional_finite_or_null(aggregate.headline); - output["values"] = std::move(values); - output["statistics"] = statistics_json(aggregate.statistics, !aggregate.values.empty()); + output["statistics"] = statistics_json(aggregate.statistics, count != 0); return output; } @@ -919,7 +908,8 @@ OrderedJson scenario_aggregate_json(const LlmScenarioAggregate& aggregate) { OrderedJson output; output["scenario"] = llm_scenario_to_string(aggregate.scenario); output["status"] = std::string(aggregate.status); - output["stability_quality"] = std::string(aggregate.stability_quality); + output["observed_cv_classification"] = std::string(aggregate.observed_cv_classification); + output["accepted_measurement_ids"] = aggregate.accepted_measurement_ids; output["cv_warning_threshold_pct"] = Constants::LLM_STREAMING_CV_WARNING_PCT; output["synthetic_work_unit_latency_seconds"] = metric_aggregate_json(aggregate.work_unit_latency_seconds, "seconds_per_synthetic_memory_work_unit"); @@ -1133,7 +1123,7 @@ OrderedJson resources_json(const LlmMemoryWorkPlan& plan, const LlmResourcePrepa OrderedJson output; output["valid"] = preparation.valid; output["reason_code"] = preparation.reason_code; - output["model_plan_identity"] = non_empty_or_null(plan.plan_identity); + output["model_ref"] = plan.plan_identity.empty() ? OrderedJson(nullptr) : OrderedJson("resolved_plan"); output["mappings"] = std::move(mappings); output["descriptors"] = std::move(descriptors); output["executor_auxiliary"] = executor_auxiliary_json(preparation.auxiliary); @@ -1170,13 +1160,10 @@ OrderedJson model_work_plan_json(const LlmMemoryWorkPlan& plan) { OrderedJson output; output["valid"] = plan.valid; output["reason_code"] = plan.reason_code; - output["plan_identity"] = non_empty_or_null(plan.plan_identity); - output["methodology_version"] = plan.methodology_version; output["backend"] = llm_memory_backend_to_string(plan.backend); output["phase"] = llm_phase_to_string(plan.phase); output["kv_layout"] = llm_kv_layout_to_string(plan.kv_layout); output["work_unit_kind"] = llm_work_unit_kind_to_string(plan.work_unit_kind); - output["component_identity"] = non_empty_or_null(plan.component_identities.identity); output["weight_passes_per_work_unit"] = plan.weight_passes_per_work_unit; output["kv_replay_factor"] = plan.kv_replay_factor; output["requested_workers"] = number_or_null(cpu.requested_workers, cpu_available); @@ -1202,7 +1189,9 @@ OrderedJson scenario_work_plan_json(const LlmScenarioWorkPlan& plan, LlmScenario output["work_unit_kind"] = llm_work_unit_kind_to_string(plan.work_unit_kind); output["kv_write_kind"] = llm_kv_write_kind_to_string(plan.kv_write_kind); output["explicit_iterations"] = available ? OrderedJson(plan.explicit_iterations) : OrderedJson(nullptr); - output["model_plan_identity"] = available ? non_empty_or_null(plan.model_plan_identity) : OrderedJson(nullptr); + output["work_policy"] = !available ? OrderedJson(nullptr) : OrderedJson(plan.explicit_iterations ? + "explicit_fixed_work" : "automatic_calibration"); + output["model_ref"] = "resolved_plan"; output["scenario_seed_uint64_decimal"] = decimal_or_null(plan.scenario_seed, available); output["work_units"] = number_or_null(plan.work_units, available); output["weight_read_bytes_per_work_unit"] = decimal_or_null(plan.weight_read_bytes_per_work_unit, available); @@ -1229,22 +1218,6 @@ OrderedJson scenario_work_plan_json(const LlmScenarioWorkPlan& plan, LlmScenario return output; } -OrderedJson frozen_scenario_plans_json(const LlmFrozenScenarioPlans& frozen) { - OrderedJson scenarios = OrderedJson::array(); - for (LlmScenario scenario : kScenarios) { - scenarios.push_back(scenario_work_plan_json(frozen.scenarios[scenario_index(scenario)], scenario)); - } - - OrderedJson output; - output["valid"] = frozen.valid; - output["reason_code"] = frozen.reason_code; - output["explicit_iterations"] = frozen.valid ? OrderedJson(frozen.explicit_iterations) : OrderedJson(nullptr); - output["model_plan_identity"] = frozen.valid ? non_empty_or_null(frozen.model_plan_identity) : OrderedJson(nullptr); - output["plan_identity"] = frozen.valid ? non_empty_or_null(frozen.plan_identity) : OrderedJson(nullptr); - output["scenarios"] = std::move(scenarios); - return output; -} - OrderedJson run_checksum_json(const LlmRunChecksum& checksum) { return OrderedJson{{"state_a_uint64_decimal", decimal_string(checksum.state_a)}, {"state_b_uint64_decimal", decimal_string(checksum.state_b)}}; @@ -1329,7 +1302,7 @@ OrderedJson metal_grid_plan_json(const LlmMetalGridPlan& grid) { return output; } -OrderedJson metal_task_evidence_json(const LlmMetalTaskEvidence& metal) { +OrderedJson metal_task_evidence_json(const LlmMetalRuntimeEvidence& metal) { OrderedJson pipeline; pipeline["available"] = metal.timed_pipeline_available; pipeline["label"] = metal.timed_pipeline_available ? non_empty_or_null(metal.pipeline_label) : OrderedJson(nullptr); @@ -1343,8 +1316,6 @@ OrderedJson metal_task_evidence_json(const LlmMetalTaskEvidence& metal) { timing["valid"] = metal.timing_evaluated ? OrderedJson(metal.timing_valid) : OrderedJson(nullptr); timing["gpu_start_seconds"] = metal.timing_evaluated ? finite_or_null(metal.gpu_start_seconds) : OrderedJson(nullptr); timing["gpu_end_seconds"] = metal.timing_evaluated ? finite_or_null(metal.gpu_end_seconds) : OrderedJson(nullptr); - timing["gpu_elapsed_seconds"] = - positive_finite_or_null(metal.gpu_elapsed_seconds, metal.timing_evaluated && metal.timing_valid); timing["host_submit_to_completion_seconds"] = positive_finite_or_null(metal.host_submit_to_completion_seconds, metal.host_timing_evaluated); timing["host_wait_seconds"] = positive_finite_or_null(metal.host_wait_seconds, metal.host_timing_evaluated); @@ -1361,38 +1332,11 @@ OrderedJson metal_task_evidence_json(const LlmMetalTaskEvidence& metal) { commands["timed_compute_encoders"] = metal.timed_compute_encoder_count; commands["timed_workload_dispatches"] = metal.timed_workload_dispatch_count; - OrderedJson checksum; - checksum["algorithm_version"] = metal.checksum_algorithm_version; - checksum["evaluated"] = metal.checksum_evaluated; - checksum["valid"] = metal.checksum_evaluated ? OrderedJson(metal.checksum_valid) : OrderedJson(nullptr); - checksum["expected"] = - metal.checksum_evaluated ? metal_dual_mod32_checksum_json(metal.expected_checksum) : OrderedJson(nullptr); - checksum["actual"] = - metal.checksum_evaluated ? metal_dual_mod32_checksum_json(metal.actual_checksum) : OrderedJson(nullptr); - - OrderedJson validation; - validation["post_validation_evaluated"] = metal.post_validation_evaluated; - validation["post_validation_valid"] = - metal.post_validation_evaluated ? OrderedJson(metal.post_validation_valid) : OrderedJson(nullptr); - validation["kv_write_evaluated"] = metal.kv_write_validation_evaluated; - validation["kv_write_valid"] = - metal.kv_write_validation_evaluated - ? OrderedJson(metal.kv_write_validation_valid) - : OrderedJson(nullptr); - validation["padding_canary_applicable"] = metal.padding_canary_applicable; - validation["padding_canary_evaluated"] = - metal.padding_canary_applicable ? OrderedJson(metal.padding_canary_evaluated) : OrderedJson(nullptr); - validation["padding_canary_valid"] = metal.padding_canary_applicable && metal.padding_canary_evaluated - ? OrderedJson(metal.padding_canary_valid) - : OrderedJson(nullptr); - OrderedJson output; output["pipeline"] = std::move(pipeline); output["grid"] = metal.grid_plan_available ? metal_grid_plan_json(metal.grid_plan) : OrderedJson(nullptr); output["timing"] = std::move(timing); output["commands"] = std::move(commands); - output["checksum"] = std::move(checksum); - output["validation"] = std::move(validation); output["error"] = metal_error_json(metal.error); return output; } @@ -1408,12 +1352,11 @@ bool compact_checksum_evaluated(const LlmTaskExecutionEvidence& execution) noexc } bool measurement_checksum_evaluated(const LlmMeasurementState& measurement) noexcept { - const LlmExecutorResult* cpu = get_llm_cpu_task_evidence(measurement.execution); + const LlmCpuRuntimeEvidence* cpu = get_llm_cpu_task_evidence(measurement.execution); if (measurement.execution_evidence_available && cpu != nullptr) { - return cpu->checksum_evaluated && cpu->expected_checksums.size() == cpu->requested_workers && - cpu->actual_checksums.size() == cpu->requested_workers; + return cpu->checksum_evaluated && cpu->actual_checksums.size() == cpu->requested_workers; } - const LlmMetalTaskEvidence* const metal = get_llm_metal_task_evidence(measurement.execution); + const LlmMetalRuntimeEvidence* const metal = get_llm_metal_task_evidence(measurement.execution); return measurement.execution_evidence_available && metal != nullptr && metal->checksum_evaluated; } @@ -1424,10 +1367,37 @@ const char* checksum_status(bool evaluated, bool valid) noexcept { return valid ? "valid" : "invalid"; } +const char* cold_check_name(LlmColdCheckKind kind) noexcept { + switch (kind) { + case LlmColdCheckKind::PostValidationStructure: return "post-validation-structure"; + case LlmColdCheckKind::KvAppendFinal: return "kv-append-final"; + case LlmColdCheckKind::KvPrefillFinalSamples: return "kv-prefill-final-samples"; + case LlmColdCheckKind::KvAppendUnchanged: return "kv-append-unchanged"; + case LlmColdCheckKind::KvPaddingCanary: return "kv-padding-canary"; + } + return "unknown"; +} + +OrderedJson cold_checks_json(const LlmColdChecks& expected, const LlmColdChecks* observed) { + OrderedJson output = OrderedJson::array(); + for (size_t i = 0; i < expected.size(); ++i) { + const bool applicable = expected[i].applicable; + const bool matched = observed && (*observed)[i].kind == expected[i].kind && + (*observed)[i].applicable == applicable; + const bool evaluated = matched && (*observed)[i].evaluated; + output.push_back(OrderedJson{{"kind", cold_check_name(expected[i].kind)}, {"applicable", applicable}, + {"evaluated", applicable ? OrderedJson(evaluated) : OrderedJson(nullptr)}, + {"valid", applicable && evaluated ? OrderedJson((*observed)[i].valid) : OrderedJson(nullptr)}, + {"reason_code", !applicable ? "not-applicable" : !matched ? "execution-evidence-unavailable" : + std::string((*observed)[i].reason_code)}}); + } + return output; +} + OrderedJson compact_execution_json(const LlmTaskExecutionEvidence& execution, std::string_view fallback_reason_code, - std::string_view checksum_algorithm_version) { + std::string_view /*checksum_algorithm_version*/) { const bool available = execution.available; - const LlmMetalTaskEvidence* const metal = + const LlmMetalRuntimeEvidence* const metal = available && execution.metal_evidence_available && execution.metal.has_value() ? &*execution.metal : nullptr; const bool checksum_evaluated = compact_checksum_evaluated(execution); const bool checksum_valid = metal != nullptr ? metal->checksum_valid : execution.checksum_valid; @@ -1435,12 +1405,7 @@ OrderedJson compact_execution_json(const LlmTaskExecutionEvidence& execution, st OrderedJson checksum; checksum["status"] = checksum_status(checksum_evaluated, checksum_valid); checksum["reason_code"] = reason_code; - checksum["algorithm_version"] = - metal != nullptr ? metal->checksum_algorithm_version : std::string(checksum_algorithm_version); checksum["checksum_valid"] = checksum_evaluated ? OrderedJson(checksum_valid) : OrderedJson(nullptr); - checksum["expected_run_checksum"] = !checksum_evaluated ? OrderedJson(nullptr) - : metal != nullptr ? metal_dual_mod32_checksum_json(metal->expected_checksum) - : run_checksum_json(execution.expected_run_checksum); checksum["actual_run_checksum"] = !checksum_evaluated ? OrderedJson(nullptr) : metal != nullptr ? metal_dual_mod32_checksum_json(metal->actual_checksum) : run_checksum_json(execution.actual_run_checksum); @@ -1468,41 +1433,47 @@ OrderedJson compact_execution_json(const LlmTaskExecutionEvidence& execution, st return output; } +OrderedJson plan_ref_json(const LlmMemoryResult& result, LlmPlanHandle handle, + std::optional expected_scenario = std::nullopt) { + if (handle == kLlmNoTaskIndex) return nullptr; + if (handle >= result.scenario_plans.size() || result.snapshot_plan_refs.size() != result.scenario_plans.size() || + result.snapshot_plan_order.size() != result.scenario_plans.size() || + result.snapshot_plan_refs[handle] >= result.snapshot_plan_order.size() || + result.snapshot_plan_order[result.snapshot_plan_refs[handle]] != handle) { + throw std::invalid_argument("invalid-canonical-plan-reference"); + } + if (expected_scenario && result.scenario_plans[handle].plan.scenario != *expected_scenario) + throw std::invalid_argument("contradictory-canonical-plan-scenario"); + return result.snapshot_plan_refs[handle]; +} + OrderedJson calibration_attempt_json(const LlmCalibrationAttempt& attempt, size_t attempt_index, - std::string_view checksum_algorithm_version) { - const bool plan_available = !attempt.work_plan_identity.empty(); + const LlmMemoryResult& result, const LlmMemoryWorkPlan& model_plan, std::string_view checksum_algorithm_version) { OrderedJson output; output["attempt_index"] = attempt_index; output["scenario"] = llm_scenario_to_string(attempt.scenario); - output["work_unit_kind"] = llm_work_unit_kind_to_string(attempt.work_unit_kind); - output["kv_write_kind"] = llm_kv_write_kind_to_string(attempt.kv_write_kind); + output["plan_ref"] = plan_ref_json(result, attempt.plan_handle, attempt.scenario); output["purpose"] = std::string(attempt.purpose); - output["explicit_iterations"] = attempt.explicit_iterations; - output["work_units"] = number_or_null(attempt.work_units, plan_available); - output["weight_read_bytes"] = decimal_or_null(attempt.weight_read_bytes, plan_available); - output["kv_read_bytes"] = decimal_or_null(attempt.kv_read_bytes, plan_available); - output["kv_write_bytes"] = decimal_or_null(attempt.kv_write_bytes, plan_available); - output["effective_model_payload_bytes"] = decimal_or_null(attempt.effective_model_payload_bytes, plan_available); - output["layout_metadata_lookup_count"] = decimal_or_null(attempt.layout_metadata_lookup_count, plan_available); - output["layout_metadata_read_bytes"] = decimal_or_null(attempt.layout_metadata_read_bytes, plan_available); - output["task_accounted_bytes"] = decimal_or_null(attempt.task_accounted_bytes, plan_available); - output["work_plan_identity"] = non_empty_or_null(attempt.work_plan_identity); output["duration_quality"] = std::string(attempt.duration_quality); output["terminal"] = attempt.terminal; output["valid"] = attempt.valid; output["reason_code"] = std::string(attempt.reason_code); output["execution"] = compact_execution_json(attempt.execution, attempt.reason_code, checksum_algorithm_version); + const auto& evidence = attempt.execution; + const LlmColdChecks* observed = !evidence.available ? nullptr : evidence.cpu_evidence_available ? + &evidence.cpu_cold_checks : evidence.metal ? &evidence.metal->cold_checks : nullptr; + output["execution"]["validation"]["checks"] = cold_checks_json(required_llm_cold_checks(model_plan, attempt.scenario), observed); return output; } -OrderedJson excluded_calibration_json(const LlmMemoryResult& result, std::string_view checksum_algorithm_version) { +OrderedJson excluded_calibration_json(const LlmMemoryResult& result, const LlmMemoryWorkPlan& model_plan, std::string_view checksum_algorithm_version) { OrderedJson output = OrderedJson::object(); for (LlmScenario scenario : kScenarios) { const size_t index = scenario_index(scenario); OrderedJson attempts = OrderedJson::array(); for (size_t attempt_index = 0; attempt_index < result.calibration_attempts[index].size(); ++attempt_index) { attempts.push_back(calibration_attempt_json(result.calibration_attempts[index][attempt_index], attempt_index, - checksum_algorithm_version)); + result, model_plan, checksum_algorithm_version)); } output[llm_scenario_to_string(scenario)] = std::move(attempts); } @@ -1555,11 +1526,13 @@ OrderedJson counters_json(const LlmMemoryResult& result) { OrderedJson checkpoint_lifecycle_json(const LlmMemoryResult& result) { OrderedJson output; output["checkpoint_failed"] = result.checkpoint_failed; - output["logical_checkpoint_attempts"] = result.logical_checkpoint_attempts; - output["successful_logical_checkpoints"] = result.successful_logical_checkpoints; - output["terminal_checkpoint_attempted"] = result.terminal_checkpoint_attempted; - output["terminal_checkpoint_completed"] = result.terminal_checkpoint_completed; - output["checkpoint_policy"] = "after-each-terminal-measurement-and-at-command-terminal"; + output["observation_point"] = "before-current-snapshot-preparation"; + output["prior_file_writer_attempts"] = result.prior_file_writer_attempts; + output["prior_successful_file_writes"] = result.prior_successful_file_writes; + output["current_request"] = std::string(result.snapshot_request); + output["current_persistence_success"] = nullptr; + output["snapshot_interval_loops"] = result.snapshot_interval_loops; + output["checkpoint_policy"] = "bounded-loop-snapshots"; output["file_checkpoint_failure_is_terminal_and_not_retried"] = true; output["stdout_intermediate_checkpoints_are_lazy"] = true; return output; @@ -1594,9 +1567,9 @@ OrderedJson loop_records_json(const LlmMemoryResult& result) { OrderedJson measurement_execution_json(const LlmMeasurementState& measurement) { const bool attempted = measurement.attempted; const bool available = measurement.execution_evidence_available; - const LlmTaskExecutionResult& execution = measurement.execution; - const LlmExecutorResult* cpu = get_llm_cpu_task_evidence(execution); - const LlmMetalTaskEvidence* const metal = get_llm_metal_task_evidence(execution); + const LlmRetainedExecution& execution = measurement.execution; + const LlmCpuRuntimeEvidence* cpu = get_llm_cpu_task_evidence(execution); + const LlmMetalRuntimeEvidence* const metal = get_llm_metal_task_evidence(execution); const bool valid = available && execution.status == LlmTaskExecutionStatus::Complete && execution.reason_code == LlmBackendReason::VALID && execution.validation.evaluated && execution.validation.valid && execution.timing.evaluated && execution.timing.valid; @@ -1604,8 +1577,10 @@ OrderedJson measurement_execution_json(const LlmMeasurementState& measurement) { output["status"] = !attempted ? "not_run" : (!available ? "unavailable" : (valid ? "valid" : "invalid")); output["reason_code"] = available ? execution.reason_code : std::string(measurement.reason_code); output["valid"] = available ? OrderedJson(valid) : OrderedJson(nullptr); - output["elapsed_seconds"] = positive_finite_or_null( - execution.timing.elapsed_seconds, available && execution.timing.evaluated && execution.timing.valid); + output["timing"] = OrderedJson{{"evaluated", available && execution.timing.evaluated}, + {"valid", available && execution.timing.evaluated ? OrderedJson(execution.timing.valid) : OrderedJson(nullptr)}, + {"diagnostic_elapsed_seconds", measurement.status != LlmMeasurementStatus::Measured && + available && execution.timing.evaluated ? finite_or_null(execution.timing.elapsed_seconds) : OrderedJson(nullptr)}}; const bool cpu_available = available && cpu != nullptr; output["requested_workers"] = number_or_null(cpu_available ? cpu->requested_workers : 0, cpu_available); output["created_workers"] = number_or_null(cpu_available ? cpu->created_workers : 0, cpu_available); @@ -1616,152 +1591,101 @@ OrderedJson measurement_execution_json(const LlmMeasurementState& measurement) { output["kernel_succeeded"] = cpu_available ? OrderedJson(cpu->kernel_succeeded) : OrderedJson(nullptr); output["timer_started"] = cpu_available ? OrderedJson(cpu->timer_started) : OrderedJson(nullptr); output["timer_stopped"] = cpu_available ? OrderedJson(cpu->timer_stopped) : OrderedJson(nullptr); - output["post_validation_evaluated"] = - cpu_available ? OrderedJson(cpu->post_validation_evaluated) : OrderedJson(nullptr); - output["post_validation_valid"] = - cpu_available && cpu->post_validation_evaluated ? OrderedJson(cpu->post_validation_valid) : OrderedJson(nullptr); - output["kv_write_validation_applicable"] = - cpu_available ? OrderedJson(cpu->kv_write_validation_applicable) : OrderedJson(nullptr); - output["kv_write_validation_evaluated"] = - cpu_available && cpu->kv_write_validation_applicable - ? OrderedJson(cpu->kv_write_validation_evaluated) : OrderedJson(nullptr); - output["kv_write_validation_valid"] = - cpu_available && cpu->kv_write_validation_applicable && cpu->kv_write_validation_evaluated - ? OrderedJson(cpu->kv_write_validation_valid) : OrderedJson(nullptr); if (metal != nullptr) { output["metal"] = metal_task_evidence_json(*metal); } return output; } -OrderedJson measurement_checksum_json(const LlmMeasurementState& measurement, const LlmMemoryWorkPlan& model_plan) { +OrderedJson measurement_checksum_json(const LlmMeasurementState& measurement, const LlmMemoryWorkPlan& /*model_plan*/) { const bool evaluated = measurement_checksum_evaluated(measurement); - const LlmTaskExecutionResult& execution = measurement.execution; - const LlmExecutorResult* cpu = get_llm_cpu_task_evidence(execution); - const LlmMetalTaskEvidence* const metal = get_llm_metal_task_evidence(execution); + const LlmRetainedExecution& execution = measurement.execution; + const LlmCpuRuntimeEvidence* cpu = get_llm_cpu_task_evidence(execution); + const LlmMetalRuntimeEvidence* const metal = get_llm_metal_task_evidence(execution); const bool checksum_valid = evaluated && ((cpu != nullptr && cpu->checksum_valid) || (metal != nullptr && metal->checksum_valid)); OrderedJson output; output["status"] = checksum_status(evaluated, checksum_valid); output["reason_code"] = measurement.execution_evidence_available ? execution.reason_code : std::string(measurement.reason_code); - output["initialization_pattern_version"] = model_plan.component_identities.buffer_pattern_version; - output["write_pattern_version"] = model_plan.component_identities.write_pattern_version; - output["checksum_pattern_version"] = model_plan.component_identities.checksum_pattern_version; output["checksum_valid"] = evaluated ? OrderedJson(checksum_valid) : OrderedJson(nullptr); - output["expected_worker_checksums"] = - evaluated && cpu != nullptr ? worker_checksums_json(cpu->expected_checksums) : OrderedJson(nullptr); output["actual_worker_checksums"] = evaluated && cpu != nullptr ? worker_checksums_json(cpu->actual_checksums) : OrderedJson(nullptr); - output["expected_run_checksum"] = !evaluated ? OrderedJson(nullptr) - : cpu != nullptr ? run_checksum_json(cpu->expected_run_checksum) - : metal_dual_mod32_checksum_json(metal->expected_checksum); output["actual_run_checksum"] = !evaluated ? OrderedJson(nullptr) : cpu != nullptr ? run_checksum_json(cpu->actual_run_checksum) : metal_dual_mod32_checksum_json(metal->actual_checksum); return output; } -OrderedJson calibration_indexes_json(size_t count) { - OrderedJson output = OrderedJson::array(); - for (size_t index = 0; index < count; ++index) { - output.push_back(index); - } - return output; -} - -OrderedJson measurement_json(const LlmMeasurementState& measurement, const LlmMemoryResult& result, +OrderedJson measurement_json(const LlmMeasurementState& measurement, size_t id, const LlmMemoryResult& result, const LlmMemoryWorkPlan& model_plan) { - const bool plan_available = measurement.frozen_plan_index < kLlmScenarioCount && - result.frozen_scenario_plans.scenarios[measurement.frozen_plan_index].valid; - const LlmScenarioWorkPlan* frozen_plan = - plan_available ? &result.frozen_scenario_plans.scenarios[measurement.frozen_plan_index] : nullptr; - - OrderedJson working_set; - working_set["bytes"] = decimal_string(measurement.working_set_bytes); - working_set["full_size_physical_mappings"] = true; - working_set["cacheable"] = true; - working_set["kv_layout"] = llm_kv_layout_to_string(model_plan.kv_layout); - working_set["fixed_visible_context_tokens"] = model_plan.geometry.decode.has_value() - ? OrderedJson(model_plan.geometry.decode->visible_context_tokens) - : OrderedJson(nullptr); - working_set["current_token_slot_included"] = - model_plan.phase == LlmPhase::Decode ? OrderedJson(true) : OrderedJson(nullptr); - + const auto metrics = derive_llm_measurement_metrics(measurement); + const bool measured = measurement.status == LlmMeasurementStatus::Measured; + const auto& completed = measurement.execution.completion; OrderedJson output; + output["measurement_id"] = id; + output["plan_ref"] = plan_ref_json(result, measurement.plan_handle, measurement.scenario); output["scenario"] = llm_scenario_to_string(measurement.scenario); - output["work_unit_kind"] = llm_work_unit_kind_to_string(measurement.work_unit_kind); - output["kv_write_kind"] = llm_kv_write_kind_to_string(measurement.kv_write_kind); output["loop_index"] = measurement.loop_index; output["order_position"] = measurement.order_position; output["status"] = llm_measurement_status_to_string(measurement.status); output["reason_code"] = std::string(measurement.reason_code); output["attempted"] = measurement.attempted; - const bool workers_applicable = model_plan.backend == LlmMemoryBackend::Cpu; - output["requested_workers"] = number_or_null(measurement.requested_workers, workers_applicable); - output["effective_workers"] = number_or_null(measurement.effective_workers, workers_applicable); - output["qos_successful_workers"] = number_or_null(measurement.qos_successful_workers, - workers_applicable && measurement.execution_evidence_available); - output["qos_failed_workers"] = - number_or_null(measurement.qos_failed_workers, workers_applicable && measurement.execution_evidence_available); - output["frozen_plan_index"] = number_or_null(measurement.frozen_plan_index, plan_available); - output["frozen_work_plan_identity"] = - frozen_plan == nullptr ? OrderedJson(nullptr) : non_empty_or_null(frozen_plan->plan_identity); - output["scenario_seed_uint64_decimal"] = - frozen_plan == nullptr ? OrderedJson(nullptr) : decimal_string(frozen_plan->scenario_seed); - output["explicit_iterations"] = plan_available ? OrderedJson(measurement.explicit_iterations) : OrderedJson(nullptr); - output["work_policy"] = - !plan_available ? OrderedJson(nullptr) - : OrderedJson(measurement.explicit_iterations ? "explicit_fixed_work" : "automatic_calibration"); + const auto* cpu_plan = get_llm_cpu_execution_plan(model_plan); + const auto* cpu_runtime = get_llm_cpu_task_evidence(measurement.execution); + output["requested_workers"] = number_or_null(cpu_plan ? cpu_plan->requested_workers : 0, cpu_plan != nullptr); + output["effective_workers"] = number_or_null(cpu_plan ? cpu_plan->effective_workers : 0, cpu_plan != nullptr); + output["qos_successful_workers"] = number_or_null(cpu_runtime ? cpu_runtime->qos_successful_workers : 0, + measurement.execution_evidence_available && cpu_runtime != nullptr); + output["qos_failed_workers"] = number_or_null(cpu_runtime ? cpu_runtime->qos_failed_workers : 0, + measurement.execution_evidence_available && cpu_runtime != nullptr); + const size_t attempts = result.calibration_attempt_counts[scenario_index(measurement.scenario)]; + output["calibration_attempt_count"] = attempts; + OrderedJson attempt_indexes = OrderedJson::array(); + for (size_t index = 0; index < attempts; ++index) attempt_indexes.push_back(index); + output["calibration_attempt_indexes"] = std::move(attempt_indexes); + output["working_set"] = OrderedJson{{"bytes", decimal_string(model_plan.geometry.total_data_mapping_bytes)}, + {"full_size_physical_mappings", true}, {"cacheable", true}, + {"kv_layout", llm_kv_layout_to_string(model_plan.kv_layout)}, + {"fixed_visible_context_tokens", model_plan.geometry.decode ? + OrderedJson(model_plan.geometry.decode->visible_context_tokens) : OrderedJson(nullptr)}, + {"current_token_slot_included", model_plan.phase == LlmPhase::Decode ? OrderedJson(true) : OrderedJson(nullptr)}}; + const auto* canonical = find_llm_scenario_plan(result, measurement.plan_handle); + const bool fractions = canonical && measurement.scenario == LlmScenario::Mixed && + canonical->plan.effective_model_payload_bytes_per_work_unit != 0; + const auto fraction = [&](size_t bytes) -> OrderedJson { + return fractions ? finite_or_null(static_cast(static_cast(bytes) / + canonical->plan.effective_model_payload_bytes_per_work_unit)) : OrderedJson(nullptr); + }; + output["weight_payload_fraction"] = fraction(canonical ? canonical->plan.weight_read_bytes_per_work_unit : 0); + output["kv_read_payload_fraction"] = fraction(canonical ? canonical->plan.kv_read_bytes_per_work_unit : 0); + output["kv_write_payload_fraction"] = fraction(canonical ? canonical->plan.kv_write_bytes_per_work_unit : 0); output["duration_quality"] = std::string(measurement.duration_quality); - output["calibration_attempt_count"] = measurement.calibration_attempt_count; - output["calibration_attempt_indexes"] = calibration_indexes_json(measurement.calibration_attempt_count); - // Work-unit counts remain JSON integers, while all exact potentially-large - // counts and byte quantities remain decimal strings, including zero-valued - // not-run records. This keeps field types independent of run status. - output["planned_work_units"] = measurement.planned_work_units; - output["completed_work_units"] = measurement.completed_work_units; - output["weight_read_bytes_per_work_unit"] = decimal_string(measurement.weight_read_bytes_per_work_unit); - output["kv_read_bytes_per_work_unit"] = decimal_string(measurement.kv_read_bytes_per_work_unit); - output["kv_write_bytes_per_work_unit"] = decimal_string(measurement.kv_write_bytes_per_work_unit); - output["effective_model_payload_bytes_per_work_unit"] = - decimal_string(measurement.effective_model_payload_bytes_per_work_unit); - output["layout_metadata_lookup_count_per_work_unit"] = - decimal_string(measurement.layout_metadata_lookup_count_per_work_unit); - output["layout_metadata_read_bytes_per_work_unit"] = - decimal_string(measurement.layout_metadata_read_bytes_per_work_unit); - output["accounted_bytes_per_work_unit"] = decimal_string(measurement.accounted_bytes_per_work_unit); - output["planned_weight_read_bytes"] = decimal_string(measurement.planned_weight_read_bytes); - output["planned_kv_read_bytes"] = decimal_string(measurement.planned_kv_read_bytes); - output["planned_kv_write_bytes"] = decimal_string(measurement.planned_kv_write_bytes); - output["planned_effective_model_payload_bytes"] = decimal_string(measurement.planned_effective_model_payload_bytes); - output["completed_effective_model_payload_bytes"] = - decimal_string(measurement.completed_effective_model_payload_bytes); - output["planned_layout_metadata_lookup_count"] = decimal_string(measurement.planned_layout_metadata_lookup_count); - output["completed_layout_metadata_lookup_count"] = decimal_string(measurement.completed_layout_metadata_lookup_count); - output["planned_layout_metadata_read_bytes"] = decimal_string(measurement.planned_layout_metadata_read_bytes); - output["completed_layout_metadata_read_bytes"] = decimal_string(measurement.completed_layout_metadata_read_bytes); - output["planned_task_accounted_bytes"] = decimal_string(measurement.planned_task_accounted_bytes); - output["completed_task_accounted_bytes"] = decimal_string(measurement.completed_task_accounted_bytes); - output["elapsed_seconds"] = optional_finite_or_null(measurement.elapsed_seconds); - output["synthetic_work_unit_latency_seconds"] = - optional_finite_or_null(measurement.synthetic_work_unit_latency_seconds); - output["synthetic_memory_work_units_per_second"] = - optional_finite_or_null(measurement.synthetic_memory_work_units_per_second); - output["effective_model_payload_gb_s"] = optional_finite_or_null(measurement.effective_model_payload_gb_s); - output["weight_payload_fraction"] = optional_finite_or_null(measurement.weight_payload_fraction); - output["kv_read_payload_fraction"] = optional_finite_or_null(measurement.kv_read_payload_fraction); - output["kv_write_payload_fraction"] = optional_finite_or_null(measurement.kv_write_payload_fraction); - output["working_set"] = std::move(working_set); + output["completed_work_units"] = completed.completed_work_units; + output["completed_effective_model_payload_bytes"] = decimal_string(completed.completed_effective_model_payload_bytes); + output["completed_layout_metadata_lookup_count"] = decimal_string(completed.completed_layout_metadata_lookup_count); + output["completed_layout_metadata_read_bytes"] = decimal_string(completed.completed_layout_metadata_read_bytes); + output["completed_task_accounted_bytes"] = decimal_string(completed.completed_task_accounted_bytes); + output["completion_derivation"] = measured && model_plan.backend == LlmMemoryBackend::Cpu + ? OrderedJson("accepted-plan-derived") : OrderedJson(nullptr); + output["elapsed_seconds"] = positive_finite_or_null(measurement.execution.timing.elapsed_seconds, measured); + output["synthetic_work_unit_latency_seconds"] = optional_finite_or_null(metrics.latency_seconds); + output["synthetic_memory_work_units_per_second"] = optional_finite_or_null(metrics.work_units_per_second); + output["effective_model_payload_gb_s"] = optional_finite_or_null(metrics.payload_gb_s); output["execution"] = measurement_execution_json(measurement); output["checksum"] = measurement_checksum_json(measurement, model_plan); + const auto* cpu = get_llm_cpu_task_evidence(measurement.execution); + const auto* metal = get_llm_metal_task_evidence(measurement.execution); + const LlmColdChecks* observed = !measurement.execution_evidence_available ? nullptr : + cpu ? &cpu->cold_checks : metal ? &metal->cold_checks : nullptr; + output["execution"]["validation"]["checks"] = cold_checks_json(required_llm_cold_checks(model_plan, measurement.scenario), observed); return output; } OrderedJson measurements_json(const LlmMemoryResult& result, const LlmMemoryWorkPlan& model_plan) { OrderedJson output = OrderedJson::array(); - for (const LlmMeasurementState& measurement : result.measurements) { - output.push_back(measurement_json(measurement, result, model_plan)); + for (size_t id = 0; id < result.measurements.size(); ++id) { + output.push_back(measurement_json(result.measurements[id], id, result, model_plan)); } return output; } @@ -1782,24 +1706,55 @@ OrderedJson software_json(const LlmResultMetadata& metadata) { return output; } -OrderedJson resolved_plan_json(const LlmMemoryWorkPlan& plan, const LlmFrozenScenarioPlans& frozen, +OrderedJson resolved_plan_json(const LlmMemoryWorkPlan& plan, const LlmMemoryResult& result, const LlmBackendEvidence& backend_evidence) { OrderedJson output; output["valid"] = plan.valid; output["reason_code"] = plan.reason_code; output["plan_identity"] = non_empty_or_null(plan.plan_identity); - output["methodology_version"] = plan.methodology_version; output["backend"] = llm_memory_backend_to_string(plan.backend); output["phase"] = llm_phase_to_string(plan.phase); output["kv_layout"] = llm_kv_layout_to_string(plan.kv_layout); output["work_unit_kind"] = llm_work_unit_kind_to_string(plan.work_unit_kind); output["geometry"] = geometry_json(plan.geometry); + OrderedJson context = nullptr; + if (plan.geometry.prefill) { + context = OrderedJson::object(); + const auto append = [&](const char* name, const std::optional& quantity) { + context[name] = quantity ? decimal_string(*quantity) : OrderedJson(nullptr); + context[std::string(name) + "_reason_code"] = quantity ? "valid" : "arithmetic-overflow"; + }; + append("causal_token_pairs_per_sequence", plan.geometry.prefill->causal_token_pairs_per_sequence); + append("logical_attention_pairs", plan.geometry.prefill->logical_attention_pairs); + append("logical_attention_fma_terms", plan.geometry.prefill->logical_attention_fma_terms); + } + output["model_context"] = OrderedJson{{"prefill", std::move(context)}}; output["layout"] = layout_json(plan, backend_evidence); output["resources"] = resolved_resources_json(plan); output["component_identities"] = component_identities_json(plan.component_identities); + output["component_identities"]["run_policy_version"] = "llm-run-policy-bounded-loop-snapshots-v1"; output["methodology"] = methodology_json(plan); output["model_work_plan"] = model_work_plan_json(plan); - output["frozen_scenario_work_plans"] = frozen_scenario_plans_json(frozen); + OrderedJson plans = OrderedJson::array(); + for (auto handle : result.snapshot_plan_order) { + (void)plan_ref_json(result, handle); + const auto& entry = result.scenario_plans.at(handle); + OrderedJson item = scenario_work_plan_json(entry.plan, entry.plan.scenario); + const auto& expected = entry.expected; + item["expected_checksum"] = OrderedJson{ + {"status", expected.available ? "available" : "unavailable"}, + {"reason_code", expected.reason_code}, + {"expected_worker_checksums", expected.available && plan.backend == LlmMemoryBackend::Cpu + ? worker_checksums_json(expected.cpu_workers) : OrderedJson(nullptr)}, + {"expected_run_checksum", !expected.available ? OrderedJson(nullptr) : plan.backend == LlmMemoryBackend::Cpu + ? run_checksum_json(expected.cpu_run) : metal_dual_mod32_checksum_json(expected.metal_run)}}; + plans.push_back(std::move(item)); + } + output["scenario_plans"] = std::move(plans); + OrderedJson frozen = OrderedJson::object(); + for (auto scenario : kScenarios) frozen[llm_scenario_to_string(scenario)] = + plan_ref_json(result, result.frozen_plan_handles[scenario_index(scenario)], scenario); + output["frozen_plan_refs"] = std::move(frozen); return output; } @@ -2003,7 +1958,7 @@ OrderedJson backend_evidence_json(const LlmMemoryWorkPlan& plan, const LlmBacken OrderedJson calibration_json(const LlmMemoryResult& result, const LlmMemoryWorkPlan& model_plan) { OrderedJson output; output["excluded_from_results"] = true; - output["attempts"] = excluded_calibration_json(result, model_plan.component_identities.checksum_pattern_version); + output["attempts"] = excluded_calibration_json(result, model_plan, model_plan.component_identities.checksum_pattern_version); return output; } @@ -2074,7 +2029,9 @@ std::vector collect_quality_warning_tokens(const LlmMemoryWorkPlan& append_warning(warnings, "main-thread-qos-not-applied"); } if (std::any_of(result.measurements.begin(), result.measurements.end(), - [](const LlmMeasurementState& measurement) { return measurement.qos_failed_workers > 0; })) { + [](const LlmMeasurementState& measurement) { + const auto* cpu = get_llm_cpu_task_evidence(measurement.execution); + return cpu && cpu->qos_failed_workers > 0; })) { append_warning(warnings, "worker-qos-not-applied"); } if (metadata.l2_data_cache_bytes > 0 && plan.geometry.valid && @@ -2187,7 +2144,7 @@ LlmJsonPeakEstimate calculate_json_peak_from_identity_sizes(const LlmMemoryConfi } size_t measurement_identity_bytes = 0; - if (!NumericUtils::checked_multiply(config.loop_count, scenario_identity_bytes_per_loop, + if (!NumericUtils::checked_multiply(static_cast(1), scenario_identity_bytes_per_loop, measurement_identity_bytes)) { return estimate; } @@ -2221,7 +2178,9 @@ LlmJsonPeakEstimate calculate_json_peak_from_identity_sizes(const LlmMemoryConfi return estimate; } task_evidence_records = planned_measurements; - if (backend == LlmMemoryBackend::Metal) { + size_t canonical_plans = 0; + if (!NumericUtils::checked_multiply(calibration_attempts_per_scenario + 1, kLlmScenarioCount, canonical_plans)) return estimate; + { size_t calibration_task_records = 0; if (!NumericUtils::checked_multiply(calibration_attempts_per_scenario, kLlmScenarioCount, calibration_task_records) || @@ -2231,7 +2190,10 @@ LlmJsonPeakEstimate calculate_json_peak_from_identity_sizes(const LlmMemoryConfi } const size_t task_record_peak_bytes = backend == LlmMemoryBackend::Metal ? kJsonMetalTaskPeakBytes : kJsonMeasurementPeakBytes; - if (!NumericUtils::checked_multiply(planned_measurements, effective_workers, worker_checksum_records) || + size_t expected_and_actual_records = 0; + if (!NumericUtils::checked_add(planned_measurements, canonical_plans, expected_and_actual_records) || + !NumericUtils::checked_add(task_evidence_records, canonical_plans, task_evidence_records) || + !NumericUtils::checked_multiply(expected_and_actual_records, effective_workers, worker_checksum_records) || !NumericUtils::checked_multiply(raw_input_bytes, kJsonInputStringExpansionFactor, estimate.input_string_bytes) || !NumericUtils::checked_multiply(task_evidence_records, task_record_peak_bytes, estimate.measurement_record_bytes) || @@ -2330,8 +2292,12 @@ nlohmann::ordered_json build_llm_memory_json(const LlmMemoryConfig& config, cons output["kv_layout"] = llm_kv_layout_to_string(model_plan.kv_layout); output["methodology_version"] = model_plan.methodology_version; output["software"] = software_json(metadata); + output["build_manifest"] = OrderedJson{{"manifest_version", 1}, {"status", "unavailable"}, + {"reason_code", "build-provenance-not-provided"}, {"binary_sha256", nullptr}, {"git_commit", nullptr}, + {"git_dirty", nullptr}, {"compiler", nullptr}, {"compile_flags", nullptr}, {"link_flags", nullptr}, {"target_arch", nullptr}, + {"sdk", nullptr}, {"min_os", nullptr}}; output["configuration"] = configuration_json(config); - output["resolved_plan"] = resolved_plan_json(model_plan, result.frozen_scenario_plans, backend_evidence); + output["resolved_plan"] = resolved_plan_json(model_plan, result, backend_evidence); output["backend_evidence"] = backend_evidence_json(model_plan, backend_evidence, metadata.json_peak_estimate, !config.output_file.empty()); output["memory_budget"] = memory_budget_json( @@ -2343,7 +2309,7 @@ nlohmann::ordered_json build_llm_memory_json(const LlmMemoryConfig& config, cons output["status"] = llm_run_status_to_string(result.status); output["reason_code"] = result.reason_code; output["results_complete"] = result.results_complete; - output["conclusions_valid"] = result.conclusions_valid; + output["run_accepted"] = result.run_accepted; output["interpretation"] = interpretation_json(model_plan); // Additional lifecycle and audit evidence remains part of schema v1 but is diff --git a/src/llm_memory/llm_memory.cpp b/src/llm_memory/llm_memory.cpp index 358a2f9..75f92bd 100644 --- a/src/llm_memory/llm_memory.cpp +++ b/src/llm_memory/llm_memory.cpp @@ -207,10 +207,8 @@ void initialize_llm_setup_failure_result( for (size_t index = 0; index < kScenarios.size(); ++index) { result.aggregates[index].scenario = kScenarios[index]; } - result.logical_checkpoint_attempts = 1; - result.successful_logical_checkpoints = 1; + result.snapshot_request = "terminal"; result.terminal_checkpoint_attempted = true; - result.terminal_checkpoint_completed = true; } int write_llm_setup_failure_final( @@ -221,6 +219,7 @@ int write_llm_setup_failure_final( LlmMemoryResult& result) noexcept { try { initialize_llm_setup_failure_result(config, reason_code, result); + if (output_session.kind() == JsonOutputKind::Disabled) return EXIT_SUCCESS; metadata.environment_end = capture_llm_host_environment(); return output_session.write_final( build_llm_memory_json(config, model_plan, backend_evidence, metadata, @@ -244,11 +243,17 @@ int write_llm_post_run_exception_final( const LlmBackendEvidence& backend_evidence, LlmResultMetadata& metadata, LlmMemoryResult& result) noexcept { try { + if (result.checkpoint_failed || (output_session.kind() == JsonOutputKind::File && + !result.terminal_checkpoint_completed)) return EXIT_FAILURE; + result.prior_file_writer_attempts = output_session.file_writer_attempts(); + result.prior_successful_file_writes = output_session.successful_file_writes(); + result.snapshot_request = "late-command-error-correction"; + result.terminal_checkpoint_completed = false; result.status = LlmRunStatus::Failed; result.reason_code = LlmRunnerReason::RUNNER_EXCEPTION; - result.results_complete = false; - result.conclusions_valid = false; + result.run_accepted = false; result.diagnostic.clear(); + if (output_session.kind() == JsonOutputKind::Disabled) return EXIT_SUCCESS; metadata.environment_end = capture_llm_host_environment(); return output_session.write_final( build_llm_memory_json(config, model_plan, backend_evidence, metadata, @@ -1098,6 +1103,10 @@ int run_llm_memory_mode(int argc, char* argv[]) { } LlmRunnerHooks hooks; + hooks.progress_snapshots = output_session->persists_checkpoints(); + hooks.observe_file_writes = [&]() { + return std::make_pair(output_session->file_writer_attempts(), output_session->successful_file_writes()); + }; hooks.checkpoint = [&](const LlmMemoryResult& snapshot, LlmCheckpointKind kind) { static_cast(kind); return output_session->checkpoint([&]() { diff --git a/src/llm_memory/llm_output.cpp b/src/llm_memory/llm_output.cpp index 4814e9f..9672493 100644 --- a/src/llm_memory/llm_output.cpp +++ b/src/llm_memory/llm_output.cpp @@ -160,7 +160,7 @@ void print_metal_task_evidence(const LlmMemoryWorkPlan& plan, const LlmMemoryRes if (index >= printed.size() || printed[index]) { continue; } - const LlmMetalTaskEvidence* const metal = get_llm_metal_task_evidence(measurement.execution); + const LlmMetalRuntimeEvidence* const metal = get_llm_metal_task_evidence(measurement.execution); if (metal == nullptr) { continue; } @@ -169,7 +169,7 @@ void print_metal_task_evidence(const LlmMemoryWorkPlan& plan, const LlmMemoryRes std::cout << Messages::report_llm_memory_metal_task( llm_scenario_to_string(measurement.scenario), metal->pipeline_label, threadgroups, threads_per_threadgroup, metal->timing_evaluated, - metal->timing_valid, metal->gpu_elapsed_seconds, + metal->timing_valid, measurement.execution.timing.elapsed_seconds, metal->checksum_evaluated, metal->checksum_valid, measurement.scenario != LlmScenario::WeightsOnly, metal->kv_write_validation_evaluated, @@ -321,6 +321,11 @@ void print_llm_memory_console_report(const LlmMemoryWorkPlan& model_plan, const << "\n\n"; for (const LlmScenarioAggregate& aggregate : result.aggregates) { print_headline(aggregate, work_unit_name, plural_work_unit_name); + const auto& stats = aggregate.effective_model_payload_gb_s.statistics; + if (stats.sample_count != 0) { + std::cout << Messages::report_llm_memory_distribution(stats.sample_count, stats.median, + stats.min, stats.max, stats.coefficient_of_variation_pct, stats.median_absolute_deviation) << '\n'; + } } print_metal_task_evidence(model_plan, result); std::cout << Messages::report_llm_memory_interpretation_note( diff --git a/src/llm_memory/llm_prefill.cpp b/src/llm_memory/llm_prefill.cpp index 07a70f6..ae2f88c 100644 --- a/src/llm_memory/llm_prefill.cpp +++ b/src/llm_memory/llm_prefill.cpp @@ -623,6 +623,24 @@ bool checked_llm_prefill_floor_sum( return true; } +LlmPrefillModelContext calculate_llm_prefill_model_context(size_t prompt_tokens, + size_t layers, size_t batch, size_t query_heads, size_t head_dimension) noexcept { + LlmPrefillModelContext context; + size_t pairs = 0; + if (checked_llm_prefill_triangular(prompt_tokens, pairs)) { + context.causal_token_pairs_per_sequence = pairs; + size_t owners = 0; + if (NumericUtils::checked_multiply(layers, batch, owners) && + NumericUtils::checked_multiply(owners, query_heads, owners) && + NumericUtils::checked_multiply(owners, pairs, pairs)) { + context.logical_attention_pairs = pairs; + if (NumericUtils::checked_multiply(pairs, head_dimension, pairs)) + context.logical_attention_fma_terms = pairs; + } + } + return context; +} + LlmPrefillPlan resolve_llm_prefill_plan( const LlmPrefillPlanRequest& request) { LlmPrefillPlan plan; @@ -703,31 +721,14 @@ LlmPrefillPlan resolve_llm_prefill_plan( } plan.attention_prefix_token_visits_per_sequence = full_tile_visits; - if (!checked_llm_prefill_triangular( - request.prompt_tokens, - plan.causal_token_pairs_per_sequence)) { - plan.reason_code = LlmPrefillReason::CAUSAL_TOKEN_PAIRS_OVERFLOW; - return plan; - } + const auto context = calculate_llm_prefill_model_context(request.prompt_tokens, + request.layer_count, request.batch_size, request.query_head_count, request.head_dimension); + plan.causal_token_pairs_per_sequence = context.causal_token_pairs_per_sequence; + plan.logical_attention_pairs = context.logical_attention_pairs; + plan.logical_attention_fma_terms = context.logical_attention_fma_terms; size_t layer_batch_count = 0; - if (!NumericUtils::checked_multiply( - request.layer_count, request.batch_size, layer_batch_count) || - !NumericUtils::checked_multiply( - layer_batch_count, request.query_head_count, - plan.logical_attention_pairs) || - !NumericUtils::checked_multiply( - plan.logical_attention_pairs, - plan.causal_token_pairs_per_sequence, - plan.logical_attention_pairs)) { - plan.reason_code = - LlmPrefillReason::LOGICAL_ATTENTION_PAIRS_OVERFLOW; - return plan; - } - if (!NumericUtils::checked_multiply( - plan.logical_attention_pairs, request.head_dimension, - plan.logical_attention_fma_terms)) { - plan.reason_code = - LlmPrefillReason::LOGICAL_ATTENTION_FMA_TERMS_OVERFLOW; + if (!NumericUtils::checked_multiply(request.layer_count, request.batch_size, layer_batch_count)) { + plan.reason_code = LlmPrefillReason::LOGICAL_ATTENTION_PAIRS_OVERFLOW; return plan; } diff --git a/src/llm_memory/llm_prefill.h b/src/llm_memory/llm_prefill.h index c8d92b2..727a16b 100644 --- a/src/llm_memory/llm_prefill.h +++ b/src/llm_memory/llm_prefill.h @@ -28,6 +28,7 @@ #include #include +#include #include #include @@ -201,6 +202,18 @@ struct LlmPrefillPlanRequest { size_t kv_block_tokens = 0; }; +/** Theoretical context only; null fields denote arithmetic overflow, never byte work. */ +struct LlmPrefillModelContext { + std::optional causal_token_pairs_per_sequence; + std::optional logical_attention_pairs; + std::optional logical_attention_fma_terms; +}; + +/** Pure allocation-free context calculation. Each dependent overflow remains null; + * callers must independently validate payload/prefix/layout arithmetic. */ +LlmPrefillModelContext calculate_llm_prefill_model_context(size_t prompt_tokens, + size_t layers, size_t batch, size_t query_heads, size_t head_dimension) noexcept; + /** * Backend-neutral exact prefill geometry, payload, and optional paged costs. * @@ -218,13 +231,13 @@ struct LlmPrefillPlan { size_t final_query_tile_tokens = 0; size_t tile_count = 0; size_t attention_prefix_token_visits_per_sequence = 0; - size_t causal_token_pairs_per_sequence = 0; + std::optional causal_token_pairs_per_sequence; size_t layer_count = 0; size_t batch_size = 0; size_t query_head_count = 0; size_t head_dimension = 0; - size_t logical_attention_pairs = 0; - size_t logical_attention_fma_terms = 0; + std::optional logical_attention_pairs; + std::optional logical_attention_fma_terms; size_t k_or_v_record_bytes_per_layer = 0; size_t kv_record_bytes_per_layer = 0; size_t kv_bytes_per_token = 0; diff --git a/src/llm_memory/llm_runner.cpp b/src/llm_memory/llm_runner.cpp index 2c15dba..8174302 100644 --- a/src/llm_memory/llm_runner.cpp +++ b/src/llm_memory/llm_runner.cpp @@ -27,11 +27,129 @@ #include #include #include +#include +#include "llm_memory/llm_metal_backend.h" #include "core/config/constants.h" #include "core/signal/signal_handler.h" #include "utils/numeric_utils.h" +LlmColdChecks required_llm_cold_checks(const LlmMemoryWorkPlan& model, LlmScenario scenario) noexcept { + const bool writes = scenario != LlmScenario::WeightsOnly; + const bool paged = model.kv_layout == LlmKvLayout::Paged; + const bool prefill = model.phase == LlmPhase::Prefill; + bool padding = false; + if (model.backend == LlmMemoryBackend::Cpu) { + const auto* cpu = get_llm_cpu_execution_plan(model); + padding = paged && cpu && cpu->paged && + cpu->paged->layout.last_block_valid_bytes < cpu->paged->layout.block_bytes; + return make_llm_cold_checks(paged || prefill || writes, + prefill ? LlmColdCheckKind::KvPrefillFinalSamples : writes ? LlmColdCheckKind::KvAppendFinal : + LlmColdCheckKind::KvAppendUnchanged, writes || (paged && !prefill), padding); + } + const auto* metal = get_llm_metal_execution_plan(model); + padding = paged && metal && metal->resources.paged_layout && + metal->resources.paged_layout->last_block_valid_bytes < metal->resources.paged_layout->block_bytes; + return make_llm_cold_checks(writes, prefill ? LlmColdCheckKind::KvPrefillFinalSamples : + LlmColdCheckKind::KvAppendFinal, writes, writes && padding); +} + +const LlmCanonicalScenarioPlan* find_llm_scenario_plan(const LlmMemoryResult& result, + LlmPlanHandle handle) noexcept { + return handle < result.scenario_plans.size() ? &result.scenario_plans[handle] : nullptr; +} + +namespace { +auto scenario_content(const LlmScenarioWorkPlan& p) noexcept { + return std::tie(p.valid, p.reason_code, p.scenario, p.work_unit_kind, p.kv_write_kind, + p.explicit_iterations, p.model_plan_identity, p.scenario_seed, p.work_units, + p.weight_read_bytes_per_work_unit, p.kv_read_bytes_per_work_unit, p.kv_write_bytes_per_work_unit, + p.effective_model_payload_bytes_per_work_unit, p.layout_metadata_lookup_count_per_work_unit, + p.layout_metadata_read_bytes_per_work_unit, p.accounted_bytes_per_work_unit, + p.weight_read_bytes, p.kv_read_bytes, p.kv_write_bytes, p.effective_model_payload_bytes, + p.layout_metadata_lookup_count, p.layout_metadata_read_bytes, p.task_accounted_bytes, + p.maximum_work_units_by_work_unit_cap, p.maximum_work_units_by_guardrail, + p.effective_maximum_work_units, p.plan_identity); +} +} + +LlmPlanHandle register_llm_scenario_plan(LlmMemoryResult& result, + const LlmMemoryWorkPlan& model, const LlmScenarioWorkPlan& plan, LlmBackend& backend) { + if (!plan.valid || plan.model_plan_identity != model.plan_identity) + throw std::invalid_argument("invalid canonical scenario plan"); + for (size_t i = 0; i < result.scenario_plans.size(); ++i) { + const auto& prior = result.scenario_plans[i].plan; + if (prior.scenario == plan.scenario && prior.work_units == plan.work_units && + prior.explicit_iterations == plan.explicit_iterations) { + if (scenario_content(prior) != scenario_content(plan)) + throw std::invalid_argument("contradictory canonical scenario plan"); + return i; + } + } + const auto rebuilt = build_llm_scenario_work_plan(model, plan.scenario, plan.work_units, plan.explicit_iterations); + if (scenario_content(rebuilt) != scenario_content(plan)) + throw std::invalid_argument("noncanonical scenario content"); + LlmCanonicalScenarioPlan entry; + entry.plan = plan; + if (model.backend == LlmMemoryBackend::Cpu) { + auto expected = backend.expected_cpu_checksum(model, plan); + const auto* cpu = get_llm_cpu_execution_plan(model); + entry.expected.available = expected.valid && cpu && expected.workers.size() == cpu->effective_workers; + entry.expected.reason_code = expected.valid && !entry.expected.available ? + LlmExecutorReason::INVALID_DESCRIPTOR_LAYOUT : expected.reason_code; + if (entry.expected.available) { + entry.expected.cpu_workers = std::move(expected.workers); + entry.expected.cpu_run = expected.run_checksum; + } + } else { + LlmMetalChecksumOracle oracle; + if (model.kv_layout == LlmKvLayout::Contiguous) { + oracle = model.phase == LlmPhase::Decode ? calculate_llm_metal_decode_contiguous_checksum(model, plan) + : calculate_llm_metal_prefill_contiguous_checksum(model, plan); + } else { + const auto* metal = get_llm_metal_execution_plan(model); + if (metal != nullptr && metal->resources.paged_layout) { + auto table = materialize_llm_kv_block_table(*metal->resources.paged_layout, + derive_llm_kv_permutation_seed(model.base_seed)); + if (table.valid) { + if (model.phase == LlmPhase::Decode) { + const auto summary = build_llm_metal_decode_paged_checksum_summary(model, table.entries.data(), table.entries.size()); + oracle = calculate_llm_metal_decode_paged_checksum(model, plan, summary); + } else { + const auto summary = build_llm_metal_prefill_paged_checksum_summary(model, table.entries.data(), table.entries.size()); + oracle = calculate_llm_metal_prefill_paged_checksum(model, plan, summary); + } + } + } + } + entry.expected.available = oracle.valid; + entry.expected.reason_code = oracle.reason_code; + if (oracle.valid) entry.expected.metal_run = oracle.checksum; + } + result.scenario_plans.push_back(std::move(entry)); + return result.scenario_plans.size() - 1; +} + +LlmDerivedMetrics derive_llm_measurement_metrics(const LlmMeasurementState& measurement) noexcept { + LlmDerivedMetrics values; + const auto& runtime = measurement.execution; + if (measurement.status != LlmMeasurementStatus::Measured || !runtime.timing.evaluated || + !runtime.timing.valid || !std::isfinite(runtime.timing.elapsed_seconds) || + runtime.timing.elapsed_seconds <= 0 || runtime.completion.completed_work_units == 0) return values; + const long double elapsed = runtime.timing.elapsed_seconds; + const long double work = runtime.completion.completed_work_units; + const double latency = static_cast(elapsed / work); + const double rate = static_cast(work / elapsed); + const double bandwidth = static_cast(runtime.completion.completed_effective_model_payload_bytes / elapsed / 1.0e9L); + if (std::isfinite(latency) && latency > 0 && std::isfinite(rate) && rate > 0 && + std::isfinite(bandwidth) && bandwidth > 0) { + values.latency_seconds = latency; + values.work_units_per_second = rate; + values.payload_gb_s = bandwidth; + } + return values; +} + namespace { constexpr std::array kLlmScenarios = {LlmScenario::WeightsOnly, LlmScenario::KvOnly, @@ -396,6 +514,11 @@ const LlmExecutorResult* cpu_executor_evidence( return cpu == nullptr ? nullptr : &cpu->executor; } +const LlmCpuRuntimeEvidence* cpu_executor_evidence(const LlmRetainedExecution& execution) noexcept { + const auto* cpu = std::get_if(&execution.backend_evidence); + return cpu == nullptr ? nullptr : &cpu->executor; +} + LlmTaskExecutionEvidence compact_execution( const LlmTaskExecutionResult& execution) { LlmTaskExecutionEvidence evidence; @@ -428,7 +551,6 @@ LlmTaskExecutionEvidence compact_execution( evidence.checksum_evaluated = checksum_evidence_complete; evidence.checksum_valid = checksum_evidence_complete && cpu->checksum_valid; if (checksum_evidence_complete) { - evidence.expected_run_checksum = cpu->expected_run_checksum; evidence.actual_run_checksum = cpu->actual_run_checksum; } } @@ -462,10 +584,9 @@ bool task_identity_matches(const LlmTaskIdentity& identity, identity.scenario_plan_identity == task_plan.plan_identity; } -bool task_completion_matches(const LlmTaskCompletion& completion, +bool task_completion_matches(const LlmCompletedWork& completion, const LlmScenarioWorkPlan& task_plan) noexcept { - return completion.planned_work_units == task_plan.work_units && - completion.completed_work_units == task_plan.work_units && + return completion.completed_work_units == task_plan.work_units && completion.completed_effective_model_payload_bytes == task_plan.effective_model_payload_bytes && completion.completed_layout_metadata_lookup_count == @@ -476,6 +597,34 @@ bool task_completion_matches(const LlmTaskCompletion& completion, task_plan.task_accounted_bytes; } +/** Validate observed slot applicability against the admitted profile, not the + * producer's booleans alone. Missing whole backend evidence cannot certify a task. */ +bool required_checks_valid(const LlmTaskExecutionResult& execution, + const LlmMemoryWorkPlan& model, const LlmScenarioWorkPlan& plan) noexcept { + const LlmColdChecks expected = required_llm_cold_checks(model, plan.scenario); + const LlmColdChecks* actual = nullptr; + if (model.backend == LlmMemoryBackend::Cpu) { + const auto* evidence = get_llm_cpu_task_evidence(execution); + const auto* cpu = get_llm_cpu_execution_plan(model); + if (!evidence || !cpu || !evidence->checksum_evaluated || !evidence->checksum_valid || + evidence->actual_checksums.size() != cpu->effective_workers || + evidence->requested_workers != cpu->effective_workers || + evidence->created_workers != cpu->effective_workers || + evidence->completed_workers != cpu->effective_workers) return false; + actual = &evidence->cold_checks; + } else { + const auto* evidence = get_llm_metal_task_evidence(execution); + const auto* metal = get_llm_metal_execution_plan(model); + if (!evidence || !metal || !evidence->checksum_evaluated || !evidence->checksum_valid) return false; + actual = &evidence->cold_checks; + } + for (size_t i = 0; i < expected.size(); ++i) { + if ((*actual)[i].kind != expected[i].kind || (*actual)[i].applicable != expected[i].applicable || + (expected[i].applicable && (!(*actual)[i].evaluated || !(*actual)[i].valid))) return false; + } + return true; +} + bool execution_is_accepted(const LlmTaskExecutionResult& execution, const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan& task_plan, @@ -487,8 +636,10 @@ bool execution_is_accepted(const LlmTaskExecutionResult& execution, execution.timing.evaluated && execution.timing.valid && std::isfinite(execution.timing.elapsed_seconds) && execution.timing.elapsed_seconds > 0.0 && + execution.completion.planned_work_units == task_plan.work_units && task_completion_matches(execution.completion, task_plan) && - execution.validation.evaluated && execution.validation.valid; + execution.validation.evaluated && execution.validation.valid && + required_checks_valid(execution, model_plan, task_plan); } std::string_view execution_failure_reason( @@ -513,7 +664,8 @@ std::string_view execution_failure_reason( context)) { return LlmBackendReason::TASK_IDENTITY_MISMATCH; } - if (!task_completion_matches(execution.completion, task_plan)) { + if (execution.completion.planned_work_units != task_plan.work_units || + !task_completion_matches(execution.completion, task_plan)) { return LlmBackendReason::TASK_COMPLETION_MISMATCH; } if (!execution.timing.evaluated || !execution.timing.valid || @@ -524,7 +676,7 @@ std::string_view execution_failure_reason( if (!execution.validation.evaluated) { return LlmBackendReason::VALIDATION_NOT_EVALUATED; } - if (!execution.validation.valid) { + if (!execution.validation.valid || !required_checks_valid(execution, model_plan, task_plan)) { return LlmBackendReason::VALIDATION_FAILED; } return reason_code == LlmBackendReason::VALID @@ -544,65 +696,10 @@ LlmMeasurementStatus execution_failure_status( : LlmMeasurementStatus::Failed; } -void clear_measurement_values(LlmMeasurementState& measurement) { - measurement.completed_work_units = 0; - measurement.completed_effective_model_payload_bytes = 0; - measurement.completed_layout_metadata_lookup_count = 0; - measurement.completed_layout_metadata_read_bytes = 0; - measurement.completed_task_accounted_bytes = 0; - measurement.elapsed_seconds.reset(); - measurement.synthetic_work_unit_latency_seconds.reset(); - measurement.synthetic_memory_work_units_per_second.reset(); - measurement.effective_model_payload_gb_s.reset(); - measurement.checksum_valid = false; -} - -void update_metric_aggregate(LlmMetricAggregate& aggregate, LlmStatisticsWorkspace& workspace) { - aggregate.statistics = DescriptiveStatistics{}; - aggregate.headline.reset(); - if (aggregate.values.empty()) { - return; - } - aggregate.statistics = - calculate_descriptive_statistics(aggregate.values, workspace.sorted_values, workspace.absolute_deviations); - aggregate.headline = aggregate.values.size() == 1 ? aggregate.values.front() : aggregate.statistics.median; -} - -void update_scenario_aggregate(LlmMemoryResult& result, const LlmMeasurementState& measurement) { - const size_t index = scenario_index(measurement.scenario); - if (index >= result.aggregates.size() || measurement.status != LlmMeasurementStatus::Measured || - !measurement.checksum_valid || !measurement.synthetic_work_unit_latency_seconds.has_value() || - !measurement.synthetic_memory_work_units_per_second.has_value() || - !measurement.effective_model_payload_gb_s.has_value()) { - return; - } - - LlmScenarioAggregate& aggregate = result.aggregates[index]; - aggregate.work_unit_latency_seconds.values.push_back(*measurement.synthetic_work_unit_latency_seconds); - aggregate.synthetic_memory_work_units_per_second.values.push_back( - *measurement.synthetic_memory_work_units_per_second); - aggregate.effective_model_payload_gb_s.values.push_back(*measurement.effective_model_payload_gb_s); - update_metric_aggregate(aggregate.work_unit_latency_seconds, result.statistics_workspace); - update_metric_aggregate(aggregate.synthetic_memory_work_units_per_second, result.statistics_workspace); - update_metric_aggregate(aggregate.effective_model_payload_gb_s, result.statistics_workspace); - - const size_t sample_count = aggregate.effective_model_payload_gb_s.values.size(); - aggregate.status = sample_count == result.counters.planned_loops ? "complete" : "partial"; - if (sample_count < 3) { - aggregate.stability_quality = "insufficient-samples"; - } else if (aggregate.effective_model_payload_gb_s.statistics.coefficient_of_variation_defined && - aggregate.effective_model_payload_gb_s.statistics.coefficient_of_variation_pct > - Constants::LLM_STREAMING_CV_WARNING_PCT) { - aggregate.stability_quality = "noisy"; - } else { - aggregate.stability_quality = "stable"; - } -} - void rebuild_quality_warnings(LlmMemoryResult& result) { result.quality_warnings.clear(); for (const LlmScenarioAggregate& aggregate : result.aggregates) { - if (aggregate.stability_quality != "noisy") { + if (aggregate.observed_cv_classification != "above-threshold") { continue; } switch (aggregate.scenario) { @@ -670,8 +767,8 @@ void update_completion_state(LlmMemoryResult& result) { } } - result.conclusions_valid = result.status == LlmRunStatus::Complete && result.results_complete && - result.scenario_order_balance_complete && !result.checkpoint_failed; + result.run_accepted = result.status == LlmRunStatus::Complete && result.results_complete && + !result.checkpoint_failed; rebuild_quality_warnings(result); } @@ -685,7 +782,6 @@ void finalize_remaining_interrupted(LlmMemoryResult& result) { } measurement.status = LlmMeasurementStatus::Interrupted; measurement.reason_code = LlmRunnerReason::INTERRUPTION_BEFORE_TASK; - clear_measurement_values(measurement); } } } @@ -696,7 +792,6 @@ void finalize_remaining_failed(LlmMemoryResult& result) { ++result.counters.terminal_measurements; measurement.status = LlmMeasurementStatus::Failed; measurement.reason_code = LlmRunnerReason::NOT_RUN_AFTER_RUNTIME_FAILURE; - clear_measurement_values(measurement); } } } @@ -708,7 +803,6 @@ void finalize_failure(LlmMemoryResult& result, std::string_view reason_code, boo ++result.counters.terminal_measurements; measurement.status = LlmMeasurementStatus::Failed; measurement.reason_code = stable_reason_code; - clear_measurement_values(measurement); } } if (interruption_pending) { @@ -723,21 +817,27 @@ void finalize_failure(LlmMemoryResult& result, std::string_view reason_code, boo } bool invoke_checkpoint(LlmMemoryResult& result, LlmCheckpointKind kind, const LlmRunnerHooks& hooks) { + if (kind == LlmCheckpointKind::MeasurementTerminal && + (!hooks.progress_snapshots || result.counters.terminal_measurements == 0 || result.counters.terminal_measurements % kLlmScenarioCount != 0 || + result.counters.completed_loops % result.snapshot_interval_loops != 0 || + result.status == LlmRunStatus::Failed || result.status == LlmRunStatus::Interrupted)) return true; + result.snapshot_request = kind == LlmCheckpointKind::CommandTerminal ? "terminal" : "progress"; ++result.logical_checkpoint_attempts; - ++result.successful_logical_checkpoints; if (kind == LlmCheckpointKind::CommandTerminal) { result.terminal_checkpoint_attempted = true; - result.terminal_checkpoint_completed = true; + result.terminal_checkpoint_completed = false; } - if (!hooks.checkpoint) { - return true; - } - int status = EXIT_FAILURE; bool checkpoint_threw = false; std::string_view checkpoint_failure_reason = LlmRunnerReason::CHECKPOINT_WRITE_FAILED; try { - status = hooks.checkpoint(result, kind); + if (hooks.observe_file_writes) { + const auto observed = hooks.observe_file_writes(); + result.prior_file_writer_attempts = observed.first; + result.prior_successful_file_writes = observed.second; + } + prepare_llm_result_snapshot(result); + status = hooks.checkpoint ? hooks.checkpoint(result, kind) : EXIT_SUCCESS; } catch (const std::exception& error) { checkpoint_threw = true; checkpoint_failure_reason = LlmRunnerReason::RUNNER_EXCEPTION; @@ -748,7 +848,6 @@ bool invoke_checkpoint(LlmMemoryResult& result, LlmCheckpointKind kind, const Ll result.diagnostic.clear(); } if (status != EXIT_SUCCESS) { - --result.successful_logical_checkpoints; if (kind == LlmCheckpointKind::CommandTerminal) { result.terminal_checkpoint_completed = false; } @@ -758,21 +857,14 @@ bool invoke_checkpoint(LlmMemoryResult& result, LlmCheckpointKind kind, const Ll if (!checkpoint_threw) { result.diagnostic.clear(); } - result.conclusions_valid = false; + result.run_accepted = false; return false; } + ++result.successful_logical_checkpoints; + if (kind == LlmCheckpointKind::CommandTerminal) result.terminal_checkpoint_completed = true; return true; } -void refresh_calibration_references(LlmMemoryResult& result) noexcept { - for (LlmMeasurementState& measurement : result.measurements) { - const size_t index = scenario_index(measurement.scenario); - if (index < kLlmScenarioCount) { - measurement.calibration_attempt_count = result.calibration_attempt_counts[index]; - } - } -} - bool release_backend_once(LlmBackend& backend, bool& release_attempted, LlmMemoryResult& result) { if (release_attempted) { @@ -800,7 +892,6 @@ int fail_uninitialized_backend_transition( int finish_terminal(LlmMemoryResult& result, const LlmRunnerHooks& hooks, LlmBackend& backend, bool& release_attempted) { - refresh_calibration_references(result); update_completion_state(result); const bool release_succeeded = release_backend_once(backend, release_attempted, result); @@ -853,7 +944,7 @@ bool add_string_backing(const std::string& value, size_t& total) noexcept { return NumericUtils::checked_add(value.capacity(), static_cast(1), bytes) && checked_add_to(bytes, total); } -bool add_metal_task_string_backing(const LlmMetalTaskEvidence& metal, +bool add_metal_task_string_backing(const LlmMetalRuntimeEvidence& metal, size_t& total) noexcept { return add_string_backing(metal.pipeline_label, total) && add_string_backing(metal.grid_plan.reason_code, total) && @@ -864,7 +955,6 @@ bool add_metal_task_string_backing(const LlmMetalTaskEvidence& metal, add_string_backing(metal.reset_command_status, total) && add_string_backing(metal.timed_command_status, total) && add_string_backing(metal.post_validation_command_status, total) && - add_string_backing(metal.checksum_algorithm_version, total) && add_string_backing(metal.error.domain, total) && add_string_backing(metal.error.description, total); } @@ -877,79 +967,46 @@ LlmRunnerAuxiliaryEstimate calculate_actual_runner_backing(const LlmMemoryResult return actual; } - for (const std::vector& attempts : result.calibration_attempts) { + for (const auto& attempts : result.calibration_attempts) { if (!add_capacity_bytes(attempts.capacity(), sizeof(LlmCalibrationAttempt), actual.calibration_record_bytes)) { return actual; } - for (const LlmCalibrationAttempt& attempt : attempts) { - if (!add_string_backing(attempt.work_plan_identity, actual.calibration_identity_bytes)) { - return actual; - } - if (attempt.execution.metal_evidence_available && - attempt.execution.metal.has_value() && - !add_metal_task_string_backing(*attempt.execution.metal, - actual.fixed_metadata_bytes)) { - return actual; - } - } - } - - for (const LlmScenarioAggregate& aggregate : result.aggregates) { - if (!add_capacity_bytes(aggregate.work_unit_latency_seconds.values.capacity(), sizeof(double), - actual.aggregate_value_bytes) || - !add_capacity_bytes(aggregate.synthetic_memory_work_units_per_second.values.capacity(), sizeof(double), - actual.aggregate_value_bytes) || - !add_capacity_bytes(aggregate.effective_model_payload_gb_s.values.capacity(), sizeof(double), - actual.aggregate_value_bytes)) { - return actual; - } - } - if (!add_capacity_bytes(result.statistics_workspace.sorted_values.capacity(), sizeof(double), - actual.statistics_workspace_bytes) || - !add_capacity_bytes(result.statistics_workspace.absolute_deviations.capacity(), sizeof(double), - actual.statistics_workspace_bytes) || - !add_capacity_bytes(result.quality_warnings.capacity(), sizeof(std::string_view), actual.warning_record_bytes)) { - return actual; - } - - if (!add_string_backing(result.reason_code, actual.fixed_metadata_bytes) || - !add_string_backing(result.diagnostic, actual.fixed_metadata_bytes) || - !add_string_backing(result.frozen_scenario_plans.reason_code, actual.fixed_metadata_bytes) || - !add_string_backing(result.frozen_scenario_plans.model_plan_identity, actual.fixed_metadata_bytes) || - !add_string_backing(result.frozen_scenario_plans.plan_identity, actual.fixed_metadata_bytes)) { - return actual; - } - for (const LlmScenarioWorkPlan& scenario : result.frozen_scenario_plans.scenarios) { - if (!add_string_backing(scenario.reason_code, actual.fixed_metadata_bytes) || - !add_string_backing(scenario.model_plan_identity, actual.fixed_metadata_bytes) || - !add_string_backing(scenario.plan_identity, actual.fixed_metadata_bytes)) { - return actual; - } - } - for (const LlmMeasurementState& measurement : result.measurements) { - if (!add_string_backing(measurement.execution.reason_code, - actual.fixed_metadata_bytes)) { - return actual; - } - const LlmExecutorResult* cpu = - cpu_executor_evidence(measurement.execution); - if (cpu != nullptr && - (!add_string_backing(cpu->reason_code, actual.fixed_metadata_bytes) || - !add_capacity_bytes(cpu->expected_checksums.capacity(), - sizeof(LlmWorkerChecksum), - actual.retained_checksum_bytes) || - !add_capacity_bytes(cpu->actual_checksums.capacity(), - sizeof(LlmWorkerChecksum), - actual.retained_checksum_bytes))) { - return actual; - } - const LlmMetalTaskEvidence* const metal = - get_llm_metal_task_evidence(measurement.execution); - if (metal != nullptr && - !add_metal_task_string_backing(*metal, - actual.fixed_metadata_bytes)) { - return actual; - } + for (const auto& attempt : attempts) { + if (attempt.execution.metal && + !add_metal_task_string_backing(*attempt.execution.metal, actual.fixed_metadata_bytes)) return actual; + } + } + for (const auto& aggregate : result.aggregates) { + if (!add_capacity_bytes(aggregate.accepted_measurement_ids.capacity(), sizeof(size_t), + actual.aggregate_value_bytes)) return actual; + } + for (const auto* values : {&result.statistics_workspace.extracted_values, + &result.statistics_workspace.sorted_values, + &result.statistics_workspace.absolute_deviations}) { + if (!add_capacity_bytes(values->capacity(), sizeof(double), actual.statistics_workspace_bytes)) return actual; + } + if (!add_capacity_bytes(result.quality_warnings.capacity(), sizeof(std::string_view), actual.warning_record_bytes) || + !add_capacity_bytes(result.scenario_plans.capacity(), sizeof(LlmCanonicalScenarioPlan), actual.fixed_metadata_bytes) || + !add_capacity_bytes(result.snapshot_plan_order.capacity(), sizeof(size_t), actual.fixed_metadata_bytes) || + !add_capacity_bytes(result.snapshot_plan_refs.capacity(), sizeof(size_t), actual.fixed_metadata_bytes) || + !add_string_backing(result.reason_code, actual.fixed_metadata_bytes) || + !add_string_backing(result.diagnostic, actual.fixed_metadata_bytes)) return actual; + for (const auto& entry : result.scenario_plans) { + if (!add_string_backing(entry.plan.reason_code, actual.fixed_metadata_bytes) || + !add_string_backing(entry.plan.model_plan_identity, actual.calibration_identity_bytes) || + !add_string_backing(entry.plan.plan_identity, actual.calibration_identity_bytes) || + !add_string_backing(entry.expected.reason_code, actual.fixed_metadata_bytes) || + !add_capacity_bytes(entry.expected.cpu_workers.capacity(), sizeof(LlmWorkerChecksum), + actual.retained_checksum_bytes)) return actual; + } + for (const auto& measurement : result.measurements) { + if (!add_string_backing(measurement.execution.reason_code, actual.fixed_metadata_bytes)) return actual; + const auto* cpu = cpu_executor_evidence(measurement.execution); + if (cpu && (!add_string_backing(cpu->reason_code, actual.fixed_metadata_bytes) || + !add_capacity_bytes(cpu->actual_checksums.capacity(), sizeof(LlmWorkerChecksum), + actual.retained_checksum_bytes))) return actual; + const auto* metal = std::get_if(&measurement.execution.backend_evidence); + if (metal && !add_metal_task_string_backing(*metal, actual.fixed_metadata_bytes)) return actual; } actual.checksum_auxiliary_bytes = actual.retained_checksum_bytes; @@ -1001,18 +1058,15 @@ bool initialize_result(const LlmMemoryConfig& config, const LlmMemoryWorkPlan& m const size_t attempts_per_scenario = calibration_capacity(config); for (size_t index = 0; index < kLlmScenarioCount; ++index) { result.aggregates[index].scenario = kLlmScenarios[index]; - result.aggregates[index].work_unit_latency_seconds.values.reserve(config.loop_count); - result.aggregates[index].synthetic_memory_work_units_per_second.values.reserve(config.loop_count); - result.aggregates[index].effective_model_payload_gb_s.values.reserve(config.loop_count); + result.aggregates[index].accepted_measurement_ids.reserve(config.loop_count); result.calibration_attempts[index].resize(attempts_per_scenario); - for (LlmCalibrationAttempt& attempt : result.calibration_attempts[index]) { - attempt.scenario = kLlmScenarios[index]; - attempt.work_unit_kind = model_plan.work_unit_kind; - attempt.kv_write_kind = - llm_kv_write_kind_for(model_plan.phase, attempt.scenario); - attempt.work_plan_identity.reserve(identity_capacities[index]); - } + for (auto& attempt : result.calibration_attempts[index]) attempt.scenario = kLlmScenarios[index]; } + result.scenario_plans.reserve(kLlmScenarioCount * (attempts_per_scenario + 1)); + result.snapshot_plan_order.reserve(result.scenario_plans.capacity()); + result.snapshot_plan_refs.reserve(result.scenario_plans.capacity()); + result.statistics_workspace.extracted_values.reserve(config.loop_count); + result.snapshot_interval_loops = std::max(1, config.loop_count / 8 + (config.loop_count % 8 != 0)); result.loops.reserve(config.loop_count); result.measurements.reserve(result.counters.planned_measurements); const LlmCpuExecutionPlan* cpu_plan = @@ -1024,23 +1078,14 @@ bool initialize_result(const LlmMemoryConfig& config, const LlmMemoryWorkPlan& m for (size_t position = 0; position < kLlmScenarioCount; ++position) { LlmMeasurementState measurement; measurement.scenario = loop.planned_order[position]; - measurement.work_unit_kind = model_plan.work_unit_kind; - measurement.kv_write_kind = - llm_kv_write_kind_for(model_plan.phase, measurement.scenario); measurement.loop_index = loop_index; measurement.order_position = position; - measurement.requested_workers = - cpu_plan == nullptr ? 0 : cpu_plan->requested_workers; - measurement.effective_workers = - cpu_plan == nullptr ? 0 : cpu_plan->effective_workers; - measurement.working_set_bytes = model_plan.geometry.total_data_mapping_bytes; measurement.execution.reason_code.reserve(kLlmRunnerReasonCapacity); if (cpu_plan != nullptr) { - measurement.execution.backend_evidence = LlmCpuTaskEvidence{}; - auto& cpu = std::get( + measurement.execution.backend_evidence = LlmCpuRetainedEvidence{}; + auto& cpu = std::get( measurement.execution.backend_evidence).executor; cpu.reason_code.reserve(kLlmRunnerReasonCapacity); - cpu.expected_checksums.reserve(cpu_plan->effective_workers); cpu.actual_checksums.reserve(cpu_plan->effective_workers); } loop.measurement_indexes[position] = result.measurements.size(); @@ -1130,20 +1175,8 @@ ExcludedTaskOutcome execute_excluded_task(const LlmMemoryWorkPlan& model_plan, c LlmCalibrationAttempt& attempt = result.calibration_attempts[index][attempt_index]; ++result.calibration_attempt_counts[index]; attempt.scenario = task_plan.scenario; - attempt.work_unit_kind = task_plan.work_unit_kind; - attempt.kv_write_kind = task_plan.kv_write_kind; attempt.purpose = canonical_calibration_purpose(purpose); - attempt.explicit_iterations = task_plan.explicit_iterations; - attempt.work_units = task_plan.work_units; - attempt.weight_read_bytes = task_plan.weight_read_bytes; - attempt.kv_read_bytes = task_plan.kv_read_bytes; - attempt.kv_write_bytes = task_plan.kv_write_bytes; - attempt.effective_model_payload_bytes = task_plan.effective_model_payload_bytes; - attempt.layout_metadata_lookup_count = - task_plan.layout_metadata_lookup_count; - attempt.layout_metadata_read_bytes = task_plan.layout_metadata_read_bytes; - attempt.task_accounted_bytes = task_plan.task_accounted_bytes; - attempt.work_plan_identity.assign(task_plan.plan_identity); + attempt.plan_handle = register_llm_scenario_plan(result, model_plan, task_plan, backend); try { LlmTaskExecutionResult execution = backend.execute_task(model_plan, task_plan, context); @@ -1155,13 +1188,14 @@ ExcludedTaskOutcome execute_excluded_task(const LlmMemoryWorkPlan& model_plan, c model_plan.geometry, task_plan.scenario, model_plan.backend)); attempt.terminal = true; - attempt.valid = execution_is_accepted(execution, model_plan, task_plan, - context); + const auto& expected = result.scenario_plans.at(attempt.plan_handle).expected; + const bool runtime_accepted = execution_is_accepted(execution, model_plan, task_plan, context); + attempt.valid = runtime_accepted && expected.available; attempt.reason_code = attempt.valid ? std::string_view(LlmBackendReason::VALID) - : execution_failure_reason(execution, model_plan, task_plan, - context); + : runtime_accepted ? canonicalize_llm_result_reason_code(expected.reason_code) + : execution_failure_reason(execution, model_plan, task_plan, context); attempt.execution.valid = attempt.valid; attempt.execution.reason_code = attempt.reason_code; @@ -1312,14 +1346,15 @@ CalibrationOutcome calibrate_scenario(const LlmMemoryWorkPlan& model_plan, CalibrationOutcome warm_frozen_scenarios( const LlmMemoryWorkPlan& model_plan, LlmBackend& backend, LlmMemoryResult& result, const LlmRunnerHooks& hooks) { - if (!result.frozen_scenario_plans.valid) { + if (std::any_of(result.frozen_plan_handles.begin(), result.frozen_plan_handles.end(), + [&](size_t h) { return h >= result.scenario_plans.size(); })) { result.status = LlmRunStatus::Failed; result.reason_code = LlmRunnerReason::FROZEN_PLAN_MISMATCH; return CalibrationOutcome::Failed; } for (size_t index = 0; index < kLlmScenarioCount; ++index) { const LlmScenarioWorkPlan& frozen = - result.frozen_scenario_plans.scenarios[index]; + result.scenario_plans.at(result.frozen_plan_handles[index]).plan; if (!frozen.valid || frozen.scenario != kLlmScenarios[index]) { result.status = LlmRunStatus::Failed; result.reason_code = LlmRunnerReason::FROZEN_PLAN_MISMATCH; @@ -1346,42 +1381,8 @@ bool assign_frozen_plans(LlmMemoryResult& result, const LlmMemoryWorkPlan& model if (index >= kLlmScenarioCount) { return false; } - const LlmScenarioWorkPlan& plan = result.frozen_scenario_plans.scenarios[index]; - measurement.frozen_plan_index = index; - measurement.work_unit_kind = plan.work_unit_kind; - measurement.kv_write_kind = plan.kv_write_kind; - measurement.explicit_iterations = plan.explicit_iterations; - measurement.planned_work_units = plan.work_units; - measurement.weight_read_bytes_per_work_unit = plan.weight_read_bytes_per_work_unit; - measurement.kv_read_bytes_per_work_unit = plan.kv_read_bytes_per_work_unit; - measurement.kv_write_bytes_per_work_unit = plan.kv_write_bytes_per_work_unit; - measurement.effective_model_payload_bytes_per_work_unit = plan.effective_model_payload_bytes_per_work_unit; - measurement.layout_metadata_lookup_count_per_work_unit = - plan.layout_metadata_lookup_count_per_work_unit; - measurement.layout_metadata_read_bytes_per_work_unit = - plan.layout_metadata_read_bytes_per_work_unit; - measurement.accounted_bytes_per_work_unit = - plan.accounted_bytes_per_work_unit; - measurement.planned_weight_read_bytes = plan.weight_read_bytes; - measurement.planned_kv_read_bytes = plan.kv_read_bytes; - measurement.planned_kv_write_bytes = plan.kv_write_bytes; - measurement.planned_effective_model_payload_bytes = plan.effective_model_payload_bytes; - measurement.planned_layout_metadata_lookup_count = - plan.layout_metadata_lookup_count; - measurement.planned_layout_metadata_read_bytes = - plan.layout_metadata_read_bytes; - measurement.planned_task_accounted_bytes = plan.task_accounted_bytes; - measurement.calibration_attempt_count = result.calibration_attempt_counts[index]; - - if (measurement.scenario == LlmScenario::Mixed && plan.effective_model_payload_bytes_per_work_unit != 0) { - const long double total = static_cast(plan.effective_model_payload_bytes_per_work_unit); - measurement.weight_payload_fraction = - static_cast(static_cast(plan.weight_read_bytes_per_work_unit) / total); - measurement.kv_read_payload_fraction = - static_cast(static_cast(plan.kv_read_bytes_per_work_unit) / total); - measurement.kv_write_payload_fraction = - static_cast(static_cast(plan.kv_write_bytes_per_work_unit) / total); - } + const LlmScenarioWorkPlan& plan = result.scenario_plans.at(result.frozen_plan_handles[index]).plan; + measurement.plan_handle = result.frozen_plan_handles[index]; if (!checked_add_to(plan.work_units, total_work_units) || !checked_add_to(plan.effective_model_payload_bytes, @@ -1405,63 +1406,16 @@ bool assign_frozen_plans(LlmMemoryResult& result, const LlmMemoryWorkPlan& model return model_plan.valid; } -void retain_task_evidence(LlmTaskExecutionResult& retained, - LlmTaskExecutionResult& execution, - const LlmMemoryWorkPlan& model_plan) { - if (auto* metal = - std::get_if(&execution.backend_evidence)) { - retained.backend_evidence = std::move(*metal); - return; - } - auto* source = std::get_if(&execution.backend_evidence); - if (source == nullptr) { - retained.backend_evidence = std::monostate{}; - return; - } - auto* target = std::get_if(&retained.backend_evidence); - if (target == nullptr) { - retained.backend_evidence = LlmCpuTaskEvidence{}; - target = std::get_if(&retained.backend_evidence); - } - LlmExecutorResult& destination = target->executor; - LlmExecutorResult& input = source->executor; - destination.valid = input.valid; - destination.reason_code.assign( - canonicalize_llm_result_reason_code(input.reason_code)); - destination.elapsed_seconds = input.elapsed_seconds; - destination.requested_workers = input.requested_workers; - destination.created_workers = input.created_workers; - destination.completed_workers = input.completed_workers; - destination.qos_successful_workers = input.qos_successful_workers; - destination.qos_failed_workers = input.qos_failed_workers; - destination.worker_startup_failed = input.worker_startup_failed; - destination.kernel_succeeded = input.kernel_succeeded; - destination.timer_started = input.timer_started; - destination.timer_stopped = input.timer_stopped; - destination.checksum_evaluated = input.checksum_evaluated; - destination.checksum_valid = input.checksum_valid; - destination.post_validation_evaluated = - input.post_validation_evaluated; - destination.post_validation_valid = input.post_validation_valid; - destination.cold_checks = input.cold_checks; - destination.kv_write_validation_applicable = input.kv_write_validation_applicable; - destination.kv_write_validation_evaluated = input.kv_write_validation_evaluated; - destination.kv_write_validation_valid = input.kv_write_validation_valid; - destination.expected_run_checksum = input.expected_run_checksum; - destination.actual_run_checksum = input.actual_run_checksum; - const LlmCpuExecutionPlan* cpu_plan = - get_llm_cpu_execution_plan(model_plan); - const size_t effective_workers = - cpu_plan == nullptr ? 0 : cpu_plan->effective_workers; - if (input.expected_checksums.size() == effective_workers && - input.actual_checksums.size() == effective_workers) { - destination.expected_checksums.assign(input.expected_checksums.begin(), - input.expected_checksums.end()); - destination.actual_checksums.assign(input.actual_checksums.begin(), - input.actual_checksums.end()); +void retain_task_evidence(LlmRetainedExecution& retained, LlmTaskExecutionResult& execution, + const LlmMemoryWorkPlan&) { + if (auto* cpu = std::get_if(&execution.backend_evidence)) { + LlmCpuRetainedEvidence runtime; + runtime.executor = std::move(static_cast(cpu->executor)); + retained.backend_evidence = std::move(runtime); + } else if (auto* metal = std::get_if(&execution.backend_evidence)) { + retained.backend_evidence = std::move(static_cast(*metal)); } else { - destination.expected_checksums.clear(); - destination.actual_checksums.clear(); + retained.backend_evidence = std::monostate{}; } } @@ -1477,21 +1431,15 @@ void populate_measurement(LlmMeasurementState& measurement, ? std::string_view(LlmBackendReason::VALID) : execution_failure_reason(execution, model_plan, task_plan, context); - LlmTaskExecutionResult& retained = measurement.execution; + LlmRetainedExecution& retained = measurement.execution; retained.status = execution.status; retained.reason_code.assign(accepted ? LlmBackendReason::VALID : failure_reason); - retained.identity = {}; retained.timing = execution.timing; retained.completion = execution.completion; retained.validation = execution.validation; retain_task_evidence(retained, execution, model_plan); - const LlmExecutorResult* cpu = cpu_executor_evidence(retained); - if (cpu != nullptr) { - measurement.qos_successful_workers = cpu->qos_successful_workers; - measurement.qos_failed_workers = cpu->qos_failed_workers; - } measurement.duration_quality = classify_llm_duration_quality( measurement.execution.timing.elapsed_seconds, @@ -1504,7 +1452,6 @@ void populate_measurement(LlmMeasurementState& measurement, measurement.execution.status = execution.status; measurement.status = execution_failure_status(execution, measurement.reason_code); - clear_measurement_values(measurement); measurement.execution_evidence_available = true; return; } @@ -1522,25 +1469,12 @@ void populate_measurement(LlmMeasurementState& measurement, work_units_per_second_value <= 0.0 || !std::isfinite(bandwidth_value) || bandwidth_value <= 0.0) { measurement.status = LlmMeasurementStatus::Invalid; measurement.reason_code = LlmRunnerReason::INVALID_DERIVED_METRIC; - clear_measurement_values(measurement); measurement.execution_evidence_available = true; return; } measurement.status = LlmMeasurementStatus::Measured; measurement.reason_code = "measured"; - measurement.completed_work_units = task_plan.work_units; - measurement.completed_effective_model_payload_bytes = task_plan.effective_model_payload_bytes; - measurement.completed_layout_metadata_lookup_count = - task_plan.layout_metadata_lookup_count; - measurement.completed_layout_metadata_read_bytes = - task_plan.layout_metadata_read_bytes; - measurement.completed_task_accounted_bytes = task_plan.task_accounted_bytes; - measurement.elapsed_seconds = measurement.execution.timing.elapsed_seconds; - measurement.synthetic_work_unit_latency_seconds = latency_value; - measurement.synthetic_memory_work_units_per_second = work_units_per_second_value; - measurement.effective_model_payload_gb_s = bandwidth_value; - measurement.checksum_valid = true; measurement.execution_evidence_available = true; } @@ -1551,18 +1485,17 @@ void record_terminal_measurement(LlmMemoryResult& result, const LlmLoopRecord& l return; } ++result.counters.measured_measurements; - result.counters.completed_work_units += measurement.completed_work_units; - result.counters.completed_effective_model_payload_bytes += measurement.completed_effective_model_payload_bytes; + result.counters.completed_work_units += measurement.execution.completion.completed_work_units; + result.counters.completed_effective_model_payload_bytes += measurement.execution.completion.completed_effective_model_payload_bytes; result.counters.completed_layout_metadata_lookup_count += - measurement.completed_layout_metadata_lookup_count; + measurement.execution.completion.completed_layout_metadata_lookup_count; result.counters.completed_layout_metadata_read_bytes += - measurement.completed_layout_metadata_read_bytes; + measurement.execution.completion.completed_layout_metadata_read_bytes; result.counters.completed_task_accounted_bytes += - measurement.completed_task_accounted_bytes; + measurement.execution.completion.completed_task_accounted_bytes; if (loop.realized_order_count == kLlmScenarioCount) { ++result.counters.completed_loops; } - update_scenario_aggregate(result, measurement); } int fail_initialized_run(LlmMemoryResult& result, const LlmRunnerHooks& hooks, @@ -1582,7 +1515,7 @@ int run_measurements(const LlmMemoryWorkPlan& model_plan, LlmBackend& backend, } LlmMeasurementState& measurement = result.measurements[loop.measurement_indexes[position]]; const size_t index = scenario_index(measurement.scenario); - const LlmScenarioWorkPlan& task_plan = result.frozen_scenario_plans.scenarios[index]; + const LlmScenarioWorkPlan& task_plan = result.scenario_plans.at(result.frozen_plan_handles[index]).plan; if (loop.realized_order_count == 0) { ++result.counters.attempted_loops; } @@ -1751,14 +1684,11 @@ LlmRunnerAuxiliaryEstimate calculate_llm_runner_auxiliary_estimate( attempts_per_scenario, kLlmScenarioCount, total_calibration_attempts) || !NumericUtils::checked_multiply( - planned_measurements, static_cast(3), + planned_measurements, static_cast(1), aggregate_value_count) || !NumericUtils::checked_multiply( planned_measurements, preflight.effective_workers, retained_worker_checksums) || - !NumericUtils::checked_multiply( - retained_worker_checksums, static_cast(2), - retained_worker_checksums) || !NumericUtils::checked_multiply( planned_measurements, sizeof(LlmMeasurementState), estimate.measurement_record_bytes) || @@ -1769,10 +1699,10 @@ LlmRunnerAuxiliaryEstimate calculate_llm_runner_auxiliary_estimate( total_calibration_attempts, sizeof(LlmCalibrationAttempt), estimate.calibration_record_bytes) || !NumericUtils::checked_multiply( - aggregate_value_count, sizeof(double), + aggregate_value_count, sizeof(size_t), estimate.aggregate_value_bytes) || !NumericUtils::checked_multiply( - config.loop_count, static_cast(2 * sizeof(double)), + config.loop_count, static_cast(3 * sizeof(double)), estimate.statistics_workspace_bytes) || !NumericUtils::checked_multiply( kLlmRunnerMaximumWarnings, sizeof(std::string_view), @@ -1799,7 +1729,7 @@ LlmRunnerAuxiliaryEstimate calculate_llm_runner_auxiliary_estimate( identity_with_null, static_cast(2), conservative_identity) || !NumericUtils::checked_multiply( - conservative_identity, attempts_per_scenario, + conservative_identity, attempts_per_scenario + 1, scenario_identity_total) || !checked_add_to(scenario_identity_total, estimate.calibration_identity_bytes)) { @@ -1884,6 +1814,27 @@ LlmRunnerAuxiliaryEstimate calculate_llm_runner_auxiliary_estimate( } } + // Worst-case unique excluded plans plus frozen plans coexist with the active + // reconstructed expectation. Maps and expected worker vectors are capacity charged. + size_t canonical_count = 0; + size_t canonical_storage = 0; + size_t canonical_workers = 0; + size_t canonical_checksum_bytes = 0; + size_t canonical_model_strings = 0; + if (!NumericUtils::checked_add(total_calibration_attempts, kLlmScenarioCount, canonical_count) || + !NumericUtils::checked_multiply(canonical_count, + sizeof(LlmCanonicalScenarioPlan) + 2 * sizeof(size_t), canonical_storage) || + !checked_add_to(canonical_storage, estimate.fixed_metadata_bytes) || + !NumericUtils::checked_add(canonical_count, static_cast(2), canonical_workers) || + !NumericUtils::checked_multiply(canonical_workers, preflight.effective_workers, canonical_workers) || + !NumericUtils::checked_multiply(canonical_workers, sizeof(LlmWorkerChecksum), canonical_checksum_bytes) || + !checked_add_to(canonical_checksum_bytes, estimate.retained_checksum_bytes) || + !NumericUtils::checked_add(preflight.model_plan_identity_bytes, static_cast(1), canonical_model_strings) || + !NumericUtils::checked_multiply(canonical_model_strings, static_cast(2), canonical_model_strings) || + !NumericUtils::checked_multiply(canonical_model_strings, canonical_count, canonical_model_strings) || + !checked_add_to(canonical_model_strings, estimate.calibration_identity_bytes)) return estimate; + + if (!checked_add_to(preflight.canonical_expected_scratch_bytes, estimate.fixed_metadata_bytes)) return estimate; estimate.checksum_auxiliary_bytes = estimate.retained_checksum_bytes; size_t orchestration = 0; @@ -1944,7 +1895,7 @@ LlmRunnerAuxiliaryEstimate calculate_llm_runner_auxiliary_estimate(const LlmMemo size_t retained_worker_checksums = 0; if (!NumericUtils::checked_multiply(config.loop_count, kLlmScenarioCount, planned_measurements) || !NumericUtils::checked_multiply(attempts_per_scenario, kLlmScenarioCount, total_calibration_attempts) || - !NumericUtils::checked_multiply(planned_measurements, static_cast(3), aggregate_value_count)) { + !NumericUtils::checked_multiply(planned_measurements, static_cast(1), aggregate_value_count)) { return estimate; } const size_t retained_records_per_measurement = @@ -1952,14 +1903,13 @@ LlmRunnerAuxiliaryEstimate calculate_llm_runner_auxiliary_estimate(const LlmMemo if (!NumericUtils::checked_multiply(planned_measurements, retained_records_per_measurement, retained_worker_checksums) || - !NumericUtils::checked_multiply(retained_worker_checksums, static_cast(2), retained_worker_checksums) || !NumericUtils::checked_multiply(planned_measurements, sizeof(LlmMeasurementState), estimate.measurement_record_bytes) || !NumericUtils::checked_multiply(config.loop_count, sizeof(LlmLoopRecord), estimate.loop_record_bytes) || !NumericUtils::checked_multiply(total_calibration_attempts, sizeof(LlmCalibrationAttempt), estimate.calibration_record_bytes) || - !NumericUtils::checked_multiply(aggregate_value_count, sizeof(double), estimate.aggregate_value_bytes) || - !NumericUtils::checked_multiply(config.loop_count, static_cast(2 * sizeof(double)), + !NumericUtils::checked_multiply(aggregate_value_count, sizeof(size_t), estimate.aggregate_value_bytes) || + !NumericUtils::checked_multiply(config.loop_count, static_cast(3 * sizeof(double)), estimate.statistics_workspace_bytes) || !NumericUtils::checked_multiply(kLlmRunnerMaximumWarnings, sizeof(std::string_view), estimate.warning_record_bytes) || @@ -1985,7 +1935,7 @@ LlmRunnerAuxiliaryEstimate calculate_llm_runner_auxiliary_estimate(const LlmMemo size_t scenario_identity_total = 0; if (!NumericUtils::checked_add(identity_capacities[index], static_cast(1), identity_with_null) || !NumericUtils::checked_multiply(identity_with_null, static_cast(2), conservative_identity) || - !NumericUtils::checked_multiply(conservative_identity, attempts_per_scenario, scenario_identity_total) || + !NumericUtils::checked_multiply(conservative_identity, attempts_per_scenario + 1, scenario_identity_total) || !checked_add_to(scenario_identity_total, estimate.calibration_identity_bytes)) { return estimate; } @@ -2045,6 +1995,29 @@ LlmRunnerAuxiliaryEstimate calculate_llm_runner_auxiliary_estimate(const LlmMemo } } + // Worst-case unique excluded plans plus frozen plans coexist with the active + // reconstructed expectation. Maps and expected worker vectors are capacity charged. + size_t canonical_count = 0; + size_t canonical_storage = 0; + size_t canonical_workers = 0; + size_t canonical_checksum_bytes = 0; + size_t canonical_model_strings = 0; + if (!NumericUtils::checked_add(total_calibration_attempts, kLlmScenarioCount, canonical_count) || + !NumericUtils::checked_multiply(canonical_count, + sizeof(LlmCanonicalScenarioPlan) + 2 * sizeof(size_t), canonical_storage) || + !checked_add_to(canonical_storage, estimate.fixed_metadata_bytes) || + !NumericUtils::checked_add(canonical_count, static_cast(2), canonical_workers) || + !NumericUtils::checked_multiply(canonical_workers, retained_records_per_measurement, canonical_workers) || + !NumericUtils::checked_multiply(canonical_workers, sizeof(LlmWorkerChecksum), canonical_checksum_bytes) || + !checked_add_to(canonical_checksum_bytes, estimate.retained_checksum_bytes) || + !NumericUtils::checked_add(model_plan.plan_identity.size(), static_cast(1), canonical_model_strings) || + !NumericUtils::checked_multiply(canonical_model_strings, static_cast(2), canonical_model_strings) || + !NumericUtils::checked_multiply(canonical_model_strings, canonical_count, canonical_model_strings) || + !checked_add_to(canonical_model_strings, estimate.calibration_identity_bytes)) return estimate; + + if (metal_plan && metal_plan->resources.paged_layout && + (!checked_add_to(metal_plan->resources.paged_layout->memory.transient_peak_bytes, estimate.fixed_metadata_bytes) || + !checked_add_to(Constants::LLM_CANONICAL_PAGED_EXPECTED_FIXED_SCRATCH_BYTES, estimate.fixed_metadata_bytes))) return estimate; estimate.checksum_auxiliary_bytes = estimate.retained_checksum_bytes; size_t orchestration = 0; if (!checked_add_to(estimate.measurement_record_bytes, orchestration) || @@ -2273,12 +2246,12 @@ int run_llm_memory_suite(const LlmMemoryConfig& config, } } - result.frozen_scenario_plans = freeze_llm_scenario_work_plans( - model_plan, frozen_work_units, config.user_specified_iterations); - if (!result.frozen_scenario_plans.valid) { - return fail_initialized_run(result, hooks, - result.frozen_scenario_plans.reason_code, - backend, release_attempted); + for (size_t index = 0; index < kLlmScenarioCount; ++index) { + const auto frozen = build_llm_scenario_work_plan(model_plan, kLlmScenarios[index], + frozen_work_units[index], config.user_specified_iterations); + result.frozen_plan_handles[index] = register_llm_scenario_plan(result, model_plan, frozen, backend); + const auto& expected = result.scenario_plans.at(result.frozen_plan_handles[index]).expected; + if (!expected.available) return fail_initialized_run(result, hooks, expected.reason_code, backend, release_attempted); } for (size_t index = 0; !config.user_specified_iterations && index < kLlmScenarioCount; @@ -2290,8 +2263,8 @@ int run_llm_memory_suite(const LlmMemoryConfig& config, } const LlmCalibrationAttempt& latest = result.calibration_attempts[index] [result.calibration_attempt_counts[index] - 1]; - if (result.frozen_scenario_plans.scenarios[index].work_units != frozen_work_units[index] || - result.frozen_scenario_plans.scenarios[index].plan_identity != latest.work_plan_identity) { + if (result.scenario_plans.at(result.frozen_plan_handles[index]).plan.work_units != frozen_work_units[index] || + result.scenario_plans.at(result.frozen_plan_handles[index]).plan.plan_identity != result.scenario_plans.at(latest.plan_handle).plan.plan_identity) { return fail_initialized_run( result, hooks, LlmRunnerReason::FROZEN_PLAN_MISMATCH, backend, release_attempted); @@ -2319,7 +2292,6 @@ int run_llm_memory_suite(const LlmMemoryConfig& config, release_attempted); } trim_calibration_attempts(result); - refresh_calibration_references(result); if (!actual_runner_backing_is_covered(result)) { return fail_initialized_run( result, hooks, LlmRunnerReason::AUXILIARY_BUDGET_INSUFFICIENT, @@ -2353,7 +2325,7 @@ int run_llm_memory_suite(const LlmMemoryConfig& config, result.status = LlmRunStatus::Failed; result.reason_code = LlmRunnerReason::RUNNER_EXCEPTION; result.results_complete = false; - result.conclusions_valid = false; + result.run_accepted = false; release_backend_once(backend, release_attempted, result); return EXIT_FAILURE; } @@ -2378,7 +2350,7 @@ int run_llm_memory_suite(const LlmMemoryConfig& config, result.status = LlmRunStatus::Failed; result.reason_code = LlmRunnerReason::RUNNER_UNKNOWN_EXCEPTION; result.results_complete = false; - result.conclusions_valid = false; + result.run_accepted = false; release_backend_once(backend, release_attempted, result); return EXIT_FAILURE; } @@ -2406,3 +2378,79 @@ const char* llm_checkpoint_kind_to_string(LlmCheckpointKind kind) noexcept { } return "unknown"; } + +void prepare_llm_result_snapshot(LlmMemoryResult& result) { + ++result.snapshot_preparation_attempts; + const auto validate_reference = [&](LlmPlanHandle handle, LlmScenario scenario) { + if (handle == kLlmNoTaskIndex) return; + const auto* canonical = find_llm_scenario_plan(result, handle); + if (!canonical || !canonical->plan.valid || canonical->plan.scenario != scenario) + throw std::invalid_argument("contradictory canonical scenario reference"); + }; + for (size_t index = 0; index < kLlmScenarioCount; ++index) { + validate_reference(result.frozen_plan_handles[index], kLlmScenarios[index]); + if (result.calibration_attempt_counts[index] > result.calibration_attempts[index].size()) + throw std::invalid_argument("invalid calibration attempt count"); + for (size_t attempt_index = 0; attempt_index < result.calibration_attempt_counts[index]; ++attempt_index) { + const auto& attempt = result.calibration_attempts[index][attempt_index]; + if (attempt.scenario != kLlmScenarios[index]) + throw std::invalid_argument("contradictory calibration scenario"); + validate_reference(attempt.plan_handle, attempt.scenario); + } + } + for (const auto& measurement : result.measurements) + validate_reference(measurement.plan_handle, measurement.scenario); + + result.snapshot_plan_order.resize(result.scenario_plans.size()); + result.snapshot_plan_refs.resize(result.scenario_plans.size()); + for (size_t i = 0; i < result.scenario_plans.size(); ++i) result.snapshot_plan_order[i] = i; + std::sort(result.snapshot_plan_order.begin(), result.snapshot_plan_order.end(), [&](size_t a, size_t b) { + const auto& x = result.scenario_plans[a].plan; + const auto& y = result.scenario_plans[b].plan; + return std::tie(x.scenario, x.work_units, x.explicit_iterations) < std::tie(y.scenario, y.work_units, y.explicit_iterations); + }); + for (size_t index = 0; index < result.snapshot_plan_order.size(); ++index) + result.snapshot_plan_refs[result.snapshot_plan_order[index]] = index; + for (auto& aggregate : result.aggregates) { + aggregate.accepted_measurement_ids.clear(); + for (size_t index = 0; index < result.measurements.size(); ++index) { + const auto& measurement = result.measurements[index]; + if (measurement.scenario != aggregate.scenario || measurement.status != LlmMeasurementStatus::Measured) continue; + const auto* plan = find_llm_scenario_plan(result, measurement.plan_handle); + if (!plan || !plan->expected.available || plan->plan.scenario != measurement.scenario || + !measurement.execution_evidence_available || measurement.execution.status != LlmTaskExecutionStatus::Complete || + !measurement.execution.validation.evaluated || !measurement.execution.validation.valid || + !measurement.execution.timing.evaluated || !measurement.execution.timing.valid || + !task_completion_matches(measurement.execution.completion, plan->plan) || + !derive_llm_measurement_metrics(measurement).payload_gb_s) + throw std::invalid_argument("invalid accepted measurement plan or work"); + aggregate.accepted_measurement_ids.push_back(index); + } + LlmMetricAggregate* metrics[] = {&aggregate.work_unit_latency_seconds, + &aggregate.synthetic_memory_work_units_per_second, &aggregate.effective_model_payload_gb_s}; + for (size_t metric = 0; metric < 3; ++metric) { + auto& target = *metrics[metric]; + target.statistics = {}; + target.headline.reset(); + auto& values = result.statistics_workspace.extracted_values; + values.clear(); + for (size_t index : aggregate.accepted_measurement_ids) { + const auto derived = derive_llm_measurement_metrics(result.measurements[index]); + values.push_back(metric == 0 ? *derived.latency_seconds : metric == 1 ? *derived.work_units_per_second : *derived.payload_gb_s); + } + if (!values.empty()) { + ++result.exact_statistics_passes; + target.statistics = calculate_descriptive_statistics(values, result.statistics_workspace.sorted_values, + result.statistics_workspace.absolute_deviations); + target.headline = target.statistics.median; + } + } + const size_t count = aggregate.accepted_measurement_ids.size(); + aggregate.status = count == 0 ? "unavailable" : count == result.counters.planned_loops ? "complete" : "partial"; + const auto& statistics = aggregate.effective_model_payload_gb_s.statistics; + aggregate.observed_cv_classification = count < 3 ? "insufficient-samples" : + !statistics.coefficient_of_variation_defined ? "undefined" : + statistics.coefficient_of_variation_pct > Constants::LLM_STREAMING_CV_WARNING_PCT ? "above-threshold" : "below-threshold"; + } + rebuild_quality_warnings(result); +} diff --git a/src/llm_memory/llm_runner.h b/src/llm_memory/llm_runner.h index d8953c4..cfe68b4 100644 --- a/src/llm_memory/llm_runner.h +++ b/src/llm_memory/llm_runner.h @@ -62,6 +62,47 @@ enum class LlmCheckpointKind : uint8_t { CommandTerminal, }; +/** Stable insertion handle; public snapshot indices are a separate sorted map. */ +using LlmPlanHandle = size_t; + +/** Once-per-plan expected witness; no runtime-valid assertion is implied. */ +struct LlmCanonicalExpectedChecksum { + bool available = false; + std::string reason_code = "not-evaluated"; + std::vector cpu_workers; + LlmRunChecksum cpu_run{0, 0}; + LlmMetalDualMod32Checksum metal_run; +}; + +/** Unique scenario/T/explicit content with its sole retained expected witness. */ +struct LlmCanonicalScenarioPlan { + LlmScenarioWorkPlan plan; + LlmCanonicalExpectedChecksum expected; +}; + +struct LlmCpuRetainedEvidence { + LlmCpuRuntimeEvidence executor; +}; + +/** Runtime-only retained result: no copied task identity or expected checksum. */ +struct LlmRetainedExecution { + LlmTaskExecutionStatus status = LlmTaskExecutionStatus::NotStarted; + std::string reason_code = LlmBackendReason::NOT_INITIALIZED; + LlmAuthoritativeTiming timing; + LlmCompletedWork completion; + LlmTaskValidation validation; + std::variant backend_evidence; +}; + +/** Read-only runtime variant access; returned pointers borrow the result. */ +inline const LlmCpuRuntimeEvidence* get_llm_cpu_task_evidence(const LlmRetainedExecution& execution) noexcept { + const auto* cpu = std::get_if(&execution.backend_evidence); + return cpu ? &cpu->executor : nullptr; +} +inline const LlmMetalRuntimeEvidence* get_llm_metal_task_evidence(const LlmRetainedExecution& execution) noexcept { + return std::get_if(&execution.backend_evidence); +} + /** Compact excluded-task evidence without retained per-worker vectors. */ struct LlmTaskExecutionEvidence { LlmColdChecks cpu_cold_checks; ///< Compact CPU copy; Metal owns its own array. @@ -72,7 +113,7 @@ struct LlmTaskExecutionEvidence { double elapsed_seconds = 0.0; bool timing_evaluated = false; bool timing_valid = false; - LlmTaskCompletion completion; + LlmCompletedWork completion; bool validation_evaluated = false; bool validation_valid = false; bool cpu_evidence_available = false; @@ -87,28 +128,16 @@ struct LlmTaskExecutionEvidence { bool timer_stopped = false; bool checksum_evaluated = false; ///< CPU checksum detail, when applicable. bool checksum_valid = false; - LlmRunChecksum expected_run_checksum{0, 0}; LlmRunChecksum actual_run_checksum{0, 0}; bool metal_evidence_available = false; - std::optional metal; + std::optional metal; }; /** One excluded warmup, pilot, duration trial, or correction record. */ struct LlmCalibrationAttempt { + LlmPlanHandle plan_handle = kLlmNoTaskIndex; LlmScenario scenario = LlmScenario::WeightsOnly; - LlmWorkUnitKind work_unit_kind = LlmWorkUnitKind::DecodeStep; - LlmKvWriteKind kv_write_kind = LlmKvWriteKind::None; std::string_view purpose = "not-run"; - bool explicit_iterations = false; - size_t work_units = 0; - size_t weight_read_bytes = 0; - size_t kv_read_bytes = 0; - size_t kv_write_bytes = 0; - size_t effective_model_payload_bytes = 0; - size_t layout_metadata_lookup_count = 0; - size_t layout_metadata_read_bytes = 0; - size_t task_accounted_bytes = 0; - std::string work_plan_identity; LlmTaskExecutionEvidence execution; std::string_view duration_quality = "not-run"; bool terminal = false; @@ -118,62 +147,32 @@ struct LlmCalibrationAttempt { /** Raw values, shared descriptive statistics, and one headline metric. */ struct LlmMetricAggregate { - std::vector values; DescriptiveStatistics statistics; std::optional headline; }; /** Status-bearing record for one planned scenario measurement. */ struct LlmMeasurementState { + LlmPlanHandle plan_handle = kLlmNoTaskIndex; LlmScenario scenario = LlmScenario::WeightsOnly; - LlmWorkUnitKind work_unit_kind = LlmWorkUnitKind::DecodeStep; - LlmKvWriteKind kv_write_kind = LlmKvWriteKind::None; LlmMeasurementStatus status = LlmMeasurementStatus::NotRun; std::string_view reason_code = "not-run"; size_t loop_index = 0; size_t order_position = 0; bool attempted = false; - bool execution_evidence_available = false; ///< Complete generic task evidence was retained. - size_t requested_workers = 0; - size_t effective_workers = 0; - size_t qos_successful_workers = 0; - size_t qos_failed_workers = 0; - size_t frozen_plan_index = kLlmNoTaskIndex; - bool explicit_iterations = false; + bool execution_evidence_available = false; std::string_view duration_quality = "not-run"; - size_t calibration_attempt_count = 0; - size_t planned_work_units = 0; - size_t completed_work_units = 0; - size_t weight_read_bytes_per_work_unit = 0; - size_t kv_read_bytes_per_work_unit = 0; - size_t kv_write_bytes_per_work_unit = 0; - size_t effective_model_payload_bytes_per_work_unit = 0; - size_t layout_metadata_lookup_count_per_work_unit = 0; - size_t layout_metadata_read_bytes_per_work_unit = 0; - size_t accounted_bytes_per_work_unit = 0; - size_t planned_weight_read_bytes = 0; - size_t planned_kv_read_bytes = 0; - size_t planned_kv_write_bytes = 0; - size_t planned_effective_model_payload_bytes = 0; - size_t completed_effective_model_payload_bytes = 0; - size_t planned_layout_metadata_lookup_count = 0; - size_t completed_layout_metadata_lookup_count = 0; - size_t planned_layout_metadata_read_bytes = 0; - size_t completed_layout_metadata_read_bytes = 0; - size_t planned_task_accounted_bytes = 0; - size_t completed_task_accounted_bytes = 0; - std::optional elapsed_seconds; - std::optional synthetic_work_unit_latency_seconds; - std::optional synthetic_memory_work_units_per_second; - std::optional effective_model_payload_gb_s; - std::optional weight_payload_fraction; - std::optional kv_read_payload_fraction; - std::optional kv_write_payload_fraction; - size_t working_set_bytes = 0; - bool checksum_valid = false; - LlmTaskExecutionResult execution; + LlmRetainedExecution execution; }; +/** On-demand rates all derived from the same accepted duration/work/payload. */ +struct LlmDerivedMetrics { + std::optional latency_seconds; + std::optional work_units_per_second; + std::optional payload_gb_s; +}; +LlmDerivedMetrics derive_llm_measurement_metrics(const LlmMeasurementState& measurement) noexcept; + /** Planned and realized scenario order for one count-loop. */ struct LlmLoopRecord { size_t loop_index = 0; @@ -190,11 +189,13 @@ struct LlmScenarioAggregate { LlmMetricAggregate synthetic_memory_work_units_per_second; LlmMetricAggregate effective_model_payload_gb_s; std::string_view status = "unavailable"; - std::string_view stability_quality = "insufficient-samples"; + std::string_view observed_cv_classification = "insufficient-samples"; + std::vector accepted_measurement_ids; }; /** Reused sorted and deviation storage for allocation-free statistics updates. */ struct LlmStatisticsWorkspace { + std::vector extracted_values; std::vector sorted_values; std::vector absolute_deviations; }; @@ -257,16 +258,27 @@ struct LlmMemoryResult { std::string diagnostic; bool interruption_requested = false; bool results_complete = false; - bool conclusions_valid = false; + bool run_accepted = false; bool scenario_order_balance_complete = false; bool checkpoint_failed = false; bool terminal_checkpoint_attempted = false; bool terminal_checkpoint_completed = false; + size_t prior_file_writer_attempts = 0; + size_t prior_successful_file_writes = 0; + std::string_view snapshot_request = "none"; size_t logical_checkpoint_attempts = 0; size_t successful_logical_checkpoints = 0; LlmRunCounters counters; LlmRunnerAuxiliaryEstimate runner_auxiliary; - LlmFrozenScenarioPlans frozen_scenario_plans; + // Append-only canonical storage: entries become immutable after registration. + std::vector scenario_plans; + std::array frozen_plan_handles{ + kLlmNoTaskIndex, kLlmNoTaskIndex, kLlmNoTaskIndex}; + std::vector snapshot_plan_order; + std::vector snapshot_plan_refs; + size_t snapshot_preparation_attempts = 0; + size_t exact_statistics_passes = 0; + size_t snapshot_interval_loops = 1; std::array, kLlmScenarioCount> calibration_attempts; std::array calibration_attempt_counts{}; std::vector loops; @@ -276,9 +288,27 @@ struct LlmMemoryResult { std::vector quality_warnings; }; +/** Insert/reuse exact scenario content. Contradictory keys throw before publication. + * Handles survive vector growth; no pointer may be held across insertion. + * Registered entries must not be mutated or erased. The result has one writer; + * readers may inspect it only while that writer is idle. */ +LlmPlanHandle register_llm_scenario_plan(LlmMemoryResult& result, + const LlmMemoryWorkPlan& model, const LlmScenarioWorkPlan& plan, LlmBackend& backend); +const LlmCanonicalScenarioPlan* find_llm_scenario_plan(const LlmMemoryResult& result, + LlmPlanHandle handle) noexcept; +/** Applicability from admitted phase/layout/scenario, before execution exists. */ +LlmColdChecks required_llm_cold_checks(const LlmMemoryWorkPlan& model, LlmScenario scenario) noexcept; +/** Build one consistent sorted reference map and exact accepted-ID statistics. + * Requires immutable registered content and exclusive access to the result. + * Rejects unknown or scenario-contradictory references with std::invalid_argument; + * allocation failures propagate to the owning checkpoint boundary. */ +void prepare_llm_result_snapshot(LlmMemoryResult& result); + /** Deterministic stop and logical-checkpoint seams. */ struct LlmRunnerHooks { + bool progress_snapshots = true; ///< False for stdout/disabled transports; terminal statistics still finalize. std::function stop_requested; + std::function()> observe_file_writes; std::function checkpoint; }; diff --git a/src/llm_memory/llm_work_plan.cpp b/src/llm_memory/llm_work_plan.cpp index f92ce4d..bda7605 100644 --- a/src/llm_memory/llm_work_plan.cpp +++ b/src/llm_memory/llm_work_plan.cpp @@ -226,6 +226,12 @@ void append_component_identity( identity += "=null"; } +/** Preserve numeric encoding; null theoretical context has an explicit reason token. */ +void append_identity_field(std::string& identity, const char* name, const std::optional& value) { + if (value) append_identity_field(identity, name, *value); + else append_identity_field(identity, name, std::string("unavailable:arithmetic-overflow")); +} + /** Match a retained pipe-delimited identity without constructing strings. */ class IdentityMatcher { public: @@ -246,6 +252,10 @@ class IdentityMatcher { return literal("|") && literal(name) && literal("=") && literal(value); } + bool integer_field(std::string_view name, const std::optional& value) noexcept { + return value ? integer_field(name, *value) : field(name, "unavailable:arithmetic-overflow"); + } + template bool integer(Integer value) noexcept { std::array::digits10 + 3> encoded{}; @@ -1220,10 +1230,12 @@ bool validate_prefill_plan_noalloc( size_t tile_count = full_tiles; size_t full_tile_triangular = 0; size_t prefix_visits = 0; - size_t causal_pairs = 0; + const auto context = calculate_llm_prefill_model_context(prefill.prompt_tokens, + prefill.layer_count, prefill.batch_size, prefill.query_head_count, prefill.head_dimension); + const auto causal_pairs = context.causal_token_pairs_per_sequence; + const auto logical_attention_pairs = context.logical_attention_pairs; + const auto logical_attention_fma_terms = context.logical_attention_fma_terms; size_t layer_batch_count = 0; - size_t logical_attention_pairs = 0; - size_t logical_attention_fma_terms = 0; size_t kv_record_bytes = 0; size_t kv_bytes_per_token = 0; size_t logical_records = 0; @@ -1249,20 +1261,7 @@ bool validate_prefill_plan_noalloc( (final_tile != 0 && !NumericUtils::checked_add(prefix_visits, prefill.prompt_tokens, prefix_visits)) || - !checked_llm_prefill_triangular(prefill.prompt_tokens, - causal_pairs) || - !NumericUtils::checked_multiply(prefill.layer_count, - prefill.batch_size, - layer_batch_count) || - !NumericUtils::checked_multiply(layer_batch_count, - prefill.query_head_count, - logical_attention_pairs) || - !NumericUtils::checked_multiply(logical_attention_pairs, - causal_pairs, - logical_attention_pairs) || - !NumericUtils::checked_multiply(logical_attention_pairs, - prefill.head_dimension, - logical_attention_fma_terms) || + !NumericUtils::checked_multiply(prefill.layer_count, prefill.batch_size, layer_batch_count) || !NumericUtils::checked_multiply( prefill.k_or_v_record_bytes_per_layer, 2, kv_record_bytes) || !NumericUtils::checked_multiply(prefill.layer_count, @@ -2319,6 +2318,13 @@ bool build_auxiliary_preflight_view( plan.valid = true; view.backend = plan.backend; view.kv_layout = plan.kv_layout; + if (metal_plan && metal_plan->resources.paged_layout) { + if (!NumericUtils::checked_add(metal_plan->resources.paged_layout->memory.transient_peak_bytes, + Constants::LLM_CANONICAL_PAGED_EXPECTED_FIXED_SCRATCH_BYTES, view.canonical_expected_scratch_bytes)) { + plan.valid = original_valid; + return false; + } + } if (cpu_plan != nullptr) { view.effective_workers = cpu_plan->effective_workers; view.total_layer_descriptors = cpu_plan->total_layer_descriptors; @@ -2507,6 +2513,7 @@ bool auxiliary_preflight_views_match( const bool fixed_match = lhs.valid && rhs.valid && lhs.backend == rhs.backend && lhs.kv_layout == rhs.kv_layout && + lhs.canonical_expected_scratch_bytes == rhs.canonical_expected_scratch_bytes && lhs.effective_workers == rhs.effective_workers && lhs.total_layer_descriptors == rhs.total_layer_descriptors && lhs.total_sequence_descriptors == rhs.total_sequence_descriptors && @@ -2944,7 +2951,7 @@ bool validate_llm_prefill_cpu_execution_evidence( std::string build_llm_methodology_version(LlmMemoryBackend backend, LlmPhase phase, LlmKvLayout layout) { - std::string methodology = "llm-memory-v1-"; + std::string methodology = "llm-memory-v2-"; methodology += llm_memory_backend_to_string(backend); methodology += '-'; methodology += llm_phase_to_string(phase); diff --git a/src/llm_memory/llm_work_plan.h b/src/llm_memory/llm_work_plan.h index 0b02aa0..538e0e6 100644 --- a/src/llm_memory/llm_work_plan.h +++ b/src/llm_memory/llm_work_plan.h @@ -496,9 +496,9 @@ struct LlmPrefillGeometry { size_t attention_query_tile_tokens = 0; size_t tile_count = 0; size_t attention_prefix_token_visits_per_sequence = 0; - size_t causal_token_pairs_per_sequence = 0; - size_t logical_attention_pairs = 0; - size_t logical_attention_fma_terms = 0; + std::optional causal_token_pairs_per_sequence; + std::optional logical_attention_pairs; + std::optional logical_attention_fma_terms; size_t paged_prefix_block_visits_per_sequence = 0; }; @@ -955,6 +955,7 @@ struct LlmAuxiliaryPreflightView { size_t metal_persistent_resource_count = 0; size_t metal_resolved_execution_plan_backing_bytes = 0; size_t metal_resolved_plan_identity_backing_bytes = 0; + size_t canonical_expected_scratch_bytes = 0; ///< Paged Metal temporary table plus validation/hash scratch. size_t model_plan_identity_bytes = 0; std::array maximum_scenario_plan_identity_bytes{}; @@ -1226,7 +1227,7 @@ bool readmit_llm_memory_work_plan( LlmMemoryWorkPlan& plan, size_t checksum_auxiliary_bytes, size_t orchestration_auxiliary_bytes) noexcept; -/** Build `llm-memory-v1---`. */ +/** Build `llm-memory-v2---`. */ std::string build_llm_methodology_version(LlmMemoryBackend backend, LlmPhase phase, LlmKvLayout layout); diff --git a/src/output/console/messages/llm_memory_messages.cpp b/src/output/console/messages/llm_memory_messages.cpp index 2ad8665..210db5a 100644 --- a/src/output/console/messages/llm_memory_messages.cpp +++ b/src/output/console/messages/llm_memory_messages.cpp @@ -10,6 +10,7 @@ * @brief Centralized synthetic LLM memory-profile CLI text */ +#include #include "output/console/messages/messages_api.h" #include @@ -156,9 +157,10 @@ std::string llm_memory_usage_options(const std::string& prog_name) { << " -r, --count Cyclic weights/KV/mixed loops (default: " << Constants::LLM_DEFAULT_LOOP_COUNT << ").\n" << " --seed Reproducible base seed; generated once when omitted.\n" - << " -o, --output JSON schema 1 target; exact - writes one final document to\n" + << " -o, --output JSON schema 2 target; exact - writes one final document to\n" << " stdout and routes human output to stderr. Every other non-empty\n" - << " target is a file with atomic scenario and terminal checkpoints.\n" + << " target is a file with bounded loop and terminal atomic snapshots.\n" + << " K=max(1,ceil(count/8)); abrupt loss bound: 3K completed attempts.\n" << " An empty value disables JSON for this direct command.\n" << " -h, --help Show this LLM-mode help and exit.\n" << "This profile models CPU or Metal memory traffic only: it performs no Transformer math and\n" @@ -302,16 +304,16 @@ std::string report_llm_memory_decode_geometry( std::string report_llm_memory_prefill_geometry(size_t prompt_tokens, size_t attention_query_tile_tokens, size_t tile_count, size_t attention_prefix_token_visits_per_sequence, - size_t causal_token_pairs_per_sequence, size_t logical_attention_pairs, - size_t logical_attention_fma_terms) { + std::optional causal_token_pairs_per_sequence, std::optional logical_attention_pairs, + std::optional logical_attention_fma_terms) { std::ostringstream report; report << " Prompt tokens (P): " << prompt_tokens << "\n" << " Attention query tile tokens (Q): " << attention_query_tile_tokens << "\n" << " Attention query tiles (C): " << tile_count << "\n" << " Prefix token visits / sequence: " << attention_prefix_token_visits_per_sequence << "\n" - << " Causal token pairs / sequence: " << causal_token_pairs_per_sequence << "\n" - << " Logical attention pairs: " << logical_attention_pairs << "\n" - << " Logical attention FMA terms: " << logical_attention_fma_terms; + << " Causal token pairs / sequence: " << (causal_token_pairs_per_sequence ? std::to_string(*causal_token_pairs_per_sequence) : "unavailable (arithmetic-overflow)") << "\n" + << " Logical attention pairs: " << (logical_attention_pairs ? std::to_string(*logical_attention_pairs) : "unavailable (arithmetic-overflow)") << "\n" + << " Logical attention FMA terms: " << (logical_attention_fma_terms ? std::to_string(*logical_attention_fma_terms) : "unavailable (arithmetic-overflow)"); return report.str(); } @@ -371,6 +373,15 @@ std::string report_llm_memory_scenario_headline( return report.str(); } +std::string report_llm_memory_distribution(size_t count, double median, double minimum, double maximum, + double cv_pct, double mad) { + std::ostringstream report; + report << " n=" << count << std::fixed << std::setprecision(2) + << ", median=" << median << " GB/s, min=" << minimum << ", max=" << maximum + << ", CV=" << cv_pct << "%, MAD=" << mad << " GB/s"; + return report.str(); +} + std::string report_llm_memory_scenario_name( const std::string& scenario_token) { if (scenario_token == "weights_only") { diff --git a/src/output/console/messages/messages_api.h b/src/output/console/messages/messages_api.h index 2cd1a97..825a6f9 100644 --- a/src/output/console/messages/messages_api.h +++ b/src/output/console/messages/messages_api.h @@ -37,6 +37,7 @@ #ifndef MESSAGES_MESSAGES_API_H #define MESSAGES_MESSAGES_API_H +#include #include #include #include @@ -278,8 +279,8 @@ std::string report_llm_memory_decode_geometry( size_t visible_context_tokens, double traffic_crossover_context_tokens); std::string report_llm_memory_prefill_geometry(size_t prompt_tokens, size_t attention_query_tile_tokens, size_t tile_count, size_t attention_prefix_token_visits_per_sequence, - size_t causal_token_pairs_per_sequence, size_t logical_attention_pairs, - size_t logical_attention_fma_terms); + std::optional causal_token_pairs_per_sequence, std::optional logical_attention_pairs, + std::optional logical_attention_fma_terms); struct LlmPagedLayoutReportValues { size_t block_tokens = 0; size_t blocks_per_sequence = 0; @@ -314,6 +315,9 @@ std::string report_llm_memory_scenario_headline( double synthetic_memory_work_units_per_second, double effective_model_payload_gb_s, bool include_work_units_per_second); +/** Shared accepted-population payload distribution, with descriptive CV/MAD. */ +std::string report_llm_memory_distribution(size_t count, double median, double minimum, double maximum, + double cv_pct, double mad); std::string report_llm_memory_scenario_name( const std::string& scenario_token); std::string report_llm_memory_interpretation_note( diff --git a/src/output/console/messages/program_messages.cpp b/src/output/console/messages/program_messages.cpp index babca4c..a361331 100644 --- a/src/output/console/messages/program_messages.cpp +++ b/src/output/console/messages/program_messages.cpp @@ -173,10 +173,10 @@ std::string usage_options(const std::string& prog_name) { << Constants::LLM_CPU_PREFILL_CONTIGUOUS_METHODOLOGY_VERSION << ",\n" << " " << Constants::LLM_CPU_PREFILL_PAGED_METHODOLOGY_VERSION << ",\n" - << " llm-memory-v1-metal-decode-contiguous,\n" - << " llm-memory-v1-metal-decode-paged,\n" - << " llm-memory-v1-metal-prefill-contiguous, and\n" - << " llm-memory-v1-metal-prefill-paged.\n" + << " llm-memory-v2-metal-decode-contiguous,\n" + << " llm-memory-v2-metal-decode-paged,\n" + << " llm-memory-v2-metal-prefill-contiguous, and\n" + << " llm-memory-v2-metal-prefill-paged.\n" << " -i, --iterations \n" << " Exact measured R/W/Copy pass count when explicitly supplied.\n" << " When omitted, --benchmark, --patterns, --gpu-bandwidth, and\n" @@ -276,8 +276,9 @@ std::string usage_options(const std::string& prog_name) { << " an empty value disables JSON for direct commands and is invalid for sweeps.\n" << " Every other non-empty value is a file, including ./- and names such as -G.\n" << " Standard, GPU, and LLM-memory files retain mode-specific atomic\n" - << " checkpoints; LLM-memory checkpoints each terminal scenario and\n" - << " command state, while sweep files checkpoint attempts.\n" + << " checkpoints; LLM-memory snapshots every K=max(1,ceil(count/8)) loops\n" + << " and at command terminal (abrupt loss bound: 3K completed attempts).\n" + << " Sweep files checkpoint attempts.\n" << " -S, --sweep Run a Cartesian sweep over one parameter. Repeat for multiple\n" << " parameters. Supported keys: buffer-size, cache-size, threads,\n" << " latency-tlb-locality-kb, latency-stride-bytes,\n" diff --git a/src/output/json/json_output/json_output_session.cpp b/src/output/json/json_output/json_output_session.cpp index 844134f..99f3dfc 100644 --- a/src/output/json/json_output/json_output_session.cpp +++ b/src/output/json/json_output/json_output_session.cpp @@ -118,7 +118,7 @@ int JsonOutputSession::checkpoint( try { const nlohmann::ordered_json payload = build_payload(); - return write_json_to_file(target_.file_path, payload, announce_success); + return write_file(payload, announce_success); } catch (const std::exception& error) { report_payload_build_failure(target_.file_path, error.what()); } catch (...) { @@ -133,7 +133,7 @@ int JsonOutputSession::write_final(const nlohmann::ordered_json& payload, case JsonOutputKind::Disabled: return EXIT_SUCCESS; case JsonOutputKind::File: - return write_json_to_file(target_.file_path, payload, announce_success); + return write_file(payload, announce_success); case JsonOutputKind::Stdout: // A file-style save announcement is intentionally suppressed. Human // output already routes to stderr, while stdout remains JSON-only. @@ -142,6 +142,13 @@ int JsonOutputSession::write_final(const nlohmann::ordered_json& payload, return EXIT_FAILURE; } +int JsonOutputSession::write_file(const nlohmann::ordered_json& payload, bool announce_success) { + ++file_writer_attempts_; + const int status = write_json_to_file(target_.file_path, payload, announce_success); + if (status == EXIT_SUCCESS) ++successful_file_writes_; + return status; +} + int JsonOutputSession::write_stdout( const nlohmann::ordered_json& payload) noexcept { if (!stdout_routing_installed_ || original_stdout_buffer_ == nullptr) { diff --git a/src/output/json/json_output/json_output_session.h b/src/output/json/json_output/json_output_session.h index 796169d..8e4b7c6 100644 --- a/src/output/json/json_output/json_output_session.h +++ b/src/output/json/json_output/json_output_session.h @@ -112,6 +112,12 @@ class JsonOutputSession { /** @return `true` only when logical checkpoints must be written to a file. */ bool persists_checkpoints() const noexcept; + /** Completed host observations, read before preparing the next document. + * File writer entry is counted after payload construction. A successful + * no-op or stdout write does not increment either file counter. Single owner. */ + size_t file_writer_attempts() const noexcept { return file_writer_attempts_; } + size_t successful_file_writes() const noexcept { return successful_file_writes_; } + /** * Offer one logical checkpoint to the selected transport. * @@ -150,6 +156,10 @@ class JsonOutputSession { bool announce_success = true); private: + int write_file(const nlohmann::ordered_json& payload, bool announce_success); + size_t file_writer_attempts_ = 0; + size_t successful_file_writes_ = 0; + int write_stdout(const nlohmann::ordered_json& payload) noexcept; JsonOutputTarget target_; diff --git a/tests/test_executable_cli.cpp b/tests/test_executable_cli.cpp index ffe98f7..25b3399 100644 --- a/tests/test_executable_cli.cpp +++ b/tests/test_executable_cli.cpp @@ -336,25 +336,69 @@ nlohmann::json parse_single_stdout_json(const CliResult& result) { return nlohmann::json::parse(text); } +const nlohmann::json& canonical_llm_plan(const nlohmann::json& document, const nlohmann::json& measurement) { + return document.at("resolved_plan").at("scenario_plans").at(measurement.at("plan_ref").get()); +} + +void expect_complete_llm_named_checks(const nlohmann::json& document, const nlohmann::json& measurement) { + const bool cpu = document.at("backend") == "cpu"; + const bool prefill = document.at("phase") == "prefill"; + const bool paged = document.at("kv_layout") == "paged"; + const bool writes = measurement.at("scenario") != "weights_only"; + const bool padding = paged && + document.at("resolved_plan").at("resources").at("k_layout_padding_bytes") != "0"; + const std::array kinds = { + "post-validation-structure", + prefill ? "kv-prefill-final-samples" : cpu && !writes ? "kv-append-unchanged" : "kv-append-final", + "kv-padding-canary"}; + const std::array applicability = { + cpu ? paged || prefill || writes : writes, + writes || (cpu && paged && !prefill), padding && (cpu || writes)}; + const auto& checks = measurement.at("execution").at("validation").at("checks"); + ASSERT_TRUE(checks.is_array()); + ASSERT_EQ(checks.size(), kinds.size()); + for (size_t index = 0; index < kinds.size(); ++index) { + SCOPED_TRACE(kinds[index]); + const auto& check = checks.at(index); + EXPECT_EQ(check.at("kind"), kinds[index]); + EXPECT_EQ(check.at("applicable"), applicability[index]); + if (applicability[index]) { + EXPECT_EQ(check.at("evaluated"), true); + EXPECT_EQ(check.at("valid"), true); + EXPECT_EQ(check.at("reason_code"), "valid"); + } else { + EXPECT_TRUE(check.at("evaluated").is_null()); + EXPECT_TRUE(check.at("valid").is_null()); + EXPECT_EQ(check.at("reason_code"), "not-applicable"); + } + } +} + void expect_complete_llm_checkpoint_lifecycle(const nlohmann::json& json) { ASSERT_TRUE(json.contains("checkpoint_lifecycle")); - const nlohmann::json& lifecycle = json["checkpoint_lifecycle"]; + const auto& lifecycle = json["checkpoint_lifecycle"]; EXPECT_FALSE(lifecycle["checkpoint_failed"].get()); - EXPECT_EQ(lifecycle["logical_checkpoint_attempts"], 10u); - EXPECT_EQ(lifecycle["successful_logical_checkpoints"], 10u); - EXPECT_TRUE(lifecycle["terminal_checkpoint_attempted"].get()); - EXPECT_TRUE(lifecycle["terminal_checkpoint_completed"].get()); + const size_t count = json["configuration"]["loop_count"].get(); + const size_t interval = std::max(size_t{1}, count / 8 + (count % 8 != 0)); + const bool file = json["configuration"]["output_file"] != "-"; + EXPECT_EQ(lifecycle["prior_file_writer_attempts"], file ? count / interval : 0); + EXPECT_EQ(lifecycle["prior_successful_file_writes"], file ? count / interval : 0); + EXPECT_EQ(lifecycle["observation_point"], "before-current-snapshot-preparation"); + EXPECT_EQ(lifecycle["current_request"], "terminal"); + EXPECT_EQ(lifecycle["snapshot_interval_loops"], interval); + EXPECT_EQ(lifecycle["checkpoint_policy"], "bounded-loop-snapshots"); + EXPECT_TRUE(lifecycle["current_persistence_success"].is_null()); } void expect_complete_or_unsupported_metal_result( const CliResult& result, const nlohmann::json& json, const char* expected_kv_layout = "contiguous", const char* expected_methodology = - "llm-memory-v1-metal-decode-contiguous", + "llm-memory-v2-metal-decode-contiguous", const char* expected_phase = "decode", const char* expected_work_unit_kind = "decode_step") { ASSERT_TRUE(json.is_object()); - EXPECT_EQ(json["schema_version"], Constants::LLM_JSON_SCHEMA_VERSION); + EXPECT_EQ(json["schema_version"], 2); EXPECT_EQ(json["mode"], Constants::LLM_JSON_MODE_NAME); EXPECT_EQ(json["backend"], "metal"); EXPECT_EQ(json["phase"], expected_phase); @@ -363,8 +407,6 @@ void expect_complete_or_unsupported_metal_result( ASSERT_TRUE(json["resolved_plan"].is_object()); EXPECT_EQ(json["resolved_plan"]["phase"], expected_phase); EXPECT_EQ(json["resolved_plan"]["kv_layout"], expected_kv_layout); - EXPECT_EQ(json["resolved_plan"]["methodology_version"], - expected_methodology); EXPECT_EQ(json["resolved_plan"]["work_unit_kind"], expected_work_unit_kind); EXPECT_TRUE(json["backend_evidence"]["cpu"].is_null()); @@ -378,7 +420,7 @@ void expect_complete_or_unsupported_metal_result( if (status == "unsupported") { EXPECT_EQ(result.exit_code, EXIT_FAILURE) << result.output; EXPECT_FALSE(json["results_complete"].get()); - EXPECT_FALSE(json["conclusions_valid"].get()); + EXPECT_FALSE(json["run_accepted"].get()); const std::string reason_code = json["reason_code"].get(); EXPECT_TRUE( reason_code == LlmBackendReason::METAL_DEVICE_UNAVAILABLE || @@ -394,22 +436,16 @@ void expect_complete_or_unsupported_metal_result( EXPECT_EQ(json["backend_evidence"]["metal"]["lifecycle"] ["initialization"]["reason_code"], json["reason_code"]); - EXPECT_EQ(json["checkpoint_lifecycle"]["logical_checkpoint_attempts"], - 1u); - EXPECT_EQ(json["checkpoint_lifecycle"] - ["successful_logical_checkpoints"], - 1u); - EXPECT_TRUE(json["checkpoint_lifecycle"]["terminal_checkpoint_attempted"] - .get()); - EXPECT_TRUE(json["checkpoint_lifecycle"]["terminal_checkpoint_completed"] - .get()); + EXPECT_EQ(json["checkpoint_lifecycle"]["prior_file_writer_attempts"], 0u); + EXPECT_EQ(json["checkpoint_lifecycle"]["prior_successful_file_writes"], 0u); + EXPECT_TRUE(json["checkpoint_lifecycle"]["current_persistence_success"].is_null()); return; } ASSERT_EQ(status, "complete") << json.dump(2); EXPECT_EQ(result.exit_code, EXIT_SUCCESS) << result.output; EXPECT_TRUE(json["results_complete"].get()); - EXPECT_TRUE(json["conclusions_valid"].get()); + EXPECT_TRUE(json["run_accepted"].get()); EXPECT_EQ(json["reason_code"], "complete"); EXPECT_EQ(json["backend_evidence"]["metal"]["lifecycle"] ["initialization"]["status"], @@ -430,10 +466,15 @@ void expect_complete_or_unsupported_metal_result( EXPECT_EQ(task["commands"]["timed_workload_dispatches"], 1u); EXPECT_TRUE(task["timing"]["gpu_start_seconds"].is_number()); EXPECT_TRUE(task["timing"]["gpu_end_seconds"].is_number()); - EXPECT_TRUE(task["timing"]["gpu_elapsed_seconds"].is_number()); + EXPECT_TRUE(measurement["elapsed_seconds"].is_number()); EXPECT_TRUE(task["timing"]["queue_delay_seconds"].is_null()); - EXPECT_TRUE(task["checksum"]["valid"].get()); - EXPECT_TRUE(task["validation"]["post_validation_valid"].get()); + EXPECT_TRUE(measurement["checksum"]["checksum_valid"].get()); + const auto& canonical = canonical_llm_plan(json, measurement); + EXPECT_EQ(canonical["scenario"], measurement["scenario"]); + EXPECT_EQ(canonical["expected_checksum"]["status"], "available"); + EXPECT_EQ(canonical["expected_checksum"]["expected_run_checksum"], + measurement["checksum"]["actual_run_checksum"]); + expect_complete_llm_named_checks(json, measurement); } expect_complete_llm_checkpoint_lifecycle(json); } @@ -442,7 +483,7 @@ void expect_bounded_metal_prefill_result( const CliResult& result, const nlohmann::json& json, const char* expected_kv_layout = "contiguous", const char* expected_methodology = - "llm-memory-v1-metal-prefill-contiguous", + "llm-memory-v2-metal-prefill-contiguous", const char* expected_write_pattern = "llm-metal-prefill-contiguous-full-prompt-affine32-v1", const char* expected_checksum_pattern = @@ -453,14 +494,14 @@ void expect_bounded_metal_prefill_result( const nlohmann::json& resolved_plan = json["resolved_plan"]; const nlohmann::json& methodology = resolved_plan["methodology"]; - EXPECT_EQ(methodology["methodology_version"], expected_methodology); + EXPECT_EQ(json["methodology_version"], expected_methodology); EXPECT_EQ(methodology["work_unit_kind"], "prefill_operation"); EXPECT_EQ(methodology["weight_passes_per_work_unit"], 1u); EXPECT_EQ(methodology["kv_replay_factor"], 1u); EXPECT_EQ(methodology["context_policy"], "full-prompt-population-with-tiled-causal-prefix-scans"); - EXPECT_EQ(methodology["write_pattern_version"], expected_write_pattern); - EXPECT_EQ(methodology["checksum_pattern_version"], + EXPECT_EQ(resolved_plan["component_identities"]["write_pattern_version"], expected_write_pattern); + EXPECT_EQ(resolved_plan["component_identities"]["checksum_pattern_version"], expected_checksum_pattern); const nlohmann::json& geometry = resolved_plan["geometry"]; @@ -506,20 +547,20 @@ void expect_bounded_metal_prefill_result( const std::string scenario = measurement["scenario"]; const bool weights_only = scenario == "weights_only"; const bool kv_only = scenario == "kv_only"; - EXPECT_EQ(measurement["work_unit_kind"], "prefill_operation"); - EXPECT_EQ(measurement["kv_write_kind"], + EXPECT_EQ(canonical_llm_plan(json, measurement)["work_unit_kind"], "prefill_operation"); + EXPECT_EQ(canonical_llm_plan(json, measurement)["kv_write_kind"], weights_only ? "none" : "full_prompt_population"); - EXPECT_EQ(measurement["weight_read_bytes_per_work_unit"], + EXPECT_EQ(canonical_llm_plan(json, measurement)["weight_read_bytes_per_work_unit"], kv_only ? "0" : "1048576"); - EXPECT_EQ(measurement["kv_read_bytes_per_work_unit"], + EXPECT_EQ(canonical_llm_plan(json, measurement)["kv_read_bytes_per_work_unit"], weights_only ? "0" : "176"); - EXPECT_EQ(measurement["kv_write_bytes_per_work_unit"], + EXPECT_EQ(canonical_llm_plan(json, measurement)["kv_write_bytes_per_work_unit"], weights_only ? "0" : "80"); - EXPECT_EQ(measurement["effective_model_payload_bytes_per_work_unit"], + EXPECT_EQ(canonical_llm_plan(json, measurement)["effective_model_payload_bytes_per_work_unit"], weights_only ? "1048576" : kv_only ? "256" : "1048832"); const nlohmann::json& task = measurement["execution"]["metal"]; - EXPECT_EQ(task["checksum"]["algorithm_version"], + EXPECT_EQ(resolved_plan["component_identities"]["checksum_pattern_version"], expected_checksum_pattern); const bool exact_grid_cost = weights_only || @@ -541,11 +582,11 @@ void expect_bounded_metal_prefill_result( task["grid"]["threadgroup_accounted_bytes"].empty()); } if (std::string_view(expected_kv_layout) == "paged") { - EXPECT_EQ(measurement["layout_metadata_lookup_count_per_work_unit"], + EXPECT_EQ(canonical_llm_plan(json, measurement)["layout_metadata_lookup_count_per_work_unit"], weights_only ? "0" : "10"); - EXPECT_EQ(measurement["layout_metadata_read_bytes_per_work_unit"], + EXPECT_EQ(canonical_llm_plan(json, measurement)["layout_metadata_read_bytes_per_work_unit"], weights_only ? "0" : "40"); - EXPECT_EQ(measurement["accounted_bytes_per_work_unit"], + EXPECT_EQ(canonical_llm_plan(json, measurement)["accounted_bytes_per_work_unit"], weights_only ? "1048576" : kv_only ? "296" : "1048872"); EXPECT_EQ(task["grid"]["serial_range_visits_per_lane"], @@ -801,7 +842,7 @@ TEST(ExecutableCliIntegrationTest, EXPECT_NE(result.stdout_output.find( "Usage: ./memory_benchmark --llm-memory [options]"), std::string::npos); - EXPECT_NE(result.stdout_output.find("schema 1"), std::string::npos); + EXPECT_NE(result.stdout_output.find("schema 2"), std::string::npos); EXPECT_TRUE(result.stderr_output.empty()) << result.stderr_output; EXPECT_FALSE(nlohmann::json::accept(result.stdout_output)); expect_no_dash_transport_artifacts(result); @@ -887,7 +928,7 @@ TEST(ExecutableCliIntegrationTest, expect_process_completed(result); const nlohmann::json json = parse_single_stdout_json(result); expect_complete_or_unsupported_metal_result( - result, json, "paged", "llm-memory-v1-metal-decode-paged"); + result, json, "paged", "llm-memory-v2-metal-decode-paged"); EXPECT_EQ(json["phase"], "decode"); EXPECT_EQ(json["kv_layout"], "paged"); EXPECT_EQ(json["configuration"]["kv_block_tokens"], 2U); @@ -915,7 +956,7 @@ TEST(ExecutableCliIntegrationTest, expect_process_completed(result); const nlohmann::json json = parse_single_stdout_json(result); expect_bounded_metal_prefill_result( - result, json, "paged", "llm-memory-v1-metal-prefill-paged", + result, json, "paged", "llm-memory-v2-metal-prefill-paged", "llm-metal-prefill-paged-full-prompt-affine32-v1", "llm-metal-paged-prefill-dual-mod32-lookup-address-mix-v1"); expect_single_runtime_banner(result); @@ -955,7 +996,7 @@ TEST(ExecutableCliIntegrationTest, const nlohmann::json json = nlohmann::json::parse(read_file(output.path())); expect_bounded_metal_prefill_result( - result, json, "paged", "llm-memory-v1-metal-prefill-paged", + result, json, "paged", "llm-memory-v2-metal-prefill-paged", "llm-metal-prefill-paged-full-prompt-affine32-v1", "llm-metal-paged-prefill-dual-mod32-lookup-address-mix-v1"); EXPECT_EQ(json["configuration"]["output_file"], output.path()); @@ -967,7 +1008,7 @@ TEST(ExecutableCliIntegrationTest, << result.output; } -TEST(ExecutableCliIntegrationTest, LlmPrefillPagedWritesCompleteSchemaV1Integration) { +TEST(ExecutableCliIntegrationTest, LlmPrefillPagedWritesCompleteSchemaV2Integration) { std::vector arguments = bounded_llm_prefill_arguments("-"); arguments.insert(arguments.end() - 2, {"--kv-layout", "paged", "--kv-block-tokens", "4"}); @@ -977,7 +1018,7 @@ TEST(ExecutableCliIntegrationTest, LlmPrefillPagedWritesCompleteSchemaV1Integrat ASSERT_EQ(result.exit_code, EXIT_SUCCESS) << result.stderr_output; const nlohmann::json json = parse_single_stdout_json(result); ASSERT_TRUE(json.is_object()) << result.stdout_output; - EXPECT_EQ(json["schema_version"], Constants::LLM_JSON_SCHEMA_VERSION); + EXPECT_EQ(json["schema_version"], 2); EXPECT_EQ(json["mode"], Constants::LLM_JSON_MODE_NAME); EXPECT_EQ(json["backend"], "cpu"); EXPECT_EQ(json["phase"], "prefill"); @@ -985,7 +1026,7 @@ TEST(ExecutableCliIntegrationTest, LlmPrefillPagedWritesCompleteSchemaV1Integrat EXPECT_EQ(json["methodology_version"], Constants::LLM_CPU_PREFILL_PAGED_METHODOLOGY_VERSION); EXPECT_EQ(json["status"], "complete"); EXPECT_TRUE(json["results_complete"].get()); - EXPECT_TRUE(json["conclusions_valid"].get()); + EXPECT_TRUE(json["run_accepted"].get()); EXPECT_EQ(json["configuration"]["kv_block_tokens"], 4u); EXPECT_EQ(json["configuration"]["resolved_sources"]["phase"], "explicit"); EXPECT_EQ(json["configuration"]["resolved_sources"]["kv_layout"], "explicit"); @@ -996,6 +1037,11 @@ TEST(ExecutableCliIntegrationTest, LlmPrefillPagedWritesCompleteSchemaV1Integrat EXPECT_EQ(json["resolved_plan"]["layout"]["kv_block_tokens"], 4u); EXPECT_TRUE(json["backend_evidence"]["cpu"]["prefill"].is_object()); EXPECT_TRUE(json["backend_evidence"]["cpu"]["paged"].is_object()); + ASSERT_EQ(json["measurements"].size(), 9u); + for (const auto& measurement : json["measurements"]) { + EXPECT_EQ(measurement["status"], "measured"); + expect_complete_llm_named_checks(json, measurement); + } expect_complete_llm_checkpoint_lifecycle(json); expect_single_runtime_banner(result); expect_no_dash_transport_artifacts(result); @@ -1061,7 +1107,7 @@ TEST(ExecutableCliIntegrationTest, } TEST(ExecutableCliIntegrationTest, - LlmWritesSingleCompleteSchemaV1DocumentToStdoutIntegration) { + LlmWritesSingleCompleteSchemaV2DocumentToStdoutIntegration) { const CliResult result = run_memory_benchmark(bounded_llm_arguments("-")); expect_process_completed(result); @@ -1070,7 +1116,7 @@ TEST(ExecutableCliIntegrationTest, ASSERT_TRUE(json.is_object()) << result.stdout_output; EXPECT_EQ(json["software"]["version"], SOFTVERSION); EXPECT_TRUE(json["software"]["timestamp"].is_string()); - EXPECT_EQ(json["schema_version"], Constants::LLM_JSON_SCHEMA_VERSION); + EXPECT_EQ(json["schema_version"], 2); EXPECT_EQ(json["mode"], Constants::LLM_JSON_MODE_NAME); EXPECT_EQ(json["backend"], "cpu"); EXPECT_EQ(json["phase"], "decode"); @@ -1079,7 +1125,7 @@ TEST(ExecutableCliIntegrationTest, Constants::LLM_CPU_DECODE_CONTIGUOUS_METHODOLOGY_VERSION); EXPECT_EQ(json["status"], "complete"); EXPECT_TRUE(json["results_complete"].get()); - EXPECT_TRUE(json["conclusions_valid"].get()); + EXPECT_TRUE(json["run_accepted"].get()); EXPECT_TRUE(json["scenario_order_balance_complete"].get()); EXPECT_EQ(json["configuration"]["output_file"], "-"); EXPECT_EQ(json["configuration"]["base_seed_uint64_decimal"], "42"); @@ -1102,9 +1148,10 @@ TEST(ExecutableCliIntegrationTest, ASSERT_EQ(json["measurements"].size(), 9u); for (const nlohmann::json& measurement : json["measurements"]) { EXPECT_EQ(measurement["status"], "measured"); - EXPECT_EQ(measurement["work_unit_kind"], "decode_step"); - EXPECT_EQ(measurement["planned_work_units"], 1u); + EXPECT_EQ(canonical_llm_plan(json, measurement)["work_unit_kind"], "decode_step"); + EXPECT_EQ(canonical_llm_plan(json, measurement)["work_units"], 1u); EXPECT_EQ(measurement["completed_work_units"], 1u); + expect_complete_llm_named_checks(json, measurement); EXPECT_TRUE(measurement["checksum"]["checksum_valid"].get()); EXPECT_TRUE( measurement["synthetic_work_unit_latency_seconds"].is_number()); @@ -1127,23 +1174,23 @@ TEST(ExecutableCliIntegrationTest, expect_no_dash_transport_artifacts(result); } -TEST(ExecutableCliIntegrationTest, LlmPrefillWritesExactCompleteContiguousSchemaV1Integration) { +TEST(ExecutableCliIntegrationTest, LlmPrefillWritesExactCompleteContiguousSchemaV2Integration) { const CliResult result = run_memory_benchmark(bounded_llm_prefill_arguments("-")); expect_process_completed(result); ASSERT_EQ(result.exit_code, EXIT_SUCCESS) << result.stderr_output; const nlohmann::json json = parse_single_stdout_json(result); ASSERT_TRUE(json.is_object()) << result.stdout_output; - EXPECT_EQ(json["schema_version"], Constants::LLM_JSON_SCHEMA_VERSION); + EXPECT_EQ(json["schema_version"], 2); EXPECT_EQ(json["mode"], Constants::LLM_JSON_MODE_NAME); EXPECT_EQ(json["backend"], "cpu"); EXPECT_EQ(json["phase"], "prefill"); EXPECT_EQ(json["kv_layout"], "contiguous"); - EXPECT_EQ(json["methodology_version"], "llm-memory-v1-cpu-prefill-contiguous"); + EXPECT_EQ(json["methodology_version"], "llm-memory-v2-cpu-prefill-contiguous"); EXPECT_EQ(json["status"], "complete"); EXPECT_EQ(json["reason_code"], "complete"); EXPECT_TRUE(json["results_complete"].get()); - EXPECT_TRUE(json["conclusions_valid"].get()); + EXPECT_TRUE(json["run_accepted"].get()); EXPECT_TRUE(json["scenario_order_balance_complete"].get()); const nlohmann::json& configuration = json["configuration"]; @@ -1170,9 +1217,9 @@ TEST(ExecutableCliIntegrationTest, LlmPrefillWritesExactCompleteContiguousSchema EXPECT_EQ(geometry["prefill"]["attention_query_tile_tokens"], 2u); EXPECT_EQ(geometry["prefill"]["tile_count"], "3"); EXPECT_EQ(geometry["prefill"]["attention_prefix_token_visits_per_sequence"], "11"); - EXPECT_EQ(geometry["prefill"]["causal_token_pairs_per_sequence"], "15"); - EXPECT_EQ(geometry["prefill"]["logical_attention_pairs"], "15"); - EXPECT_EQ(geometry["prefill"]["logical_attention_fma_terms"], "120"); + EXPECT_EQ(resolved_plan["model_context"]["prefill"]["causal_token_pairs_per_sequence"], "15"); + EXPECT_EQ(resolved_plan["model_context"]["prefill"]["logical_attention_pairs"], "15"); + EXPECT_EQ(resolved_plan["model_context"]["prefill"]["logical_attention_fma_terms"], "120"); EXPECT_EQ(geometry["k_or_v_sequence_visible_bytes"], "40"); EXPECT_EQ(geometry["k_mapping_bytes"], "40"); EXPECT_EQ(geometry["v_mapping_bytes"], "40"); @@ -1218,7 +1265,7 @@ TEST(ExecutableCliIntegrationTest, LlmPrefillWritesExactCompleteContiguousSchema } EXPECT_TRUE(json["backend_evidence"]["cpu"]["paged"].is_null()); - const nlohmann::json& scenarios = resolved_plan["frozen_scenario_work_plans"]["scenarios"]; + const nlohmann::json& scenarios = resolved_plan["scenario_plans"]; ASSERT_EQ(scenarios.size(), 3u); EXPECT_EQ(scenarios[0]["scenario"], "weights_only"); EXPECT_EQ(scenarios[0]["work_unit_kind"], "prefill_operation"); @@ -1253,25 +1300,22 @@ TEST(ExecutableCliIntegrationTest, LlmPrefillWritesExactCompleteContiguousSchema for (const nlohmann::json& measurement : json["measurements"]) { EXPECT_EQ(measurement["status"], "measured"); EXPECT_EQ(measurement["reason_code"], "measured"); - EXPECT_EQ(measurement["work_unit_kind"], "prefill_operation"); - EXPECT_EQ(measurement["planned_work_units"], 1u); + EXPECT_EQ(canonical_llm_plan(json, measurement)["work_unit_kind"], "prefill_operation"); + EXPECT_EQ(canonical_llm_plan(json, measurement)["work_units"], 1u); EXPECT_EQ(measurement["completed_work_units"], 1u); EXPECT_TRUE(measurement["working_set"]["fixed_visible_context_tokens"].is_null()); EXPECT_TRUE(measurement["working_set"]["current_token_slot_included"].is_null()); EXPECT_EQ(measurement["execution"]["status"], "valid"); - EXPECT_TRUE(measurement["execution"]["post_validation_evaluated"].get()); - EXPECT_TRUE(measurement["execution"]["post_validation_valid"].get()); + expect_complete_llm_named_checks(json, measurement); const nlohmann::json& checksum = measurement["checksum"]; EXPECT_EQ(checksum["status"], "valid"); EXPECT_TRUE(checksum["checksum_valid"].get()); - EXPECT_EQ(checksum["write_pattern_version"], "llm-prefill-kv-affine64-v1"); - EXPECT_EQ(checksum["checksum_pattern_version"], "llm-prefill-affine64-parity-sum-v1"); - EXPECT_FALSE(checksum.contains("append_pattern_version")); - EXPECT_FALSE(checksum.contains("read_checksum_version")); - EXPECT_EQ(checksum["expected_worker_checksums"], checksum["actual_worker_checksums"]); - EXPECT_EQ(checksum["expected_run_checksum"], checksum["actual_run_checksum"]); - ASSERT_EQ(checksum["expected_worker_checksums"].size(), 1u); - const nlohmann::json& worker = checksum["expected_worker_checksums"][0]; + EXPECT_EQ(resolved_plan["component_identities"]["write_pattern_version"], "llm-prefill-kv-affine64-v1"); + EXPECT_EQ(resolved_plan["component_identities"]["checksum_pattern_version"], "llm-prefill-affine64-parity-sum-v1"); + EXPECT_EQ(canonical_llm_plan(json, measurement)["expected_checksum"]["expected_worker_checksums"], checksum["actual_worker_checksums"]); + EXPECT_EQ(canonical_llm_plan(json, measurement)["expected_checksum"]["expected_run_checksum"], checksum["actual_run_checksum"]); + ASSERT_EQ(canonical_llm_plan(json, measurement)["expected_checksum"]["expected_worker_checksums"].size(), 1u); + const nlohmann::json& worker = canonical_llm_plan(json, measurement)["expected_checksum"]["expected_worker_checksums"][0]; const std::string scenario = measurement["scenario"]; EXPECT_EQ(worker["weight"]["exact_bytes_read"], scenario == "kv_only" ? "0" : "1048576"); EXPECT_EQ(worker["k"]["exact_bytes_read"], scenario == "weights_only" ? "0" : "88"); @@ -1294,8 +1338,8 @@ TEST(ExecutableCliIntegrationTest, LlmPrefillWritesExactCompleteContiguousSchema } TEST(ExecutableCliIntegrationTest, - LlmWritesCheckpointedCompleteSchemaV1FileIntegration) { - const TemporaryJsonFile output("llm_schema_v1"); + LlmWritesCheckpointedCompleteSchemaV2FileIntegration) { + const TemporaryJsonFile output("llm_schema_v2"); const CliResult result = run_memory_benchmark(bounded_llm_arguments(output.path())); @@ -1306,15 +1350,14 @@ TEST(ExecutableCliIntegrationTest, EXPECT_EQ(access((output.path() + ".tmp").c_str(), F_OK), -1); const nlohmann::json json = nlohmann::json::parse(read_file(output.path())); - EXPECT_EQ(json["schema_version"], Constants::LLM_JSON_SCHEMA_VERSION); + EXPECT_EQ(json["schema_version"], 2); EXPECT_EQ(json["mode"], Constants::LLM_JSON_MODE_NAME); EXPECT_EQ(json["status"], "complete"); EXPECT_TRUE(json["results_complete"].get()); - EXPECT_TRUE(json["conclusions_valid"].get()); + EXPECT_TRUE(json["run_accepted"].get()); EXPECT_EQ(json["configuration"]["output_file"], output.path()); - EXPECT_EQ(json["resolved_plan"]["model_work_plan"]["plan_identity"], - json["resolved_plan"]["frozen_scenario_work_plans"] - ["model_plan_identity"]); + EXPECT_FALSE(json["resolved_plan"]["plan_identity"].get().empty()); + for (const auto& plan : json["resolved_plan"]["scenario_plans"]) EXPECT_EQ(plan["model_ref"], "resolved_plan"); expect_complete_llm_checkpoint_lifecycle(json); EXPECT_EQ(count_occurrences( result.stdout_output, @@ -1444,7 +1487,7 @@ TEST(ExecutableCliIntegrationTest, EXPECT_TRUE(components["msl_source_sha256"].is_null()); const nlohmann::json& scenarios = - resolved_plan["frozen_scenario_work_plans"]["scenarios"]; + resolved_plan["scenario_plans"]; ASSERT_EQ(scenarios.size(), 3u); EXPECT_EQ(scenarios[0]["scenario"], "weights_only"); EXPECT_EQ(scenarios[0]["work_unit_kind"], "decode_step"); @@ -1490,12 +1533,12 @@ TEST(ExecutableCliIntegrationTest, EXPECT_EQ(first["seeds"], second["seeds"]); EXPECT_EQ(first["resolved_plan"]["model_work_plan"], second["resolved_plan"]["model_work_plan"]); - EXPECT_EQ(first["resolved_plan"]["frozen_scenario_work_plans"], - second["resolved_plan"]["frozen_scenario_work_plans"]); + EXPECT_EQ(first["resolved_plan"]["scenario_plans"], + second["resolved_plan"]["scenario_plans"]); ASSERT_EQ(first["measurements"].size(), second["measurements"].size()); for (size_t index = 0; index < first["measurements"].size(); ++index) { - EXPECT_EQ(first["measurements"][index]["frozen_work_plan_identity"], - second["measurements"][index]["frozen_work_plan_identity"]); + EXPECT_EQ(first["measurements"][index]["plan_ref"], + second["measurements"][index]["plan_ref"]); EXPECT_EQ(first["measurements"][index]["checksum"], second["measurements"][index]["checksum"]); } @@ -1517,10 +1560,7 @@ TEST(ExecutableCliIntegrationTest, EXPECT_TRUE(document["results_complete"].get()); ASSERT_EQ(document["measurements"].size(), 9u); for (const nlohmann::json& measurement : document["measurements"]) { - EXPECT_TRUE( - measurement["execution"]["post_validation_evaluated"].get()); - EXPECT_TRUE( - measurement["execution"]["post_validation_valid"].get()); + expect_complete_llm_named_checks(document, measurement); } } @@ -1544,7 +1584,7 @@ TEST(ExecutableCliIntegrationTest, EXPECT_EQ(json["mode"], Constants::LLM_JSON_MODE_NAME); EXPECT_EQ(json["status"], "complete"); EXPECT_TRUE(json["results_complete"].get()); - EXPECT_FALSE(json["conclusions_valid"].get()); + EXPECT_TRUE(json["run_accepted"].get()); EXPECT_FALSE(json["scenario_order_balance_complete"].get()); } } diff --git a/tests/test_json_output_session.cpp b/tests/test_json_output_session.cpp index 586f7b9..3a6570a 100644 --- a/tests/test_json_output_session.cpp +++ b/tests/test_json_output_session.cpp @@ -467,3 +467,40 @@ TEST(JsonOutputSessionTest, FileCheckpointBuilderExceptionIsContained) { "injected builder exception")) + "\n"); } + +TEST(JsonOutputSessionTest, FileObservationsCountWriterEntryAfterBuilderAndOnlySuccessfulReturns) { + TemporaryDirectory temporary("writer_observations"); + JsonOutputSession session(make_json_output_target((temporary.path() / "result.json").string())); + EXPECT_EQ(session.file_writer_attempts(), 0u); + EXPECT_EQ(session.successful_file_writes(), 0u); + EXPECT_EQ(session.checkpoint([]() -> nlohmann::ordered_json { throw std::runtime_error("builder failure"); }), + EXIT_FAILURE); + EXPECT_EQ(session.file_writer_attempts(), 0u); + EXPECT_EQ(session.successful_file_writes(), 0u); + EXPECT_EQ(session.checkpoint([&]() { + EXPECT_EQ(session.file_writer_attempts(), 0u); + return nlohmann::ordered_json{{"prior_attempts", session.file_writer_attempts()}, + {"prior_successes", session.successful_file_writes()}}; + }), EXIT_SUCCESS); + EXPECT_EQ(session.file_writer_attempts(), 1u); + EXPECT_EQ(session.successful_file_writes(), 1u); + std::ifstream input(temporary.path() / "result.json"); + const auto document = nlohmann::ordered_json::parse(input); + EXPECT_EQ(document["prior_attempts"], 0); + EXPECT_EQ(document["prior_successes"], 0); + JsonOutputSession failing(make_json_output_target(temporary.path().string())); + EXPECT_EQ(failing.checkpoint([]() { return nlohmann::ordered_json::object(); }), EXIT_FAILURE); + EXPECT_EQ(failing.file_writer_attempts(), 1u); + EXPECT_EQ(failing.successful_file_writes(), 0u); +} + +TEST(JsonOutputSessionTest, DisabledAndStdoutNoOpsNeverReportFilePersistence) { + for (const std::string& target : {std::string{}, std::string{"-"}}) { + JsonOutputSession session(make_json_output_target(target)); + size_t builders = 0; + EXPECT_EQ(session.checkpoint([&]() { ++builders; return nlohmann::ordered_json::object(); }), EXIT_SUCCESS); + EXPECT_EQ(builders, 0u); + EXPECT_EQ(session.file_writer_attempts(), 0u); + EXPECT_EQ(session.successful_file_writes(), 0u); + } +} diff --git a/tests/test_llm_memory_config.cpp b/tests/test_llm_memory_config.cpp index 2fae6a2..548f4d8 100644 --- a/tests/test_llm_memory_config.cpp +++ b/tests/test_llm_memory_config.cpp @@ -1593,8 +1593,13 @@ TEST(LlmMemoryConfigIntegrationTest, ASSERT_NO_THROW(document = nlohmann::ordered_json::parse(stdout_output)); EXPECT_EQ(document.at("status"), "failed"); EXPECT_EQ(document.at("reason_code"), LlmRunnerReason::RUNNER_EXCEPTION); - EXPECT_FALSE(document.at("results_complete").get()); - EXPECT_FALSE(document.at("conclusions_valid").get()); + EXPECT_TRUE(document.at("results_complete").get()); + EXPECT_FALSE(document.at("run_accepted").get()); + const auto& lifecycle = document.at("checkpoint_lifecycle"); + EXPECT_EQ(lifecycle.at("current_request"), "late-command-error-correction"); + EXPECT_EQ(lifecycle.at("prior_file_writer_attempts"), 0); + EXPECT_EQ(lifecycle.at("prior_successful_file_writes"), 0); + EXPECT_TRUE(lifecycle.at("current_persistence_success").is_null()); EXPECT_NE(stderr_output.find("injected post-run command exception"), std::string::npos); } @@ -1635,8 +1640,13 @@ TEST(LlmMemoryConfigIntegrationTest, read_llm_command_output_file(output.path())); EXPECT_EQ(document.at("status"), "failed"); EXPECT_EQ(document.at("reason_code"), LlmRunnerReason::RUNNER_EXCEPTION); - EXPECT_FALSE(document.at("results_complete").get()); - EXPECT_FALSE(document.at("conclusions_valid").get()); + EXPECT_TRUE(document.at("results_complete").get()); + EXPECT_FALSE(document.at("run_accepted").get()); + const auto& lifecycle = document.at("checkpoint_lifecycle"); + EXPECT_EQ(lifecycle.at("current_request"), "late-command-error-correction"); + EXPECT_EQ(lifecycle.at("prior_file_writer_attempts"), 2); + EXPECT_EQ(lifecycle.at("prior_successful_file_writes"), 2); + EXPECT_TRUE(lifecycle.at("current_persistence_success").is_null()); EXPECT_EQ(stdout_output.find("Results saved"), std::string::npos); EXPECT_NE(stderr_output.find("injected post-run command exception"), std::string::npos); @@ -1684,18 +1694,18 @@ TEST(LlmMemoryConfigTest, ResultFoundationKeepsUnavailableValuesAbsent) { const LlmMeasurementState measurement; EXPECT_EQ(measurement.status, LlmMeasurementStatus::NotRun); EXPECT_EQ(measurement.reason_code, "not-run"); - EXPECT_EQ(measurement.planned_work_units, 0u); - EXPECT_EQ(measurement.completed_work_units, 0u); - EXPECT_FALSE(measurement.elapsed_seconds.has_value()); - EXPECT_FALSE(measurement.effective_model_payload_gb_s.has_value()); - EXPECT_FALSE(measurement.checksum_valid); + EXPECT_EQ(measurement.plan_handle, kLlmNoTaskIndex); + EXPECT_EQ(measurement.execution.completion.completed_work_units, 0u); + EXPECT_FALSE(derive_llm_measurement_metrics(measurement).latency_seconds.has_value()); + EXPECT_FALSE(derive_llm_measurement_metrics(measurement).payload_gb_s.has_value()); + EXPECT_FALSE(measurement.execution_evidence_available); const LlmMemoryResult result; EXPECT_EQ(result.status, LlmRunStatus::NotStarted); EXPECT_EQ(result.reason_code, "not-started"); EXPECT_FALSE(result.interruption_requested); EXPECT_FALSE(result.results_complete); - EXPECT_FALSE(result.conclusions_valid); + EXPECT_FALSE(result.run_accepted); EXPECT_FALSE(result.scenario_order_balance_complete); EXPECT_TRUE(result.measurements.empty()); EXPECT_EQ(result.counters.planned_measurements, 0u); diff --git a/tests/test_llm_memory_contract.cpp b/tests/test_llm_memory_contract.cpp index 412c67f..b1f66df 100644 --- a/tests/test_llm_memory_contract.cpp +++ b/tests/test_llm_memory_contract.cpp @@ -102,26 +102,6 @@ enum class ContractMeasurementStatus { Failed, }; -enum class SchemaValueKind { - Integer, - IntegerOrNull, - DecimalString, - DecimalStringOrNull, - Boolean, - String, - StringOrNull, - FiniteNumberOrNull, - Object, - ObjectOrNull, - Array, -}; - -struct SchemaFieldContract { - std::string_view section; - std::string_view name; - SchemaValueKind kind; -}; - struct PrefillPayloadContract { std::vector tile_ends; uint64_t attention_prefix_token_visits = 0; @@ -226,15 +206,6 @@ struct RunChecksum { uint64_t state_b = kRunInitialB; }; -struct PreExpansionDecodeIdentity { - std::string_view mode; - std::string_view backend; - int schema_version; - std::string_view methodology; - std::string_view descriptor_abi; - std::string_view append_identity; - std::string_view checksum_identity; -}; struct GenericResultIdentity { std::string_view mode; @@ -248,7 +219,7 @@ struct GenericResultIdentity { std::string_view methodology_version; std::string_view status; bool results_complete; - bool conclusions_valid; + bool run_accepted; bool all_planned_measurements_measured; }; @@ -379,38 +350,10 @@ std::string_view contract_token(ContractMeasurementStatus status) { return {}; } -std::string_view schema_kind_token(SchemaValueKind kind) { - switch (kind) { - case SchemaValueKind::Integer: - return "integer"; - case SchemaValueKind::IntegerOrNull: - return "integer_or_null"; - case SchemaValueKind::DecimalString: - return "decimal_string"; - case SchemaValueKind::DecimalStringOrNull: - return "decimal_string_or_null"; - case SchemaValueKind::Boolean: - return "boolean"; - case SchemaValueKind::String: - return "string"; - case SchemaValueKind::StringOrNull: - return "string_or_null"; - case SchemaValueKind::FiniteNumberOrNull: - return "finite_number_or_null"; - case SchemaValueKind::Object: - return "object"; - case SchemaValueKind::ObjectOrNull: - return "object_or_null"; - case SchemaValueKind::Array: - return "array"; - } - return {}; -} - std::string methodology_token(ContractBackend backend, ContractPhase phase, ContractKvLayout layout) { - return "llm-memory-v1-" + std::string(contract_token(backend)) + "-" + + return "llm-memory-v2-" + std::string(contract_token(backend)) + "-" + std::string(contract_token(phase)) + "-" + std::string(contract_token(layout)); } @@ -864,204 +807,6 @@ std::string serialize_component_identity( return identity; } -std::vector minimum_generic_schema_v1_vocabulary() { - return { - {"top_level", "schema_version", SchemaValueKind::Integer}, - {"top_level", "mode", SchemaValueKind::String}, - {"top_level", "backend", SchemaValueKind::String}, - {"top_level", "phase", SchemaValueKind::String}, - {"top_level", "kv_layout", SchemaValueKind::String}, - {"top_level", "methodology_version", SchemaValueKind::String}, - {"top_level", "software", SchemaValueKind::Object}, - {"top_level", "configuration", SchemaValueKind::Object}, - {"top_level", "resolved_plan", SchemaValueKind::Object}, - {"top_level", "backend_evidence", SchemaValueKind::Object}, - {"top_level", "memory_budget", SchemaValueKind::Object}, - {"top_level", "calibration", SchemaValueKind::Object}, - {"top_level", "measurements", SchemaValueKind::Array}, - {"top_level", "aggregates", SchemaValueKind::Object}, - {"top_level", "status", SchemaValueKind::String}, - {"top_level", "reason_code", SchemaValueKind::String}, - {"top_level", "results_complete", SchemaValueKind::Boolean}, - {"top_level", "conclusions_valid", SchemaValueKind::Boolean}, - {"top_level", "interpretation", SchemaValueKind::Object}, - - {"configuration", "argv", SchemaValueKind::Array}, - {"configuration", "resolved_sources", SchemaValueKind::Object}, - - {"resolved_plan", "geometry", SchemaValueKind::Object}, - {"resolved_plan", "layout", SchemaValueKind::Object}, - {"resolved_plan", "resources", SchemaValueKind::Object}, - {"resolved_plan", "component_identities", SchemaValueKind::Object}, - {"resolved_plan.geometry", "decode", SchemaValueKind::ObjectOrNull}, - {"resolved_plan.geometry", "prefill", SchemaValueKind::ObjectOrNull}, - - {"resolved_plan.geometry.decode", "visible_context_tokens", - SchemaValueKind::Integer}, - {"resolved_plan.geometry.prefill", "prompt_tokens", - SchemaValueKind::Integer}, - {"resolved_plan.geometry.prefill", "attention_query_tile_tokens", - SchemaValueKind::Integer}, - {"resolved_plan.geometry.prefill", "tile_count", - SchemaValueKind::DecimalString}, - {"resolved_plan.geometry.prefill", - "attention_prefix_token_visits_per_sequence", - SchemaValueKind::DecimalString}, - {"resolved_plan.geometry.prefill", "causal_token_pairs_per_sequence", - SchemaValueKind::DecimalString}, - {"resolved_plan.geometry.prefill", "logical_attention_pairs", - SchemaValueKind::DecimalString}, - {"resolved_plan.geometry.prefill", "logical_attention_fma_terms", - SchemaValueKind::DecimalString}, - - {"resolved_plan.layout", "kv_layout", SchemaValueKind::String}, - {"resolved_plan.layout", "kv_block_tokens", - SchemaValueKind::IntegerOrNull}, - {"resolved_plan.layout", "blocks_per_sequence", - SchemaValueKind::DecimalStringOrNull}, - {"resolved_plan.layout", "physical_blocks_per_layer", - SchemaValueKind::DecimalStringOrNull}, - {"resolved_plan.layout", "last_block_tokens", - SchemaValueKind::DecimalStringOrNull}, - {"resolved_plan.layout", "last_block_valid_bytes", - SchemaValueKind::DecimalStringOrNull}, - {"resolved_plan.layout", "block_table_entries", - SchemaValueKind::DecimalStringOrNull}, - {"resolved_plan.layout", "block_table_bytes", - SchemaValueKind::DecimalStringOrNull}, - {"resolved_plan.layout", "permutation_domain_uint64_hex", - SchemaValueKind::StringOrNull}, - {"resolved_plan.layout", "permutation_seed_uint64_decimal", - SchemaValueKind::DecimalStringOrNull}, - {"resolved_plan.layout", "permutation_algorithm_version", - SchemaValueKind::StringOrNull}, - {"resolved_plan.layout", "permutation_sha256", - SchemaValueKind::StringOrNull}, - - {"resolved_plan.resources", "weight_logical_bytes", - SchemaValueKind::DecimalString}, - {"resolved_plan.resources", "k_logical_bytes", - SchemaValueKind::DecimalString}, - {"resolved_plan.resources", "v_logical_bytes", - SchemaValueKind::DecimalString}, - {"resolved_plan.resources", "k_physical_length_bytes", - SchemaValueKind::DecimalString}, - {"resolved_plan.resources", "v_physical_length_bytes", - SchemaValueKind::DecimalString}, - {"resolved_plan.resources", "k_layout_padding_bytes", - SchemaValueKind::DecimalString}, - {"resolved_plan.resources", "v_layout_padding_bytes", - SchemaValueKind::DecimalString}, - {"resolved_plan.resources", "block_table_bytes", - SchemaValueKind::DecimalStringOrNull}, - - {"memory_budget", "resource_rounding_bytes", - SchemaValueKind::DecimalString}, - {"memory_budget", "transient_peak_bytes", - SchemaValueKind::DecimalString}, - {"memory_budget", "known_owned_peak_bytes", - SchemaValueKind::DecimalString}, - {"memory_budget", "admitted_budget_bytes", - SchemaValueKind::DecimalString}, - - {"resolved_plan.component_identities", "logical_profile_version", - SchemaValueKind::String}, - {"resolved_plan.component_identities", "kv_layout_version", - SchemaValueKind::String}, - {"resolved_plan.component_identities", "permutation_version", - SchemaValueKind::StringOrNull}, - {"resolved_plan.component_identities", "backend_executor_version", - SchemaValueKind::String}, - {"resolved_plan.component_identities", "resource_abi_version", - SchemaValueKind::String}, - {"resolved_plan.component_identities", "schedule_version", - SchemaValueKind::String}, - {"resolved_plan.component_identities", "timer_policy_version", - SchemaValueKind::String}, - {"resolved_plan.component_identities", "buffer_pattern_version", - SchemaValueKind::String}, - {"resolved_plan.component_identities", "write_pattern_version", - SchemaValueKind::String}, - {"resolved_plan.component_identities", "checksum_pattern_version", - SchemaValueKind::String}, - {"resolved_plan.component_identities", "msl_revision", - SchemaValueKind::StringOrNull}, - {"resolved_plan.component_identities", "msl_source_sha256", - SchemaValueKind::StringOrNull}, - - {"measurements.*", "work_unit_kind", SchemaValueKind::String}, - {"measurements.*", "planned_work_units", SchemaValueKind::Integer}, - {"measurements.*", "completed_work_units", - SchemaValueKind::Integer}, - {"measurements.*", "weight_read_bytes_per_work_unit", - SchemaValueKind::DecimalString}, - {"measurements.*", "kv_read_bytes_per_work_unit", - SchemaValueKind::DecimalString}, - {"measurements.*", "kv_write_bytes_per_work_unit", - SchemaValueKind::DecimalString}, - {"measurements.*", "kv_write_kind", SchemaValueKind::String}, - {"measurements.*", "effective_model_payload_bytes_per_work_unit", - SchemaValueKind::DecimalString}, - {"measurements.*", "layout_metadata_lookup_count_per_work_unit", - SchemaValueKind::DecimalString}, - {"measurements.*", "layout_metadata_read_bytes_per_work_unit", - SchemaValueKind::DecimalString}, - {"measurements.*", "accounted_bytes_per_work_unit", - SchemaValueKind::DecimalString}, - {"measurements.*", "planned_effective_model_payload_bytes", - SchemaValueKind::DecimalString}, - {"measurements.*", "completed_effective_model_payload_bytes", - SchemaValueKind::DecimalString}, - {"measurements.*", "planned_layout_metadata_lookup_count", - SchemaValueKind::DecimalString}, - {"measurements.*", "completed_layout_metadata_lookup_count", - SchemaValueKind::DecimalString}, - {"measurements.*", "planned_layout_metadata_read_bytes", - SchemaValueKind::DecimalString}, - {"measurements.*", "completed_layout_metadata_read_bytes", - SchemaValueKind::DecimalString}, - {"measurements.*", "planned_task_accounted_bytes", - SchemaValueKind::DecimalString}, - {"measurements.*", "completed_task_accounted_bytes", - SchemaValueKind::DecimalString}, - {"measurements.*", "synthetic_work_unit_latency_seconds", - SchemaValueKind::FiniteNumberOrNull}, - {"measurements.*", "synthetic_memory_work_units_per_second", - SchemaValueKind::FiniteNumberOrNull}, - {"measurements.*", "effective_model_payload_gb_s", - SchemaValueKind::FiniteNumberOrNull}, - - {"backend_evidence", "cpu", SchemaValueKind::ObjectOrNull}, - {"backend_evidence", "metal", SchemaValueKind::ObjectOrNull}, - }; -} - -std::string serialize_schema_vocabulary( - const std::vector& vocabulary) { - std::string result = "llm-memory-schema-v1"; - for (const SchemaFieldContract& field : vocabulary) { - result += '|'; - result += field.section; - result += '.'; - result += field.name; - result += ':'; - result += schema_kind_token(field.kind); - } - return result; -} - -const SchemaFieldContract* find_schema_field( - const std::vector& vocabulary, - std::string_view section, - std::string_view name) { - const auto found = std::find_if( - vocabulary.begin(), vocabulary.end(), - [section, name](const SchemaFieldContract& field) { - return field.section == section && field.name == name; - }); - return found == vocabulary.end() ? nullptr : &*found; -} - uint64_t rotate_left(uint64_t value, unsigned int shift) { return (value << shift) | (value >> (64 - shift)); } @@ -1204,26 +949,6 @@ struct alignas(16) KvSequenceDescriptorAbiV1 { uint64_t append_record_byte_offset; }; -bool matches_pre_expansion_decode_identity( - const PreExpansionDecodeIdentity& identity) { - return identity.mode == "llm_memory" && identity.backend == "cpu" && - identity.schema_version == 1 && - identity.methodology == - "llm-memory-v1-cpu-fixed-context-warm-layer-interleaved" && - identity.descriptor_abi == "llm-memory-descriptor-abi-v1" && - identity.append_identity == "llm-kv-append-affine64-v1" && - identity.checksum_identity == "llm-read-checksum-v1"; -} - -bool accepted_pre_expansion_completion(std::string_view mode, - int schema_version, - std::string_view status, - bool results_complete, - bool conclusions_valid) { - return mode == "llm_memory" && schema_version == 1 && - status == "complete" && results_complete && conclusions_valid; -} - bool accepted_generic_result(const GenericResultIdentity& identity) { const bool backend_is_known = identity.backend == "cpu" || identity.backend == "metal"; @@ -1232,16 +957,16 @@ bool accepted_generic_result(const GenericResultIdentity& identity) { const bool layout_is_known = identity.kv_layout == "contiguous" || identity.kv_layout == "paged"; const std::string expected_methodology = - "llm-memory-v1-" + std::string(identity.backend) + "-" + + "llm-memory-v2-" + std::string(identity.backend) + "-" + std::string(identity.phase) + "-" + std::string(identity.kv_layout); - return identity.mode == "llm_memory" && identity.schema_version == 1 && + return identity.mode == "llm_memory" && identity.schema_version == 2 && backend_is_known && phase_is_known && layout_is_known && identity.backend == identity.requested_backend && identity.phase == identity.requested_phase && identity.kv_layout == identity.requested_kv_layout && identity.methodology_version == expected_methodology && identity.status == "complete" && identity.results_complete && - identity.conclusions_valid && + identity.run_accepted && identity.all_planned_measurements_measured; } @@ -1450,67 +1175,17 @@ TEST(LlmMemoryContractTest, DescriptorAbiLayoutGolden) { offsetof(KvSequenceDescriptorAbiV1, append_record_byte_offset), 72u); } -TEST(LlmMemoryContractTest, - PreExpansionDecodeIdentityAndCompletionRemainRegressionGoldens) { - const PreExpansionDecodeIdentity frozen = { - "llm_memory", - "cpu", - 1, - "llm-memory-v1-cpu-fixed-context-warm-layer-interleaved", - "llm-memory-descriptor-abi-v1", - "llm-kv-append-affine64-v1", - "llm-read-checksum-v1", - }; - EXPECT_TRUE(matches_pre_expansion_decode_identity(frozen)); - - PreExpansionDecodeIdentity candidate = frozen; - candidate.mode = "benchmark"; - EXPECT_FALSE(matches_pre_expansion_decode_identity(candidate)); - candidate = frozen; - candidate.backend = "gpu"; - EXPECT_FALSE(matches_pre_expansion_decode_identity(candidate)); - candidate = frozen; - candidate.schema_version = 2; - EXPECT_FALSE(matches_pre_expansion_decode_identity(candidate)); - candidate = frozen; - candidate.methodology = "llm-memory-v2"; - EXPECT_FALSE(matches_pre_expansion_decode_identity(candidate)); - candidate = frozen; - candidate.descriptor_abi = "llm-memory-descriptor-abi-v2"; - EXPECT_FALSE(matches_pre_expansion_decode_identity(candidate)); - candidate = frozen; - candidate.append_identity = "llm-kv-append-affine64-v2"; - EXPECT_FALSE(matches_pre_expansion_decode_identity(candidate)); - candidate = frozen; - candidate.checksum_identity = "llm-read-checksum-v2"; - EXPECT_FALSE(matches_pre_expansion_decode_identity(candidate)); - - EXPECT_TRUE(accepted_pre_expansion_completion( - "llm_memory", 1, "complete", true, true)); - EXPECT_FALSE( - accepted_pre_expansion_completion("benchmark", 1, "complete", true, - true)); - EXPECT_FALSE(accepted_pre_expansion_completion( - "llm_memory", 2, "complete", true, true)); - EXPECT_FALSE(accepted_pre_expansion_completion( - "llm_memory", 1, "partial", true, true)); - EXPECT_FALSE(accepted_pre_expansion_completion( - "llm_memory", 1, "complete", false, true)); - EXPECT_FALSE(accepted_pre_expansion_completion( - "llm_memory", 1, "complete", true, false)); -} - -TEST(LlmMemoryContractTest, GenericV1AcceptancePredicateIsExact) { +TEST(LlmMemoryContractTest, GenericV2AcceptancePredicateIsExact) { const GenericResultIdentity accepted = { "llm_memory", - 1, + 2, "metal", "metal", "prefill", "prefill", "paged", "paged", - "llm-memory-v1-metal-prefill-paged", + "llm-memory-v2-metal-prefill-paged", "complete", true, true, @@ -1522,7 +1197,7 @@ TEST(LlmMemoryContractTest, GenericV1AcceptancePredicateIsExact) { candidate.mode = "benchmark"; EXPECT_FALSE(accepted_generic_result(candidate)); candidate = accepted; - candidate.schema_version = 2; + candidate.schema_version = 1; EXPECT_FALSE(accepted_generic_result(candidate)); candidate = accepted; candidate.backend = "cpu"; @@ -1536,21 +1211,21 @@ TEST(LlmMemoryContractTest, GenericV1AcceptancePredicateIsExact) { candidate = accepted; candidate.requested_backend = "gpu"; candidate.backend = "gpu"; - candidate.methodology_version = "llm-memory-v1-gpu-prefill-paged"; + candidate.methodology_version = "llm-memory-v2-gpu-prefill-paged"; EXPECT_FALSE(accepted_generic_result(candidate)); candidate = accepted; candidate.requested_phase = "train"; candidate.phase = "train"; - candidate.methodology_version = "llm-memory-v1-metal-train-paged"; + candidate.methodology_version = "llm-memory-v2-metal-train-paged"; EXPECT_FALSE(accepted_generic_result(candidate)); candidate = accepted; candidate.requested_kv_layout = "sparse"; candidate.kv_layout = "sparse"; - candidate.methodology_version = "llm-memory-v1-metal-prefill-sparse"; + candidate.methodology_version = "llm-memory-v2-metal-prefill-sparse"; EXPECT_FALSE(accepted_generic_result(candidate)); candidate = accepted; candidate.methodology_version = - "llm-memory-v1-cpu-fixed-context-warm-layer-interleaved"; + "llm-memory-v2-cpu-fixed-context-warm-layer-interleaved"; EXPECT_FALSE(accepted_generic_result(candidate)); candidate = accepted; candidate.status = "partial"; @@ -1559,7 +1234,7 @@ TEST(LlmMemoryContractTest, GenericV1AcceptancePredicateIsExact) { candidate.results_complete = false; EXPECT_FALSE(accepted_generic_result(candidate)); candidate = accepted; - candidate.conclusions_valid = false; + candidate.run_accepted = false; EXPECT_FALSE(accepted_generic_result(candidate)); candidate = accepted; candidate.all_planned_measurements_measured = false; @@ -1656,14 +1331,14 @@ TEST(LlmMemoryContractTest, const std::array layouts = { ContractKvLayout::Contiguous, ContractKvLayout::Paged}; const std::array expected_methodologies = { - "llm-memory-v1-cpu-decode-contiguous", - "llm-memory-v1-cpu-decode-paged", - "llm-memory-v1-cpu-prefill-contiguous", - "llm-memory-v1-cpu-prefill-paged", - "llm-memory-v1-metal-decode-contiguous", - "llm-memory-v1-metal-decode-paged", - "llm-memory-v1-metal-prefill-contiguous", - "llm-memory-v1-metal-prefill-paged", + "llm-memory-v2-cpu-decode-contiguous", + "llm-memory-v2-cpu-decode-paged", + "llm-memory-v2-cpu-prefill-contiguous", + "llm-memory-v2-cpu-prefill-paged", + "llm-memory-v2-metal-decode-contiguous", + "llm-memory-v2-metal-decode-paged", + "llm-memory-v2-metal-prefill-contiguous", + "llm-memory-v2-metal-prefill-paged", }; size_t methodology_index = 0; for (ContractBackend backend : backends) { @@ -2206,159 +1881,6 @@ TEST(LlmMemoryContractTest, "|msl_revision=null|msl_source_sha256=null"); } -TEST(LlmMemoryContractTest, - MinimumGenericSchemaV1FieldVocabularyAndTypesAreFrozen) { - const std::vector vocabulary = - minimum_generic_schema_v1_vocabulary(); - ASSERT_EQ(vocabulary.size(), 95u); - - const auto names_in_section = - [&vocabulary](std::string_view section) { - std::vector names; - for (const SchemaFieldContract& field : vocabulary) { - if (field.section == section) { - names.push_back(field.name); - } - } - return names; - }; - EXPECT_EQ( - names_in_section("top_level"), - (std::vector{ - "schema_version", "mode", "backend", "phase", "kv_layout", - "methodology_version", "software", "configuration", - "resolved_plan", "backend_evidence", "memory_budget", - "calibration", "measurements", "aggregates", "status", - "reason_code", "results_complete", "conclusions_valid", - "interpretation"})); - EXPECT_EQ(names_in_section("configuration"), - (std::vector{"argv", "resolved_sources"})); - EXPECT_EQ(names_in_section("resolved_plan"), - (std::vector{ - "geometry", "layout", "resources", - "component_identities"})); - EXPECT_EQ(names_in_section("resolved_plan.geometry"), - (std::vector{"decode", "prefill"})); - EXPECT_EQ(names_in_section("resolved_plan.geometry.decode"), - (std::vector{"visible_context_tokens"})); - EXPECT_EQ( - names_in_section("resolved_plan.geometry.prefill"), - (std::vector{ - "prompt_tokens", "attention_query_tile_tokens", "tile_count", - "attention_prefix_token_visits_per_sequence", - "causal_token_pairs_per_sequence", "logical_attention_pairs", - "logical_attention_fma_terms"})); - EXPECT_EQ( - names_in_section("resolved_plan.layout"), - (std::vector{ - "kv_layout", "kv_block_tokens", "blocks_per_sequence", - "physical_blocks_per_layer", "last_block_tokens", - "last_block_valid_bytes", "block_table_entries", - "block_table_bytes", "permutation_domain_uint64_hex", - "permutation_seed_uint64_decimal", - "permutation_algorithm_version", "permutation_sha256"})); - EXPECT_EQ( - names_in_section("resolved_plan.resources"), - (std::vector{ - "weight_logical_bytes", "k_logical_bytes", "v_logical_bytes", - "k_physical_length_bytes", "v_physical_length_bytes", - "k_layout_padding_bytes", "v_layout_padding_bytes", - "block_table_bytes"})); - EXPECT_EQ(names_in_section("memory_budget"), - (std::vector{ - "resource_rounding_bytes", "transient_peak_bytes", - "known_owned_peak_bytes", "admitted_budget_bytes"})); - EXPECT_EQ( - names_in_section("resolved_plan.component_identities"), - (std::vector{ - "logical_profile_version", "kv_layout_version", - "permutation_version", "backend_executor_version", - "resource_abi_version", "schedule_version", - "timer_policy_version", "buffer_pattern_version", - "write_pattern_version", "checksum_pattern_version", - "msl_revision", "msl_source_sha256"})); - EXPECT_EQ( - names_in_section("measurements.*"), - (std::vector{ - "work_unit_kind", "planned_work_units", "completed_work_units", - "weight_read_bytes_per_work_unit", - "kv_read_bytes_per_work_unit", "kv_write_bytes_per_work_unit", - "kv_write_kind", "effective_model_payload_bytes_per_work_unit", - "layout_metadata_lookup_count_per_work_unit", - "layout_metadata_read_bytes_per_work_unit", - "accounted_bytes_per_work_unit", - "planned_effective_model_payload_bytes", - "completed_effective_model_payload_bytes", - "planned_layout_metadata_lookup_count", - "completed_layout_metadata_lookup_count", - "planned_layout_metadata_read_bytes", - "completed_layout_metadata_read_bytes", - "planned_task_accounted_bytes", "completed_task_accounted_bytes", - "synthetic_work_unit_latency_seconds", - "synthetic_memory_work_units_per_second", - "effective_model_payload_gb_s"})); - EXPECT_EQ(names_in_section("backend_evidence"), - (std::vector{"cpu", "metal"})); - - for (size_t left = 0; left < vocabulary.size(); ++left) { - for (size_t right = left + 1; right < vocabulary.size(); ++right) { - EXPECT_FALSE(vocabulary[left].section == vocabulary[right].section && - vocabulary[left].name == vocabulary[right].name); - } - } - - ASSERT_NE(find_schema_field(vocabulary, "resolved_plan.geometry.prefill", - "prompt_tokens"), - nullptr); - EXPECT_EQ(find_schema_field(vocabulary, "resolved_plan.geometry.prefill", - "prompt_tokens") - ->kind, - SchemaValueKind::Integer); - EXPECT_EQ(find_schema_field(vocabulary, "resolved_plan.geometry.prefill", - "attention_prefix_token_visits_per_sequence") - ->kind, - SchemaValueKind::DecimalString); - EXPECT_EQ(find_schema_field(vocabulary, "resolved_plan.layout", - "kv_block_tokens") - ->kind, - SchemaValueKind::IntegerOrNull); - EXPECT_EQ(find_schema_field(vocabulary, "resolved_plan.layout", - "permutation_seed_uint64_decimal") - ->kind, - SchemaValueKind::DecimalStringOrNull); - EXPECT_EQ(find_schema_field(vocabulary, "measurements.*", - "layout_metadata_lookup_count_per_work_unit") - ->kind, - SchemaValueKind::DecimalString); - EXPECT_EQ(find_schema_field(vocabulary, "measurements.*", - "effective_model_payload_gb_s") - ->kind, - SchemaValueKind::FiniteNumberOrNull); - EXPECT_EQ(find_schema_field(vocabulary, "backend_evidence", "cpu") - ->kind, - SchemaValueKind::ObjectOrNull); - EXPECT_EQ(find_schema_field(vocabulary, "memory_budget", - "known_owned_peak_bytes") - ->kind, - SchemaValueKind::DecimalString); - EXPECT_EQ(find_schema_field(vocabulary, "memory_budget", - "resource_rounding_bytes") - ->kind, - SchemaValueKind::DecimalString); - EXPECT_EQ(find_schema_field(vocabulary, "top_level", "reason_code") - ->kind, - SchemaValueKind::String); - EXPECT_EQ(find_schema_field(vocabulary, "top_level", - "conclusions_valid") - ->kind, - SchemaValueKind::Boolean); - - const std::string serialized = serialize_schema_vocabulary(vocabulary); - EXPECT_EQ(serialized.substr(0, 20), "llm-memory-schema-v1"); - EXPECT_EQ(sha256_text(serialized), - "77fbcf8e13b7399cff685c41854d5d93e36b18cb4103cb490cf52e52829b2442"); -} - TEST(LlmMemoryContractTest, FinalSupportMatrixAndPublicActivationOrderAreFrozen) { constexpr std::array profiles = {{ diff --git a/tests/test_llm_memory_json.cpp b/tests/test_llm_memory_json.cpp index 4fcf59d..38326ab 100644 --- a/tests/test_llm_memory_json.cpp +++ b/tests/test_llm_memory_json.cpp @@ -466,16 +466,22 @@ LlmMemoryResult metal_result_with_measurement_and_calibration(const LlmMemoryWor result.status = LlmRunStatus::Complete; result.reason_code = LlmBackendReason::VALID; result.results_complete = true; - result.conclusions_valid = true; - - const LlmScenarioWorkPlan scenario_plan = - build_llm_scenario_work_plan(plan, LlmScenario::Mixed, 1, true); + result.run_accepted = true; + const auto scenario = build_llm_scenario_work_plan(plan, LlmScenario::Mixed, 1, true); + // These are controlled serialization witnesses, not independently validated GPU results. + LlmCanonicalScenarioPlan canonical; + canonical.plan = scenario; + canonical.expected.available = true; + canonical.expected.reason_code = "valid"; + canonical.expected.metal_run = {{1, 2}, {3, 4}, {5, 6}}; + result.scenario_plans.push_back(std::move(canonical)); + result.frozen_plan_handles[static_cast(LlmScenario::Mixed)] = 0; LlmMetalTaskEvidence task = complete_metal_task_evidence(plan); + task.cold_checks = required_llm_cold_checks(plan, LlmScenario::Mixed); + for (size_t slot = 0; slot < task.cold_checks.size(); ++slot) resolve_llm_cold_check(task.cold_checks, slot, true); LlmMeasurementState measurement; + measurement.plan_handle = 0; measurement.scenario = LlmScenario::Mixed; - measurement.work_unit_kind = plan.work_unit_kind; - measurement.kv_write_kind = - llm_kv_write_kind_for(plan.phase, measurement.scenario); measurement.status = LlmMeasurementStatus::Measured; measurement.reason_code = LlmBackendReason::VALID; measurement.attempted = true; @@ -484,61 +490,14 @@ LlmMemoryResult metal_result_with_measurement_and_calibration(const LlmMemoryWor measurement.execution.reason_code = LlmBackendReason::VALID; measurement.execution.timing = {true, true, 0.0025}; measurement.execution.validation = {true, true}; - measurement.execution.backend_evidence = task; - measurement.elapsed_seconds = 0.0025; - measurement.checksum_valid = true; - if (scenario_plan.valid) { - measurement.planned_work_units = scenario_plan.work_units; - measurement.completed_work_units = scenario_plan.work_units; - measurement.weight_read_bytes_per_work_unit = - scenario_plan.weight_read_bytes_per_work_unit; - measurement.kv_read_bytes_per_work_unit = - scenario_plan.kv_read_bytes_per_work_unit; - measurement.kv_write_bytes_per_work_unit = - scenario_plan.kv_write_bytes_per_work_unit; - measurement.effective_model_payload_bytes_per_work_unit = - scenario_plan.effective_model_payload_bytes_per_work_unit; - measurement.layout_metadata_lookup_count_per_work_unit = - scenario_plan.layout_metadata_lookup_count_per_work_unit; - measurement.layout_metadata_read_bytes_per_work_unit = - scenario_plan.layout_metadata_read_bytes_per_work_unit; - measurement.accounted_bytes_per_work_unit = - scenario_plan.accounted_bytes_per_work_unit; - measurement.planned_weight_read_bytes = scenario_plan.weight_read_bytes; - measurement.planned_kv_read_bytes = scenario_plan.kv_read_bytes; - measurement.planned_kv_write_bytes = scenario_plan.kv_write_bytes; - measurement.planned_effective_model_payload_bytes = - scenario_plan.effective_model_payload_bytes; - measurement.completed_effective_model_payload_bytes = - scenario_plan.effective_model_payload_bytes; - measurement.planned_layout_metadata_lookup_count = - scenario_plan.layout_metadata_lookup_count; - measurement.completed_layout_metadata_lookup_count = - scenario_plan.layout_metadata_lookup_count; - measurement.planned_layout_metadata_read_bytes = - scenario_plan.layout_metadata_read_bytes; - measurement.completed_layout_metadata_read_bytes = - scenario_plan.layout_metadata_read_bytes; - measurement.planned_task_accounted_bytes = - scenario_plan.task_accounted_bytes; - measurement.completed_task_accounted_bytes = - scenario_plan.task_accounted_bytes; - measurement.execution.completion = { - scenario_plan.work_units, - scenario_plan.work_units, - scenario_plan.effective_model_payload_bytes, - scenario_plan.layout_metadata_lookup_count, - scenario_plan.layout_metadata_read_bytes, - scenario_plan.task_accounted_bytes}; - } + measurement.execution.backend_evidence = static_cast(task); + measurement.execution.completion = {scenario.work_units, scenario.effective_model_payload_bytes, + scenario.layout_metadata_lookup_count, scenario.layout_metadata_read_bytes, scenario.task_accounted_bytes}; result.measurements.push_back(std::move(measurement)); - task.queue_delay_available = false; LlmCalibrationAttempt attempt; + attempt.plan_handle = 0; attempt.scenario = LlmScenario::Mixed; - attempt.work_unit_kind = plan.work_unit_kind; - attempt.kv_write_kind = - llm_kv_write_kind_for(plan.phase, attempt.scenario); attempt.reason_code = LlmBackendReason::VALID; attempt.terminal = true; attempt.valid = true; @@ -552,23 +511,10 @@ LlmMemoryResult metal_result_with_measurement_and_calibration(const LlmMemoryWor attempt.execution.validation_evaluated = true; attempt.execution.validation_valid = true; attempt.execution.metal_evidence_available = true; - attempt.execution.metal = std::move(task); - if (scenario_plan.valid) { - attempt.explicit_iterations = true; - attempt.work_units = scenario_plan.work_units; - attempt.weight_read_bytes = scenario_plan.weight_read_bytes; - attempt.kv_read_bytes = scenario_plan.kv_read_bytes; - attempt.kv_write_bytes = scenario_plan.kv_write_bytes; - attempt.effective_model_payload_bytes = - scenario_plan.effective_model_payload_bytes; - attempt.layout_metadata_lookup_count = - scenario_plan.layout_metadata_lookup_count; - attempt.layout_metadata_read_bytes = - scenario_plan.layout_metadata_read_bytes; - attempt.task_accounted_bytes = scenario_plan.task_accounted_bytes; - attempt.work_plan_identity = scenario_plan.plan_identity; - } + attempt.execution.metal = static_cast(task); + attempt.execution.completion = result.measurements.front().execution.completion; result.calibration_attempts[static_cast(LlmScenario::Mixed)].push_back(std::move(attempt)); + prepare_llm_result_snapshot(result); return result; } @@ -633,6 +579,7 @@ LlmResourcePreparationResult preparation_for(const LlmMemoryWorkPlan& plan) { class FakeLlmBackend final : public LlmBackend { public: + bool omit_measured_cold_checks = false; using CpuTaskExecutor = std::function; @@ -673,10 +620,30 @@ class FakeLlmBackend final : public LlmBackend { return evidence_.preparation; } + LlmExpectedChecksumResult expected_cpu_checksum(const LlmMemoryWorkPlan& model, + const LlmScenarioWorkPlan&) const noexcept override { + const auto controlled = successful_execution(model); + LlmExpectedChecksumResult expected; + expected.valid = true; + expected.reason_code = LlmExecutorReason::VALID; + expected.workers = controlled.expected_checksums; + expected.run_checksum = controlled.expected_run_checksum; + return expected; + } + LlmTaskExecutionResult execute_task(const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan& scenario_plan, const LlmRunnerTaskContext& context) override { LlmExecutorResult execution = executor_ ? executor_(model_plan, scenario_plan, context) : successful_execution(model_plan); + execution.cold_checks = required_llm_cold_checks(model_plan, scenario_plan.scenario); + if (execution.post_validation_evaluated) { + for (size_t slot = 0; slot < execution.cold_checks.size(); ++slot) { + resolve_llm_cold_check(execution.cold_checks, slot, execution.post_validation_valid, + execution.post_validation_valid ? std::string_view("valid") : + canonicalize_llm_result_reason_code(execution.reason_code)); + } + } + if (omit_measured_cold_checks && context.kind == LlmRunnerTaskKind::Measurement) execution.cold_checks = {}; execution.kv_write_validation_applicable = scenario_plan.scenario != LlmScenario::WeightsOnly; execution.kv_write_validation_evaluated = execution.kv_write_validation_applicable && execution.post_validation_evaluated; execution.kv_write_validation_valid = execution.kv_write_validation_evaluated && execution.post_validation_valid; @@ -733,6 +700,23 @@ std::vector json_string_array(const OrderedJson& array) { return output; } +const OrderedJson& referenced_plan(const OrderedJson& document, const OrderedJson& record) { + const auto& reference = record.at("plan_ref"); + if (!reference.is_number_unsigned() && !reference.is_number_integer()) + throw std::invalid_argument("expected integer plan_ref"); + const auto index = reference.get(); + if (index < 0 || static_cast(index) >= document.at("resolved_plan").at("scenario_plans").size()) + throw std::out_of_range("plan_ref"); + return document.at("resolved_plan").at("scenario_plans").at(static_cast(index)); +} + +const OrderedJson& named_check(const OrderedJson& execution, const char* kind) { + for (const auto& check : execution.at("validation").at("checks")) { + if (check.at("kind") == kind) return check; + } + throw std::invalid_argument("missing named check"); +} + void expect_exact_keys(const OrderedJson& object, std::initializer_list keys) { ASSERT_TRUE(object.is_object()); EXPECT_EQ(object.size(), keys.size()); @@ -758,7 +742,7 @@ TEST(LlmMemoryJsonTest, MetalDocumentPublishesSegmentationBackendTaskAndCompactE const OrderedJson document = build_llm_memory_json(config, plan, backend, metadata, result); EXPECT_EQ(document["backend"], "metal"); - EXPECT_EQ(document["methodology_version"], "llm-memory-v1-metal-decode-contiguous"); + EXPECT_EQ(document["methodology_version"], "llm-memory-v2-metal-decode-contiguous"); EXPECT_EQ(document["configuration"]["resolved_sources"]["backend"], "explicit"); EXPECT_TRUE(document["configuration"]["requested_workers"].is_null()); EXPECT_TRUE(document["configuration"]["available_workers"].is_null()); @@ -806,17 +790,14 @@ TEST(LlmMemoryJsonTest, MetalDocumentPublishesSegmentationBackendTaskAndCompactE std::to_string(metal_plan->resources.admitted_budget_bytes)); ASSERT_EQ(document["measurements"].size(), 1U); - EXPECT_TRUE(document["measurements"][0]["requested_workers"].is_null()); - EXPECT_TRUE(document["measurements"][0]["effective_workers"].is_null()); - EXPECT_TRUE(document["measurements"][0]["qos_successful_workers"].is_null()); - EXPECT_TRUE(document["measurements"][0]["qos_failed_workers"].is_null()); + EXPECT_TRUE(document["measurements"][0]["execution"]["requested_workers"].is_null()); + EXPECT_TRUE(document["resolved_plan"]["model_work_plan"]["effective_workers"].is_null()); + EXPECT_TRUE(document["measurements"][0]["execution"]["qos_successful_workers"].is_null()); + EXPECT_TRUE(document["measurements"][0]["execution"]["qos_failed_workers"].is_null()); const OrderedJson& execution = document["measurements"][0]["execution"]; EXPECT_TRUE(execution["requested_workers"].is_null()); - EXPECT_TRUE(execution["kv_write_validation_applicable"].is_null()); - EXPECT_TRUE(execution["kv_write_validation_evaluated"].is_null()); - EXPECT_TRUE(execution["kv_write_validation_valid"].is_null()); EXPECT_TRUE(execution["qos_successful_workers"].is_null()); - EXPECT_DOUBLE_EQ(execution["elapsed_seconds"].get(), 0.0025); + EXPECT_DOUBLE_EQ(document["measurements"][0]["elapsed_seconds"].get(), 0.0025); const OrderedJson& task = execution["metal"]; EXPECT_EQ(task["pipeline"]["label"], "membenchmark.llm-metal.pipeline.decode-contiguous.mixed"); EXPECT_EQ(task["grid"]["actual_threadgroups"], 2U); @@ -828,38 +809,32 @@ TEST(LlmMemoryJsonTest, MetalDocumentPublishesSegmentationBackendTaskAndCompactE EXPECT_TRUE( task["grid"]["threadgroup_accounted_imbalance_bytes"].is_null()); EXPECT_TRUE(task["grid"]["threadgroup_accounted_bytes"].empty()); - EXPECT_DOUBLE_EQ(task["timing"]["gpu_elapsed_seconds"].get(), 0.0025); + EXPECT_TRUE(task["timing"]["evaluated"].get()); + EXPECT_TRUE(task["timing"]["valid"].get()); EXPECT_DOUBLE_EQ(task["timing"]["queue_delay_seconds"].get(), 0.0005); EXPECT_EQ(task["commands"]["reset_status"], "completed"); EXPECT_EQ(task["commands"]["timed_status"], "completed"); EXPECT_EQ(task["commands"]["post_validation_status"], "completed"); EXPECT_EQ(task["commands"]["timed_workload_dispatches"], 1U); - EXPECT_EQ(task["checksum"]["algorithm_version"], "llm-metal-dual-mod32-v1"); - EXPECT_EQ(task["checksum"]["expected"]["weight"]["a_uint32_decimal"], "1"); - expect_exact_keys(task["validation"], - {"post_validation_evaluated", - "post_validation_valid", "kv_write_evaluated", - "kv_write_valid", "padding_canary_applicable", - "padding_canary_evaluated", "padding_canary_valid"}); - EXPECT_TRUE(task["validation"]["kv_write_evaluated"].get()); - EXPECT_TRUE(task["validation"]["kv_write_valid"].get()); - EXPECT_FALSE(task["validation"].contains("append_evaluated")); - EXPECT_FALSE(task["validation"].contains("append_valid")); - EXPECT_FALSE(task["validation"]["padding_canary_applicable"].get()); - EXPECT_TRUE(task["validation"]["padding_canary_evaluated"].is_null()); const OrderedJson& checksum = document["measurements"][0]["checksum"]; EXPECT_EQ(checksum["status"], "valid"); - EXPECT_EQ(checksum["checksum_pattern_version"], "llm-metal-dual-mod32-v1"); - EXPECT_TRUE(checksum["expected_worker_checksums"].is_null()); EXPECT_EQ(checksum["actual_run_checksum"]["v"]["b_uint32_decimal"], "6"); const OrderedJson& compact = document["calibration"]["attempts"]["mixed"][0]["execution"]; EXPECT_TRUE(compact["requested_workers"].is_null()); - EXPECT_EQ(compact["checksum"]["algorithm_version"], "llm-metal-dual-mod32-v1"); - EXPECT_EQ(compact["checksum"]["expected_run_checksum"]["k"]["b_uint32_decimal"], "4"); + EXPECT_EQ(document["resolved_plan"]["scenario_plans"][0]["expected_checksum"]["expected_run_checksum"]["k"]["b_uint32_decimal"], "4"); EXPECT_EQ(compact["metal"]["commands"]["timed_status"], "completed"); EXPECT_TRUE(compact["metal"]["timing"]["queue_delay_seconds"].is_null()); + const auto& checks = document["measurements"][0]["execution"]; + EXPECT_TRUE(named_check(checks, "kv-append-final")["applicable"]); + EXPECT_TRUE(named_check(checks, "kv-append-final")["evaluated"]); + EXPECT_TRUE(named_check(checks, "kv-append-final")["valid"]); + const auto& padding = named_check(checks, "kv-padding-canary"); + EXPECT_FALSE(padding["applicable"]); + EXPECT_TRUE(padding["evaluated"].is_null()); + EXPECT_TRUE(padding["valid"].is_null()); + EXPECT_EQ(document["measurements"][0]["checksum"]["actual_run_checksum"]["weight"]["a_uint32_decimal"], "1"); } TEST(LlmMemoryJsonTest, @@ -888,7 +863,7 @@ TEST(LlmMemoryJsonTest, EXPECT_EQ(document["resolved_plan"]["work_unit_kind"], "prefill_operation"); EXPECT_EQ(document["methodology_version"], - "llm-memory-v1-metal-prefill-contiguous"); + "llm-memory-v2-metal-prefill-contiguous"); EXPECT_EQ( document["resolved_plan"]["methodology"] ["maximum_serial_range_visits_per_lane_per_task"], @@ -920,8 +895,6 @@ TEST(LlmMemoryJsonTest, LlmMetalPrefillContiguousVersion::SCHEDULE); EXPECT_EQ(components["write_pattern_version"], LlmMetalPrefillContiguousVersion::WRITE_PATTERN); - EXPECT_EQ(components["checksum_pattern_version"], - LlmMetalPrefillContiguousVersion::CHECKSUM); EXPECT_EQ(document["resolved_plan"]["resources"]["metal"] ["table_segments"], nullptr); @@ -932,33 +905,28 @@ TEST(LlmMemoryJsonTest, ASSERT_EQ(document["measurements"].size(), 1u); const OrderedJson& measurement = document["measurements"][0]; - EXPECT_EQ(measurement["work_unit_kind"], "prefill_operation"); - EXPECT_EQ(measurement["kv_write_kind"], "full_prompt_population"); + EXPECT_EQ(referenced_plan(document, measurement)["work_unit_kind"], "prefill_operation"); + EXPECT_EQ(referenced_plan(document, measurement)["kv_write_kind"], "full_prompt_population"); const OrderedJson& task = measurement["execution"]["metal"]; ASSERT_TRUE(task.is_object()) << task.dump(2); - ASSERT_TRUE(task.contains("validation")) << task.dump(2); EXPECT_EQ(task["pipeline"]["label"], "membenchmark.llm-metal.pipeline.prefill-contiguous.mixed"); EXPECT_EQ(task["grid"]["serial_range_visits_per_lane"], "34"); EXPECT_TRUE(task["grid"]["cost_unit"].is_null()); EXPECT_TRUE(task["grid"]["threadgroup_accounted_bytes"].empty()); - expect_exact_keys(task["validation"], - {"post_validation_evaluated", - "post_validation_valid", "kv_write_evaluated", - "kv_write_valid", "padding_canary_applicable", - "padding_canary_evaluated", "padding_canary_valid"}); - EXPECT_TRUE(task["validation"]["kv_write_evaluated"].get()); - EXPECT_TRUE(task["validation"]["kv_write_valid"].get()); - EXPECT_FALSE(task["validation"].contains("append_evaluated")); - EXPECT_FALSE(task["validation"].contains("append_valid")); - EXPECT_FALSE( - task["validation"]["padding_canary_applicable"].get()); - EXPECT_TRUE( - task["validation"]["padding_canary_evaluated"].is_null()); const std::string serialized = document.dump(); EXPECT_EQ(serialized.find("tokens/s"), std::string::npos); EXPECT_EQ(serialized.find("tokens_per_second"), std::string::npos); + const auto& checks = document["measurements"][0]["execution"]; + EXPECT_TRUE(named_check(checks, "kv-prefill-final-samples")["applicable"]); + EXPECT_TRUE(named_check(checks, "kv-prefill-final-samples")["evaluated"]); + EXPECT_TRUE(named_check(checks, "kv-prefill-final-samples")["valid"]); + const auto& padding = named_check(checks, "kv-padding-canary"); + EXPECT_FALSE(padding["applicable"]); + EXPECT_TRUE(padding["evaluated"].is_null()); + EXPECT_TRUE(padding["valid"].is_null()); + EXPECT_EQ(document["measurements"][0]["checksum"]["actual_run_checksum"]["weight"]["a_uint32_decimal"], "1"); } TEST(LlmMemoryJsonTest, @@ -989,7 +957,7 @@ TEST(LlmMemoryJsonTest, EXPECT_EQ(document["phase"], "prefill"); EXPECT_EQ(document["kv_layout"], "paged"); EXPECT_EQ(document["methodology_version"], - "llm-memory-v1-metal-prefill-paged"); + "llm-memory-v2-metal-prefill-paged"); EXPECT_EQ(document["resolved_plan"]["work_unit_kind"], "prefill_operation"); EXPECT_EQ(document["resolved_plan"]["methodology"] @@ -1016,8 +984,6 @@ TEST(LlmMemoryJsonTest, LlmMetalPrefillPagedVersion::BUFFER_PATTERN); EXPECT_EQ(components["write_pattern_version"], LlmMetalPrefillPagedVersion::WRITE_PATTERN); - EXPECT_EQ(components["checksum_pattern_version"], - LlmMetalPrefillPagedVersion::CHECKSUM); EXPECT_EQ(components["msl_revision"], "llm-metal-decode-prefill-contiguous-paged-msl23-v5"); EXPECT_EQ(components["msl_source_sha256"], @@ -1097,20 +1063,20 @@ TEST(LlmMemoryJsonTest, ASSERT_EQ(document["measurements"].size(), 1U); const OrderedJson& measurement = document["measurements"][0]; EXPECT_EQ(measurement["scenario"], "mixed"); - EXPECT_EQ(measurement["work_unit_kind"], "prefill_operation"); - EXPECT_EQ(measurement["kv_write_kind"], "full_prompt_population"); - EXPECT_EQ(measurement["planned_work_units"], 1U); + EXPECT_EQ(referenced_plan(document, measurement)["work_unit_kind"], "prefill_operation"); + EXPECT_EQ(referenced_plan(document, measurement)["kv_write_kind"], "full_prompt_population"); + EXPECT_EQ(referenced_plan(document, measurement)["work_units"], 1U); EXPECT_EQ(measurement["completed_work_units"], 1U); - EXPECT_EQ(measurement["weight_read_bytes_per_work_unit"], "1048576"); - EXPECT_EQ(measurement["kv_read_bytes_per_work_unit"], "1408"); - EXPECT_EQ(measurement["kv_write_bytes_per_work_unit"], "640"); - EXPECT_EQ(measurement["effective_model_payload_bytes_per_work_unit"], + EXPECT_EQ(referenced_plan(document, measurement)["weight_read_bytes_per_work_unit"], "1048576"); + EXPECT_EQ(referenced_plan(document, measurement)["kv_read_bytes_per_work_unit"], "1408"); + EXPECT_EQ(referenced_plan(document, measurement)["kv_write_bytes_per_work_unit"], "640"); + EXPECT_EQ(referenced_plan(document, measurement)["effective_model_payload_bytes_per_work_unit"], "1050624"); - EXPECT_EQ(measurement["layout_metadata_lookup_count_per_work_unit"], + EXPECT_EQ(referenced_plan(document, measurement)["layout_metadata_lookup_count_per_work_unit"], "30"); - EXPECT_EQ(measurement["layout_metadata_read_bytes_per_work_unit"], + EXPECT_EQ(referenced_plan(document, measurement)["layout_metadata_read_bytes_per_work_unit"], "120"); - EXPECT_EQ(measurement["accounted_bytes_per_work_unit"], "1050744"); + EXPECT_EQ(referenced_plan(document, measurement)["accounted_bytes_per_work_unit"], "1050744"); const OrderedJson& task = measurement["execution"]["metal"]; EXPECT_EQ(task["pipeline"]["label"], @@ -1140,34 +1106,25 @@ TEST(LlmMemoryJsonTest, EXPECT_EQ(task["grid"]["threadgroup_accounted_imbalance_bytes"], "0"); EXPECT_EQ(task["grid"]["threadgroup_accounted_bytes"], (OrderedJson::array({"525372", "525372"}))); - EXPECT_TRUE(task["checksum"]["valid"].get()); - EXPECT_EQ(task["checksum"]["algorithm_version"], - LlmMetalPrefillPagedVersion::CHECKSUM); - expect_exact_keys(task["validation"], - {"post_validation_evaluated", - "post_validation_valid", "kv_write_evaluated", - "kv_write_valid", "padding_canary_applicable", - "padding_canary_evaluated", "padding_canary_valid"}); - EXPECT_TRUE(task["validation"]["kv_write_evaluated"].get()); - EXPECT_TRUE(task["validation"]["kv_write_valid"].get()); - EXPECT_TRUE( - task["validation"]["padding_canary_applicable"].get()); - EXPECT_TRUE(task["validation"]["padding_canary_evaluated"].get()); - EXPECT_TRUE(task["validation"]["padding_canary_valid"].get()); - EXPECT_FALSE(task["validation"].contains("append_evaluated")); - EXPECT_FALSE(task["validation"].contains("append_valid")); const OrderedJson& calibration_execution = document["calibration"]["attempts"]["mixed"][0]["execution"]; - EXPECT_EQ(calibration_execution["checksum"]["algorithm_version"], - LlmMetalPrefillPagedVersion::CHECKSUM); - EXPECT_EQ(calibration_execution["metal"]["checksum"] - ["algorithm_version"], + EXPECT_EQ(document["resolved_plan"]["component_identities"]["checksum_pattern_version"], LlmMetalPrefillPagedVersion::CHECKSUM); + EXPECT_EQ(calibration_execution["checksum"]["actual_run_checksum"]["k"]["b_uint32_decimal"], "4"); const std::string serialized = document.dump(); EXPECT_EQ(serialized.find("tokens/s"), std::string::npos); EXPECT_EQ(serialized.find("tokens_per_second"), std::string::npos); + const auto& checks = document["measurements"][0]["execution"]; + EXPECT_TRUE(named_check(checks, "kv-prefill-final-samples")["applicable"]); + EXPECT_TRUE(named_check(checks, "kv-prefill-final-samples")["evaluated"]); + EXPECT_TRUE(named_check(checks, "kv-prefill-final-samples")["valid"]); + const auto& padding = named_check(checks, "kv-padding-canary"); + EXPECT_TRUE(padding["applicable"]); + EXPECT_TRUE(padding["evaluated"]); + EXPECT_TRUE(padding["valid"]); + EXPECT_EQ(document["measurements"][0]["checksum"]["actual_run_checksum"]["weight"]["a_uint32_decimal"], "1"); } TEST(LlmMemoryJsonTest, MetalPagedDocumentPublishesLayoutTableLookupAndPaddingEvidence) { @@ -1188,7 +1145,7 @@ TEST(LlmMemoryJsonTest, MetalPagedDocumentPublishesLayoutTableLookupAndPaddingEv EXPECT_EQ(document["backend"], "metal"); EXPECT_EQ(document["phase"], "decode"); EXPECT_EQ(document["kv_layout"], "paged"); - EXPECT_EQ(document["methodology_version"], "llm-memory-v1-metal-decode-paged"); + EXPECT_EQ(document["methodology_version"], "llm-memory-v2-metal-decode-paged"); EXPECT_EQ(document["resolved_plan"]["component_identities"]["msl_revision"], "llm-metal-decode-prefill-contiguous-paged-msl23-v5"); EXPECT_EQ(document["configuration"]["resolved_sources"]["kv_layout"], "explicit"); @@ -1239,17 +1196,15 @@ TEST(LlmMemoryJsonTest, MetalPagedDocumentPublishesLayoutTableLookupAndPaddingEv EXPECT_EQ(task["grid"]["paged_semantic_lookups"], "10"); EXPECT_EQ(task["commands"]["timed_workload_dispatches"], 1U); EXPECT_TRUE(task["timing"]["valid"].get()); - EXPECT_TRUE(task["checksum"]["valid"].get()); - expect_exact_keys(task["validation"], - {"post_validation_evaluated", - "post_validation_valid", "kv_write_evaluated", - "kv_write_valid", "padding_canary_applicable", - "padding_canary_evaluated", "padding_canary_valid"}); - EXPECT_TRUE(task["validation"]["kv_write_evaluated"].get()); - EXPECT_TRUE(task["validation"]["kv_write_valid"].get()); - EXPECT_TRUE(task["validation"]["padding_canary_applicable"].get()); - EXPECT_TRUE(task["validation"]["padding_canary_evaluated"].get()); - EXPECT_TRUE(task["validation"]["padding_canary_valid"].get()); + const auto& checks = document["measurements"][0]["execution"]; + EXPECT_TRUE(named_check(checks, "kv-append-final")["applicable"]); + EXPECT_TRUE(named_check(checks, "kv-append-final")["evaluated"]); + EXPECT_TRUE(named_check(checks, "kv-append-final")["valid"]); + const auto& padding = named_check(checks, "kv-padding-canary"); + EXPECT_TRUE(padding["applicable"]); + EXPECT_TRUE(padding["evaluated"]); + EXPECT_TRUE(padding["valid"]); + EXPECT_EQ(document["measurements"][0]["checksum"]["actual_run_checksum"]["weight"]["a_uint32_decimal"], "1"); } TEST(LlmMemoryJsonTest, UnsupportedMetalPagedDocumentKeepsPlannedGeometryButNotRuntimePermutation) { @@ -1374,13 +1329,13 @@ TEST(LlmMemoryJsonTest, FailedMetalRuntimeSetupRetainsTerminalSchemaWithoutCpuFa result.reason_code = LlmBackendReason::METAL_PIPELINE_CREATION_FAILED; const OrderedJson document = build_llm_memory_json(config, plan, backend, fixed_metadata(config, plan), result); - EXPECT_EQ(document["schema_version"], 1); + EXPECT_EQ(document["schema_version"], 2); EXPECT_EQ(document["mode"], "llm_memory"); EXPECT_EQ(document["backend"], "metal"); EXPECT_EQ(document["status"], "failed"); EXPECT_EQ(document["reason_code"], LlmBackendReason::METAL_PIPELINE_CREATION_FAILED); EXPECT_FALSE(document["results_complete"].get()); - EXPECT_FALSE(document["conclusions_valid"].get()); + EXPECT_FALSE(document["run_accepted"].get()); EXPECT_TRUE(document["backend_evidence"]["cpu"].is_null()); EXPECT_EQ(document["backend_evidence"]["metal"]["lifecycle"]["plan_resolution"]["status"], "failed"); EXPECT_EQ(document["backend_evidence"]["metal"]["lifecycle"]["plan_resolution"]["reason_code"], @@ -1398,7 +1353,7 @@ TEST(LlmMemoryJsonTest, CompleteDocumentHasExactTopLevelIdentityAndAuditableNest const OrderedJson document = build_llm_memory_json(config, plan, preparation, metadata, result); - const std::array top_level_keys = {"schema_version", + const std::array top_level_keys = {"schema_version", "mode", "backend", "phase", @@ -1415,7 +1370,7 @@ TEST(LlmMemoryJsonTest, CompleteDocumentHasExactTopLevelIdentityAndAuditableNest "status", "reason_code", "results_complete", - "conclusions_valid", + "run_accepted", "interpretation", "diagnostic", "interruption_requested", @@ -1425,12 +1380,26 @@ TEST(LlmMemoryJsonTest, CompleteDocumentHasExactTopLevelIdentityAndAuditableNest "checkpoint_lifecycle", "loop_records", "environment", - "quality_warnings"}; + "quality_warnings", "build_manifest"}; ASSERT_EQ(document.size(), top_level_keys.size()); for (const char* key : top_level_keys) { EXPECT_TRUE(document.contains(key)) << key; } + EXPECT_TRUE(document["resolved_plan"]["methodology"] + ["maximum_serial_range_visits_per_lane_per_task"] + .is_null()); + ASSERT_TRUE(document["backend_evidence"]["cpu"].is_object()); + EXPECT_TRUE(document["backend_evidence"]["metal"].is_null()); + EXPECT_TRUE(document["backend_evidence"]["cpu"]["prefill"].is_null()); + EXPECT_TRUE(document["backend_evidence"]["cpu"]["paged"].is_null()); + const OrderedJson& cpu_resources = document["backend_evidence"]["cpu"]["resources"]; + EXPECT_TRUE(cpu_resources["mappings"]["block_table"].is_null()); + EXPECT_TRUE(cpu_resources["initialization"]["block_table_logical_bytes"].is_null()); + EXPECT_TRUE(cpu_resources["initialization"]["block_table_page_rounded_bytes"].is_null()); + EXPECT_TRUE(cpu_resources["initialization"]["block_table_read_only"].is_null()); + EXPECT_TRUE(document["resolved_plan"]["geometry"]["prefill"].is_null()); + expect_exact_keys(document["configuration"], {"backend", "phase", "kv_layout", @@ -1459,38 +1428,6 @@ TEST(LlmMemoryJsonTest, CompleteDocumentHasExactTopLevelIdentityAndAuditableNest expect_exact_keys(document["configuration"]["resolved_sources"], {"backend", "phase", "kv_layout", "kv_block_tokens", "visible_context_tokens", "prompt_tokens", "attention_query_tile_tokens", "workers", "iterations", "seed"}); - expect_exact_keys(document["resolved_plan"]["methodology"], {"methodology_version", - "backend", - "phase", - "kv_layout", - "work_unit_kind", - "weight_passes_per_work_unit", - "kv_replay_factor", - "schedule_version", - "warmup_policy", - "context_policy", - "scenario_order_policy", - "timing_policy", - "cache_policy", - "calibration_policy", - "calibration_target_seconds", - "calibration_min_seconds", - "calibration_max_seconds", - "calibration_max_corrections", - "calibration_min_pilot_accounted_bytes", - "maximum_work_units_per_measurement", - "maximum_serial_range_visits_per_lane_per_task", - "maximum_accounted_bytes_per_task", - "repeatability_cv_warning_threshold_pct", - "calibration_excluded_from_results", - "timed_region_exclusions", - "resource_abi_version", - "buffer_pattern_version", - "write_pattern_version", - "checksum_pattern_version"}); - EXPECT_TRUE(document["resolved_plan"]["methodology"] - ["maximum_serial_range_visits_per_lane_per_task"] - .is_null()); expect_exact_keys(document["resolved_plan"]["geometry"], {"valid", "reason_code", "phase", @@ -1523,64 +1460,9 @@ TEST(LlmMemoryJsonTest, CompleteDocumentHasExactTopLevelIdentityAndAuditableNest "traffic_crossover_numerator", "traffic_crossover_denominator", "traffic_crossover_context_tokens"}); - expect_exact_keys(document["aggregates"]["traffic_diagnostics"], - {"classification_version", "traffic_crossover_numerator", "traffic_crossover_denominator", - "traffic_crossover_context_tokens", "current_visible_context_tokens", - "current_weight_read_payload_bytes_per_work_unit", "current_kv_read_payload_bytes_per_work_unit", - "current_weight_to_kv_read_payload_ratio", "current_context_classification", - "classification_is_payload_only", "scenario_headlines"}); - expect_exact_keys(document["aggregates"]["traffic_diagnostics"]["scenario_headlines"]["mixed"], - {"synthetic_work_unit_latency_seconds", "synthetic_memory_work_units_per_second", - "effective_model_payload_gb_s"}); - expect_exact_keys(document["memory_budget"], - {"resource_rounding_bytes", "transient_peak_bytes", "layout_transient_bytes", "setup_peak_bytes", - "runtime_peak_bytes", "known_owned_peak_bytes", "admitted_budget_bytes", "valid", "reason_code", - "request", "available_memory_bytes", "allowed_memory_bytes", "used_fallback"}); - expect_exact_keys(document["memory_budget"]["request"], {"valid", - "reason_code", - "mapping_granularity_bytes", - "requested_weight_mapping_bytes", - "requested_k_mapping_bytes", - "requested_v_mapping_bytes", - "committed_weight_mapping_bytes", - "committed_k_mapping_bytes", - "committed_v_mapping_bytes", - "requested_block_table_mapping_bytes", - "committed_block_table_mapping_bytes", - "requested_data_bytes", - "committed_data_bytes", - "layout_transient_bytes", - "setup_peak_bytes", - "runtime_peak_bytes", - "descriptor_bytes", - "planner_storage_bytes", - "checksum_auxiliary_bytes", - "orchestration_auxiliary_bytes", - "auxiliary_bytes", - "required_total_bytes"}); - expect_exact_keys(document["backend_evidence"], {"cpu", "metal"}); - ASSERT_TRUE(document["backend_evidence"]["cpu"].is_object()); - EXPECT_TRUE(document["backend_evidence"]["metal"].is_null()); - expect_exact_keys(document["backend_evidence"]["cpu"], - {"requested_workers", "available_workers", "effective_workers", "resource_abi_version", - "schedule_version", "timer_policy_version", "prefill", "paged", "resources"}); - EXPECT_TRUE(document["backend_evidence"]["cpu"]["prefill"].is_null()); - EXPECT_TRUE(document["backend_evidence"]["cpu"]["paged"].is_null()); - const OrderedJson& cpu_resources = document["backend_evidence"]["cpu"]["resources"]; - expect_exact_keys(cpu_resources, - {"valid", "reason_code", "model_plan_identity", "mappings", "descriptors", "executor_auxiliary", - "json_output_peak_estimate", "allocation_memory_budget", "initialization"}); expect_exact_keys(cpu_resources["mappings"], {"policy", "full_size_physical_mappings", "weight", "k", "v", "block_table", "requested_data_bytes", "committed_data_bytes"}); - EXPECT_TRUE(cpu_resources["mappings"]["block_table"].is_null()); expect_exact_keys(cpu_resources["mappings"]["weight"], {"requested_bytes", "committed_bytes"}); - expect_exact_keys(cpu_resources["descriptors"], - {"abi_version", "layer_descriptors_per_worker", "sequence_descriptors_per_worker", - "total_layer_descriptors", "total_sequence_descriptors", "descriptor_bytes"}); - expect_exact_keys(cpu_resources["executor_auxiliary"], - {"valid", "reason_code", "static_reference_bytes", "expected_checksum_bytes", - "actual_checksum_bytes", "run_checksum_bytes", "worker_status_bytes", "thread_handle_bytes", - "checksum_auxiliary_bytes", "orchestration_auxiliary_bytes", "total_auxiliary_bytes"}); expect_exact_keys(cpu_resources["json_output_peak_estimate"], {"enabled", "valid", "reason_code", "policy", "fixed_schema_bytes", "input_string_bytes", "measurement_record_bytes", "worker_checksum_bytes", "total_bytes"}); @@ -1590,20 +1472,12 @@ TEST(LlmMemoryJsonTest, CompleteDocumentHasExactTopLevelIdentityAndAuditableNest "static_references_accumulated_during_initialization", "weight_bytes", "k_bytes", "v_bytes", "total_bytes", "non_empty_weight_spans", "non_empty_k_spans", "non_empty_v_spans", "block_table_logical_bytes", "block_table_page_rounded_bytes", "block_table_read_only", "k_layout_padding_bytes", "v_layout_padding_bytes"}); - EXPECT_TRUE(cpu_resources["initialization"]["block_table_logical_bytes"].is_null()); - EXPECT_TRUE(cpu_resources["initialization"]["block_table_page_rounded_bytes"].is_null()); - EXPECT_TRUE(cpu_resources["initialization"]["block_table_read_only"].is_null()); expect_exact_keys(document["seeds"], {"base_seed_uint64_decimal", "source", "buffer_domain_seeds", "scenario_domain_seeds"}); expect_exact_keys(document["seeds"]["buffer_domain_seeds"], {"weight_uint64_decimal", "k_uint64_decimal", "v_uint64_decimal"}); expect_exact_keys(document["seeds"]["scenario_domain_seeds"], {"weights_only", "kv_only", "mixed"}); - expect_exact_keys(document["resolved_plan"], - {"valid", "reason_code", "plan_identity", "methodology_version", "backend", "phase", "kv_layout", - "work_unit_kind", "geometry", "layout", "resources", "component_identities", "methodology", - "model_work_plan", "frozen_scenario_work_plans"}); expect_exact_keys(document["resolved_plan"]["geometry"]["decode"], {"visible_context_tokens"}); - EXPECT_TRUE(document["resolved_plan"]["geometry"]["prefill"].is_null()); expect_exact_keys( document["resolved_plan"]["layout"], {"kv_layout", "kv_block_tokens", "blocks_per_sequence", "physical_blocks_per_layer", "total_physical_blocks", @@ -1615,195 +1489,6 @@ TEST(LlmMemoryJsonTest, CompleteDocumentHasExactTopLevelIdentityAndAuditableNest document["resolved_plan"]["resources"], {"weight_logical_bytes", "k_logical_bytes", "v_logical_bytes", "k_physical_length_bytes", "v_physical_length_bytes", "k_layout_padding_bytes", "v_layout_padding_bytes", "block_table_bytes"}); - expect_exact_keys( - document["resolved_plan"]["component_identities"], - {"logical_profile_version", "kv_layout_version", "permutation_version", "backend_executor_version", - "resource_abi_version", "schedule_version", "timer_policy_version", "buffer_pattern_version", - "write_pattern_version", "checksum_pattern_version", "msl_revision", "msl_source_sha256", "identity"}); - expect_exact_keys(document["resolved_plan"]["model_work_plan"], {"valid", - "reason_code", - "plan_identity", - "methodology_version", - "backend", - "phase", - "kv_layout", - "work_unit_kind", - "component_identity", - "weight_passes_per_work_unit", - "kv_replay_factor", - "requested_workers", - "available_workers", - "effective_workers", - "worker_plan_count", - "weight_layer_count", - "layer_descriptors_per_worker", - "sequence_descriptors_per_worker", - "total_layer_descriptors", - "total_sequence_descriptors", - "descriptor_bytes", - "planner_storage_bytes"}); - expect_exact_keys( - document["resolved_plan"]["frozen_scenario_work_plans"], - {"valid", "reason_code", "explicit_iterations", "model_plan_identity", "plan_identity", "scenarios"}); - expect_exact_keys(document["resolved_plan"]["frozen_scenario_work_plans"]["scenarios"][0], - {"valid", - "reason_code", - "scenario", - "work_unit_kind", - "kv_write_kind", - "explicit_iterations", - "model_plan_identity", - "scenario_seed_uint64_decimal", - "work_units", - "weight_read_bytes_per_work_unit", - "kv_read_bytes_per_work_unit", - "kv_write_bytes_per_work_unit", - "effective_model_payload_bytes_per_work_unit", - "layout_metadata_lookup_count_per_work_unit", - "layout_metadata_read_bytes_per_work_unit", - "accounted_bytes_per_work_unit", - "weight_read_bytes", - "kv_read_bytes", - "kv_write_bytes", - "effective_model_payload_bytes", - "layout_metadata_lookup_count", - "layout_metadata_read_bytes", - "task_accounted_bytes", - "maximum_work_units_by_work_unit_cap", - "maximum_work_units_by_guardrail", - "effective_maximum_work_units", - "plan_identity"}); - expect_exact_keys(document["calibration"], {"excluded_from_results", "attempts"}); - expect_exact_keys(document["calibration"]["attempts"], {"weights_only", "kv_only", "mixed"}); - expect_exact_keys(document["calibration"]["attempts"]["weights_only"][0], {"attempt_index", - "scenario", - "work_unit_kind", - "kv_write_kind", - "purpose", - "explicit_iterations", - "work_units", - "weight_read_bytes", - "kv_read_bytes", - "kv_write_bytes", - "effective_model_payload_bytes", - "layout_metadata_lookup_count", - "layout_metadata_read_bytes", - "task_accounted_bytes", - "work_plan_identity", - "duration_quality", - "terminal", - "valid", - "reason_code", - "execution"}); - expect_exact_keys(document["calibration"]["attempts"]["weights_only"][0]["execution"], - {"status", "reason_code", "valid", "elapsed_seconds", "requested_workers", "created_workers", - "completed_workers", "qos_successful_workers", "qos_failed_workers", "worker_startup_failed", - "kernel_succeeded", "timer_started", "timer_stopped", "checksum"}); - expect_exact_keys( - document["calibration"]["attempts"]["weights_only"][0]["execution"]["checksum"], - {"status", "reason_code", "algorithm_version", "checksum_valid", "expected_run_checksum", "actual_run_checksum"}); - expect_exact_keys( - document["counters"], - {"planned_loops", "attempted_loops", "completed_loops", "planned_measurements", "attempted_measurements", - "terminal_measurements", "measured_measurements", "planned_work_units", "completed_work_units", - "planned_effective_model_payload_bytes", "completed_effective_model_payload_bytes", - "planned_layout_metadata_lookup_count", "completed_layout_metadata_lookup_count", - "planned_layout_metadata_read_bytes", "completed_layout_metadata_read_bytes", "planned_task_accounted_bytes", - "completed_task_accounted_bytes", "runner_auxiliary"}); - expect_exact_keys( - document["counters"]["runner_auxiliary"], - {"valid", "reason_code", "measurement_record_bytes", "loop_record_bytes", "calibration_record_bytes", - "calibration_identity_bytes", "aggregate_value_bytes", "statistics_workspace_bytes", "warning_record_bytes", - "fixed_metadata_bytes", "retained_checksum_bytes", "checksum_auxiliary_bytes", "orchestration_auxiliary_bytes", - "total_auxiliary_bytes"}); - expect_exact_keys( - document["checkpoint_lifecycle"], - {"checkpoint_failed", "logical_checkpoint_attempts", "successful_logical_checkpoints", - "terminal_checkpoint_attempted", "terminal_checkpoint_completed", "checkpoint_policy", - "file_checkpoint_failure_is_terminal_and_not_retried", "stdout_intermediate_checkpoints_are_lazy"}); - expect_exact_keys(document["loop_records"][0], - {"loop_index", "planned_order", "realized_order", "realized_order_count", "measurement_indexes"}); - expect_exact_keys(document["measurements"][0], {"scenario", - "work_unit_kind", - "kv_write_kind", - "loop_index", - "order_position", - "status", - "reason_code", - "attempted", - "requested_workers", - "effective_workers", - "qos_successful_workers", - "qos_failed_workers", - "frozen_plan_index", - "frozen_work_plan_identity", - "scenario_seed_uint64_decimal", - "explicit_iterations", - "work_policy", - "duration_quality", - "calibration_attempt_count", - "calibration_attempt_indexes", - "planned_work_units", - "completed_work_units", - "weight_read_bytes_per_work_unit", - "kv_read_bytes_per_work_unit", - "kv_write_bytes_per_work_unit", - "effective_model_payload_bytes_per_work_unit", - "layout_metadata_lookup_count_per_work_unit", - "layout_metadata_read_bytes_per_work_unit", - "accounted_bytes_per_work_unit", - "planned_weight_read_bytes", - "planned_kv_read_bytes", - "planned_kv_write_bytes", - "planned_effective_model_payload_bytes", - "completed_effective_model_payload_bytes", - "planned_layout_metadata_lookup_count", - "completed_layout_metadata_lookup_count", - "planned_layout_metadata_read_bytes", - "completed_layout_metadata_read_bytes", - "planned_task_accounted_bytes", - "completed_task_accounted_bytes", - "elapsed_seconds", - "synthetic_work_unit_latency_seconds", - "synthetic_memory_work_units_per_second", - "effective_model_payload_gb_s", - "weight_payload_fraction", - "kv_read_payload_fraction", - "kv_write_payload_fraction", - "working_set", - "execution", - "checksum"}); - expect_exact_keys(document["measurements"][0]["working_set"], - {"bytes", "full_size_physical_mappings", "cacheable", "kv_layout", "fixed_visible_context_tokens", - "current_token_slot_included"}); - expect_exact_keys( - document["measurements"][0]["execution"], - {"status", "reason_code", "valid", "elapsed_seconds", "requested_workers", "created_workers", "completed_workers", - "qos_successful_workers", "qos_failed_workers", "worker_startup_failed", "kernel_succeeded", "timer_started", - "timer_stopped", "post_validation_evaluated", "post_validation_valid", - "kv_write_validation_applicable", "kv_write_validation_evaluated", "kv_write_validation_valid"}); - expect_exact_keys(document["measurements"][0]["checksum"], - {"status", "reason_code", "initialization_pattern_version", "write_pattern_version", - "checksum_pattern_version", "checksum_valid", "expected_worker_checksums", - "actual_worker_checksums", "expected_run_checksum", "actual_run_checksum"}); - expect_exact_keys(document["measurements"][0]["checksum"]["expected_worker_checksums"][0], - {"worker_index", "weight", "k", "v"}); - expect_exact_keys( - document["measurements"][0]["checksum"]["expected_worker_checksums"][0]["weight"], - {"state_a_uint64_decimal", "state_b_uint64_decimal", "exact_bytes_read", "span_count_uint64_decimal"}); - expect_exact_keys(document["measurements"][0]["checksum"]["expected_run_checksum"], - {"state_a_uint64_decimal", "state_b_uint64_decimal"}); - expect_exact_keys(document["aggregates"], {"scenarios", "traffic_diagnostics"}); - expect_exact_keys(document["aggregates"]["scenarios"], {"weights_only", "kv_only", "mixed"}); - expect_exact_keys( - document["aggregates"]["scenarios"]["mixed"], - {"scenario", "status", "stability_quality", "cv_warning_threshold_pct", "synthetic_work_unit_latency_seconds", - "synthetic_memory_work_units_per_second", "effective_model_payload_gb_s"}); - expect_exact_keys(document["aggregates"]["scenarios"]["mixed"]["effective_model_payload_gb_s"], - {"units", "sample_count", "headline_semantics", "headline", "values", "statistics"}); - expect_exact_keys(document["aggregates"]["scenarios"]["mixed"]["effective_model_payload_gb_s"]["statistics"], - {"sample_count", "average", "min", "max", "median", "p90", "p95", "p99", "stddev", - "coefficient_of_variation_pct", "median_absolute_deviation"}); expect_exact_keys(document["environment"], {"processor_name", "macos_version", "performance_core_count", "efficiency_core_count", "logical_core_count", "page_size_bytes", "l1_data_cache_bytes", "l2_data_cache_bytes", @@ -1832,10 +1517,9 @@ TEST(LlmMemoryJsonTest, CompleteDocumentHasExactTopLevelIdentityAndAuditableNest "traffic_classification_semantics", "full_size_working_set_reduces_but_does_not_prove_dram_residency", "comparability_requires"}); - EXPECT_EQ(document["software"]["version"], SOFTVERSION); EXPECT_EQ(document["software"]["timestamp"], metadata.timestamp); - EXPECT_EQ(document["schema_version"], 1); + EXPECT_EQ(document["schema_version"], 2); EXPECT_EQ(document["mode"], "llm_memory"); EXPECT_EQ(document["backend"], "cpu"); EXPECT_EQ(document["phase"], "decode"); @@ -1843,7 +1527,7 @@ TEST(LlmMemoryJsonTest, CompleteDocumentHasExactTopLevelIdentityAndAuditableNest EXPECT_EQ(document["methodology_version"], Constants::LLM_CPU_DECODE_CONTIGUOUS_METHODOLOGY_VERSION); EXPECT_EQ(document["status"], "complete"); EXPECT_TRUE(document["results_complete"]); - EXPECT_TRUE(document["conclusions_valid"]); + EXPECT_TRUE(document["run_accepted"]); EXPECT_TRUE(document["scenario_order_balance_complete"]); EXPECT_TRUE(document["diagnostic"].is_null()); @@ -1864,7 +1548,7 @@ TEST(LlmMemoryJsonTest, CompleteDocumentHasExactTopLevelIdentityAndAuditableNest EXPECT_EQ(document["resolved_plan"]["geometry"]["kv_capacity_bytes"], std::to_string(plan.geometry.kv_capacity_bytes)); EXPECT_EQ(document["resolved_plan"]["geometry"]["kv_element_bytes"], "2"); - EXPECT_EQ(cpu_resources["model_plan_identity"], plan.plan_identity); + EXPECT_EQ(cpu_resources["model_ref"], "resolved_plan"); EXPECT_TRUE(cpu_resources["json_output_peak_estimate"]["enabled"]); EXPECT_TRUE(cpu_resources["json_output_peak_estimate"]["valid"]); EXPECT_EQ(cpu_resources["json_output_peak_estimate"]["reason_code"], LlmJsonReason::VALID); @@ -1872,8 +1556,8 @@ TEST(LlmMemoryJsonTest, CompleteDocumentHasExactTopLevelIdentityAndAuditableNest std::to_string(metadata.json_peak_estimate.total_bytes)); EXPECT_EQ(cpu_resources["initialization"]["pattern_version"], Constants::LLM_BUFFER_PATTERN_VERSION); EXPECT_FALSE(cpu_resources["initialization"]["separate_reference_read_pass"]); - EXPECT_EQ(document["resolved_plan"]["model_work_plan"]["plan_identity"], plan.plan_identity); - EXPECT_EQ(document["resolved_plan"]["frozen_scenario_work_plans"]["scenarios"].size(), kLlmScenarioCount); + EXPECT_EQ(document["resolved_plan"]["plan_identity"], plan.plan_identity); + EXPECT_EQ(document["resolved_plan"]["scenario_plans"].size(), kLlmScenarioCount); EXPECT_EQ(document["calibration"]["attempts"]["weights_only"].size(), 1u); EXPECT_EQ(document["loop_records"].size(), 3u); EXPECT_EQ(document["measurements"].size(), 9u); @@ -1884,6 +1568,197 @@ TEST(LlmMemoryJsonTest, CompleteDocumentHasExactTopLevelIdentityAndAuditableNest EXPECT_FALSE(document["interpretation"]["physical_dram_traffic_measured"]); } +TEST(LlmMemoryJsonTest, CanonicalPlansOwnExpectedWitnessesAndEveryRuntimeReferenceResolves) { + const auto config = explicit_config(); + const auto plan = admitted_plan(config); + ASSERT_TRUE(plan.valid); + const auto result = complete_result(config, plan); + const auto doc = build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); + const auto& resolved = doc.at("resolved_plan"); + expect_exact_keys(resolved, {"valid", "reason_code", "plan_identity", "backend", "phase", "kv_layout", + "work_unit_kind", "geometry", "model_context", "layout", "resources", "component_identities", "methodology", + "model_work_plan", "scenario_plans", "frozen_plan_refs"}); + EXPECT_EQ(resolved["component_identities"]["run_policy_version"], "llm-run-policy-bounded-loop-snapshots-v1"); + ASSERT_EQ(resolved["scenario_plans"].size(), 3u); + expect_exact_keys(resolved["frozen_plan_refs"], {"weights_only", "kv_only", "mixed"}); + for (size_t i = 0; i < 3; ++i) { + const auto& canonical = resolved["scenario_plans"][i]; + EXPECT_EQ(canonical["model_ref"], "resolved_plan"); + EXPECT_EQ(canonical["work_units"], 4u); + EXPECT_EQ(canonical["work_policy"], "explicit_fixed_work"); + const auto& expected = canonical["expected_checksum"]; + expect_exact_keys(expected, {"status", "reason_code", "expected_worker_checksums", "expected_run_checksum"}); + EXPECT_EQ(expected["status"], "available"); + ASSERT_EQ(expected["expected_worker_checksums"].size(), 2u); + expect_exact_keys(expected["expected_worker_checksums"][0], {"worker_index", "weight", "k", "v"}); + EXPECT_EQ(expected["expected_worker_checksums"][0]["weight"]["state_b_uint64_decimal"], "9007199254740993"); + EXPECT_EQ(expected["expected_run_checksum"]["state_a_uint64_decimal"], "18446744073709551615"); + } + for (size_t id = 0; id < doc["measurements"].size(); ++id) { + const auto& measurement = doc["measurements"][id]; + EXPECT_EQ(measurement["measurement_id"], id); + const auto& canonical = referenced_plan(doc, measurement); + EXPECT_EQ(canonical["scenario"], measurement["scenario"]); + EXPECT_EQ(measurement["completion_derivation"], "accepted-plan-derived"); + EXPECT_EQ(measurement["requested_workers"], 2u); + EXPECT_EQ(measurement["effective_workers"], 2u); + EXPECT_EQ(measurement["qos_successful_workers"], 2u); + EXPECT_EQ(measurement["qos_failed_workers"], 0u); + EXPECT_EQ(measurement["calibration_attempt_count"], 1u); + EXPECT_EQ(measurement["calibration_attempt_indexes"], OrderedJson::array({0})); + EXPECT_EQ(measurement["working_set"]["bytes"], "1048960"); + if (measurement["scenario"] == "mixed") { + EXPECT_DOUBLE_EQ(measurement["weight_payload_fraction"].get(), 1048576.0 / 1049088.0); + EXPECT_DOUBLE_EQ(measurement["kv_read_payload_fraction"].get(), 384.0 / 1049088.0); + EXPECT_DOUBLE_EQ(measurement["kv_write_payload_fraction"].get(), 128.0 / 1049088.0); + } else { + EXPECT_TRUE(measurement["weight_payload_fraction"].is_null()); + } + + expect_exact_keys(measurement["checksum"], {"status", "reason_code", "checksum_valid", "actual_worker_checksums", "actual_run_checksum"}); + EXPECT_EQ(measurement["checksum"]["actual_run_checksum"], canonical["expected_checksum"]["expected_run_checksum"]); + for (const auto& check : measurement["execution"]["validation"]["checks"]) { + expect_exact_keys(check, {"kind", "applicable", "evaluated", "valid", "reason_code"}); + if (check["applicable"] == true) { EXPECT_TRUE(check["evaluated"]); EXPECT_TRUE(check["valid"]); } + else { EXPECT_TRUE(check["evaluated"].is_null()); EXPECT_TRUE(check["valid"].is_null()); } + } + } + for (const char* scenario : {"weights_only", "kv_only", "mixed"}) { + const auto& attempt = doc["calibration"]["attempts"][scenario][0]; + EXPECT_EQ(referenced_plan(doc, attempt)["scenario"], scenario); + EXPECT_EQ(attempt["execution"]["checksum"]["actual_run_checksum"], + referenced_plan(doc, attempt)["expected_checksum"]["expected_run_checksum"]); + ASSERT_EQ(attempt["execution"]["validation"]["checks"].size(), 3u); + } + const auto& manifest = doc["build_manifest"]; + expect_exact_keys(manifest, {"manifest_version", "status", "reason_code", "binary_sha256", "git_commit", "git_dirty", + "compiler", "compile_flags", "link_flags", "target_arch", "sdk", "min_os"}); + EXPECT_EQ(manifest["manifest_version"], 1); + EXPECT_EQ(manifest["status"], "unavailable"); + EXPECT_EQ(manifest["reason_code"], "build-provenance-not-provided"); + for (const char* key : {"binary_sha256", "git_commit", "git_dirty", "compiler", "compile_flags", "link_flags", "target_arch", "sdk", "min_os"}) + EXPECT_TRUE(manifest[key].is_null()) << key; + EXPECT_TRUE(resolved["model_context"]["prefill"].is_null()); +} + +TEST(LlmMemoryJsonTest, UnknownAndContradictorySnapshotReferencesAreRejected) { + const auto config = explicit_config(1); + const auto plan = admitted_plan(config); + ASSERT_TRUE(plan.valid); + auto result = complete_result(config, plan); + const auto saved = result.measurements[0].plan_handle; + result.measurements[0].plan_handle = result.scenario_plans.size(); + EXPECT_THROW(build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result), std::invalid_argument); + result.measurements[0].plan_handle = saved; + result.snapshot_plan_refs[saved] = result.snapshot_plan_order.size(); + EXPECT_THROW(build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result), std::invalid_argument); + prepare_llm_result_snapshot(result); + auto swapped = result; + std::swap(swapped.frozen_plan_handles[0], swapped.frozen_plan_handles[1]); + EXPECT_THROW(build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), swapped), std::invalid_argument); + swapped = result; + swapped.calibration_attempts[0][0].plan_handle = swapped.frozen_plan_handles[1]; + EXPECT_THROW(build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), swapped), std::invalid_argument); + swapped = result; + swapped.measurements[0].plan_handle = swapped.frozen_plan_handles[1]; + EXPECT_THROW(build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), swapped), std::invalid_argument); + result.calibration_attempts[0][0].plan_handle = result.scenario_plans.size(); + EXPECT_THROW(build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result), std::invalid_argument); +} + +TEST(LlmMemoryJsonTest, CanonicalInsertionDeduplicatesAndRejectsContradictoryWork) { + const auto config = explicit_config(1); + const auto plan = admitted_plan(config); + ASSERT_TRUE(plan.valid); + FakeLlmBackend backend; + LlmMemoryResult result; + const auto mixed = build_llm_scenario_work_plan(plan, LlmScenario::Mixed, 4, true); + const auto handle = register_llm_scenario_plan(result, plan, mixed, backend); + EXPECT_EQ(register_llm_scenario_plan(result, plan, mixed, backend), handle); + auto changed = mixed; + ++changed.effective_model_payload_bytes; + EXPECT_THROW(register_llm_scenario_plan(result, plan, changed, backend), std::invalid_argument); + const auto warm = build_llm_scenario_work_plan(plan, LlmScenario::WeightsOnly, 1, false); + const auto warm_handle = register_llm_scenario_plan(result, plan, warm, backend); + prepare_llm_result_snapshot(result); + EXPECT_EQ(result.snapshot_plan_refs[warm_handle], 0u); + EXPECT_EQ(result.snapshot_plan_refs[handle], 1u); + EXPECT_EQ(result.scenario_plans.size(), 2u); +} + +TEST(LlmMemoryJsonTest, OneLoopAcceptanceIsIndependentOfBalanceAndObservedCvQuality) { + const auto config = explicit_config(1); + const auto plan = admitted_plan(config); + ASSERT_TRUE(plan.valid); + const auto result = complete_result(config, plan); + const auto doc = build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); + EXPECT_TRUE(doc["run_accepted"]); + EXPECT_FALSE(doc["scenario_order_balance_complete"]); + EXPECT_EQ(doc["aggregates"]["scenarios"]["weights_only"]["observed_cv_classification"], "insufficient-samples"); + EXPECT_EQ(doc["aggregates"]["scenarios"]["weights_only"]["accepted_measurement_ids"], OrderedJson::array({0})); + for (const char* metric : {"synthetic_work_unit_latency_seconds", "synthetic_memory_work_units_per_second", + "effective_model_payload_gb_s"}) { + const auto& statistics = doc["aggregates"]["scenarios"]["weights_only"][metric]["statistics"]; + EXPECT_EQ(statistics["sample_count"], 1u); + EXPECT_DOUBLE_EQ(statistics["stddev"].get(), 0.0); + EXPECT_DOUBLE_EQ(statistics["coefficient_of_variation_pct"].get(), 0.0); + EXPECT_DOUBLE_EQ(statistics["median_absolute_deviation"].get(), 0.0); + } +} + +TEST(LlmMemoryJsonTest, SortedSnapshotRemapsFrozenMeasurementAndCalibrationHandlesTogether) { + const auto config = explicit_config(1); + const auto plan = admitted_plan(config); + ASSERT_TRUE(plan.valid); + FakeLlmBackend backend; + LlmMemoryResult result; + const auto t8 = build_llm_scenario_work_plan(plan, LlmScenario::Mixed, 8, false); + const auto t2 = build_llm_scenario_work_plan(plan, LlmScenario::Mixed, 2, false); + const auto handle8 = register_llm_scenario_plan(result, plan, t8, backend); + const auto handle2 = register_llm_scenario_plan(result, plan, t2, backend); + EXPECT_EQ(register_llm_scenario_plan(result, plan, t8, backend), handle8); + result.frozen_plan_handles[static_cast(LlmScenario::Mixed)] = handle8; + LlmMeasurementState measurement; + measurement.scenario = LlmScenario::Mixed; + measurement.plan_handle = handle8; + result.measurements.push_back(std::move(measurement)); + LlmCalibrationAttempt attempt; + attempt.scenario = LlmScenario::Mixed; + attempt.plan_handle = handle2; + result.calibration_attempts[static_cast(LlmScenario::Mixed)].push_back(std::move(attempt)); + prepare_llm_result_snapshot(result); + const auto doc = build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); + ASSERT_EQ(doc["resolved_plan"]["scenario_plans"].size(), 2u); + EXPECT_EQ(doc["resolved_plan"]["scenario_plans"][0]["work_units"], 2u); + EXPECT_EQ(doc["resolved_plan"]["scenario_plans"][1]["work_units"], 8u); + EXPECT_EQ(doc["resolved_plan"]["frozen_plan_refs"]["mixed"], 1u); + EXPECT_EQ(doc["measurements"][0]["plan_ref"], 1u); + EXPECT_EQ(doc["calibration"]["attempts"]["mixed"][0]["plan_ref"], 0u); + EXPECT_EQ(result.measurements[0].plan_handle, handle8); + EXPECT_EQ(result.calibration_attempts[static_cast(LlmScenario::Mixed)][0].plan_handle, handle2); +} + +TEST(LlmMemoryJsonTest, SnapshotStatisticsUseTheSameAcceptedIdsAndMedianOfDerivedRates) { + const auto config = explicit_config(3); + const auto plan = admitted_plan(config); + ASSERT_TRUE(plan.valid); + auto result = complete_result(config, plan); + ASSERT_EQ(result.measurements[0].scenario, LlmScenario::WeightsOnly); + ASSERT_EQ(result.measurements[5].scenario, LlmScenario::WeightsOnly); + ASSERT_EQ(result.measurements[7].scenario, LlmScenario::WeightsOnly); + result.measurements[0].execution.timing.elapsed_seconds = 1.0; + result.measurements[5].execution.timing.elapsed_seconds = 3.0; + result.measurements[7].status = LlmMeasurementStatus::Invalid; + prepare_llm_result_snapshot(result); + const auto doc = build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); + const auto& aggregate = doc["aggregates"]["scenarios"]["weights_only"]; + EXPECT_EQ(aggregate["accepted_measurement_ids"], OrderedJson::array({0, 5})); + for (const char* metric : {"synthetic_work_unit_latency_seconds", "synthetic_memory_work_units_per_second", "effective_model_payload_gb_s"}) + EXPECT_EQ(aggregate[metric]["sample_count"], 2u); + EXPECT_DOUBLE_EQ(aggregate["synthetic_memory_work_units_per_second"]["statistics"]["median"].get(), (4.0 + 4.0 / 3.0) / 2.0); + EXPECT_TRUE(doc["measurements"][7]["effective_model_payload_gb_s"].is_null()); +} + TEST(LlmMemoryJsonTest, CompleteContiguousPrefillDocumentHasExactGeometryAndPartitionEvidence) { const LlmMemoryConfig config = prefill_config(); const LlmMemoryWorkPlan plan = admitted_plan(config); @@ -1897,10 +1772,10 @@ TEST(LlmMemoryJsonTest, CompleteContiguousPrefillDocumentHasExactGeometryAndPart EXPECT_EQ(document["status"], "complete"); EXPECT_TRUE(document["results_complete"]); - EXPECT_TRUE(document["conclusions_valid"]); + EXPECT_TRUE(document["run_accepted"]); EXPECT_EQ(document["phase"], "prefill"); EXPECT_EQ(document["kv_layout"], "contiguous"); - EXPECT_EQ(document["methodology_version"], "llm-memory-v1-cpu-prefill-contiguous"); + EXPECT_EQ(document["methodology_version"], "llm-memory-v2-cpu-prefill-contiguous"); const OrderedJson& configuration = document["configuration"]; EXPECT_EQ(configuration["phase"], "prefill"); @@ -1916,15 +1791,14 @@ TEST(LlmMemoryJsonTest, CompleteContiguousPrefillDocumentHasExactGeometryAndPart EXPECT_TRUE(geometry["decode"].is_null()); expect_exact_keys( geometry["prefill"], - {"prompt_tokens", "attention_query_tile_tokens", "tile_count", "attention_prefix_token_visits_per_sequence", - "causal_token_pairs_per_sequence", "logical_attention_pairs", "logical_attention_fma_terms"}); + {"prompt_tokens", "attention_query_tile_tokens", "tile_count", "attention_prefix_token_visits_per_sequence"}); EXPECT_EQ(geometry["prefill"]["prompt_tokens"], 5u); EXPECT_EQ(geometry["prefill"]["attention_query_tile_tokens"], 2u); EXPECT_EQ(geometry["prefill"]["tile_count"], "3"); EXPECT_EQ(geometry["prefill"]["attention_prefix_token_visits_per_sequence"], "11"); - EXPECT_EQ(geometry["prefill"]["causal_token_pairs_per_sequence"], "15"); - EXPECT_EQ(geometry["prefill"]["logical_attention_pairs"], "120"); - EXPECT_EQ(geometry["prefill"]["logical_attention_fma_terms"], "960"); + EXPECT_EQ(document["resolved_plan"]["model_context"]["prefill"]["causal_token_pairs_per_sequence"], "15"); + EXPECT_EQ(document["resolved_plan"]["model_context"]["prefill"]["logical_attention_pairs"], "120"); + EXPECT_EQ(document["resolved_plan"]["model_context"]["prefill"]["logical_attention_fma_terms"], "960"); EXPECT_EQ(geometry["k_mapping_bytes"], "320"); EXPECT_EQ(geometry["v_mapping_bytes"], "320"); EXPECT_EQ(geometry["kv_capacity_bytes"], "640"); @@ -1950,7 +1824,6 @@ TEST(LlmMemoryJsonTest, CompleteContiguousPrefillDocumentHasExactGeometryAndPart EXPECT_EQ(components["backend_executor_version"], Constants::LLM_PREFILL_CPU_EXECUTOR_VERSION); EXPECT_EQ(components["resource_abi_version"], Constants::LLM_PREFILL_DESCRIPTOR_ABI_VERSION); EXPECT_EQ(components["write_pattern_version"], "llm-prefill-kv-affine64-v1"); - EXPECT_EQ(components["checksum_pattern_version"], "llm-prefill-affine64-parity-sum-v1"); const OrderedJson& prefill = document["backend_evidence"]["cpu"]["prefill"]; expect_exact_keys(prefill, {"cost_unit", "sequence_descriptors_per_scenario_per_worker", "scenarios", "identity"}); @@ -1981,7 +1854,7 @@ TEST(LlmMemoryJsonTest, CompleteContiguousPrefillDocumentHasExactGeometryAndPart } EXPECT_TRUE(document["backend_evidence"]["cpu"]["paged"].is_null()); - const OrderedJson& scenarios = document["resolved_plan"]["frozen_scenario_work_plans"]["scenarios"]; + const OrderedJson& scenarios = document["resolved_plan"]["scenario_plans"]; ASSERT_EQ(scenarios.size(), 3u); EXPECT_EQ(scenarios[0]["work_unit_kind"], "prefill_operation"); EXPECT_EQ(scenarios[0]["kv_write_kind"], "none"); @@ -1993,13 +1866,11 @@ TEST(LlmMemoryJsonTest, CompleteContiguousPrefillDocumentHasExactGeometryAndPart ASSERT_EQ(document["measurements"].size(), 9u); for (const OrderedJson& measurement : document["measurements"]) { EXPECT_EQ(measurement["status"], "measured"); - EXPECT_EQ(measurement["work_unit_kind"], "prefill_operation"); + EXPECT_EQ(referenced_plan(document, measurement)["work_unit_kind"], "prefill_operation"); EXPECT_TRUE(measurement["working_set"]["fixed_visible_context_tokens"].is_null()); EXPECT_TRUE(measurement["working_set"]["current_token_slot_included"].is_null()); EXPECT_EQ(measurement["checksum"]["status"], "valid"); EXPECT_TRUE(measurement["checksum"]["checksum_valid"]); - EXPECT_FALSE(measurement["checksum"].contains("append_pattern_version")); - EXPECT_FALSE(measurement["checksum"].contains("read_checksum_version")); } EXPECT_TRUE(document["interpretation"]["fixed_context_includes_current_token_slot"].is_null()); EXPECT_FALSE(document["interpretation"]["prefill_transformer_compute_or_ttft_prediction_included"]); @@ -2026,7 +1897,7 @@ TEST(LlmMemoryJsonTest, CompletePagedPrefillPublishesLayoutAndPrefillEvidenceWit const OrderedJson document = build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); - EXPECT_EQ(document["schema_version"], 1); + EXPECT_EQ(document["schema_version"], 2); EXPECT_EQ(document["status"], "complete"); EXPECT_EQ(document["phase"], "prefill"); EXPECT_EQ(document["kv_layout"], "paged"); @@ -2035,8 +1906,6 @@ TEST(LlmMemoryJsonTest, CompletePagedPrefillPublishesLayoutAndPrefillEvidenceWit Constants::LLM_PREFILL_PAGED_CPU_EXECUTOR_VERSION); EXPECT_EQ(document["resolved_plan"]["component_identities"]["resource_abi_version"], Constants::LLM_PREFILL_PAGED_DESCRIPTOR_ABI_VERSION); - EXPECT_EQ(document["resolved_plan"]["component_identities"]["checksum_pattern_version"], - LlmPrefillVersion::PAGED_CHECKSUM_ORACLE); const OrderedJson& layout = document["resolved_plan"]["layout"]; EXPECT_EQ(layout["kv_layout"], "paged"); @@ -2065,16 +1934,15 @@ TEST(LlmMemoryJsonTest, CompletePagedPrefillPublishesLayoutAndPrefillEvidenceWit ASSERT_TRUE(kv_only_limits.valid) << kv_only_limits.reason_code; ASSERT_GE(document["measurements"].size(), 2u); const OrderedJson& kv_only = document["measurements"][1]; - EXPECT_EQ(kv_only["work_unit_kind"], "prefill_operation"); - EXPECT_EQ(kv_only["kv_write_kind"], "full_prompt_population"); - EXPECT_EQ(kv_only["layout_metadata_lookup_count_per_work_unit"], + EXPECT_EQ(referenced_plan(document, kv_only)["work_unit_kind"], "prefill_operation"); + EXPECT_EQ(referenced_plan(document, kv_only)["kv_write_kind"], "full_prompt_population"); + EXPECT_EQ(referenced_plan(document, kv_only)["layout_metadata_lookup_count_per_work_unit"], std::to_string(kv_only_limits.layout_metadata_lookup_count_per_work_unit)); - EXPECT_EQ(kv_only["layout_metadata_read_bytes_per_work_unit"], + EXPECT_EQ(referenced_plan(document, kv_only)["layout_metadata_read_bytes_per_work_unit"], std::to_string(kv_only_limits.layout_metadata_read_bytes_per_work_unit)); - EXPECT_EQ(kv_only["accounted_bytes_per_work_unit"], std::to_string(kv_only_limits.accounted_bytes_per_work_unit)); - EXPECT_EQ(kv_only["checksum"]["initialization_pattern_version"], Constants::LLM_PAGED_BUFFER_PATTERN_VERSION); - EXPECT_EQ(kv_only["checksum"]["write_pattern_version"], LlmPrefillVersion::WRITE_PATTERN); - EXPECT_EQ(kv_only["checksum"]["checksum_pattern_version"], LlmPrefillVersion::PAGED_CHECKSUM_ORACLE); + EXPECT_EQ(referenced_plan(document, kv_only)["accounted_bytes_per_work_unit"], std::to_string(kv_only_limits.accounted_bytes_per_work_unit)); + EXPECT_EQ(document["resolved_plan"]["component_identities"]["buffer_pattern_version"], Constants::LLM_PAGED_BUFFER_PATTERN_VERSION); + EXPECT_EQ(document["resolved_plan"]["component_identities"]["write_pattern_version"], LlmPrefillVersion::WRITE_PATTERN); } TEST(LlmMemoryJsonTest, PagedDocumentPublishesExactLayoutResourceAndOwnershipProvenance) { @@ -2089,7 +1957,7 @@ TEST(LlmMemoryJsonTest, PagedDocumentPublishesExactLayoutResourceAndOwnershipPro const LlmResourcePreparationResult preparation = preparation_for(plan); const OrderedJson document = build_llm_memory_json(config, plan, preparation, fixed_metadata(config, plan), result); - EXPECT_EQ(document["schema_version"], 1); + EXPECT_EQ(document["schema_version"], 2); EXPECT_EQ(document["kv_layout"], "paged"); EXPECT_EQ(document["configuration"]["kv_layout"], "paged"); EXPECT_EQ(document["configuration"]["kv_block_tokens"], 2u); @@ -2186,16 +2054,13 @@ TEST(LlmMemoryJsonTest, PagedDocumentPublishesExactLayoutResourceAndOwnershipPro ASSERT_GE(document["measurements"].size(), 2u); const OrderedJson& kv_only = document["measurements"][1]; ASSERT_EQ(kv_only["scenario"], "kv_only"); - EXPECT_EQ(kv_only["layout_metadata_lookup_count_per_work_unit"], "10"); - EXPECT_EQ(kv_only["layout_metadata_read_bytes_per_work_unit"], "40"); - EXPECT_EQ(kv_only["effective_model_payload_bytes_per_work_unit"], "512"); - EXPECT_EQ(kv_only["accounted_bytes_per_work_unit"], "552"); - EXPECT_TRUE(kv_only["execution"]["post_validation_evaluated"]); - EXPECT_TRUE(kv_only["execution"]["post_validation_valid"]); - EXPECT_EQ(kv_only["checksum"]["initialization_pattern_version"], Constants::LLM_PAGED_BUFFER_PATTERN_VERSION); - EXPECT_EQ(kv_only["checksum"]["checksum_pattern_version"], Constants::LLM_PAGED_READ_CHECKSUM_VERSION); + EXPECT_EQ(referenced_plan(document, kv_only)["layout_metadata_lookup_count_per_work_unit"], "10"); + EXPECT_EQ(referenced_plan(document, kv_only)["layout_metadata_read_bytes_per_work_unit"], "40"); + EXPECT_EQ(referenced_plan(document, kv_only)["effective_model_payload_bytes_per_work_unit"], "512"); + EXPECT_EQ(referenced_plan(document, kv_only)["accounted_bytes_per_work_unit"], "552"); + EXPECT_EQ(document["resolved_plan"]["component_identities"]["buffer_pattern_version"], Constants::LLM_PAGED_BUFFER_PATTERN_VERSION); ASSERT_FALSE(document["calibration"]["attempts"]["weights_only"].empty()); - EXPECT_EQ(document["calibration"]["attempts"]["weights_only"][0]["execution"]["checksum"]["algorithm_version"], + EXPECT_EQ(document["resolved_plan"]["component_identities"]["checksum_pattern_version"], Constants::LLM_PAGED_READ_CHECKSUM_VERSION); } @@ -2331,7 +2196,7 @@ TEST(LlmMemoryJsonTest, PreflightAuxiliaryEstimatesExactlyMatchFinalizedPlanForA } } -TEST(LlmMemoryJsonTest, SyntheticPreflightIdentityCopiesScaleByLoopAndCalibrationCapacity) { +TEST(LlmMemoryJsonTest, CanonicalIdentityBudgetIsIndependentOfLoopCountAndReservesCalibrationPlans) { LlmAuxiliaryPreflightView preflight; preflight.valid = true; preflight.backend = LlmMemoryBackend::Cpu; @@ -2364,7 +2229,7 @@ TEST(LlmMemoryJsonTest, SyntheticPreflightIdentityCopiesScaleByLoopAndCalibratio config.user_specified_iterations ? 1 : 4 + Constants::LLM_CALIBRATION_MAX_CORRECTIONS; const size_t raw_identity_and_input_bytes = config.output_file.size() + argv_bytes + preflight.json_identity_string_bytes + frozen_identity_bytes + - config.loop_count * scenario_identity_bytes + calibration_attempts * scenario_identity_bytes; + scenario_identity_bytes + calibration_attempts * scenario_identity_bytes; return raw_identity_and_input_bytes * kDomTransportExpansionFactor; }; @@ -2380,7 +2245,7 @@ TEST(LlmMemoryJsonTest, SyntheticPreflightIdentityCopiesScaleByLoopAndCalibratio EXPECT_EQ(four_loop_estimate.input_string_bytes, expected_input_string_bytes(four_loops)); constexpr size_t kScenarioIdentityBytes = 2 + 3 + 5; EXPECT_EQ(four_loop_estimate.input_string_bytes - one_loop_estimate.input_string_bytes, - 3 * kScenarioIdentityBytes * kDomTransportExpansionFactor); + 0u); LlmMemoryConfig automatic = four_loops; automatic.user_specified_iterations = false; @@ -2503,7 +2368,8 @@ TEST(LlmMemoryJsonTest, JsonPeakEstimateAccountsForRetainedMetalMeasurementAndCa const LlmJsonPeakEstimate one_loop_estimate = calculate_llm_json_peak_estimate(one_loop, plan); ASSERT_TRUE(one_loop_estimate.valid) << one_loop_estimate.reason_code; - constexpr size_t kExplicitTaskRecords = 2 * kLlmScenarioCount; + // One measurement, one excluded task and two canonical-plan capacity slots per scenario. + constexpr size_t kExplicitTaskRecords = 4 * kLlmScenarioCount; ASSERT_EQ(one_loop_estimate.measurement_record_bytes % kExplicitTaskRecords, 0u); const size_t metal_task_record_bytes = one_loop_estimate.measurement_record_bytes / kExplicitTaskRecords; EXPECT_GT(metal_task_record_bytes, 64u * 1024u); @@ -2520,7 +2386,7 @@ TEST(LlmMemoryJsonTest, JsonPeakEstimateAccountsForRetainedMetalMeasurementAndCa const LlmJsonPeakEstimate automatic_estimate = calculate_llm_json_peak_estimate(automatic, plan); ASSERT_TRUE(automatic_estimate.valid) << automatic_estimate.reason_code; EXPECT_EQ(automatic_estimate.measurement_record_bytes - four_loop_estimate.measurement_record_bytes, - (3 + Constants::LLM_CALIBRATION_MAX_CORRECTIONS) * kLlmScenarioCount * metal_task_record_bytes); + 2 * (3 + Constants::LLM_CALIBRATION_MAX_CORRECTIONS) * kLlmScenarioCount * metal_task_record_bytes); } TEST(LlmMemoryJsonTest, JsonPeakEstimateRejectsCountArithmeticOverflow) { @@ -2546,30 +2412,21 @@ TEST(LlmMemoryJsonTest, ExactByteSeedAndChecksumIntegersAreCanonicalDecimalStrin const OrderedJson document = build_llm_memory_json(config, plan, preparation_for(plan), metadata, result); const OrderedJson& measurement = document["measurements"][0]; - EXPECT_EQ(measurement["work_unit_kind"], "decode_step"); - EXPECT_EQ(measurement["kv_write_kind"], "none"); - EXPECT_TRUE(measurement["planned_work_units"].is_number_unsigned()); + EXPECT_EQ(referenced_plan(document, measurement)["work_unit_kind"], "decode_step"); + EXPECT_EQ(referenced_plan(document, measurement)["kv_write_kind"], "none"); + EXPECT_TRUE(referenced_plan(document, measurement)["work_units"].is_number_unsigned()); EXPECT_TRUE(measurement["completed_work_units"].is_number_unsigned()); - const std::array decimal_measurement_fields = {"weight_read_bytes_per_work_unit", - "kv_read_bytes_per_work_unit", - "kv_write_bytes_per_work_unit", - "effective_model_payload_bytes_per_work_unit", - "layout_metadata_lookup_count_per_work_unit", - "layout_metadata_read_bytes_per_work_unit", - "accounted_bytes_per_work_unit", - "planned_effective_model_payload_bytes", - "completed_effective_model_payload_bytes", - "planned_layout_metadata_lookup_count", - "completed_layout_metadata_lookup_count", - "planned_layout_metadata_read_bytes", - "completed_layout_metadata_read_bytes", - "planned_task_accounted_bytes", - "completed_task_accounted_bytes"}; - for (const char* field : decimal_measurement_fields) { + for (const char* field : {"completed_effective_model_payload_bytes", "completed_layout_metadata_lookup_count", + "completed_layout_metadata_read_bytes", "completed_task_accounted_bytes"}) { EXPECT_TRUE(measurement[field].is_string()) << field; } - EXPECT_EQ(measurement["layout_metadata_lookup_count_per_work_unit"], "0"); - EXPECT_EQ(measurement["layout_metadata_read_bytes_per_work_unit"], "0"); + for (const char* field : {"weight_read_bytes_per_work_unit", "kv_read_bytes_per_work_unit", "kv_write_bytes_per_work_unit", + "effective_model_payload_bytes", "layout_metadata_lookup_count", "layout_metadata_read_bytes", + "task_accounted_bytes"}) { + EXPECT_TRUE(referenced_plan(document, measurement)[field].is_string()) << field; + } + EXPECT_EQ(referenced_plan(document, measurement)["layout_metadata_lookup_count_per_work_unit"], "0"); + EXPECT_EQ(referenced_plan(document, measurement)["layout_metadata_read_bytes_per_work_unit"], "0"); EXPECT_TRUE(measurement["synthetic_work_unit_latency_seconds"].is_number()); EXPECT_TRUE(measurement["synthetic_memory_work_units_per_second"].is_number()); EXPECT_TRUE(measurement["effective_model_payload_gb_s"].is_number()); @@ -2608,12 +2465,10 @@ TEST(LlmMemoryJsonTest, ExactByteSeedAndChecksumIntegersAreCanonicalDecimalStrin const OrderedJson& checksum = document["measurements"][0]["checksum"]; ASSERT_EQ(checksum["status"], "valid"); ASSERT_TRUE(checksum["checksum_valid"]); - EXPECT_TRUE(document["measurements"][0]["execution"]["post_validation_evaluated"]); - EXPECT_TRUE(document["measurements"][0]["execution"]["post_validation_valid"]); - EXPECT_EQ(checksum["expected_worker_checksums"][0]["weight"]["state_a_uint64_decimal"], "18446744073709551615"); - EXPECT_EQ(checksum["expected_worker_checksums"][0]["weight"]["state_b_uint64_decimal"], "9007199254740993"); - EXPECT_EQ(checksum["expected_worker_checksums"][0]["weight"]["exact_bytes_read"], "9007199254741093"); - EXPECT_TRUE(checksum["expected_worker_checksums"][0]["weight"]["span_count_uint64_decimal"].is_string()); + EXPECT_EQ(referenced_plan(document, measurement)["expected_checksum"]["expected_worker_checksums"][0]["weight"]["state_a_uint64_decimal"], "18446744073709551615"); + EXPECT_EQ(referenced_plan(document, measurement)["expected_checksum"]["expected_worker_checksums"][0]["weight"]["state_b_uint64_decimal"], "9007199254740993"); + EXPECT_EQ(referenced_plan(document, measurement)["expected_checksum"]["expected_worker_checksums"][0]["weight"]["exact_bytes_read"], "9007199254741093"); + EXPECT_TRUE(referenced_plan(document, measurement)["expected_checksum"]["expected_worker_checksums"][0]["weight"]["span_count_uint64_decimal"].is_string()); EXPECT_EQ(checksum["actual_run_checksum"]["state_a_uint64_decimal"], "18446744073709551615"); EXPECT_TRUE(document["counters"]["planned_work_units"].is_string()); EXPECT_TRUE(document["counters"]["completed_effective_model_payload_bytes"].is_string()); @@ -2637,13 +2492,13 @@ TEST(LlmMemoryJsonTest, InterruptedRunnerSerializesUnavailableMetricsExecutionQo build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); EXPECT_EQ(document["status"], "interrupted"); EXPECT_FALSE(document["results_complete"]); - EXPECT_FALSE(document["conclusions_valid"]); + EXPECT_FALSE(document["run_accepted"]); const OrderedJson& serialized = document["measurements"][0]; EXPECT_EQ(serialized["status"], "interrupted"); EXPECT_EQ(serialized["reason_code"], LlmRunnerReason::INTERRUPTION_BEFORE_TASK); EXPECT_FALSE(serialized["attempted"]); - EXPECT_TRUE(serialized["qos_successful_workers"].is_null()); - EXPECT_TRUE(serialized["qos_failed_workers"].is_null()); + EXPECT_TRUE(serialized["execution"]["qos_successful_workers"].is_null()); + EXPECT_TRUE(serialized["execution"]["qos_failed_workers"].is_null()); EXPECT_EQ(serialized["completed_work_units"], 0u); EXPECT_EQ(serialized["completed_effective_model_payload_bytes"], "0"); EXPECT_TRUE(serialized["elapsed_seconds"].is_null()); @@ -2652,18 +2507,12 @@ TEST(LlmMemoryJsonTest, InterruptedRunnerSerializesUnavailableMetricsExecutionQo EXPECT_TRUE(serialized["effective_model_payload_gb_s"].is_null()); EXPECT_EQ(serialized["execution"]["status"], "not_run"); EXPECT_TRUE(serialized["execution"]["valid"].is_null()); - EXPECT_TRUE(serialized["execution"]["post_validation_evaluated"].is_null()); - EXPECT_TRUE(serialized["execution"]["post_validation_valid"].is_null()); - EXPECT_TRUE(serialized["execution"]["kv_write_validation_applicable"].is_null()); - EXPECT_TRUE(serialized["execution"]["kv_write_validation_evaluated"].is_null()); - EXPECT_TRUE(serialized["execution"]["kv_write_validation_valid"].is_null()); EXPECT_EQ(serialized["checksum"]["status"], "not_evaluated"); EXPECT_EQ(serialized["checksum"]["reason_code"], LlmRunnerReason::INTERRUPTION_BEFORE_TASK); EXPECT_TRUE(serialized["checksum"]["checksum_valid"].is_null()); - EXPECT_TRUE(serialized["checksum"]["expected_worker_checksums"].is_null()); EXPECT_TRUE(serialized["checksum"]["actual_run_checksum"].is_null()); // The immutable frozen plan remains available despite the interruption. - EXPECT_TRUE(serialized["planned_effective_model_payload_bytes"].is_string()); + EXPECT_TRUE(referenced_plan(document, serialized)["effective_model_payload_bytes"].is_string()); } TEST(LlmMemoryJsonTest, ExecutorExceptionUsesRunnerReasonAndNullUnavailableExecutionEvidence) { @@ -2690,18 +2539,13 @@ TEST(LlmMemoryJsonTest, ExecutorExceptionUsesRunnerReasonAndNullUnavailableExecu EXPECT_TRUE(measurement["attempted"]); EXPECT_EQ(measurement["status"], "failed"); EXPECT_EQ(measurement["reason_code"], LlmRunnerReason::RUNNER_EXCEPTION); - EXPECT_TRUE(measurement["qos_successful_workers"].is_null()); - EXPECT_TRUE(measurement["qos_failed_workers"].is_null()); + EXPECT_TRUE(measurement["execution"]["qos_successful_workers"].is_null()); + EXPECT_TRUE(measurement["execution"]["qos_failed_workers"].is_null()); EXPECT_EQ(measurement["execution"]["status"], "unavailable"); EXPECT_EQ(measurement["execution"]["reason_code"], LlmRunnerReason::RUNNER_EXCEPTION); EXPECT_TRUE(measurement["execution"]["valid"].is_null()); EXPECT_TRUE(measurement["execution"]["requested_workers"].is_null()); EXPECT_TRUE(measurement["execution"]["kernel_succeeded"].is_null()); - EXPECT_TRUE(measurement["execution"]["post_validation_evaluated"].is_null()); - EXPECT_TRUE(measurement["execution"]["post_validation_valid"].is_null()); - EXPECT_TRUE(measurement["execution"]["kv_write_validation_applicable"].is_null()); - EXPECT_TRUE(measurement["execution"]["kv_write_validation_evaluated"].is_null()); - EXPECT_TRUE(measurement["execution"]["kv_write_validation_valid"].is_null()); EXPECT_EQ(measurement["checksum"]["status"], "not_evaluated"); EXPECT_EQ(measurement["checksum"]["reason_code"], LlmRunnerReason::RUNNER_EXCEPTION); EXPECT_TRUE(measurement["checksum"]["checksum_valid"].is_null()); @@ -2764,9 +2608,7 @@ TEST(LlmMemoryJsonTest, InvalidElapsedMeasurementLeavesChecksumEvidenceNotEvalua EXPECT_EQ(measurement["checksum"]["status"], "not_evaluated"); EXPECT_EQ(measurement["checksum"]["reason_code"], LlmExecutorReason::INVALID_ELAPSED_TIME); EXPECT_TRUE(measurement["checksum"]["checksum_valid"].is_null()); - EXPECT_TRUE(measurement["checksum"]["expected_worker_checksums"].is_null()); EXPECT_TRUE(measurement["checksum"]["actual_worker_checksums"].is_null()); - EXPECT_TRUE(measurement["checksum"]["expected_run_checksum"].is_null()); EXPECT_TRUE(measurement["checksum"]["actual_run_checksum"].is_null()); } @@ -2798,7 +2640,6 @@ TEST(LlmMemoryJsonTest, InvalidElapsedExcludedTaskLeavesCompactChecksumEvidenceN EXPECT_EQ(execution["checksum"]["status"], "not_evaluated"); EXPECT_EQ(execution["checksum"]["reason_code"], LlmExecutorReason::INVALID_ELAPSED_TIME); EXPECT_TRUE(execution["checksum"]["checksum_valid"].is_null()); - EXPECT_TRUE(execution["checksum"]["expected_run_checksum"].is_null()); EXPECT_TRUE(execution["checksum"]["actual_run_checksum"].is_null()); } @@ -2831,7 +2672,6 @@ TEST(LlmMemoryJsonTest, MalformedExcludedChecksumCardinalityDoesNotPublishCompac EXPECT_EQ(execution["checksum"]["status"], "not_evaluated"); EXPECT_EQ(execution["checksum"]["reason_code"], LlmExecutorReason::INVALID_RESOURCES); EXPECT_TRUE(execution["checksum"]["checksum_valid"].is_null()); - EXPECT_TRUE(execution["checksum"]["expected_run_checksum"].is_null()); EXPECT_TRUE(execution["checksum"]["actual_run_checksum"].is_null()); } @@ -2858,7 +2698,7 @@ TEST(LlmMemoryJsonTest, ChecksumMismatchSerializesEvaluatedFalseInsteadOfMissing EXPECT_EQ(measurement["checksum"]["status"], "invalid"); EXPECT_EQ(measurement["checksum"]["reason_code"], LlmExecutorReason::CHECKSUM_MISMATCH); EXPECT_FALSE(measurement["checksum"]["checksum_valid"]); - EXPECT_TRUE(measurement["checksum"]["expected_worker_checksums"].is_array()); + EXPECT_TRUE(referenced_plan(document, measurement)["expected_checksum"]["expected_worker_checksums"].is_array()); EXPECT_TRUE(measurement["checksum"]["actual_worker_checksums"].is_array()); } @@ -2884,8 +2724,19 @@ TEST(LlmMemoryJsonTest, PagedPostValidationFailureSerializesEvaluatedInvalidEvid const OrderedJson& measurement = document["measurements"][0]; EXPECT_EQ(measurement["status"], "invalid"); EXPECT_EQ(measurement["reason_code"], LlmExecutorReason::PAGED_POST_VALIDATION_FAILED); - EXPECT_TRUE(measurement["execution"]["post_validation_evaluated"]); - EXPECT_FALSE(measurement["execution"]["post_validation_valid"]); + // This fixture supplies a conclusive failure for each applicable cold check. + // The serializer must retain those verdicts even though the task is excluded. + for (const char* kind : {"post-validation-structure", "kv-append-unchanged", "kv-padding-canary"}) { + const auto& check = named_check(measurement["execution"], kind); + EXPECT_TRUE(check["applicable"]); + EXPECT_TRUE(check["evaluated"]); + EXPECT_FALSE(check["valid"]); + EXPECT_EQ(check["reason_code"], LlmExecutorReason::PAGED_POST_VALIDATION_FAILED); + } + EXPECT_TRUE(measurement["elapsed_seconds"].is_null()); + EXPECT_DOUBLE_EQ(measurement["execution"]["timing"]["diagnostic_elapsed_seconds"].get(), 0.150); + EXPECT_TRUE(measurement["checksum"]["checksum_valid"]); + EXPECT_TRUE(measurement["effective_model_payload_gb_s"].is_null()); } TEST(LlmMemoryJsonTest, CommonAcceptanceFailureCannotSerializeCompleteBackendEvidenceAsValid) { @@ -2908,8 +2759,8 @@ TEST(LlmMemoryJsonTest, CommonAcceptanceFailureCannotSerializeCompleteBackendEvi EXPECT_FALSE(execution["valid"]); } -TEST(LlmMemoryJsonTest, MeasurementCheckpointSerializesPartialStatusAndUnavailableTailContract) { - const LlmMemoryConfig config = explicit_config(1); +TEST(LlmMemoryJsonTest, LoopSnapshotSerializesPartialStatusAndUnavailableTailContract) { + const LlmMemoryConfig config = explicit_config(2); const LlmMemoryWorkPlan plan = admitted_plan(config); ASSERT_TRUE(plan.valid) << plan.reason_code; FakeLlmBackend backend; @@ -2933,16 +2784,15 @@ TEST(LlmMemoryJsonTest, MeasurementCheckpointSerializesPartialStatusAndUnavailab EXPECT_EQ(document["reason_code"], LlmRunnerReason::PARTIAL_RESULTS); EXPECT_FALSE(document["interruption_requested"]); EXPECT_FALSE(document["results_complete"]); - EXPECT_FALSE(document["conclusions_valid"]); + EXPECT_FALSE(document["run_accepted"]); EXPECT_FALSE(document["scenario_order_balance_complete"]); - EXPECT_EQ(document["counters"]["planned_measurements"], 3u); - EXPECT_EQ(document["counters"]["attempted_measurements"], 1u); - EXPECT_EQ(document["counters"]["terminal_measurements"], 1u); - EXPECT_EQ(document["counters"]["measured_measurements"], 1u); - EXPECT_EQ(document["checkpoint_lifecycle"]["logical_checkpoint_attempts"], 1u); - EXPECT_EQ(document["checkpoint_lifecycle"]["successful_logical_checkpoints"], 1u); - EXPECT_FALSE(document["checkpoint_lifecycle"]["terminal_checkpoint_attempted"]); - EXPECT_FALSE(document["checkpoint_lifecycle"]["terminal_checkpoint_completed"]); + EXPECT_EQ(document["counters"]["planned_measurements"], 6u); + EXPECT_EQ(document["counters"]["attempted_measurements"], 3u); + EXPECT_EQ(document["counters"]["terminal_measurements"], 3u); + EXPECT_EQ(document["counters"]["measured_measurements"], 3u); + EXPECT_EQ(document["checkpoint_lifecycle"]["current_request"], "progress"); + EXPECT_EQ(document["checkpoint_lifecycle"]["snapshot_interval_loops"], 1u); + EXPECT_TRUE(document["checkpoint_lifecycle"]["current_persistence_success"].is_null()); size_t measured_count = 0; size_t not_run_count = 0; @@ -2963,7 +2813,7 @@ TEST(LlmMemoryJsonTest, MeasurementCheckpointSerializesPartialStatusAndUnavailab ++not_run_count; EXPECT_EQ(measurement["reason_code"], "not-run"); EXPECT_FALSE(measurement["attempted"]); - EXPECT_TRUE(measurement["qos_successful_workers"].is_null()); + EXPECT_TRUE(measurement["execution"]["qos_successful_workers"].is_null()); EXPECT_EQ(measurement["completed_work_units"], 0u); EXPECT_EQ(measurement["completed_effective_model_payload_bytes"], "0"); EXPECT_TRUE(measurement["elapsed_seconds"].is_null()); @@ -2976,12 +2826,12 @@ TEST(LlmMemoryJsonTest, MeasurementCheckpointSerializesPartialStatusAndUnavailab EXPECT_EQ(measurement["checksum"]["reason_code"], "not-run"); EXPECT_TRUE(measurement["checksum"]["checksum_valid"].is_null()); } - EXPECT_EQ(measured_count, 1u); - EXPECT_EQ(not_run_count, 2u); + EXPECT_EQ(measured_count, 3u); + EXPECT_EQ(not_run_count, 3u); } TEST(LlmMemoryJsonTest, CheckpointFailureRetainsMeasuredPrefixAndNullFailedTailWithoutRetry) { - const LlmMemoryConfig config = explicit_config(1); + const LlmMemoryConfig config = explicit_config(2); const LlmMemoryWorkPlan plan = admitted_plan(config); ASSERT_TRUE(plan.valid) << plan.reason_code; FakeLlmBackend backend; @@ -2995,15 +2845,11 @@ TEST(LlmMemoryJsonTest, CheckpointFailureRetainsMeasuredPrefixAndNullFailedTailW EXPECT_EQ(document["status"], "failed"); EXPECT_EQ(document["reason_code"], LlmRunnerReason::CHECKPOINT_WRITE_FAILED); EXPECT_TRUE(document["checkpoint_lifecycle"]["checkpoint_failed"]); - EXPECT_EQ(document["checkpoint_lifecycle"]["logical_checkpoint_attempts"], 1u); - EXPECT_EQ(document["checkpoint_lifecycle"]["successful_logical_checkpoints"], 0u); - EXPECT_FALSE(document["checkpoint_lifecycle"]["terminal_checkpoint_attempted"]); - EXPECT_FALSE(document["checkpoint_lifecycle"]["terminal_checkpoint_completed"]); EXPECT_EQ(document["measurements"][0]["status"], "measured"); - EXPECT_EQ(document["measurements"][1]["status"], "failed"); - EXPECT_TRUE(document["measurements"][1]["qos_successful_workers"].is_null()); - EXPECT_TRUE(document["measurements"][1]["effective_model_payload_gb_s"].is_null()); - EXPECT_TRUE(document["measurements"][1]["checksum"]["checksum_valid"].is_null()); + EXPECT_EQ(document["measurements"][3]["status"], "failed"); + EXPECT_TRUE(document["measurements"][3]["execution"]["qos_successful_workers"].is_null()); + EXPECT_TRUE(document["measurements"][3]["effective_model_payload_gb_s"].is_null()); + EXPECT_TRUE(document["measurements"][3]["checksum"]["checksum_valid"].is_null()); } TEST(LlmMemoryJsonTest, QualityWarningsMergeAndDeduplicateInStableConsoleAgreementOrder) { @@ -3015,7 +2861,7 @@ TEST(LlmMemoryJsonTest, QualityWarningsMergeAndDeduplicateInStableConsoleAgreeme for (LlmMeasurementState& measurement : result.measurements) { measurement.duration_quality = "above-target-single-work-unit"; } - result.measurements[0].qos_failed_workers = 1; + std::get(result.measurements[0].execution.backend_evidence).executor.qos_failed_workers = 1; LlmResultMetadata metadata = fixed_metadata(config, plan); metadata.environment_end.thermal_state = "serious"; @@ -3066,15 +2912,18 @@ TEST(LlmMemoryJsonTest, DecodeWriteValidationFailureRetainsInvalidAttemptAndPopu ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_FAILURE); const auto document = build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); const auto& weights = document["measurements"][0]; - EXPECT_FALSE(weights["execution"]["kv_write_validation_applicable"]); - EXPECT_TRUE(weights["execution"]["kv_write_validation_evaluated"].is_null()); - EXPECT_TRUE(weights["execution"]["kv_write_validation_valid"].is_null()); const auto& measurement = document["measurements"][1]; + const auto& absent_write = named_check(weights["execution"], "kv-append-unchanged"); + EXPECT_FALSE(absent_write["applicable"]); + EXPECT_TRUE(absent_write["evaluated"].is_null()); + EXPECT_TRUE(absent_write["valid"].is_null()); + const auto& failed_write = named_check(measurement["execution"], "kv-append-final"); + EXPECT_TRUE(failed_write["applicable"]); + EXPECT_TRUE(failed_write["evaluated"]); + EXPECT_FALSE(failed_write["valid"]); + EXPECT_EQ(measurement["status"], "invalid"); EXPECT_EQ(measurement["reason_code"], LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); - EXPECT_TRUE(measurement["execution"]["kv_write_validation_applicable"]); - EXPECT_TRUE(measurement["execution"]["kv_write_validation_evaluated"]); - EXPECT_FALSE(measurement["execution"]["kv_write_validation_valid"]); EXPECT_TRUE(measurement["checksum"]["checksum_valid"]); EXPECT_TRUE(measurement["synthetic_memory_work_units_per_second"].is_null()); EXPECT_TRUE(measurement["effective_model_payload_gb_s"].is_null()); @@ -3097,10 +2946,165 @@ TEST(LlmMemoryJsonTest, RequiredCpuWriteValidationNotRunIsFailedWithNullVerdict) ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_FAILURE); const auto document = build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); const auto& measurement = document["measurements"][1]; + const auto& unresolved = named_check(measurement["execution"], "kv-append-final"); + EXPECT_TRUE(unresolved["applicable"]); + EXPECT_FALSE(unresolved["evaluated"]); + EXPECT_TRUE(unresolved["valid"].is_null()); + EXPECT_EQ(measurement["status"], "failed"); - EXPECT_TRUE(measurement["execution"]["kv_write_validation_applicable"]); - EXPECT_FALSE(measurement["execution"]["kv_write_validation_evaluated"]); - EXPECT_TRUE(measurement["execution"]["kv_write_validation_valid"].is_null()); EXPECT_TRUE(measurement["synthetic_memory_work_units_per_second"].is_null()); EXPECT_TRUE(measurement["effective_model_payload_gb_s"].is_null()); } + +TEST(LlmMemoryJsonTest, UnresolvedPlanAndUnavailableExpectedWitnessHaveExplicitNulls) { + const auto config = explicit_config(1); + const auto plan = admitted_plan(config); + ASSERT_TRUE(plan.valid); + LlmMemoryResult result; + LlmCanonicalScenarioPlan entry; + entry.plan = build_llm_scenario_work_plan(plan, LlmScenario::KvOnly, 4, true); + result.scenario_plans.push_back(std::move(entry)); + LlmMeasurementState measurement; + measurement.scenario = LlmScenario::KvOnly; + result.measurements.push_back(std::move(measurement)); + prepare_llm_result_snapshot(result); + const auto doc = build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); + const auto& expected = doc["resolved_plan"]["scenario_plans"][0]["expected_checksum"]; + EXPECT_EQ(expected["status"], "unavailable"); + EXPECT_EQ(expected["reason_code"], "not-evaluated"); + EXPECT_TRUE(expected["expected_worker_checksums"].is_null()); + EXPECT_TRUE(expected["expected_run_checksum"].is_null()); + EXPECT_TRUE(doc["measurements"][0]["plan_ref"].is_null()); + EXPECT_TRUE(doc["resolved_plan"]["frozen_plan_refs"]["kv_only"].is_null()); + const auto& check = named_check(doc["measurements"][0]["execution"], "kv-append-final"); + EXPECT_TRUE(check["applicable"]); + EXPECT_FALSE(check["evaluated"]); + EXPECT_TRUE(check["valid"].is_null()); + EXPECT_FALSE(doc["run_accepted"]); +} + +TEST(LlmMemoryJsonTest, PrefillTheoreticalOverflowKeepsByteGeometryAndNullableReason) { + auto config = prefill_config(1); + config.query_head_count = size_t{1} << 52; + config.head_dimension = 256; + const auto plan = admitted_plan(config); + ASSERT_TRUE(plan.valid) << plan.reason_code; + LlmMemoryResult result; + const auto doc = build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); + const auto& context = doc["resolved_plan"]["model_context"]["prefill"]; + expect_exact_keys(context, {"causal_token_pairs_per_sequence", "causal_token_pairs_per_sequence_reason_code", + "logical_attention_pairs", "logical_attention_pairs_reason_code", "logical_attention_fma_terms", + "logical_attention_fma_terms_reason_code"}); + EXPECT_EQ(context["causal_token_pairs_per_sequence"], "15"); + EXPECT_EQ(context["logical_attention_pairs"], "135107988821114880"); + EXPECT_EQ(context["logical_attention_pairs_reason_code"], "valid"); + EXPECT_TRUE(context["logical_attention_fma_terms"].is_null()); + EXPECT_EQ(context["logical_attention_fma_terms_reason_code"], "arithmetic-overflow"); + EXPECT_EQ(doc["resolved_plan"]["geometry"]["prefill"]["attention_prefix_token_visits_per_sequence"], "11"); + EXPECT_EQ(doc["resolved_plan"]["geometry"]["kv_read_bytes_per_work_unit"], "45056"); +} + +TEST(LlmMemoryJsonTest, FileLifecycleReportsPriorObservationsAndUnresolvedCurrentPersistence) { + const auto config = explicit_config(1); + const auto plan = admitted_plan(config); + ASSERT_TRUE(plan.valid); + auto result = complete_result(config, plan); + result.prior_file_writer_attempts = 3; + result.prior_successful_file_writes = 2; + const auto doc = build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); + const auto& lifecycle = doc["checkpoint_lifecycle"]; + EXPECT_EQ(lifecycle["observation_point"], "before-current-snapshot-preparation"); + EXPECT_EQ(lifecycle["prior_file_writer_attempts"], 3u); + EXPECT_EQ(lifecycle["prior_successful_file_writes"], 2u); + EXPECT_TRUE(lifecycle["current_persistence_success"].is_null()); + EXPECT_EQ(lifecycle["checkpoint_policy"], "bounded-loop-snapshots"); +} + +TEST(LlmMemoryJsonTest, MissingNamedEvidenceCannotPassThroughSuccessfulAggregateFlags) { + const auto config = explicit_config(1); + const auto plan = admitted_plan(config); + ASSERT_TRUE(plan.valid); + FakeLlmBackend backend; + backend.omit_measured_cold_checks = true; + LlmMemoryResult result; + EXPECT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_FAILURE); + const auto doc = build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); + EXPECT_FALSE(doc["run_accepted"]); + const auto& kv = doc["measurements"][1]; + EXPECT_NE(kv["status"], "measured"); + const auto& check = named_check(kv["execution"], "kv-append-final"); + EXPECT_TRUE(check["applicable"]); + EXPECT_FALSE(check["evaluated"]); + EXPECT_TRUE(check["valid"].is_null()); + EXPECT_TRUE(kv["effective_model_payload_gb_s"].is_null()); +} + +TEST(LlmMemoryJsonTest, AutomaticCalibrationReferencesEveryDistinctPlanAndExpectedWitness) { + auto config = explicit_config(1); + config.user_specified_iterations = false; + config.iterations = 0; + const auto plan = admitted_plan(config); + ASSERT_TRUE(plan.valid); + FakeLlmBackend backend([](const LlmMemoryWorkPlan& model, const LlmScenarioWorkPlan& task, + const LlmRunnerTaskContext&) { + return successful_execution(model, static_cast(task.work_units) * 0.01); + }); + LlmMemoryResult result; + ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_SUCCESS); + const auto doc = build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); + ASSERT_GT(doc["resolved_plan"]["scenario_plans"].size(), 3u); + for (const char* scenario : {"weights_only", "kv_only", "mixed"}) { + const auto& attempts = doc["calibration"]["attempts"][scenario]; + ASSERT_GT(attempts.size(), 1u); + for (const auto& attempt : attempts) { + const auto& canonical = referenced_plan(doc, attempt); + EXPECT_EQ(canonical["scenario"], scenario); + EXPECT_EQ(canonical["work_policy"], "automatic_calibration"); + EXPECT_EQ(canonical["expected_checksum"]["status"], "available"); + ASSERT_EQ(canonical["expected_checksum"]["expected_worker_checksums"].size(), 2u); + EXPECT_EQ(canonical["expected_checksum"]["expected_run_checksum"], attempt["execution"]["checksum"]["actual_run_checksum"]); + } + } +} + +TEST(LlmMemoryJsonTest, AllEightProfilesPublishExactSchemaTwoSelectorsAndNamedKinds) { + using ConfigFactory = LlmMemoryConfig (*)(); + const std::array factories = { + +[]() { return explicit_config(1); }, +[]() { return paged_config(1); }, + +[]() { return prefill_config(1); }, +[]() { return paged_prefill_config(1); }, + explicit_metal_config, explicit_metal_paged_config, explicit_metal_prefill_config, + explicit_metal_paged_prefill_config}; + const std::array selectors = { + "llm-memory-v2-cpu-decode-contiguous", "llm-memory-v2-cpu-decode-paged", + "llm-memory-v2-cpu-prefill-contiguous", "llm-memory-v2-cpu-prefill-paged", + "llm-memory-v2-metal-decode-contiguous", "llm-memory-v2-metal-decode-paged", + "llm-memory-v2-metal-prefill-contiguous", "llm-memory-v2-metal-prefill-paged"}; + for (size_t i = 0; i < factories.size(); ++i) { + SCOPED_TRACE(selectors[i]); + const auto config = factories[i](); + const bool metal = config.backend == LlmMemoryBackend::Metal; + const auto plan = metal ? admitted_metal_plan(config) : admitted_plan(config); + ASSERT_TRUE(plan.valid) << plan.reason_code; + const auto result = metal ? metal_result_with_measurement_and_calibration(plan) : complete_result(config, plan); + const auto doc = metal + ? build_llm_memory_json(config, plan, complete_metal_backend_evidence(plan), fixed_metadata(config, plan), result) + : build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); + EXPECT_EQ(doc["schema_version"], 2); + EXPECT_EQ(doc["methodology_version"], selectors[i]); + EXPECT_EQ(doc["resolved_plan"]["component_identities"]["run_policy_version"], "llm-run-policy-bounded-loop-snapshots-v1"); + const auto& measurement = doc["measurements"][metal ? 0 : 1]; + const char* write_kind = config.phase == LlmPhase::Decode ? "kv-append-final" : "kv-prefill-final-samples"; + EXPECT_TRUE(named_check(measurement["execution"], write_kind)["valid"]); + EXPECT_TRUE(named_check(measurement["execution"], "post-validation-structure")["valid"]); + const auto& expected = referenced_plan(doc, measurement)["expected_checksum"]; + EXPECT_EQ(expected["status"], "available"); + if (metal) EXPECT_TRUE(expected["expected_worker_checksums"].is_null()); + else ASSERT_EQ(expected["expected_worker_checksums"].size(), 2u); + if (!metal && config.kv_layout == LlmKvLayout::Paged && config.phase == LlmPhase::Decode) { + const auto& unchanged = named_check(doc["measurements"][0]["execution"], "kv-append-unchanged"); + EXPECT_TRUE(unchanged["applicable"]); + EXPECT_TRUE(unchanged["evaluated"]); + EXPECT_TRUE(unchanged["valid"]); + } + } +} diff --git a/tests/test_llm_memory_output.cpp b/tests/test_llm_memory_output.cpp index 451e913..e5bcd1c 100644 --- a/tests/test_llm_memory_output.cpp +++ b/tests/test_llm_memory_output.cpp @@ -422,9 +422,23 @@ class FakeLlmBackend final : public LlmBackend { return evidence_.preparation; } + LlmExpectedChecksumResult expected_cpu_checksum(const LlmMemoryWorkPlan& model_plan, + const LlmScenarioWorkPlan&) const noexcept override { + LlmExpectedChecksumResult expected; + expected.valid = true; + expected.reason_code = LlmExecutorReason::VALID; + expected.workers.resize(cpu_execution_plan(model_plan).effective_workers); + expected.run_checksum = {11, 22}; + return expected; + } + LlmTaskExecutionResult execute_task(const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan& scenario_plan, const LlmRunnerTaskContext& context) override { auto execution = successful_fake_execution(model_plan); + execution.cold_checks = required_llm_cold_checks(model_plan, scenario_plan.scenario); + for (size_t slot = 0; slot < execution.cold_checks.size(); ++slot) { + resolve_llm_cold_check(execution.cold_checks, slot, true); + } execution.kv_write_validation_applicable = scenario_plan.scenario != LlmScenario::WeightsOnly; execution.kv_write_validation_evaluated = execution.kv_write_validation_applicable; execution.kv_write_validation_valid = execution.kv_write_validation_applicable; @@ -535,6 +549,7 @@ TEST(LlmMemoryOutputTest, MetalReportPrintsCapabilitySegmentsAndTaskValidationEv weights_task.timing_evaluated = true; weights_task.timing_valid = true; weights_task.gpu_elapsed_seconds = 0.001; + weights_measurement.execution.timing = {true, true, 0.001}; weights_task.checksum_evaluated = true; weights_task.checksum_valid = true; weights_task.kv_write_validation_evaluated = false; @@ -552,6 +567,7 @@ TEST(LlmMemoryOutputTest, MetalReportPrintsCapabilitySegmentsAndTaskValidationEv task.timing_evaluated = true; task.timing_valid = true; task.gpu_elapsed_seconds = 0.0025; + measurement.execution.timing = {true, true, 0.0025}; task.checksum_evaluated = true; task.checksum_valid = true; task.kv_write_validation_evaluated = true; @@ -624,7 +640,7 @@ TEST(LlmMemoryOutputTest, MetalPagedReportPrintsTableLookupPaddingAndCanaryEvide task.grid_plan.paged_semantic_lookups = 10; task.timing_evaluated = true; task.timing_valid = true; - task.gpu_elapsed_seconds = 0.0025; + measurement.execution.timing = {true, true, 0.0025}; task.checksum_evaluated = true; task.checksum_valid = true; task.kv_write_validation_evaluated = true; @@ -682,7 +698,7 @@ TEST(LlmMemoryOutputTest, task.grid_plan.threads_per_threadgroup = 64; task.timing_evaluated = true; task.timing_valid = true; - task.gpu_elapsed_seconds = 0.004; + measurement.execution.timing = {true, true, 0.004}; task.checksum_evaluated = true; task.checksum_valid = true; task.kv_write_validation_evaluated = true; @@ -752,8 +768,6 @@ TEST(LlmMemoryOutputTest, set_headline(result, LlmScenario::Mixed, 0.004, 250.0, 75.0); LlmMeasurementState measurement; measurement.scenario = LlmScenario::Mixed; - measurement.work_unit_kind = LlmWorkUnitKind::PrefillOperation; - measurement.kv_write_kind = LlmKvWriteKind::FullPromptPopulation; LlmMetalTaskEvidence task; task.pipeline_label = "membenchmark.llm-metal.pipeline.prefill-paged.mixed"; @@ -769,7 +783,7 @@ TEST(LlmMemoryOutputTest, task.grid_plan.threadgroup_accounted_bytes = {1596, 1596}; task.timing_evaluated = true; task.timing_valid = true; - task.gpu_elapsed_seconds = 0.004; + measurement.execution.timing = {true, true, 0.004}; task.checksum_evaluated = true; task.checksum_valid = true; task.kv_write_validation_evaluated = true; @@ -1015,7 +1029,9 @@ TEST(LlmMemoryOutputTest, EmitsDeduplicatedWarningsInContractOrder) { LlmMeasurementState mixed; mixed.scenario = LlmScenario::Mixed; mixed.status = LlmMeasurementStatus::Measured; - mixed.qos_failed_workers = 1; + LlmCpuRetainedEvidence cpu; + cpu.executor.qos_failed_workers = 1; + mixed.execution.backend_evidence = cpu; mixed.duration_quality = "above-target-single-work-unit"; result.measurements.push_back(mixed); result.measurements.push_back(mixed); diff --git a/tests/test_llm_memory_runner.cpp b/tests/test_llm_memory_runner.cpp index b37c601..8af73a9 100644 --- a/tests/test_llm_memory_runner.cpp +++ b/tests/test_llm_memory_runner.cpp @@ -29,6 +29,33 @@ namespace { +const LlmScenarioWorkPlan& frozen_plan(const LlmMemoryResult& result, size_t index) { + return result.scenario_plans.at(result.frozen_plan_handles.at(index)).plan; +} +const LlmScenarioWorkPlan& measurement_plan(const LlmMemoryResult& result, const LlmMeasurementState& measurement) { + return result.scenario_plans.at(measurement.plan_handle).plan; +} +bool frozen_plans_valid(const LlmMemoryResult& result) { + return std::all_of(result.frozen_plan_handles.begin(), result.frozen_plan_handles.end(), + [&](size_t handle) { return handle < result.scenario_plans.size(); }); +} +LlmColdChecks fake_passed_checks(const LlmMemoryWorkPlan& model, LlmScenario scenario) { + const bool kv = scenario != LlmScenario::WeightsOnly; + const bool paged = model.kv_layout == LlmKvLayout::Paged; + const bool prefill = model.phase == LlmPhase::Prefill; + const size_t tokens = prefill ? model.geometry.prefill->prompt_tokens : model.geometry.decode->visible_context_tokens; + const bool padding = paged && tokens % model.geometry.kv_block_tokens != 0; + const bool cpu = model.backend == LlmMemoryBackend::Cpu; + LlmColdChecks checks = {{{LlmColdCheckKind::PostValidationStructure, cpu ? paged || prefill || kv : kv}, + {prefill ? LlmColdCheckKind::KvPrefillFinalSamples : kv || !cpu ? LlmColdCheckKind::KvAppendFinal : + LlmColdCheckKind::KvAppendUnchanged, kv || (cpu && paged && !prefill)}, + {LlmColdCheckKind::KvPaddingCanary, padding && (cpu || kv)}}}; + for (auto& check : checks) if (check.applicable) { + check.evaluated = true; check.valid = true; check.reason_code = "valid"; + } + return checks; +} + struct TaskRecord { LlmRunnerTaskContext context; size_t work_units = 0; @@ -307,6 +334,20 @@ LlmTaskExecutionResult successful_execution(const LlmMemoryWorkPlan& model_plan, task_plan.task_accounted_bytes; result.validation.evaluated = true; result.validation.valid = true; + if (model_plan.backend == LlmMemoryBackend::Cpu) { + LlmExecutorResult cpu; + const size_t workers = get_llm_cpu_execution_plan(model_plan)->effective_workers; + cpu.valid = true; + cpu.reason_code = LlmExecutorReason::VALID; + cpu.requested_workers = workers; cpu.created_workers = workers; cpu.completed_workers = workers; + cpu.qos_successful_workers = workers; + cpu.kernel_succeeded = true; cpu.timer_started = true; cpu.timer_stopped = true; + cpu.checksum_evaluated = true; cpu.checksum_valid = true; + cpu.post_validation_evaluated = true; cpu.post_validation_valid = true; + cpu.actual_checksums.resize(workers); cpu.expected_checksums.resize(workers); + cpu.cold_checks = fake_passed_checks(model_plan, task_plan.scenario); + result.backend_evidence = LlmCpuTaskEvidence{std::move(cpu)}; + } return result; } @@ -428,6 +469,17 @@ class FakeLlmBackend final : public LlmBackend { return preparation; } + mutable size_t expected_calls = 0; + LlmExpectedChecksumResult expected_cpu_checksum(const LlmMemoryWorkPlan& model, + const LlmScenarioWorkPlan&) const noexcept override { + ++expected_calls; + LlmExpectedChecksumResult expected; + expected.valid = true; + expected.reason_code = LlmExecutorReason::VALID; + expected.workers.resize(get_llm_cpu_execution_plan(model)->effective_workers); + return expected; + } + LlmTaskExecutionResult execute_task( const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan& task_plan, @@ -440,6 +492,7 @@ class FakeLlmBackend final : public LlmBackend { successful_execution(model_plan, task_plan, context, 0.150); if (backend_ == LlmMemoryBackend::Metal) { LlmMetalTaskEvidence metal; + metal.cold_checks = fake_passed_checks(model_plan, task_plan.scenario); metal.timed_pipeline_available = true; metal.pipeline_label = model_plan.phase == LlmPhase::Prefill && @@ -566,12 +619,12 @@ void expect_interrupted_tail(const LlmMemoryResult& result, size_t measured_pref const LlmMeasurementState& measurement = result.measurements[index]; if (index < measured_prefix) { EXPECT_EQ(measurement.status, LlmMeasurementStatus::Measured) << index; - EXPECT_TRUE(measurement.elapsed_seconds.has_value()) << index; + EXPECT_TRUE(derive_llm_measurement_metrics(measurement).latency_seconds.has_value()) << index; } else { EXPECT_EQ(measurement.status, LlmMeasurementStatus::Interrupted) << index; EXPECT_EQ(measurement.reason_code, LlmRunnerReason::INTERRUPTION_BEFORE_TASK) << index; - EXPECT_FALSE(measurement.elapsed_seconds.has_value()) << index; - EXPECT_EQ(measurement.completed_work_units, 0u) << index; + EXPECT_FALSE(derive_llm_measurement_metrics(measurement).latency_seconds.has_value()) << index; + EXPECT_EQ(measurement.execution.completion.completed_work_units, 0u) << index; } } } @@ -600,13 +653,13 @@ TEST(LlmMemoryRunnerTest, ASSERT_TRUE(result.initialized); EXPECT_EQ(result.status, LlmRunStatus::Complete); EXPECT_TRUE(result.results_complete); - EXPECT_TRUE(result.conclusions_valid); + EXPECT_TRUE(result.run_accepted); EXPECT_TRUE(result.scenario_order_balance_complete); EXPECT_FALSE(result.interruption_requested); ASSERT_EQ(executor.calls.size(), 12u); for (size_t index = 0; index < kLlmScenarioCount; ++index) { const TaskRecord& warmup = executor.calls[index]; - const LlmScenarioWorkPlan& frozen = result.frozen_scenario_plans.scenarios[index]; + const LlmScenarioWorkPlan& frozen = frozen_plan(result, index); EXPECT_EQ(warmup.context.kind, LlmRunnerTaskKind::Warmup); EXPECT_EQ(warmup.context.purpose, "frozen_measurement_warmup"); EXPECT_EQ(warmup.context.scenario, static_cast(index)); @@ -637,8 +690,7 @@ TEST(LlmMemoryRunnerTest, EXPECT_EQ(measurement.context.loop_index, loop_index); EXPECT_EQ(measurement.context.order_position, position); const LlmScenarioWorkPlan& frozen = - result.frozen_scenario_plans - .scenarios[static_cast(expected_order[position])]; + frozen_plan(result, static_cast(expected_order[position])); EXPECT_EQ(measurement.work_units, frozen.work_units); EXPECT_EQ(measurement.payload_bytes, frozen.effective_model_payload_bytes); @@ -673,7 +725,8 @@ TEST(LlmMemoryRunnerTest, EXPECT_EQ(result.counters.planned_layout_metadata_read_bytes, 0u); EXPECT_EQ(result.counters.completed_layout_metadata_read_bytes, 0u); size_t expected_payload_per_loop = 0; - for (const LlmScenarioWorkPlan& scenario : result.frozen_scenario_plans.scenarios) { + for (size_t handle : result.frozen_plan_handles) { + const auto& scenario = result.scenario_plans.at(handle).plan; EXPECT_EQ(scenario.work_unit_kind, LlmWorkUnitKind::DecodeStep); EXPECT_EQ(scenario.kv_write_kind, scenario.scenario == LlmScenario::WeightsOnly @@ -696,59 +749,59 @@ TEST(LlmMemoryRunnerTest, EXPECT_EQ(result.counters.completed_task_accounted_bytes, expected_payload_per_loop * 3); for (const LlmMeasurementState& measurement : result.measurements) { - ASSERT_LT(measurement.frozen_plan_index, kLlmScenarioCount); - const LlmScenarioWorkPlan& frozen = result.frozen_scenario_plans.scenarios[measurement.frozen_plan_index]; + ASSERT_LT(static_cast(measurement.scenario), kLlmScenarioCount); + const LlmScenarioWorkPlan& frozen = frozen_plan(result, static_cast(measurement.scenario)); EXPECT_EQ(measurement.scenario, frozen.scenario); - EXPECT_EQ(measurement.work_unit_kind, frozen.work_unit_kind); - EXPECT_EQ(measurement.kv_write_kind, frozen.kv_write_kind); - EXPECT_EQ(measurement.planned_work_units, frozen.work_units); - EXPECT_EQ(measurement.planned_effective_model_payload_bytes, frozen.effective_model_payload_bytes); - EXPECT_EQ(measurement.completed_effective_model_payload_bytes, + EXPECT_EQ(measurement_plan(result, measurement).work_unit_kind, frozen.work_unit_kind); + EXPECT_EQ(measurement_plan(result, measurement).kv_write_kind, frozen.kv_write_kind); + EXPECT_EQ(measurement_plan(result, measurement).work_units, frozen.work_units); + EXPECT_EQ(measurement_plan(result, measurement).effective_model_payload_bytes, frozen.effective_model_payload_bytes); + EXPECT_EQ(measurement.execution.completion.completed_effective_model_payload_bytes, frozen.effective_model_payload_bytes); - EXPECT_EQ(measurement.layout_metadata_lookup_count_per_work_unit, 0u); - EXPECT_EQ(measurement.layout_metadata_read_bytes_per_work_unit, 0u); - EXPECT_EQ(measurement.accounted_bytes_per_work_unit, + EXPECT_EQ(measurement_plan(result, measurement).layout_metadata_lookup_count_per_work_unit, 0u); + EXPECT_EQ(measurement_plan(result, measurement).layout_metadata_read_bytes_per_work_unit, 0u); + EXPECT_EQ(measurement_plan(result, measurement).accounted_bytes_per_work_unit, frozen.effective_model_payload_bytes_per_work_unit); - EXPECT_EQ(measurement.planned_task_accounted_bytes, + EXPECT_EQ(measurement_plan(result, measurement).task_accounted_bytes, frozen.effective_model_payload_bytes); - EXPECT_EQ(measurement.completed_task_accounted_bytes, + EXPECT_EQ(measurement.execution.completion.completed_task_accounted_bytes, frozen.effective_model_payload_bytes); EXPECT_EQ(measurement.execution.status, LlmTaskExecutionStatus::Complete); EXPECT_EQ(measurement.execution.reason_code, LlmBackendReason::VALID); EXPECT_TRUE(measurement.execution.timing.valid); EXPECT_TRUE(measurement.execution.validation.valid); - EXPECT_TRUE(std::holds_alternative( + EXPECT_TRUE(std::holds_alternative( measurement.execution.backend_evidence)); } - ASSERT_EQ(checkpoints.size(), 10u); - for (size_t index = 0; index < 9; ++index) { + ASSERT_EQ(checkpoints.size(), 4u); + for (size_t index = 0; index < 3; ++index) { EXPECT_EQ(checkpoints[index].kind, LlmCheckpointKind::MeasurementTerminal); EXPECT_EQ(checkpoints[index].planned_loops, 3u); - EXPECT_EQ(checkpoints[index].attempted_loops, index / kLlmScenarioCount + 1); - EXPECT_EQ(checkpoints[index].completed_loops, (index + 1) / kLlmScenarioCount); + EXPECT_EQ(checkpoints[index].attempted_loops, index + 1); + EXPECT_EQ(checkpoints[index].completed_loops, index + 1); EXPECT_EQ(checkpoints[index].planned_measurements, 9u); - EXPECT_EQ(checkpoints[index].attempted_measurements, index + 1); - EXPECT_EQ(checkpoints[index].terminal_measurements, index + 1); - EXPECT_EQ(checkpoints[index].measured_measurements, index + 1); + EXPECT_EQ(checkpoints[index].attempted_measurements, (index + 1) * 3); + EXPECT_EQ(checkpoints[index].terminal_measurements, (index + 1) * 3); + EXPECT_EQ(checkpoints[index].measured_measurements, (index + 1) * 3); EXPECT_EQ(checkpoints[index].logical_checkpoint_attempts, index + 1); - EXPECT_EQ(checkpoints[index].successful_logical_checkpoints, index + 1); + EXPECT_EQ(checkpoints[index].successful_logical_checkpoints, index); EXPECT_FALSE(checkpoints[index].terminal_checkpoint_attempted); EXPECT_FALSE(checkpoints[index].terminal_checkpoint_completed); EXPECT_FALSE(checkpoints[index].interruption_requested); - EXPECT_EQ(checkpoints[index].status, index == 8 ? LlmRunStatus::Complete : LlmRunStatus::Partial); + EXPECT_EQ(checkpoints[index].status, index == 2 ? LlmRunStatus::Complete : LlmRunStatus::Partial); } EXPECT_EQ(checkpoints.back().kind, LlmCheckpointKind::CommandTerminal); EXPECT_EQ(checkpoints.back().status, LlmRunStatus::Complete); EXPECT_EQ(checkpoints.back().attempted_measurements, 9u); EXPECT_EQ(checkpoints.back().terminal_measurements, 9u); EXPECT_EQ(checkpoints.back().measured_measurements, 9u); - EXPECT_EQ(checkpoints.back().logical_checkpoint_attempts, 10u); - EXPECT_EQ(checkpoints.back().successful_logical_checkpoints, 10u); + EXPECT_EQ(checkpoints.back().logical_checkpoint_attempts, 4u); + EXPECT_EQ(checkpoints.back().successful_logical_checkpoints, 3u); EXPECT_TRUE(checkpoints.back().terminal_checkpoint_attempted); - EXPECT_TRUE(checkpoints.back().terminal_checkpoint_completed); - EXPECT_EQ(result.logical_checkpoint_attempts, 10u); - EXPECT_EQ(result.successful_logical_checkpoints, 10u); + EXPECT_FALSE(checkpoints.back().terminal_checkpoint_completed); + EXPECT_EQ(result.logical_checkpoint_attempts, 4u); + EXPECT_EQ(result.successful_logical_checkpoints, 4u); EXPECT_TRUE(result.terminal_checkpoint_completed); EXPECT_EQ(executor.release_calls, 1u); ASSERT_EQ(release_completed_at_checkpoint.size(), checkpoints.size()); @@ -760,13 +813,13 @@ TEST(LlmMemoryRunnerTest, for (const LlmScenarioAggregate& aggregate : result.aggregates) { EXPECT_EQ(aggregate.status, "complete"); - EXPECT_EQ(aggregate.stability_quality, "stable"); - EXPECT_EQ(aggregate.effective_model_payload_gb_s.values.size(), 3u); + EXPECT_EQ(aggregate.observed_cv_classification, "below-threshold"); + EXPECT_EQ(aggregate.accepted_measurement_ids.size(), 3u); EXPECT_TRUE(aggregate.effective_model_payload_gb_s.headline.has_value()); } } -TEST(LlmMemoryRunnerTest, CompleteSingleLoopIsInspectableButComparativeConclusionsAreInvalid) { +TEST(LlmMemoryRunnerTest, CompleteSingleLoopIsAcceptedWithSeparateBalanceAndSampleQuality) { const LlmMemoryConfig config = explicit_config(1); const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); ASSERT_TRUE(plan.valid) << plan.reason_code; @@ -777,13 +830,13 @@ TEST(LlmMemoryRunnerTest, CompleteSingleLoopIsInspectableButComparativeConclusio EXPECT_EQ(result.status, LlmRunStatus::Complete); EXPECT_TRUE(result.results_complete); EXPECT_FALSE(result.scenario_order_balance_complete); - EXPECT_FALSE(result.conclusions_valid); + EXPECT_TRUE(result.run_accepted); EXPECT_EQ(result.quality_warnings, (std::vector{"scenario-order-not-balanced"})); - EXPECT_EQ(result.logical_checkpoint_attempts, 4u); + EXPECT_EQ(result.logical_checkpoint_attempts, 2u); for (const LlmScenarioAggregate& aggregate : result.aggregates) { - EXPECT_EQ(aggregate.stability_quality, "insufficient-samples"); - ASSERT_EQ(aggregate.effective_model_payload_gb_s.values.size(), 1u); - EXPECT_EQ(aggregate.effective_model_payload_gb_s.headline, aggregate.effective_model_payload_gb_s.values.front()); + EXPECT_EQ(aggregate.observed_cv_classification, "insufficient-samples"); + ASSERT_EQ(aggregate.accepted_measurement_ids.size(), 1u); + EXPECT_EQ(aggregate.effective_model_payload_gb_s.headline, derive_llm_measurement_metrics(result.measurements.at(aggregate.accepted_measurement_ids.front())).payload_gb_s); } } @@ -815,8 +868,7 @@ TEST(LlmMemoryRunnerTest, EXPECT_TRUE(result.results_complete); ASSERT_EQ(result.measurements.size(), kLlmScenarioCount); for (const LlmMeasurementState& measurement : result.measurements) { - const auto* const metal = std::get_if( - &measurement.execution.backend_evidence); + const auto* const metal = std::get_if(&measurement.execution.backend_evidence); ASSERT_NE(metal, nullptr); EXPECT_TRUE(metal->timed_pipeline_available); EXPECT_TRUE(metal->grid_plan_available); @@ -826,8 +878,8 @@ TEST(LlmMemoryRunnerTest, EXPECT_TRUE(metal->checksum_valid); EXPECT_TRUE(metal->kv_write_validation_valid); EXPECT_TRUE(metal->post_validation_valid); - EXPECT_EQ(measurement.requested_workers, 0u); - EXPECT_EQ(measurement.effective_workers, 0u); + EXPECT_EQ((get_llm_cpu_task_evidence(measurement.execution) ? get_llm_cpu_task_evidence(measurement.execution)->requested_workers : 0), 0u); + EXPECT_EQ((get_llm_cpu_task_evidence(measurement.execution) ? get_llm_cpu_task_evidence(measurement.execution)->requested_workers : 0), 0u); } for (size_t index = 0; index < kLlmScenarioCount; ++index) { ASSERT_EQ(result.calibration_attempt_counts[index], 1u); @@ -848,7 +900,7 @@ TEST(LlmMemoryRunnerTest, const LlmMemoryWorkPlan plan = build_metal_runner_plan(config, true); ASSERT_TRUE(plan.valid) << plan.reason_code; EXPECT_EQ(plan.methodology_version, - "llm-memory-v1-metal-decode-paged"); + "llm-memory-v2-metal-decode-paged"); const LlmMetalExecutionPlan* const execution_plan = get_llm_metal_execution_plan(plan); ASSERT_NE(execution_plan, nullptr); @@ -872,13 +924,13 @@ TEST(LlmMemoryRunnerTest, for (const LlmMeasurementState& measurement : result.measurements) { const size_t expected_per_work_unit = measurement.scenario == LlmScenario::WeightsOnly ? 0 : 3; - EXPECT_EQ(measurement.layout_metadata_lookup_count_per_work_unit, + EXPECT_EQ(measurement_plan(result, measurement).layout_metadata_lookup_count_per_work_unit, expected_per_work_unit); - EXPECT_EQ(measurement.layout_metadata_read_bytes_per_work_unit, + EXPECT_EQ(measurement_plan(result, measurement).layout_metadata_read_bytes_per_work_unit, expected_per_work_unit * sizeof(uint32_t)); - EXPECT_EQ(measurement.planned_layout_metadata_lookup_count, + EXPECT_EQ(measurement_plan(result, measurement).layout_metadata_lookup_count, expected_per_work_unit * config.iterations); - EXPECT_EQ(measurement.completed_layout_metadata_lookup_count, + EXPECT_EQ(measurement.execution.completion.completed_layout_metadata_lookup_count, expected_per_work_unit * config.iterations); expected_task_lookups += expected_per_work_unit * config.iterations; } @@ -895,7 +947,7 @@ TEST(LlmMemoryRunnerTest, const LlmMemoryWorkPlan plan = build_metal_runner_plan(config, true); ASSERT_TRUE(plan.valid) << plan.reason_code; EXPECT_EQ(plan.methodology_version, - "llm-memory-v1-metal-prefill-contiguous"); + "llm-memory-v2-metal-prefill-contiguous"); EXPECT_EQ(plan.phase, LlmPhase::Prefill); EXPECT_EQ(plan.kv_layout, LlmKvLayout::Contiguous); EXPECT_EQ(plan.work_unit_kind, LlmWorkUnitKind::PrefillOperation); @@ -914,13 +966,13 @@ TEST(LlmMemoryRunnerTest, ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_SUCCESS) << result.reason_code << ": " << result.diagnostic; - ASSERT_TRUE(result.frozen_scenario_plans.valid); + ASSERT_TRUE(frozen_plans_valid(result)); ASSERT_EQ(result.measurements.size(), kLlmScenarioCount); ASSERT_EQ(backend.calls.size(), 2 * kLlmScenarioCount); for (size_t index = 0; index < kLlmScenarioCount; ++index) { const LlmScenarioWorkPlan& frozen = - result.frozen_scenario_plans.scenarios[index]; + frozen_plan(result, index); EXPECT_EQ(frozen.work_units, 1u); EXPECT_EQ(frozen.work_unit_kind, LlmWorkUnitKind::PrefillOperation); @@ -931,15 +983,15 @@ TEST(LlmMemoryRunnerTest, ASSERT_EQ(result.calibration_attempt_counts[index], 1u); } for (const LlmMeasurementState& measurement : result.measurements) { - EXPECT_EQ(measurement.planned_work_units, 1u); - EXPECT_EQ(measurement.completed_work_units, 1u); - EXPECT_EQ(measurement.work_unit_kind, + EXPECT_EQ(measurement_plan(result, measurement).work_units, 1u); + EXPECT_EQ(measurement.execution.completion.completed_work_units, 1u); + EXPECT_EQ(measurement_plan(result, measurement).work_unit_kind, LlmWorkUnitKind::PrefillOperation); - EXPECT_EQ(measurement.kv_write_kind, + EXPECT_EQ(measurement_plan(result, measurement).kv_write_kind, measurement.scenario == LlmScenario::WeightsOnly ? LlmKvWriteKind::None : LlmKvWriteKind::FullPromptPopulation); - const LlmMetalTaskEvidence* const metal = + const LlmMetalRuntimeEvidence* const metal = get_llm_metal_task_evidence(measurement.execution); ASSERT_NE(metal, nullptr); EXPECT_EQ( @@ -957,7 +1009,7 @@ TEST(LlmMemoryRunnerTest, const LlmMemoryWorkPlan plan = build_metal_runner_plan(config, true); ASSERT_TRUE(plan.valid) << plan.reason_code; EXPECT_EQ(plan.methodology_version, - "llm-memory-v1-metal-prefill-paged"); + "llm-memory-v2-metal-prefill-paged"); EXPECT_EQ(plan.phase, LlmPhase::Prefill); EXPECT_EQ(plan.kv_layout, LlmKvLayout::Paged); ASSERT_TRUE(plan.prefill_plan.has_value()); @@ -977,7 +1029,7 @@ TEST(LlmMemoryRunnerTest, ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_SUCCESS) << result.reason_code << ": " << result.diagnostic; - ASSERT_TRUE(result.frozen_scenario_plans.valid); + ASSERT_TRUE(frozen_plans_valid(result)); ASSERT_EQ(result.measurements.size(), kLlmScenarioCount); size_t expected_total_lookups = 0; @@ -985,25 +1037,25 @@ TEST(LlmMemoryRunnerTest, const bool weights_only = measurement.scenario == LlmScenario::WeightsOnly; const size_t expected_lookups = weights_only ? 0 : 15; - EXPECT_EQ(measurement.work_unit_kind, + EXPECT_EQ(measurement_plan(result, measurement).work_unit_kind, LlmWorkUnitKind::PrefillOperation); - EXPECT_EQ(measurement.kv_write_kind, + EXPECT_EQ(measurement_plan(result, measurement).kv_write_kind, weights_only ? LlmKvWriteKind::None : LlmKvWriteKind::FullPromptPopulation); - EXPECT_EQ(measurement.layout_metadata_lookup_count_per_work_unit, + EXPECT_EQ(measurement_plan(result, measurement).layout_metadata_lookup_count_per_work_unit, expected_lookups); - EXPECT_EQ(measurement.layout_metadata_read_bytes_per_work_unit, + EXPECT_EQ(measurement_plan(result, measurement).layout_metadata_read_bytes_per_work_unit, expected_lookups * sizeof(uint32_t)); - EXPECT_EQ(measurement.completed_layout_metadata_lookup_count, + EXPECT_EQ(measurement.execution.completion.completed_layout_metadata_lookup_count, expected_lookups); expected_total_lookups += expected_lookups; - const LlmMetalTaskEvidence* const metal = + const LlmMetalRuntimeEvidence* const metal = get_llm_metal_task_evidence(measurement.execution); ASSERT_NE(metal, nullptr); EXPECT_EQ(metal->pipeline_label, "membenchmark.llm-metal.pipeline.prefill-paged.fake"); - EXPECT_EQ(metal->checksum_algorithm_version, + EXPECT_EQ(plan.component_identities.checksum_pattern_version, LlmMetalPrefillPagedVersion::CHECKSUM); EXPECT_EQ(metal->grid_plan.paged_semantic_lookups, expected_lookups); @@ -1052,9 +1104,9 @@ TEST(LlmMemoryRunnerTest, ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_SUCCESS) << result.reason_code << ": " << result.diagnostic; - ASSERT_TRUE(result.frozen_scenario_plans.valid); - for (const LlmScenarioWorkPlan& frozen : - result.frozen_scenario_plans.scenarios) { + ASSERT_TRUE(frozen_plans_valid(result)); + for (size_t handle : result.frozen_plan_handles) { + const auto& frozen = result.scenario_plans.at(handle).plan; EXPECT_EQ(frozen.work_units, 1u); EXPECT_EQ(frozen.work_unit_kind, LlmWorkUnitKind::PrefillOperation); @@ -1070,8 +1122,8 @@ TEST(LlmMemoryRunnerTest, EXPECT_EQ(confirmation->work_units, 1u); ASSERT_EQ(result.measurements.size(), kLlmScenarioCount); for (const LlmMeasurementState& measurement : result.measurements) { - EXPECT_EQ(measurement.planned_work_units, 1u); - EXPECT_EQ(measurement.completed_work_units, 1u); + EXPECT_EQ(measurement_plan(result, measurement).work_units, 1u); + EXPECT_EQ(measurement.execution.completion.completed_work_units, 1u); EXPECT_NE(get_llm_metal_task_evidence(measurement.execution), nullptr); EXPECT_EQ(get_llm_cpu_task_evidence(measurement.execution), nullptr); } @@ -1102,7 +1154,7 @@ TEST(LlmMemoryRunnerTest, ASSERT_EQ(checkpoints.size(), 1u); EXPECT_EQ(checkpoints.front().kind, LlmCheckpointKind::CommandTerminal); - EXPECT_TRUE(checkpoints.front().terminal_checkpoint_completed); + EXPECT_FALSE(checkpoints.front().terminal_checkpoint_completed); } TEST(LlmMemoryRunnerTest, @@ -1146,8 +1198,7 @@ TEST(LlmMemoryRunnerTest, LlmRunnerReason::AUXILIARY_BUDGET_INSUFFICIENT); ASSERT_EQ(result.measurements.size(), kLlmScenarioCount); for (const LlmMeasurementState& measurement : result.measurements) { - const auto* const metal = std::get_if( - &measurement.execution.backend_evidence); + const auto* const metal = std::get_if(&measurement.execution.backend_evidence); ASSERT_NE(metal, nullptr); EXPECT_EQ(metal->grid_plan.identity.size(), 4096 + @@ -1242,7 +1293,7 @@ TEST(LlmMemoryRunnerTest, EXPECT_EQ(checkpoints.front().kind, LlmCheckpointKind::CommandTerminal); EXPECT_EQ(checkpoints.front().status, LlmRunStatus::Failed); - EXPECT_TRUE(checkpoints.front().terminal_checkpoint_completed); + EXPECT_FALSE(checkpoints.front().terminal_checkpoint_completed); } TEST(LlmMemoryRunnerTest, @@ -1304,7 +1355,7 @@ TEST(LlmMemoryRunnerTest, EXPECT_EQ(checkpoints.front().kind, LlmCheckpointKind::CommandTerminal); EXPECT_EQ(checkpoints.front().status, LlmRunStatus::Failed); - EXPECT_TRUE(checkpoints.front().terminal_checkpoint_completed); + EXPECT_FALSE(checkpoints.front().terminal_checkpoint_completed); } TEST(LlmMemoryRunnerTest, InitializationUnsupportedAndFailedDoNotExecuteOrFallback) { @@ -1423,7 +1474,7 @@ TEST(LlmMemoryRunnerTest, LlmCheckpointKind::CommandTerminal); EXPECT_EQ(checkpoints.front().status, LlmRunStatus::Failed); EXPECT_TRUE(checkpoints.front().terminal_checkpoint_attempted); - EXPECT_TRUE(checkpoints.front().terminal_checkpoint_completed); + EXPECT_FALSE(checkpoints.front().terminal_checkpoint_completed); EXPECT_EQ(result.logical_checkpoint_attempts, 1u); EXPECT_EQ(result.successful_logical_checkpoints, 1u); EXPECT_TRUE(result.terminal_checkpoint_completed); @@ -1456,7 +1507,7 @@ TEST(LlmMemoryRunnerTest, OversizedGenericReasonIsCanonicalizedIntoFrozenRunnerC for (const LlmMeasurementState& measurement : result.measurements) { EXPECT_EQ(measurement.execution.reason_code, LlmBackendReason::VALID); EXPECT_LT(measurement.execution.reason_code.capacity(), oversized_capacity); - EXPECT_TRUE(std::holds_alternative( + EXPECT_TRUE(std::holds_alternative( measurement.execution.backend_evidence)); } EXPECT_EQ(result.measurements.capacity(), result.counters.planned_measurements); @@ -1464,9 +1515,9 @@ TEST(LlmMemoryRunnerTest, OversizedGenericReasonIsCanonicalizedIntoFrozenRunnerC EXPECT_EQ(result.statistics_workspace.sorted_values.capacity(), config.loop_count); EXPECT_EQ(result.statistics_workspace.absolute_deviations.capacity(), config.loop_count); for (const LlmScenarioAggregate& aggregate : result.aggregates) { - EXPECT_EQ(aggregate.work_unit_latency_seconds.values.capacity(), config.loop_count); - EXPECT_EQ(aggregate.synthetic_memory_work_units_per_second.values.capacity(), config.loop_count); - EXPECT_EQ(aggregate.effective_model_payload_gb_s.values.capacity(), config.loop_count); + EXPECT_EQ(aggregate.accepted_measurement_ids.capacity(), config.loop_count); + EXPECT_EQ(aggregate.accepted_measurement_ids.capacity(), config.loop_count); + EXPECT_EQ(aggregate.accepted_measurement_ids.capacity(), config.loop_count); } } @@ -1488,7 +1539,7 @@ TEST(LlmMemoryRunnerTest, LlmMemoryResult result; ASSERT_EQ(run_llm_memory_suite(config, plan, executor, result), EXIT_SUCCESS); - ASSERT_TRUE(result.frozen_scenario_plans.valid); + ASSERT_TRUE(frozen_plans_valid(result)); ASSERT_EQ(executor.calls.size(), 18u); constexpr std::array kScenarios = { LlmScenario::WeightsOnly, LlmScenario::KvOnly, LlmScenario::Mixed}; @@ -1519,13 +1570,13 @@ TEST(LlmMemoryRunnerTest, calculate_llm_calibrated_work_units( 0.010 * (index + 1), pilot.work_units, limits); EXPECT_EQ(correction.work_units, expected_work_units); - EXPECT_EQ(result.frozen_scenario_plans.scenarios[index].work_units, + EXPECT_EQ(frozen_plan(result, index).work_units, expected_work_units); EXPECT_EQ(correction.payload_bytes, - result.frozen_scenario_plans.scenarios[index] + frozen_plan(result, index) .effective_model_payload_bytes); EXPECT_EQ(correction.plan_identity, - result.frozen_scenario_plans.scenarios[index].plan_identity); + frozen_plan(result, index).plan_identity); } EXPECT_EQ(call_index, 3u * kLlmScenarioCount); @@ -1533,7 +1584,7 @@ TEST(LlmMemoryRunnerTest, const size_t index = static_cast(scenario); const TaskRecord& frozen_warmup = executor.calls[call_index++]; const LlmScenarioWorkPlan& frozen = - result.frozen_scenario_plans.scenarios[index]; + frozen_plan(result, index); EXPECT_EQ(frozen_warmup.context.kind, LlmRunnerTaskKind::Warmup); EXPECT_EQ(frozen_warmup.context.purpose, "frozen_measurement_warmup"); @@ -1551,8 +1602,7 @@ TEST(LlmMemoryRunnerTest, for (size_t position = 0; position < kLlmScenarioCount; ++position) { const LlmScenario scenario = expected_order[position]; const LlmScenarioWorkPlan& frozen = - result.frozen_scenario_plans - .scenarios[static_cast(scenario)]; + frozen_plan(result, static_cast(scenario)); const TaskRecord& measurement = executor.calls[call_index++]; EXPECT_EQ(measurement.context.kind, LlmRunnerTaskKind::Measurement); @@ -1606,10 +1656,10 @@ TEST(LlmMemoryRunnerTest, EXPECT_EQ(excluded[4].context.purpose, "frozen_measurement_warmup"); EXPECT_EQ(excluded[4].context.kind, LlmRunnerTaskKind::Warmup); - EXPECT_EQ(result.frozen_scenario_plans.scenarios[index].work_units, + EXPECT_EQ(frozen_plan(result, index).work_units, excluded[3].work_units); EXPECT_EQ(excluded[4].plan_identity, - result.frozen_scenario_plans.scenarios[index].plan_identity); + frozen_plan(result, index).plan_identity); EXPECT_EQ(std::count_if(excluded.begin(), excluded.end(), [](const TaskRecord& record) { return record.context.kind == LlmRunnerTaskKind::Warmup; }), 2); @@ -1660,9 +1710,9 @@ TEST(LlmMemoryRunnerTest, ASSERT_EQ(run_llm_memory_suite(config, plan, executor, result), EXIT_SUCCESS); - ASSERT_TRUE(result.frozen_scenario_plans.valid); - for (const LlmScenarioWorkPlan& frozen : - result.frozen_scenario_plans.scenarios) { + ASSERT_TRUE(frozen_plans_valid(result)); + for (size_t handle : result.frozen_plan_handles) { + const auto& frozen = result.scenario_plans.at(handle).plan; EXPECT_EQ(frozen.work_units, 1u); EXPECT_EQ(frozen.work_unit_kind, LlmWorkUnitKind::PrefillOperation); const LlmKvWriteKind expected_write = @@ -1735,7 +1785,7 @@ TEST(LlmMemoryRunnerTest, ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_SUCCESS) << result.reason_code << ": " << result.diagnostic; - ASSERT_TRUE(result.frozen_scenario_plans.valid); + ASSERT_TRUE(frozen_plans_valid(result)); ASSERT_EQ(result.measurements.size(), config.loop_count * kLlmScenarioCount); ASSERT_EQ(backend.calls.size(), @@ -1744,7 +1794,7 @@ TEST(LlmMemoryRunnerTest, size_t payload_bytes_per_loop = 0; for (size_t index = 0; index < kLlmScenarioCount; ++index) { const LlmScenarioWorkPlan& frozen = - result.frozen_scenario_plans.scenarios[index]; + frozen_plan(result, index); const LlmKvWriteKind expected_write = frozen.scenario == LlmScenario::WeightsOnly ? LlmKvWriteKind::None @@ -1784,11 +1834,11 @@ TEST(LlmMemoryRunnerTest, measurement.scenario == LlmScenario::WeightsOnly ? LlmKvWriteKind::None : LlmKvWriteKind::FullPromptPopulation; - EXPECT_EQ(measurement.work_unit_kind, + EXPECT_EQ(measurement_plan(result, measurement).work_unit_kind, LlmWorkUnitKind::PrefillOperation); - EXPECT_EQ(measurement.kv_write_kind, expected_write); - EXPECT_EQ(measurement.planned_work_units, config.iterations); - EXPECT_EQ(measurement.completed_work_units, config.iterations); + EXPECT_EQ(measurement_plan(result, measurement).kv_write_kind, expected_write); + EXPECT_EQ(measurement_plan(result, measurement).work_units, config.iterations); + EXPECT_EQ(measurement.execution.completion.completed_work_units, config.iterations); } } @@ -1917,13 +1967,13 @@ TEST(LlmMemoryRunnerTest, StopBeforeFirstTaskInterruptsAllSlotsWithoutExecutorWo EXPECT_EQ(result.counters.attempted_loops, 0u); EXPECT_EQ(result.counters.attempted_measurements, 0u); EXPECT_EQ(result.counters.terminal_measurements, 6u); - EXPECT_TRUE(result.frozen_scenario_plans.valid); + EXPECT_TRUE(frozen_plans_valid(result)); EXPECT_GT(result.counters.planned_work_units, 0u); EXPECT_GT(result.counters.planned_effective_model_payload_bytes, 0u); for (const LlmMeasurementState& measurement : result.measurements) { - EXPECT_LT(measurement.frozen_plan_index, kLlmScenarioCount); - EXPECT_EQ(measurement.planned_work_units, config.iterations); - EXPECT_GT(measurement.planned_effective_model_payload_bytes, 0u); + EXPECT_LT(static_cast(measurement.scenario), kLlmScenarioCount); + EXPECT_EQ(measurement_plan(result, measurement).work_units, config.iterations); + EXPECT_GT(measurement_plan(result, measurement).effective_model_payload_bytes, 0u); } expect_interrupted_tail(result, 0); ASSERT_EQ(checkpoints.size(), 1u); @@ -1961,10 +2011,9 @@ TEST(LlmMemoryRunnerTest, StopDuringStartedMeasurementKeepsCurrentTaskAndInterru EXPECT_EQ(result.counters.measured_measurements, 1u); EXPECT_EQ(result.counters.completed_work_units, config.iterations); EXPECT_EQ(result.counters.completed_effective_model_payload_bytes, - result.measurements[0].planned_effective_model_payload_bytes); - ASSERT_EQ(checkpoints.size(), 2u); - EXPECT_EQ(checkpoints[0].kind, LlmCheckpointKind::MeasurementTerminal); - EXPECT_EQ(checkpoints[1].kind, LlmCheckpointKind::CommandTerminal); + measurement_plan(result, result.measurements[0]).effective_model_payload_bytes); + ASSERT_EQ(checkpoints.size(), 1u); + EXPECT_EQ(checkpoints[0].kind, LlmCheckpointKind::CommandTerminal); } TEST(LlmMemoryRunnerTest, StopRaisedByMeasurementCheckpointAddsOnlyCommandTerminalSnapshot) { @@ -1991,8 +2040,8 @@ TEST(LlmMemoryRunnerTest, StopRaisedByMeasurementCheckpointAddsOnlyCommandTermin EXPECT_EQ(checkpoints[0].status, LlmRunStatus::Partial); EXPECT_EQ(checkpoints[1].kind, LlmCheckpointKind::CommandTerminal); EXPECT_EQ(checkpoints[1].status, LlmRunStatus::Interrupted); - EXPECT_EQ(result.counters.attempted_measurements, 1u); - expect_interrupted_tail(result, 1); + EXPECT_EQ(result.counters.attempted_measurements, 3u); + expect_interrupted_tail(result, 3); } TEST(LlmMemoryRunnerTest, StopAfterLastSuccessfulTaskRetainsCompleteAcceptedEvidence) { @@ -2018,10 +2067,10 @@ TEST(LlmMemoryRunnerTest, StopAfterLastSuccessfulTaskRetainsCompleteAcceptedEvid EXPECT_EQ(result.status, LlmRunStatus::Complete); EXPECT_TRUE(result.results_complete); EXPECT_TRUE(result.scenario_order_balance_complete); - EXPECT_TRUE(result.conclusions_valid); + EXPECT_TRUE(result.run_accepted); EXPECT_EQ(result.counters.measured_measurements, 9u); - EXPECT_EQ(result.logical_checkpoint_attempts, 10u); - ASSERT_EQ(checkpoints.size(), 10u); + EXPECT_EQ(result.logical_checkpoint_attempts, 4u); + ASSERT_EQ(checkpoints.size(), 4u); EXPECT_FALSE(checkpoints[8].interruption_requested); EXPECT_TRUE(checkpoints.back().interruption_requested); } @@ -2055,9 +2104,9 @@ TEST(LlmMemoryRunnerTest, StopHookExceptionAfterLastMeasurementInvalidatesOtherw EXPECT_EQ(result.diagnostic, "injected stop failure"); EXPECT_TRUE(result.results_complete); EXPECT_TRUE(result.scenario_order_balance_complete); - EXPECT_FALSE(result.conclusions_valid); + EXPECT_FALSE(result.run_accepted); EXPECT_EQ(result.counters.measured_measurements, 9u); - ASSERT_EQ(checkpoints.size(), 10u); + ASSERT_EQ(checkpoints.size(), 4u); EXPECT_EQ(checkpoints[8].kind, LlmCheckpointKind::MeasurementTerminal); EXPECT_EQ(checkpoints.back().kind, LlmCheckpointKind::CommandTerminal); EXPECT_EQ(checkpoints.back().status, LlmRunStatus::Failed); @@ -2114,8 +2163,8 @@ TEST(LlmMemoryRunnerTest, ValidationFailureWinsSimultaneousStopAndIsExcludedFrom EXPECT_EQ(result.reason_code, LlmBackendReason::VALIDATION_FAILED); EXPECT_TRUE(result.interruption_requested); EXPECT_EQ(result.measurements[0].status, LlmMeasurementStatus::Invalid); - EXPECT_FALSE(result.measurements[0].elapsed_seconds.has_value()); - EXPECT_TRUE(result.aggregates[0].effective_model_payload_gb_s.values.empty()); + EXPECT_FALSE(derive_llm_measurement_metrics(result.measurements[0]).latency_seconds.has_value()); + EXPECT_TRUE(result.aggregates[0].accepted_measurement_ids.empty()); } TEST(LlmMemoryRunnerTest, MeasurementCheckpointFailureIsTerminalAndNeverRetried) { @@ -2138,30 +2187,32 @@ TEST(LlmMemoryRunnerTest, MeasurementCheckpointFailureIsTerminalAndNeverRetried) EXPECT_EQ(result.reason_code, LlmRunnerReason::CHECKPOINT_WRITE_FAILED); EXPECT_TRUE(result.checkpoint_failed); EXPECT_FALSE(result.terminal_checkpoint_attempted); - EXPECT_EQ(result.counters.attempted_measurements, 1u); + EXPECT_EQ(result.counters.attempted_measurements, 3u); EXPECT_EQ(result.measurements[0].status, LlmMeasurementStatus::Measured); - EXPECT_TRUE(result.measurements[0].elapsed_seconds.has_value()); - EXPECT_TRUE(result.measurements[0].synthetic_work_unit_latency_seconds.has_value()); - EXPECT_TRUE(result.measurements[0].synthetic_memory_work_units_per_second.has_value()); - EXPECT_TRUE(result.measurements[0].effective_model_payload_gb_s.has_value()); - EXPECT_TRUE(result.measurements[0].checksum_valid); + EXPECT_TRUE(derive_llm_measurement_metrics(result.measurements[0]).latency_seconds.has_value()); + EXPECT_TRUE(derive_llm_measurement_metrics(result.measurements[0]).latency_seconds.has_value()); + EXPECT_TRUE(derive_llm_measurement_metrics(result.measurements[0]).work_units_per_second.has_value()); + EXPECT_TRUE(derive_llm_measurement_metrics(result.measurements[0]).payload_gb_s.has_value()); + EXPECT_TRUE(get_llm_cpu_task_evidence(result.measurements[0].execution)->checksum_valid); EXPECT_EQ(result.measurements[0].execution.status, LlmTaskExecutionStatus::Complete); EXPECT_TRUE(result.measurements[0].execution.timing.valid); EXPECT_TRUE(result.measurements[0].execution.validation.valid); - EXPECT_TRUE(std::holds_alternative( + EXPECT_TRUE(std::holds_alternative( result.measurements[0].execution.backend_evidence)); - EXPECT_EQ(result.aggregates[0].effective_model_payload_gb_s.values.size(), 1u); + EXPECT_EQ(result.aggregates[0].accepted_measurement_ids.size(), 1u); EXPECT_EQ(result.counters.planned_loops, 2u); EXPECT_EQ(result.counters.attempted_loops, 1u); - EXPECT_EQ(result.counters.completed_loops, 0u); + EXPECT_EQ(result.counters.completed_loops, 1u); EXPECT_EQ(result.counters.planned_measurements, 6u); EXPECT_EQ(result.counters.terminal_measurements, 6u); - EXPECT_EQ(result.counters.measured_measurements, 1u); - EXPECT_EQ(result.counters.completed_work_units, config.iterations); + EXPECT_EQ(result.counters.measured_measurements, 3u); + EXPECT_EQ(result.counters.completed_work_units, 3 * config.iterations); EXPECT_EQ(result.counters.completed_effective_model_payload_bytes, - result.measurements[0].planned_effective_model_payload_bytes); - for (size_t index = 1; index < result.measurements.size(); ++index) { + measurement_plan(result, result.measurements[0]).effective_model_payload_bytes + + measurement_plan(result, result.measurements[1]).effective_model_payload_bytes + + measurement_plan(result, result.measurements[2]).effective_model_payload_bytes); + for (size_t index = 3; index < result.measurements.size(); ++index) { EXPECT_EQ(result.measurements[index].status, LlmMeasurementStatus::Failed); } @@ -2261,7 +2312,7 @@ TEST(LlmMemoryRunnerTest, EXPECT_EQ(measurement.execution.reason_code, LlmExecutorReason::PAGED_POST_VALIDATION_FAILED); const auto* retained = - std::get_if(&measurement.execution.backend_evidence); + std::get_if(&measurement.execution.backend_evidence); ASSERT_NE(retained, nullptr); EXPECT_EQ(retained->executor.reason_code, LlmExecutorReason::PAGED_POST_VALIDATION_FAILED); @@ -2309,7 +2360,7 @@ TEST(LlmMemoryRunnerTest, if (context.kind != LlmRunnerTaskKind::Measurement) { return; } - LlmCpuTaskEvidence evidence; + LlmCpuTaskEvidence evidence = std::get(execution.backend_evidence); evidence.executor.valid = true; evidence.executor.reason_code = LlmExecutorReason::VALID; evidence.executor.elapsed_seconds = execution.timing.elapsed_seconds; @@ -2336,8 +2387,7 @@ TEST(LlmMemoryRunnerTest, << "scenario=" << static_cast(measurement.scenario)); ASSERT_EQ(measurement.status, LlmMeasurementStatus::Measured); ASSERT_TRUE(measurement.execution_evidence_available); - const auto* retained = std::get_if( - &measurement.execution.backend_evidence); + const auto* retained = std::get_if(&measurement.execution.backend_evidence); ASSERT_NE(retained, nullptr); EXPECT_EQ(retained->executor.reason_code, LlmExecutorReason::VALID); EXPECT_TRUE(retained->executor.post_validation_evaluated); @@ -2411,12 +2461,12 @@ TEST(LlmMemoryRunnerTest, CheckpointFailureWinsPendingStopAndInterruptsOnlyUnsta EXPECT_EQ(result.measurements[0].status, LlmMeasurementStatus::Measured); EXPECT_EQ(result.counters.planned_loops, 2u); EXPECT_EQ(result.counters.attempted_loops, 1u); - EXPECT_EQ(result.counters.completed_loops, 0u); + EXPECT_EQ(result.counters.completed_loops, 1u); EXPECT_EQ(result.counters.planned_measurements, 6u); - EXPECT_EQ(result.counters.attempted_measurements, 1u); + EXPECT_EQ(result.counters.attempted_measurements, 3u); EXPECT_EQ(result.counters.terminal_measurements, 6u); - EXPECT_EQ(result.counters.measured_measurements, 1u); - for (size_t index = 1; index < result.measurements.size(); ++index) { + EXPECT_EQ(result.counters.measured_measurements, 3u); + for (size_t index = 3; index < result.measurements.size(); ++index) { EXPECT_EQ(result.measurements[index].status, LlmMeasurementStatus::Interrupted) << index; EXPECT_EQ(result.measurements[index].reason_code, LlmRunnerReason::INTERRUPTION_BEFORE_TASK) << index; } @@ -2451,7 +2501,7 @@ TEST(LlmMemoryRunnerTest, TypedAndUnknownCheckpointExceptionsAreTerminalAndNever EXPECT_EQ(result.successful_logical_checkpoints, 0u); EXPECT_FALSE(result.terminal_checkpoint_attempted); EXPECT_EQ(result.measurements[0].status, LlmMeasurementStatus::Measured); - for (size_t index = 1; index < result.measurements.size(); ++index) { + for (size_t index = 3; index < result.measurements.size(); ++index) { EXPECT_EQ(result.measurements[index].status, LlmMeasurementStatus::Failed) << index; } if (typed) { @@ -2482,14 +2532,14 @@ TEST(LlmMemoryRunnerTest, CheckpointReturnFailureOverridesEarlierExecutorExcepti LlmMemoryResult result; EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result, hooks), EXIT_FAILURE); - EXPECT_EQ(kinds, (std::vector{LlmCheckpointKind::MeasurementTerminal})); + EXPECT_EQ(kinds, (std::vector{LlmCheckpointKind::CommandTerminal})); EXPECT_EQ(result.status, LlmRunStatus::Failed); EXPECT_EQ(result.reason_code, LlmRunnerReason::CHECKPOINT_WRITE_FAILED); EXPECT_TRUE(result.diagnostic.empty()); EXPECT_TRUE(result.checkpoint_failed); EXPECT_EQ(result.logical_checkpoint_attempts, 1u); EXPECT_EQ(result.successful_logical_checkpoints, 0u); - EXPECT_FALSE(result.terminal_checkpoint_attempted); + EXPECT_TRUE(result.terminal_checkpoint_attempted); } TEST(LlmMemoryRunnerTest, CommandTerminalCheckpointFailureIsNotRetried) { @@ -2507,16 +2557,16 @@ TEST(LlmMemoryRunnerTest, CommandTerminalCheckpointFailureIsNotRetried) { LlmMemoryResult result; EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result, hooks), EXIT_FAILURE); - ASSERT_EQ(kinds.size(), 4u); + ASSERT_EQ(kinds.size(), 2u); EXPECT_EQ(kinds.back(), LlmCheckpointKind::CommandTerminal); - EXPECT_EQ(result.logical_checkpoint_attempts, 4u); - EXPECT_EQ(result.successful_logical_checkpoints, 3u); + EXPECT_EQ(result.logical_checkpoint_attempts, 2u); + EXPECT_EQ(result.successful_logical_checkpoints, 1u); EXPECT_TRUE(result.terminal_checkpoint_attempted); EXPECT_FALSE(result.terminal_checkpoint_completed); EXPECT_EQ(result.status, LlmRunStatus::Failed); EXPECT_EQ(result.reason_code, LlmRunnerReason::CHECKPOINT_WRITE_FAILED); EXPECT_TRUE(result.results_complete); - EXPECT_FALSE(result.conclusions_valid); + EXPECT_FALSE(result.run_accepted); EXPECT_EQ(result.counters.measured_measurements, 3u); } @@ -2540,21 +2590,21 @@ TEST(LlmMemoryRunnerTest, MeasuredOnlyStatisticsKeepRawValuesAndClassifyCvThresh ASSERT_EQ(run_llm_memory_suite(config, plan, executor, result), EXIT_SUCCESS); const LlmScenarioAggregate& weights = result.aggregates[0]; - EXPECT_EQ(weights.effective_model_payload_gb_s.values.size(), 3u); + EXPECT_EQ(weights.accepted_measurement_ids.size(), 3u); EXPECT_NEAR(weights.effective_model_payload_gb_s.statistics.coefficient_of_variation_pct, 5.0, 1e-10); - EXPECT_EQ(weights.stability_quality, "stable"); + EXPECT_EQ(weights.observed_cv_classification, "below-threshold"); EXPECT_DOUBLE_EQ(*weights.effective_model_payload_gb_s.headline, 100.0); for (size_t index : {1u, 2u}) { const LlmScenarioAggregate& aggregate = result.aggregates[index]; - EXPECT_EQ(aggregate.effective_model_payload_gb_s.values.size(), 3u); + EXPECT_EQ(aggregate.accepted_measurement_ids.size(), 3u); EXPECT_GT(aggregate.effective_model_payload_gb_s.statistics.coefficient_of_variation_pct, Constants::LLM_STREAMING_CV_WARNING_PCT); - EXPECT_EQ(aggregate.stability_quality, "noisy"); + EXPECT_EQ(aggregate.observed_cv_classification, "above-threshold"); } EXPECT_EQ(result.quality_warnings, (std::vector{"kv_only-high-cv", "mixed-high-cv"})); EXPECT_TRUE(result.results_complete); - EXPECT_TRUE(result.conclusions_valid); + EXPECT_TRUE(result.run_accepted); } TEST(LlmMemoryRunnerTest, PartialAggregatesRetainEarlierMeasuredValueAndExcludeLaterInvalidValue) { @@ -2579,14 +2629,14 @@ TEST(LlmMemoryRunnerTest, PartialAggregatesRetainEarlierMeasuredValueAndExcludeL EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result), EXIT_FAILURE); EXPECT_EQ(result.status, LlmRunStatus::Failed); EXPECT_EQ(result.reason_code, LlmBackendReason::VALIDATION_FAILED); - ASSERT_EQ(result.aggregates[0].effective_model_payload_gb_s.values.size(), 1u); - EXPECT_NEAR(result.aggregates[0].effective_model_payload_gb_s.values.front(), 100.0, 1e-10); + ASSERT_EQ(result.aggregates[0].accepted_measurement_ids.size(), 1u); + EXPECT_NEAR(*derive_llm_measurement_metrics(result.measurements.at(result.aggregates[0].accepted_measurement_ids.front())).payload_gb_s, 100.0, 1e-10); EXPECT_EQ(result.aggregates[0].status, "partial"); - EXPECT_EQ(result.aggregates[0].stability_quality, "insufficient-samples"); + EXPECT_EQ(result.aggregates[0].observed_cv_classification, "insufficient-samples"); EXPECT_EQ(result.counters.measured_measurements, 5u); EXPECT_EQ(result.counters.attempted_measurements, 6u); EXPECT_EQ(result.measurements[5].status, LlmMeasurementStatus::Invalid); - EXPECT_FALSE(result.measurements[5].effective_model_payload_gb_s.has_value()); + EXPECT_FALSE(derive_llm_measurement_metrics(result.measurements[5]).payload_gb_s.has_value()); } TEST(LlmMemoryRunnerTest, InvalidAuthoritativeElapsedIsRejectedByCommonAcceptance) { @@ -2613,8 +2663,8 @@ TEST(LlmMemoryRunnerTest, InvalidAuthoritativeElapsedIsRejectedByCommonAcceptanc EXPECT_EQ(result.measurements[0].status, LlmMeasurementStatus::Invalid); EXPECT_EQ(result.measurements[0].reason_code, LlmBackendReason::INVALID_AUTHORITATIVE_ELAPSED); - EXPECT_FALSE(result.measurements[0].elapsed_seconds.has_value()); - EXPECT_TRUE(result.aggregates[0].effective_model_payload_gb_s.values.empty()); + EXPECT_FALSE(derive_llm_measurement_metrics(result.measurements[0]).latency_seconds.has_value()); + EXPECT_TRUE(result.aggregates[0].accepted_measurement_ids.empty()); } TEST(LlmMemoryRunnerTest, MeasurementCompletionMismatchIsRejectedWithoutAggregate) { @@ -2637,9 +2687,9 @@ TEST(LlmMemoryRunnerTest, MeasurementCompletionMismatchIsRejectedWithoutAggregat EXPECT_EQ(result.measurements[0].status, LlmMeasurementStatus::Failed); EXPECT_EQ(result.measurements[0].reason_code, LlmBackendReason::TASK_COMPLETION_MISMATCH); - EXPECT_TRUE(std::holds_alternative( + EXPECT_TRUE(std::holds_alternative( result.measurements[0].execution.backend_evidence)); - EXPECT_TRUE(result.aggregates[0].effective_model_payload_gb_s.values.empty()); + EXPECT_TRUE(result.aggregates[0].accepted_measurement_ids.empty()); EXPECT_EQ(result.counters.measured_measurements, 0u); } @@ -2765,14 +2815,14 @@ TEST(LlmMemoryRunnerTest, ExcludedTaskFailureWinsSimultaneousStopAndRetainsAttem EXPECT_FALSE(result.calibration_attempts[0][0].valid); EXPECT_EQ(result.calibration_attempts[0][0].reason_code, LlmBackendReason::RESOURCES_NOT_PREPARED); - EXPECT_EQ(result.calibration_attempts[0][0].work_units, config.iterations); - EXPECT_TRUE(result.frozen_scenario_plans.valid); + EXPECT_EQ(result.scenario_plans.at(result.calibration_attempts[0][0].plan_handle).plan.work_units, config.iterations); + EXPECT_TRUE(frozen_plans_valid(result)); EXPECT_GT(result.counters.planned_work_units, 0u); EXPECT_GT(result.counters.planned_effective_model_payload_bytes, 0u); for (const LlmMeasurementState& measurement : result.measurements) { - EXPECT_LT(measurement.frozen_plan_index, kLlmScenarioCount); - EXPECT_EQ(measurement.planned_work_units, config.iterations); - EXPECT_GT(measurement.planned_effective_model_payload_bytes, 0u); + EXPECT_LT(static_cast(measurement.scenario), kLlmScenarioCount); + EXPECT_EQ(measurement_plan(result, measurement).work_units, config.iterations); + EXPECT_GT(measurement_plan(result, measurement).effective_model_payload_bytes, 0u); } expect_interrupted_tail(result, 0); } @@ -2802,7 +2852,7 @@ TEST(LlmMemoryRunnerTest, ExcludedExecutorExceptionRetainsTerminalAttemptBeforeC EXPECT_FALSE(result.calibration_attempts[0][0].valid); EXPECT_EQ(result.calibration_attempts[0][0].reason_code, LlmRunnerReason::RUNNER_EXCEPTION); - EXPECT_EQ(result.calibration_attempts[0][0].work_plan_identity, + EXPECT_EQ(result.scenario_plans.at(result.calibration_attempts[0][0].plan_handle).plan.plan_identity, build_llm_scenario_work_plan(plan, LlmScenario::WeightsOnly, config.iterations, true).plan_identity); ASSERT_EQ(checkpoints.size(), 1u); EXPECT_EQ(checkpoints[0].kind, LlmCheckpointKind::CommandTerminal); @@ -2842,28 +2892,28 @@ TEST(LlmMemoryRunnerTest, TypedAndUnknownExecutorExceptionsRemainInsideRunnerBou for (const LlmMeasurementState& measurement : result.measurements) { EXPECT_EQ(measurement.status, LlmMeasurementStatus::Failed); } - ASSERT_EQ(checkpoints.size(), 2u); - EXPECT_EQ(checkpoints[0].kind, LlmCheckpointKind::MeasurementTerminal); + ASSERT_EQ(checkpoints.size(), 1u); + EXPECT_EQ(checkpoints[0].kind, LlmCheckpointKind::CommandTerminal); EXPECT_EQ(checkpoints[0].attempted_measurements, 1u); EXPECT_EQ(checkpoints[0].terminal_measurements, 3u); EXPECT_EQ(checkpoints[0].measured_measurements, 0u); - EXPECT_EQ(checkpoints[1].kind, LlmCheckpointKind::CommandTerminal); - EXPECT_EQ(result.logical_checkpoint_attempts, 2u); - EXPECT_EQ(result.successful_logical_checkpoints, 2u); + EXPECT_EQ(result.logical_checkpoint_attempts, 1u); + EXPECT_EQ(result.successful_logical_checkpoints, 1u); EXPECT_TRUE(result.terminal_checkpoint_completed); } } -TEST(LlmMemoryRunnerTest, ExecutorExceptionStillObservesStopRaisedByItsMeasurementCheckpoint) { +TEST(LlmMemoryRunnerTest, ExecutorExceptionObservesStopAtTaskBoundaryWithoutProgressSnapshot) { const LlmMemoryConfig config = explicit_config(2); const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); ASSERT_TRUE(plan.valid) << plan.reason_code; bool stop = false; FakeLlmBackend executor; - executor.mutate = [](const LlmMemoryWorkPlan&, const LlmScenarioWorkPlan&, + executor.mutate = [&stop](const LlmMemoryWorkPlan&, const LlmScenarioWorkPlan&, const LlmRunnerTaskContext& context, size_t, LlmTaskExecutionResult&) { if (context.kind == LlmRunnerTaskKind::Measurement) { + stop = true; throw std::runtime_error("injected executor failure"); } }; @@ -2880,8 +2930,7 @@ TEST(LlmMemoryRunnerTest, ExecutorExceptionStillObservesStopRaisedByItsMeasureme LlmMemoryResult result; EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result, hooks), EXIT_FAILURE); - EXPECT_EQ(kinds, (std::vector{LlmCheckpointKind::MeasurementTerminal, - LlmCheckpointKind::CommandTerminal})); + EXPECT_EQ(kinds, (std::vector{LlmCheckpointKind::CommandTerminal})); EXPECT_EQ(result.status, LlmRunStatus::Failed); EXPECT_EQ(result.reason_code, LlmRunnerReason::RUNNER_EXCEPTION); EXPECT_TRUE(result.interruption_requested); @@ -2981,6 +3030,15 @@ TEST(LlmMemoryRunnerTest, } LlmMemoryConfig config = explicit_config(); + config.loop_count = 0; + EXPECT_FALSE(calculate_llm_runner_auxiliary_estimate(config, plan).valid); + FakeLlmBackend zero_executor; + LlmMemoryResult zero_result; + EXPECT_EQ(run_llm_memory_suite(config, plan, zero_executor, zero_result), EXIT_FAILURE); + EXPECT_FALSE(zero_result.initialized); + EXPECT_TRUE(zero_executor.calls.empty()); + EXPECT_EQ(zero_executor.initialize_calls, 0u); + EXPECT_EQ(zero_result.logical_checkpoint_attempts, 0u); config.loop_count = std::numeric_limits::max(); const LlmRunnerAuxiliaryEstimate estimate = calculate_llm_runner_auxiliary_estimate(config, plan); EXPECT_FALSE(estimate.valid); @@ -3015,14 +3073,39 @@ TEST(LlmMemoryRunnerTest, RunnerAuxiliaryEstimateHasExactCategoryBreakdownForExp EXPECT_EQ(estimate.loop_record_bytes, config.loop_count * sizeof(LlmLoopRecord)); EXPECT_EQ(estimate.calibration_record_bytes, attempts_per_scenario * kLlmScenarioCount * sizeof(LlmCalibrationAttempt)); - EXPECT_GT(estimate.calibration_identity_bytes, 0u); - EXPECT_EQ(estimate.aggregate_value_bytes, planned_measurements * 3 * sizeof(double)); - EXPECT_EQ(estimate.statistics_workspace_bytes, config.loop_count * 2 * sizeof(double)); + // Independently enumerate the bounded plan shapes. The estimate's category + // values are not used to derive the expected identity/storage budget. + std::array max_work{}; + size_t expected_identity_bytes = 0; + size_t largest_identity = 0; + for (size_t index = 0; index < 3; ++index) { + const auto scenario = static_cast(index); + max_work[index] = calculate_llm_scenario_limits(plan.geometry, scenario, plan.backend).effective_maximum_work_units; + const auto maximum = build_llm_scenario_work_plan(plan, scenario, max_work[index], config.user_specified_iterations); + ASSERT_TRUE(maximum.valid); + largest_identity = std::max(largest_identity, maximum.plan_identity.size()); + expected_identity_bytes += 2 * (maximum.plan_identity.size() + 1) * (attempts_per_scenario + 1); + } + const size_t canonical_count = 3 * (attempts_per_scenario + 1); + expected_identity_bytes += 2 * (plan.plan_identity.size() + 1) * canonical_count; + EXPECT_EQ(estimate.calibration_identity_bytes, expected_identity_bytes); + const auto maximum_frozen = freeze_llm_scenario_work_plans(plan, max_work, config.user_specified_iterations); + ASSERT_TRUE(maximum_frozen.valid); + size_t frozen_strings = maximum_frozen.reason_code.size() + maximum_frozen.model_plan_identity.size() + + maximum_frozen.plan_identity.size() + 3; + for (const auto& scenario : maximum_frozen.scenarios) + frozen_strings += scenario.reason_code.size() + scenario.model_plan_identity.size() + scenario.plan_identity.size() + 3; + const size_t expected_fixed_bytes = 2 * frozen_strings + + 4 * (largest_identity + plan.plan_identity.size() + 2) + + 2 * 65 * (planned_measurements + 1) + 2 * 513 + + canonical_count * (sizeof(LlmCanonicalScenarioPlan) + 2 * sizeof(size_t)); + EXPECT_EQ(estimate.fixed_metadata_bytes, expected_fixed_bytes); + EXPECT_EQ(estimate.aggregate_value_bytes, planned_measurements * sizeof(size_t)); + EXPECT_EQ(estimate.statistics_workspace_bytes, config.loop_count * 3 * sizeof(double)); EXPECT_EQ(estimate.warning_record_bytes, (kLlmScenarioCount + 1) * sizeof(std::string_view)); - EXPECT_GT(estimate.fixed_metadata_bytes, 0u); EXPECT_EQ(estimate.retained_checksum_bytes, - planned_measurements * cpu_plan->effective_workers * 2 * - sizeof(LlmWorkerChecksum)); + (planned_measurements + (attempts_per_scenario + 1) * kLlmScenarioCount + 2) * + cpu_plan->effective_workers * sizeof(LlmWorkerChecksum)); EXPECT_EQ(estimate.checksum_auxiliary_bytes, estimate.retained_checksum_bytes); EXPECT_EQ(estimate.orchestration_auxiliary_bytes, estimate.measurement_record_bytes + estimate.loop_record_bytes + estimate.calibration_record_bytes + @@ -3033,8 +3116,8 @@ TEST(LlmMemoryRunnerTest, RunnerAuxiliaryEstimateHasExactCategoryBreakdownForExp } EXPECT_EQ(estimates[1].calibration_record_bytes, estimates[0].calibration_record_bytes * (4 + Constants::LLM_CALIBRATION_MAX_CORRECTIONS)); - EXPECT_EQ(estimates[1].calibration_identity_bytes, - estimates[0].calibration_identity_bytes * (4 + Constants::LLM_CALIBRATION_MAX_CORRECTIONS)); + EXPECT_EQ(estimates[1].calibration_identity_bytes * 2, + estimates[0].calibration_identity_bytes * (5 + Constants::LLM_CALIBRATION_MAX_CORRECTIONS)); } TEST(LlmMemoryRunnerTest, RunnerAuxiliaryBudgetAcceptsExactBoundaryAndRejectsOneByteLess) { @@ -3058,6 +3141,18 @@ TEST(LlmMemoryRunnerTest, RunnerAuxiliaryBudgetAcceptsExactBoundaryAndRejectsOne EXPECT_EQ(run_llm_memory_suite(config, exact, exact_executor, exact_result), EXIT_SUCCESS); EXPECT_TRUE(exact_result.results_complete); + for (bool checksum_extra : {false, true}) { + auto above_request = request; + if (checksum_extra) ++above_request.checksum_auxiliary_bytes; + else ++above_request.orchestration_auxiliary_bytes; + const auto above = build_llm_memory_work_plan(above_request); + ASSERT_TRUE(above.valid); + FakeLlmBackend above_executor; + LlmMemoryResult above_result; + EXPECT_EQ(run_llm_memory_suite(config, above, above_executor, above_result), EXIT_SUCCESS); + EXPECT_TRUE(above_result.run_accepted); + } + ASSERT_GT(request.orchestration_auxiliary_bytes, 0u); ASSERT_GT(request.checksum_auxiliary_bytes, 0u); for (bool checksum_short : {false, true}) { @@ -3158,7 +3253,7 @@ TEST(LlmMemoryRunnerTest, EXPECT_EQ(measurement.execution.reason_code, LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); const auto* retained = - std::get_if(&measurement.execution.backend_evidence); + std::get_if(&measurement.execution.backend_evidence); ASSERT_NE(retained, nullptr); EXPECT_EQ(retained->executor.reason_code, LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); @@ -3172,3 +3267,147 @@ TEST(LlmMemoryRunnerTest, EXPECT_FALSE(retained->executor.cold_checks[1].valid); EXPECT_EQ(retained->executor.cold_checks[1].reason_code, LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); } + +TEST(LlmMemoryRunnerTest, CanonicalInsertionReusesExactPlanAndSnapshotRemapsStableHandles) { + const auto config = explicit_config(1, 8); + const auto model = build_runner_admitted_plan(config); + ASSERT_TRUE(model.valid); + FakeLlmBackend backend; + LlmMemoryResult result; + const auto eight = build_llm_scenario_work_plan(model, LlmScenario::Mixed, 8, true); + const auto two = build_llm_scenario_work_plan(model, LlmScenario::Mixed, 2, true); + const auto first = register_llm_scenario_plan(result, model, eight, backend); + const auto second = register_llm_scenario_plan(result, model, two, backend); + EXPECT_EQ(register_llm_scenario_plan(result, model, eight, backend), first); + EXPECT_EQ(backend.expected_calls, 2u); + EXPECT_EQ(result.scenario_plans.size(), 2u); + result.frozen_plan_handles[2] = first; + for (size_t t = 3; t < 20; ++t) { + register_llm_scenario_plan(result, model, + build_llm_scenario_work_plan(model, LlmScenario::WeightsOnly, t, true), backend); + } + EXPECT_EQ(result.scenario_plans.at(first).plan.work_units, 8u); + EXPECT_EQ(result.scenario_plans.at(second).plan.work_units, 2u); + prepare_llm_result_snapshot(result); + EXPECT_LT(result.snapshot_plan_refs.at(second), result.snapshot_plan_refs.at(first)); + EXPECT_EQ(result.snapshot_plan_order.at(result.snapshot_plan_refs.at(first)), first); + EXPECT_EQ(result.frozen_plan_handles[2], first); + const size_t calls = backend.expected_calls; + prepare_llm_result_snapshot(result); + EXPECT_EQ(backend.expected_calls, calls); + auto contradiction = eight; + ++contradiction.weight_read_bytes; + EXPECT_THROW(register_llm_scenario_plan(result, model, contradiction, backend), std::invalid_argument); + EXPECT_EQ(backend.expected_calls, calls); + EXPECT_EQ(find_llm_scenario_plan(result, kLlmNoTaskIndex), nullptr); +} + +TEST(LlmMemoryRunnerTest, RequiredColdChecksAndWholeBackendEvidenceCannotBeOmitted) { + const auto config = explicit_config(1, 1); + const auto model = build_runner_admitted_plan(config); + ASSERT_TRUE(model.valid); + for (bool missing_whole : {false, true}) { + FakeLlmBackend backend; + backend.mutate = [missing_whole](const auto&, const auto&, const LlmRunnerTaskContext& context, + size_t, LlmTaskExecutionResult& execution) { + if (context.kind != LlmRunnerTaskKind::Measurement || context.scenario != LlmScenario::KvOnly) return; + if (missing_whole) execution.backend_evidence = std::monostate{}; + else std::get(execution.backend_evidence).executor.cold_checks[1].evaluated = false; + }; + LlmMemoryResult result; + EXPECT_EQ(run_llm_memory_suite(config, model, backend, result), EXIT_FAILURE); + EXPECT_FALSE(result.run_accepted); + ASSERT_EQ(result.measurements.size(), 3u); + EXPECT_EQ(result.measurements[1].status, LlmMeasurementStatus::Invalid); + EXPECT_EQ(result.measurements[1].reason_code, LlmBackendReason::VALIDATION_FAILED); + EXPECT_FALSE(derive_llm_measurement_metrics(result.measurements[1]).payload_gb_s); + EXPECT_TRUE(result.aggregates[1].accepted_measurement_ids.empty()); + EXPECT_TRUE(result.measurements[1].execution.timing.valid); + } +} + +TEST(LlmMemoryRunnerTest, BoundedSnapshotsObservePriorWritersAndDoNotSortBetweenSnapshots) { + for (size_t count : {size_t{1}, size_t{3}, size_t{12}, size_t{48}}) { + const auto config = explicit_config(count, 1); + const auto model = build_runner_admitted_plan(config); + ASSERT_TRUE(model.valid); + for (bool progress : {false, true}) { + FakeLlmBackend backend; + LlmMemoryResult result; + size_t callbacks = 0; + size_t previous_statistics = 0; + LlmRunnerHooks hooks; + hooks.progress_snapshots = progress; + hooks.observe_file_writes = [&]() { return std::make_pair(callbacks, callbacks); }; + hooks.checkpoint = [&](const LlmMemoryResult& snapshot, LlmCheckpointKind kind) { + EXPECT_EQ(snapshot.prior_file_writer_attempts, callbacks); + EXPECT_EQ(snapshot.prior_successful_file_writes, callbacks); + EXPECT_EQ(snapshot.snapshot_preparation_attempts, callbacks + 1); + EXPECT_EQ(snapshot.exact_statistics_passes, previous_statistics + 9); + previous_statistics = snapshot.exact_statistics_passes; + if (kind == LlmCheckpointKind::MeasurementTerminal) { + EXPECT_EQ(snapshot.counters.completed_loops % snapshot.snapshot_interval_loops, 0u); + EXPECT_EQ(snapshot.counters.terminal_measurements, snapshot.counters.completed_loops * 3); + } else EXPECT_FALSE(snapshot.terminal_checkpoint_completed); + ++callbacks; + return EXIT_SUCCESS; + }; + EXPECT_EQ(run_llm_memory_suite(config, model, backend, result, hooks), EXIT_SUCCESS); + const size_t interval = std::max(size_t{1}, count / 8 + (count % 8 != 0)); + EXPECT_EQ(callbacks, (progress ? count / interval : 0) + 1); + EXPECT_LE(callbacks, 9u); + EXPECT_TRUE(result.run_accepted); + EXPECT_EQ(backend.expected_calls, 3u); + for (const auto& aggregate : result.aggregates) { + EXPECT_EQ(aggregate.accepted_measurement_ids.size(), count); + EXPECT_EQ(aggregate.effective_model_payload_gb_s.statistics.sample_count, count); + } + } + } +} + +TEST(LlmMemoryRunnerTest, SnapshotRejectsUnresolvedPlanAndContradictoryCompletedWork) { + const auto config = explicit_config(1, 1); + const auto model = build_runner_admitted_plan(config); + FakeLlmBackend backend; + LlmMemoryResult result; + ASSERT_EQ(run_llm_memory_suite(config, model, backend, result), EXIT_SUCCESS); + auto invalid = result; + invalid.measurements[0].plan_handle = kLlmNoTaskIndex; + EXPECT_THROW(prepare_llm_result_snapshot(invalid), std::invalid_argument); + invalid = result; + ++invalid.measurements[0].execution.completion.completed_work_units; + EXPECT_THROW(prepare_llm_result_snapshot(invalid), std::invalid_argument); +} + +TEST(LlmMemoryRunnerTest, CanonicalCpuExpectationRequiresPreparedMatchingBackendResources) { + const auto config = explicit_config(1, 1); + const auto model = build_runner_admitted_plan(config); + const auto plan = build_llm_scenario_work_plan(model, LlmScenario::Mixed, 1, true); + auto backend = create_llm_cpu_backend(); + const auto unavailable = backend->expected_cpu_checksum(model, plan); + EXPECT_FALSE(unavailable.valid); + EXPECT_TRUE(unavailable.workers.empty()); + LlmMemoryResult result; + const auto handle = register_llm_scenario_plan(result, model, plan, *backend); + EXPECT_FALSE(result.scenario_plans.at(handle).expected.available); + EXPECT_TRUE(result.scenario_plans.at(handle).expected.cpu_workers.empty()); +} + +TEST(LlmMemoryRunnerTest, SnapshotRejectsSwappedFrozenAndCalibrationScenarioReferences) { + const auto config = explicit_config(1, 1); + const auto model = build_runner_admitted_plan(config); + FakeLlmBackend backend; + LlmMemoryResult result; + ASSERT_EQ(run_llm_memory_suite(config, model, backend, result), EXIT_SUCCESS); + auto invalid = result; + std::swap(invalid.frozen_plan_handles[0], invalid.frozen_plan_handles[1]); + EXPECT_THROW(prepare_llm_result_snapshot(invalid), std::invalid_argument); + invalid = result; + ASSERT_GT(invalid.calibration_attempt_counts[0], 0u); + invalid.calibration_attempts[0][0].plan_handle = invalid.frozen_plan_handles[1]; + EXPECT_THROW(prepare_llm_result_snapshot(invalid), std::invalid_argument); + invalid = result; + invalid.calibration_attempts[0][0].scenario = LlmScenario::KvOnly; + EXPECT_THROW(prepare_llm_result_snapshot(invalid), std::invalid_argument); +} diff --git a/tests/test_llm_memory_work_plan.cpp b/tests/test_llm_memory_work_plan.cpp index b32dcbc..d29ca81 100644 --- a/tests/test_llm_memory_work_plan.cpp +++ b/tests/test_llm_memory_work_plan.cpp @@ -33,6 +33,7 @@ #include "core/config/constants.h" #include "llm_memory/llm_kv_layout.h" #include "llm_memory/llm_metal_backend.h" +#include "llm_memory/llm_runner.h" #include "llm_memory/llm_work_plan.h" #include "test_memory_system_calls.h" #include "utils/numeric_utils.h" @@ -605,14 +606,14 @@ TEST(LlmMemoryWorkPlanTest, ConstantsAndIdentitiesMatchFrozenContract) { EXPECT_EQ(Constants::LLM_MAX_ACCOUNTED_BYTES_PER_TASK, 64ULL * 1024ULL * Constants::BYTES_PER_MB); EXPECT_DOUBLE_EQ(Constants::LLM_STREAMING_CV_WARNING_PCT, 5.0); - EXPECT_EQ(Constants::LLM_JSON_SCHEMA_VERSION, 1); + EXPECT_EQ(Constants::LLM_JSON_SCHEMA_VERSION, 2); EXPECT_STREQ(Constants::LLM_JSON_MODE_NAME, "llm_memory"); EXPECT_EQ(Constants::LLM_WEIGHT_PASSES_PER_WORK_UNIT, 1u); EXPECT_EQ(Constants::LLM_KV_REPLAY_FACTOR, 1u); EXPECT_STREQ(Constants::LLM_CPU_DECODE_CONTIGUOUS_METHODOLOGY_VERSION, - "llm-memory-v1-cpu-decode-contiguous"); + "llm-memory-v2-cpu-decode-contiguous"); EXPECT_STREQ(Constants::LLM_CPU_PREFILL_PAGED_METHODOLOGY_VERSION, - "llm-memory-v1-cpu-prefill-paged"); + "llm-memory-v2-cpu-prefill-paged"); EXPECT_STREQ(Constants::LLM_COMPONENT_IDENTITY_VERSION, "llm-memory-components-v1"); EXPECT_STREQ(Constants::LLM_LOGICAL_PROFILE_VERSION, @@ -699,15 +700,15 @@ TEST(LlmMemoryWorkPlanTest, ConstantsAndIdentitiesMatchFrozenContract) { EXPECT_EQ(build_llm_methodology_version( LlmMemoryBackend::Cpu, LlmPhase::Decode, LlmKvLayout::Paged), - "llm-memory-v1-cpu-decode-paged"); + "llm-memory-v2-cpu-decode-paged"); EXPECT_EQ(build_llm_methodology_version( LlmMemoryBackend::Metal, LlmPhase::Prefill, LlmKvLayout::Paged), - "llm-memory-v1-metal-prefill-paged"); + "llm-memory-v2-metal-prefill-paged"); EXPECT_EQ(build_llm_methodology_version( LlmMemoryBackend::Metal, LlmPhase::Decode, LlmKvLayout::Paged), - "llm-memory-v1-metal-decode-paged"); + "llm-memory-v2-metal-decode-paged"); EXPECT_EQ(llm_seed_domain_value(LlmSeedDomain::WeightBuffer), 0x4C4C4D5745494748ULL); @@ -1423,7 +1424,7 @@ TEST(LlmMemoryWorkPlanTest, LlmMemoryBackend::Metal); EXPECT_EQ(draft.auxiliary_preflight.effective_workers, 0u); EXPECT_EQ(draft.candidate.methodology_version, - "llm-memory-v1-metal-decode-contiguous"); + "llm-memory-v2-metal-decode-contiguous"); EXPECT_EQ(draft.candidate.component_identities.backend_executor_version, LlmMetalDecodeContiguousVersion::EXECUTOR); EXPECT_EQ(draft.candidate.component_identities.schedule_version, @@ -1462,7 +1463,7 @@ TEST(LlmMemoryWorkPlanTest, EXPECT_FALSE(draft.candidate.valid); ASSERT_TRUE(draft.candidate.geometry.valid); EXPECT_EQ(draft.candidate.methodology_version, - "llm-memory-v1-metal-decode-paged"); + "llm-memory-v2-metal-decode-paged"); ASSERT_TRUE( draft.candidate.component_identities.permutation_version.has_value()); EXPECT_EQ( @@ -1597,7 +1598,7 @@ TEST(LlmMemoryWorkPlanTest, EXPECT_EQ(logical.layout_metadata_read_bytes_per_work_unit, 0u); EXPECT_EQ(candidate.methodology_version, - "llm-memory-v1-metal-prefill-contiguous"); + "llm-memory-v2-metal-prefill-contiguous"); EXPECT_EQ(candidate.component_identities.logical_profile_version, Constants::LLM_PREFILL_LOGICAL_PROFILE_VERSION); EXPECT_EQ(candidate.component_identities.kv_layout_version, @@ -2079,7 +2080,7 @@ TEST(LlmMemoryWorkPlanTest, EXPECT_EQ(logical.layout_metadata_read_bytes_per_work_unit, 240u); EXPECT_EQ(candidate.methodology_version, - "llm-memory-v1-metal-prefill-paged"); + "llm-memory-v2-metal-prefill-paged"); EXPECT_EQ(candidate.component_identities.logical_profile_version, Constants::LLM_PREFILL_LOGICAL_PROFILE_VERSION); EXPECT_EQ(candidate.component_identities.kv_layout_version, @@ -2938,7 +2939,7 @@ TEST(LlmMemoryWorkPlanTest, EXPECT_EQ(plan.backend, LlmMemoryBackend::Cpu); EXPECT_EQ(plan.phase, LlmPhase::Decode); EXPECT_EQ(plan.kv_layout, LlmKvLayout::Paged); - EXPECT_EQ(plan.methodology_version, "llm-memory-v1-cpu-decode-paged"); + EXPECT_EQ(plan.methodology_version, "llm-memory-v2-cpu-decode-paged"); const LlmGeometry& geometry = plan.geometry; ASSERT_TRUE(geometry.valid) << geometry.reason_code; @@ -5045,9 +5046,9 @@ TEST(LlmMemoryWorkPlanTest, : LlmPrefillVersion::CHECKSUM_ORACLE); } EXPECT_EQ(contiguous.methodology_version, - "llm-memory-v1-cpu-prefill-contiguous"); + "llm-memory-v2-cpu-prefill-contiguous"); EXPECT_EQ(paged.methodology_version, - "llm-memory-v1-cpu-prefill-paged"); + "llm-memory-v2-cpu-prefill-paged"); EXPECT_FALSE(contiguous.component_identities.permutation_version.has_value()); ASSERT_TRUE(paged.component_identities.permutation_version.has_value()); EXPECT_EQ(*paged.component_identities.permutation_version, @@ -5672,3 +5673,128 @@ TEST(LlmMemoryWorkPlanTest, kExpected[index].visit_tokens); } } + +TEST(LlmMemoryWorkPlanTest, TheoreticalPrefillOverflowIsNullableWithoutRejectingByteWork) { + auto request = exact_prefill_request(128, 128); + request.query_head_count = size_t{1} << 40; + request.head_dimension = 4096; + const auto plan = resolve_llm_prefill_plan(request); + ASSERT_TRUE(plan.valid) << plan.reason_code; + EXPECT_EQ(plan.causal_token_pairs_per_sequence, 8256u); + ASSERT_TRUE(plan.logical_attention_pairs.has_value()); + EXPECT_EQ(*plan.logical_attention_pairs, 8256ULL * (size_t{1} << 40) * request.layer_count * request.batch_size); + EXPECT_FALSE(plan.logical_attention_fma_terms.has_value()); + EXPECT_EQ(plan.kv_write_bytes_per_work_unit, + 128 * request.layer_count * request.batch_size * request.k_or_v_record_bytes_per_layer * 2); + auto overflowing_bytes = request; + overflowing_bytes.k_or_v_record_bytes_per_layer = std::numeric_limits::max(); + EXPECT_FALSE(resolve_llm_prefill_plan(overflowing_bytes).valid); + const auto none = calculate_llm_prefill_model_context(std::numeric_limits::max(), 1, 1, 1, 1); + EXPECT_FALSE(none.causal_token_pairs_per_sequence); + EXPECT_FALSE(none.logical_attention_pairs); + EXPECT_FALSE(none.logical_attention_fma_terms); + const auto pairs_overflow = calculate_llm_prefill_model_context(3, 1, 1, std::numeric_limits::max(), 1); + EXPECT_EQ(pairs_overflow.causal_token_pairs_per_sequence, 6u); + EXPECT_FALSE(pairs_overflow.logical_attention_pairs); + EXPECT_FALSE(pairs_overflow.logical_attention_fma_terms); +} + +TEST(LlmMemoryWorkPlanTest, MetalPagedCanonicalExpectedScratchMatchesBothEstimatorPathsAndRejectsOverflow) { + for (const LlmPhase phase : {LlmPhase::Decode, LlmPhase::Prefill}) { + for (const bool explicit_work : {false, true}) { + SCOPED_TRACE(phase == LlmPhase::Decode ? "decode" : "prefill"); + SCOPED_TRACE(explicit_work); + auto request = phase == LlmPhase::Decode ? metal_paged_work_plan_request() + : metal_prefill_work_plan_request(LlmKvLayout::Paged); + auto draft = prepare_llm_memory_work_plan(request); + ASSERT_TRUE(draft.valid) << draft.reason_code; + auto provisional = build_llm_metal_execution_plan(metal_resource_request(draft.candidate)); + ASSERT_TRUE(provisional.valid) << provisional.reason_code; + ASSERT_TRUE(attach_llm_metal_execution_plan(draft, std::move(provisional))); + const auto preflight = draft.auxiliary_preflight; + // Both fixtures contain six uint32 table entries; the validation bitset needs one byte. + // The fixed envelope covers the 4 KiB hash chunk and bounded identity/stack workspace. + constexpr size_t kTableBytes = 6 * sizeof(uint32_t); + constexpr size_t kBitsetBytes = 1; + constexpr size_t kFixedEnvelope = 16 * 1024; + EXPECT_EQ(draft.candidate.geometry.block_table_entries, 6u); + EXPECT_EQ(Constants::LLM_CANONICAL_PAGED_EXPECTED_FIXED_SCRATCH_BYTES, kFixedEnvelope); + EXPECT_EQ(preflight.canonical_expected_scratch_bytes, kTableBytes + kBitsetBytes + kFixedEnvelope); + const auto* runtime = get_llm_metal_execution_plan(draft.candidate); + ASSERT_NE(runtime, nullptr); + ASSERT_TRUE(runtime->resources.paged_layout.has_value()); + const auto& layout = *runtime->resources.paged_layout; + EXPECT_EQ(layout.memory.validation_bitset_bytes, kBitsetBytes); + EXPECT_EQ(layout.memory.transient_peak_bytes, kTableBytes + kBitsetBytes); + const auto table = materialize_llm_kv_block_table(layout, 42); + ASSERT_TRUE(table.valid) << table.reason_code; + EXPECT_EQ(table.entries.size(), 6u); + EXPECT_EQ(table.entries.capacity() * sizeof(uint32_t), kTableBytes); + + LlmMemoryConfig config; + config.backend = LlmMemoryBackend::Metal; + config.phase = phase; + config.kv_layout = LlmKvLayout::Paged; + config.loop_count = 12; + config.user_specified_iterations = explicit_work; + const auto estimated = calculate_llm_runner_auxiliary_estimate(config, preflight); + ASSERT_TRUE(estimated.valid) << estimated.reason_code; + auto without_scratch = preflight; + without_scratch.canonical_expected_scratch_bytes = 0; + const auto control = calculate_llm_runner_auxiliary_estimate(config, without_scratch); + ASSERT_TRUE(control.valid); + EXPECT_EQ(estimated.fixed_metadata_bytes - control.fixed_metadata_bytes, + kTableBytes + kBitsetBytes + kFixedEnvelope); + EXPECT_EQ(estimated.orchestration_auxiliary_bytes - control.orchestration_auxiliary_bytes, + kTableBytes + kBitsetBytes + kFixedEnvelope); + EXPECT_EQ(estimated.total_auxiliary_bytes - control.total_auxiliary_bytes, + kTableBytes + kBitsetBytes + kFixedEnvelope); + EXPECT_EQ(estimated.checksum_auxiliary_bytes, control.checksum_auxiliary_bytes); + + auto exact = build_llm_metal_execution_plan( + metal_resource_request(draft.candidate, estimated.total_auxiliary_bytes)); + ASSERT_TRUE(exact.valid) << exact.reason_code; + auto plan = finalize_llm_memory_work_plan(std::move(draft), std::move(exact), + estimated.checksum_auxiliary_bytes, estimated.orchestration_auxiliary_bytes); + ASSERT_TRUE(plan.valid) << plan.reason_code; + const auto finalized = calculate_llm_runner_auxiliary_estimate(config, plan); + ASSERT_TRUE(finalized.valid) << finalized.reason_code; + for (const auto field : {&LlmRunnerAuxiliaryEstimate::measurement_record_bytes, + &LlmRunnerAuxiliaryEstimate::loop_record_bytes, + &LlmRunnerAuxiliaryEstimate::calibration_record_bytes, + &LlmRunnerAuxiliaryEstimate::calibration_identity_bytes, + &LlmRunnerAuxiliaryEstimate::aggregate_value_bytes, + &LlmRunnerAuxiliaryEstimate::statistics_workspace_bytes, + &LlmRunnerAuxiliaryEstimate::warning_record_bytes, + &LlmRunnerAuxiliaryEstimate::fixed_metadata_bytes, + &LlmRunnerAuxiliaryEstimate::retained_checksum_bytes, + &LlmRunnerAuxiliaryEstimate::checksum_auxiliary_bytes, + &LlmRunnerAuxiliaryEstimate::orchestration_auxiliary_bytes, + &LlmRunnerAuxiliaryEstimate::total_auxiliary_bytes}) { + // Preflight additionally reserves 64 KiB for each runtime-dependent + // identity. Each canonical record charges two doubled identities; + // fixed metadata charges eight doubled frozen strings and two active + // strings at four times capacity. This reserve is deliberately absent + // from the finalized estimate; the canonical scratch itself is equal. + constexpr size_t kIdentityGrowth = 64 * 1024; + EXPECT_EQ(LlmMetalPlannerAccounting::RUNTIME_IDENTITY_GROWTH_RESERVE_BYTES, kIdentityGrowth); + const size_t canonical_count = 3 * (explicit_work ? 2 : 5 + Constants::LLM_CALIBRATION_MAX_CORRECTIONS); + const size_t identity_reserve = 4 * kIdentityGrowth * canonical_count; + const size_t fixed_reserve = 24 * kIdentityGrowth; + const size_t expected_reserve = field == &LlmRunnerAuxiliaryEstimate::calibration_identity_bytes ? identity_reserve : + field == &LlmRunnerAuxiliaryEstimate::fixed_metadata_bytes ? fixed_reserve : + field == &LlmRunnerAuxiliaryEstimate::orchestration_auxiliary_bytes || + field == &LlmRunnerAuxiliaryEstimate::total_auxiliary_bytes ? identity_reserve + fixed_reserve : 0; + EXPECT_EQ(estimated.*field - finalized.*field, expected_reserve); + } + auto overflowing = preflight; + overflowing.canonical_expected_scratch_bytes = std::numeric_limits::max(); + EXPECT_FALSE(calculate_llm_runner_auxiliary_estimate(config, overflowing).valid); + auto* metal = get_llm_metal_execution_plan(plan); + ASSERT_NE(metal, nullptr); + ASSERT_TRUE(metal->resources.paged_layout.has_value()); + metal->resources.paged_layout->memory.transient_peak_bytes = std::numeric_limits::max(); + EXPECT_FALSE(calculate_llm_runner_auxiliary_estimate(config, plan).valid); + } + } +} diff --git a/tests/test_messages.cpp b/tests/test_messages.cpp index 9d1f120..968f68d 100644 --- a/tests/test_messages.cpp +++ b/tests/test_messages.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -252,9 +253,10 @@ TEST(MessagesTest, LlmMemoryCliMessagesHaveExactOutput) { std::to_string(Constants::LLM_DEFAULT_LOOP_COUNT) + ").\n" " --seed Reproducible base seed; generated once when omitted.\n" - " -o, --output JSON schema 1 target; exact - writes one final document to\n" + " -o, --output JSON schema 2 target; exact - writes one final document to\n" " stdout and routes human output to stderr. Every other non-empty\n" - " target is a file with atomic scenario and terminal checkpoints.\n" + " target is a file with bounded loop and terminal atomic snapshots.\n" + " K=max(1,ceil(count/8)); abrupt loss bound: 3K completed attempts.\n" " An empty value disables JSON for this direct command.\n" " -h, --help Show this LLM-mode help and exit.\n" "This profile models CPU or Metal memory traffic only: it performs no Transformer math and\n" @@ -494,6 +496,34 @@ TEST(MessagesTest, LlmMemoryCliMessagesHaveExactOutput) { expect_exact_messages(cases); } +TEST(MessagesTest, LlmDistributionReportsAcceptedPopulationAndDescriptiveSpread) { + EXPECT_EQ(Messages::report_llm_memory_distribution(7, 123.456, 98.766, 150.126, 6.257, 3.141), + " n=7, median=123.46 GB/s, min=98.77, max=150.13, CV=6.26%, MAD=3.14 GB/s"); + EXPECT_EQ(Messages::report_llm_memory_distribution(1, 42.0, 42.0, 42.0, 0.0, 0.0), + " n=1, median=42.00 GB/s, min=42.00, max=42.00, CV=0.00%, MAD=0.00 GB/s"); +} + +TEST(MessagesTest, LlmPrefillModelContextDistinguishesUnavailableFromKnownZero) { + const std::string prefix = + " Prompt tokens (P): 5\n" + " Attention query tile tokens (Q): 2\n" + " Attention query tiles (C): 3\n" + " Prefix token visits / sequence: 11\n"; + EXPECT_EQ(Messages::report_llm_memory_prefill_geometry(5, 2, 3, 11, 15, 60, std::nullopt), + prefix + " Causal token pairs / sequence: 15\n" + " Logical attention pairs: 60\n" + " Logical attention FMA terms: unavailable (arithmetic-overflow)"); + EXPECT_EQ(Messages::report_llm_memory_prefill_geometry(5, 2, 3, 11, std::nullopt, std::nullopt, std::nullopt), + prefix + " Causal token pairs / sequence: unavailable (arithmetic-overflow)\n" + " Logical attention pairs: unavailable (arithmetic-overflow)\n" + " Logical attention FMA terms: unavailable (arithmetic-overflow)"); + // Formatting helper receives known zero separately from absent theoretical evidence. + EXPECT_EQ(Messages::report_llm_memory_prefill_geometry(5, 2, 3, 11, 0, 0, 0), + prefix + " Causal token pairs / sequence: 0\n" + " Logical attention pairs: 0\n" + " Logical attention FMA terms: 0"); +} + TEST(MessagesTest, GeneralHelpAdvertisesTheLlmBoundaryExactlyOnce) { const std::string usage = Messages::usage_options("memory_benchmark"); EXPECT_NE(usage.find("Platform: macOS 26 or later on Apple Silicon (ARM64)."), @@ -516,24 +546,21 @@ TEST(MessagesTest, GeneralHelpAdvertisesTheLlmBoundaryExactlyOnce) { EXPECT_NE(usage.find("--query-heads"), std::string::npos); EXPECT_NE(usage.find("--context-tokens"), std::string::npos); EXPECT_NE(usage.find("memory-only interpretation"), std::string::npos); - EXPECT_NE(usage.find("JSON uses schema 1 methodologies "), - std::string::npos); - EXPECT_NE(usage.find( - Constants::LLM_CPU_DECODE_CONTIGUOUS_METHODOLOGY_VERSION), - std::string::npos); - EXPECT_NE(usage.find(Constants::LLM_CPU_DECODE_PAGED_METHODOLOGY_VERSION), - std::string::npos); - EXPECT_NE(usage.find("llm-memory-v1-metal-decode-contiguous"), - std::string::npos); - EXPECT_NE(usage.find("llm-memory-v1-metal-decode-paged"), - std::string::npos); - EXPECT_NE(usage.find("llm-memory-v1-metal-prefill-contiguous"), - std::string::npos); - EXPECT_NE(usage.find("llm-memory-v1-metal-prefill-paged"), + EXPECT_NE(usage.find("JSON uses schema 2 methodologies "), std::string::npos); + for (const char* selector : { + "llm-memory-v2-cpu-decode-contiguous", "llm-memory-v2-cpu-decode-paged", + "llm-memory-v2-cpu-prefill-contiguous", "llm-memory-v2-cpu-prefill-paged", + "llm-memory-v2-metal-decode-contiguous", "llm-memory-v2-metal-decode-paged", + "llm-memory-v2-metal-prefill-contiguous", "llm-memory-v2-metal-prefill-paged"}) { + SCOPED_TRACE(selector); + EXPECT_NE(usage.find(selector), std::string::npos); + } EXPECT_NE(usage.find("Metal LLM-memory rejects --threads"), std::string::npos); - EXPECT_NE(usage.find("checkpoints each terminal scenario"), + EXPECT_NE(usage.find("LLM-memory snapshots every K=max(1,ceil(count/8)) loops"), + std::string::npos); + EXPECT_NE(usage.find("and at command terminal (abrupt loss bound: 3K completed attempts)."), std::string::npos); } @@ -968,7 +995,7 @@ TEST(MessagesFormattingTest, UsageOptions) { EXPECT_NE(msg.find("prefill requires --phase prefill, --prompt-tokens"), std::string::npos); EXPECT_NE(msg.find("--attention-query-tile-tokens"), std::string::npos); EXPECT_NE(msg.find("Both phases support contiguous"), std::string::npos); - EXPECT_NE(msg.find("llm-memory-v1-cpu-prefill-contiguous"), std::string::npos); + EXPECT_NE(msg.find("llm-memory-v2-cpu-prefill-contiguous"), std::string::npos); EXPECT_NE(msg.find(Constants::LLM_CPU_PREFILL_PAGED_METHODOLOGY_VERSION), std::string::npos); EXPECT_NE(msg.find("--analyze-core2core"), std::string::npos); EXPECT_NE(msg.find("acquire/release token-handoff"), std::string::npos); @@ -1003,9 +1030,9 @@ TEST(MessagesFormattingTest, UsageOptions) { std::string::npos); EXPECT_NE(msg.find("Standard, GPU, and LLM-memory files retain"), std::string::npos); - EXPECT_NE(msg.find("LLM-memory checkpoints each terminal scenario"), + EXPECT_NE(msg.find("LLM-memory snapshots every K=max(1,ceil(count/8)) loops"), std::string::npos); - EXPECT_NE(msg.find("sweep files checkpoint attempts"), std::string::npos); + EXPECT_NE(msg.find("Sweep files checkpoint attempts"), std::string::npos); EXPECT_NE(msg.find("Requires --output "), std::string::npos); EXPECT_NE(msg.find("-h"), std::string::npos); // Check that default values are included From d92a918a000b8a17106bf09f10b8af4f2ca2dcdc Mon Sep 17 00:00:00 2001 From: timoheimonen Date: Sat, 5 Sep 2026 23:26:59 +0300 Subject: [PATCH 08/16] Validate LLM CPU task inputs once before oracle calculation --- documents/LLM_MEMORY_PROFILE_WHITEPAPER.md | 6 +++ src/llm_memory/llm_executor.cpp | 56 ++++++++++++++-------- tests/test_llm_memory_executor.cpp | 49 +++++++++++++++++++ 3 files changed, 91 insertions(+), 20 deletions(-) diff --git a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md index ab1c553..23f60cd 100644 --- a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md +++ b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md @@ -811,6 +811,12 @@ A CPU scenario task follows this boundary: 6. stop the timer at last-worker completion; 7. join workers and validate/fold checksums outside the elapsed interval. +Each executor task validates its borrowed plan and materialized resources before oracle generation. The private +oracle calculation consumes that same checked input synchronously, without an intervening callback or mutation; +it does not repeat the descriptor walk. A separate public oracle call validates its own inputs. No validation verdict +is retained across tasks: plan identities alone do not make caller-owned structures immutable. Mutable K/V reset and +all applicable post-execution checks still run for each task. + Paged table loads, ID-dependent address formation, model reads/writes, and timed checksum accumulation are inside the primary elapsed interval. Thread creation, QoS calls, allocation, initialization/pre-touch, permutation generation, table validation/hash/protection, preparation page faults, descriptor validation, expected-oracle generation, diff --git a/src/llm_memory/llm_executor.cpp b/src/llm_memory/llm_executor.cpp index 7093337..345ade5 100644 --- a/src/llm_memory/llm_executor.cpp +++ b/src/llm_memory/llm_executor.cpp @@ -2414,9 +2414,7 @@ LlmExpectedChecksumResult calculate_prefill_expected_checksums( const LlmCpuExecutionPlan* const cpu_plan = get_llm_cpu_execution_plan(model_plan); if (cpu_plan == nullptr || !cpu_plan->prefill.has_value() || - !model_plan.geometry.prefill.has_value() || - !materialized_resources_match_plan(model_plan, resources) || - !scenario_plan_matches_model(model_plan, scenario_plan)) { + !model_plan.geometry.prefill.has_value()) { result.reason_code = LlmExecutorReason::INVALID_RESOURCES; return result; } @@ -2609,9 +2607,7 @@ LlmExpectedChecksumResult calculate_paged_prefill_expected_checksums( get_llm_cpu_execution_plan(model_plan); if (cpu_plan == nullptr || !cpu_plan->prefill.has_value() || !cpu_plan->paged.has_value() || - !model_plan.geometry.prefill.has_value() || - !materialized_resources_match_plan(model_plan, resources) || - !scenario_plan_matches_model(model_plan, scenario_plan)) { + !model_plan.geometry.prefill.has_value()) { result.reason_code = LlmExecutorReason::INVALID_RESOURCES; return result; } @@ -2853,9 +2849,7 @@ LlmExpectedChecksumResult calculate_paged_expected_checksums( LlmExpectedChecksumResult result; const LlmCpuExecutionPlan* const cpu_plan = get_llm_cpu_execution_plan(model_plan); - if (cpu_plan == nullptr || !cpu_plan->paged.has_value() || - !materialized_resources_match_plan(model_plan, resources) || - !scenario_plan_matches_model(model_plan, scenario_plan)) { + if (cpu_plan == nullptr || !cpu_plan->paged.has_value()) { result.reason_code = LlmExecutorReason::INVALID_RESOURCES; return result; } @@ -3997,22 +3991,20 @@ LlmResourcePreparationResult prepare_llm_execution_resources(const LlmMemoryWork } } -LlmExpectedChecksumResult calculate_llm_expected_checksums(const LlmMemoryWorkPlan& model_plan, +namespace { + +/** + * Compute the oracle immediately after the caller validated this task's plan + * and materialized resources. No callback or mutation may intervene. This + * private boundary retains no validation verdict across public calls or tasks. + */ +LlmExpectedChecksumResult calculate_validated_expected_checksums(const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan& scenario_plan, const LlmExecutionResources& resources) noexcept { LlmExpectedChecksumResult result; try { const LlmCpuExecutionPlan* const cpu_plan = get_llm_cpu_execution_plan(model_plan); - if (cpu_plan == nullptr || - !materialized_resources_match_plan(model_plan, resources)) { - result.reason_code = LlmExecutorReason::INVALID_RESOURCES; - return result; - } - if (!scenario_plan_matches_model(model_plan, scenario_plan)) { - result.reason_code = LlmExecutorReason::SCENARIO_PLAN_MISMATCH; - return result; - } if (model_plan.phase == LlmPhase::Prefill && model_plan.kv_layout == LlmKvLayout::Paged) { return calculate_paged_prefill_expected_checksums( @@ -4119,6 +4111,30 @@ LlmExpectedChecksumResult calculate_llm_expected_checksums(const LlmMemoryWorkPl } } +} // namespace + +LlmExpectedChecksumResult calculate_llm_expected_checksums(const LlmMemoryWorkPlan& model_plan, + const LlmScenarioWorkPlan& scenario_plan, + const LlmExecutionResources& resources) noexcept { + LlmExpectedChecksumResult result; + try { + if (!materialized_resources_match_plan(model_plan, resources)) { + result.reason_code = LlmExecutorReason::INVALID_RESOURCES; + return result; + } + if (!scenario_plan_matches_model(model_plan, scenario_plan)) { + result.reason_code = LlmExecutorReason::SCENARIO_PLAN_MISMATCH; + return result; + } + return calculate_validated_expected_checksums(model_plan, scenario_plan, resources); + } catch (const std::bad_alloc&) { + result.reason_code = LlmExecutorReason::EXPECTED_CHECKSUM_ALLOCATION_FAILED; + } catch (...) { + result.reason_code = LlmExecutorReason::INVALID_SCENARIO_PLAN; + } + return result; +} + LlmKernelAdapter production_llm_kernel_adapter() noexcept { return {production_kernel_invoke, nullptr}; } LlmExecutorResult execute_llm_scenario(const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan& scenario_plan, @@ -4155,7 +4171,7 @@ LlmExecutorResult execute_llm_scenario(const LlmMemoryWorkPlan& model_plan, cons return result; } - LlmExpectedChecksumResult expected = calculate_llm_expected_checksums(model_plan, scenario_plan, resources); + LlmExpectedChecksumResult expected = calculate_validated_expected_checksums(model_plan, scenario_plan, resources); if (!expected.valid) { result.reason_code = expected.reason_code; return result; diff --git a/tests/test_llm_memory_executor.cpp b/tests/test_llm_memory_executor.cpp index a65cc5a..1d40bce 100644 --- a/tests/test_llm_memory_executor.cpp +++ b/tests/test_llm_memory_executor.cpp @@ -2522,3 +2522,52 @@ TEST_F(LlmMemoryExecutorTest, ColdApplicabilitySurvivesResourceFailure) { } } } + +TEST_F(LlmMemoryExecutorTest, EachTaskRevalidatesBorrowedPlansAndMaterializedResources) { + ScopedExecutorTimer timer_calls; + auto timer = HighResTimer::create(); + ASSERT_TRUE(timer.has_value()); + size_t kernel_calls = 0; + const auto count_kernel = +[](void* context, const LlmKernelInvocation&) { + ++*static_cast(context); + return false; + }; + for (const auto& geometry : {LlmGeometryRequest{257, 2, 1, 1, 33, 1, 5, 3}, + paged_geometry(5, 4), prefill_geometry(), paged_prefill_geometry()}) { + auto plan = build_executor_ready_plan(geometry, 1); + ASSERT_TRUE(plan.valid); + LlmExecutionResources resources; + ASSERT_TRUE(prepare_llm_execution_resources(plan, resources).valid); + auto task = build_llm_scenario_work_plan(plan, LlmScenario::WeightsOnly, 1, true); + ASSERT_TRUE(calculate_llm_expected_checksums(plan, task, resources).valid); + const auto reject = [&](const char* reason) { + EXPECT_EQ(calculate_llm_expected_checksums(plan, task, resources).reason_code, reason); + const auto result = execute_llm_scenario(plan, task, resources, *timer, {count_kernel, &kernel_calls}); + EXPECT_FALSE(result.valid); + EXPECT_EQ(result.reason_code, reason); + EXPECT_FALSE(result.timer_started); + EXPECT_EQ(result.created_workers, 0u); + EXPECT_EQ(kernel_calls, 0u); + }; + // The retained identity is deliberately unchanged in each mutation. + ++plan.geometry.v_mapping_bytes; + reject(LlmExecutorReason::INVALID_RESOURCES); + --plan.geometry.v_mapping_bytes; + ASSERT_TRUE(calculate_llm_expected_checksums(plan, task, resources).valid); + ++task.weight_read_bytes; + reject(LlmExecutorReason::SCENARIO_PLAN_MISMATCH); + --task.weight_read_bytes; + ASSERT_TRUE(calculate_llm_expected_checksums(plan, task, resources).valid); + --resources.worker_count; + reject(LlmExecutorReason::INVALID_RESOURCES); + ++resources.worker_count; + ASSERT_TRUE(calculate_llm_expected_checksums(plan, task, resources).valid); + if (plan.phase == LlmPhase::Decode && plan.kv_layout == LlmKvLayout::Contiguous) { + const auto pointer = resources.layer_descriptors[0].weight_ptr; + resources.layer_descriptors[0].weight_ptr = pointer + 1; + reject(LlmExecutorReason::INVALID_RESOURCES); + resources.layer_descriptors[0].weight_ptr = pointer; + ASSERT_TRUE(calculate_llm_expected_checksums(plan, task, resources).valid); + } + } +} From 7d316c549a583f5b3b7a5af133a1e68b49542594 Mon Sep 17 00:00:00 2001 From: timoheimonen Date: Sun, 6 Sep 2026 07:39:37 +0300 Subject: [PATCH 09/16] Cover Metal reduction and prefill optimization boundaries --- tests/test_llm_metal_backend.cpp | 51 +++++++++++++++++ tests/test_llm_metal_checksum.mm | 98 ++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+) diff --git a/tests/test_llm_metal_backend.cpp b/tests/test_llm_metal_backend.cpp index 36de318..1c6ec3e 100644 --- a/tests/test_llm_metal_backend.cpp +++ b/tests/test_llm_metal_backend.cpp @@ -4768,3 +4768,54 @@ TEST(LlmMetalBackendTest, PendingColdChecksPreserveAllApplicabilityCombinations) } } } + +TEST_F(LlmMetalBackendIntegrationTest, PrefillElementWidthsAndWideSingleTokenPreserveTaskContractIntegration) { + struct Case { + size_t prompt_tokens; + size_t query_tile_tokens; + size_t head_dimension; + size_t block_tokens; + }; + // Include a wide P=Q=1 record: replacing token control with vector control + // must preserve this supported case as well as narrow, split-token tails. + constexpr std::array kCases = {{{1, 1, 131072, 1}, + {17, 1, 3, 4}, + {257, 257, 64, 128}, + {129, 16, 5, 16}}}; + for (LlmKvLayout layout : {LlmKvLayout::Contiguous, LlmKvLayout::Paged}) { + for (size_t element_bytes : {1U, 2U, 4U}) { + for (const Case& test_case : kCases) { + SCOPED_TRACE("P=" + std::to_string(test_case.prompt_tokens) + + ", E=" + std::to_string(element_bytes) + + ", layout=" + std::to_string(static_cast(layout))); + LlmMemoryConfig config = metal_config(); + config.phase = LlmPhase::Prefill; + config.kv_layout = layout; + backend_ = create_llm_metal_backend(); + ASSERT_EQ(backend_->initialize(config).status, LlmBackendStatus::Ready); + LlmGeometryRequest request; + request.active_weight_bytes = 4097; + request.layer_count = 2; + request.batch_size = 2; + request.query_head_count = 1; + request.kv_head_count = 1; + request.head_dimension = test_case.head_dimension; + request.kv_element_bytes = element_bytes; + request.phase = LlmPhase::Prefill; + request.kv_layout = layout; + request.prompt_tokens = test_case.prompt_tokens; + request.attention_query_tile_tokens = test_case.query_tile_tokens; + request.kv_block_tokens = layout == LlmKvLayout::Paged ? test_case.block_tokens : 0; + const LlmGeometry geometry = resolve_llm_geometry(request); + ASSERT_TRUE(geometry.valid) << geometry.reason_code; + ASSERT_EQ(geometry.k_or_v_record_bytes_per_layer, test_case.head_dimension * element_bytes); + LlmMemoryWorkPlan plan = build_device_plan(geometry, "prefill-element-width-and-record-boundary"); + resolve_and_prepare(plan); + for (LlmScenario scenario : {LlmScenario::WeightsOnly, LlmScenario::KvOnly, LlmScenario::Mixed}) { + expect_complete_scenario_task(plan, scenario, 1); + expect_complete_scenario_task(plan, scenario, 3); + } + } + } + } +} diff --git a/tests/test_llm_metal_checksum.mm b/tests/test_llm_metal_checksum.mm index 0ee366d..0580e2b 100644 --- a/tests/test_llm_metal_checksum.mm +++ b/tests/test_llm_metal_checksum.mm @@ -106,3 +106,101 @@ kernel void llm_checksum_helper_probe(device const uint* input [[buffer(0)]], EXPECT_NE(actual[1], actual[7]); } } + +TEST(LlmMetalChecksumHelperIntegrationTest, ReductionPreservesSixModuloSumsAcrossPartialGroups) { + @autoreleasepool { + id device = MTLCreateSystemDefaultDevice(); + if (device == nil) { + GTEST_SKIP() << "metal-device-unavailable"; + } + // Exercise the production reduction with independently generated lane values. + // This is a helper/dispatch contract test, not evidence of payload loads. + std::string source(LlmMetalKernelContract::kSource); + source += R"MSL( +kernel void llm_reduction_probe( + constant LlmMetalResources& resources [[buffer(0)]], + threadgroup uint* reduction [[threadgroup(0)]], + uint global_id [[thread_position_in_grid]], + uint thread_index [[thread_index_in_threadgroup]], + uint threads_per_threadgroup [[threads_per_threadgroup]], + uint simd_width [[threads_per_simdgroup]], + uint simd_lane [[thread_index_in_simdgroup]]) { + const uint n = global_id + 1u; + publish_task_checksum(resources, uint2(n, 0xffffffffu), + uint2(n * 0x9e3779b1u, n * n), uint2(0x80000000u + n, 7u), + reduction, thread_index, threads_per_threadgroup); +} +)MSL"; + NSString* source_string = [[NSString alloc] initWithBytes:source.data() + length:source.size() + encoding:NSUTF8StringEncoding]; + ASSERT_NE(source_string, nil); + MTLCompileOptions* options = [[MTLCompileOptions alloc] init]; + options.languageVersion = MTLLanguageVersion2_3; + options.preprocessorMacros = @{@"LLM_METAL_DECODE_CONTIGUOUS" : @1, + @"LLM_METAL_DECODE_PAGED" : @0, + @"LLM_METAL_PREFILL_CONTIGUOUS" : @0, + @"LLM_METAL_PREFILL_PAGED" : @0}; + NSError* error = nil; + id library = [device newLibraryWithSource:source_string options:options error:&error]; + ASSERT_NE(library, nil) << (error == nil ? "no compiler diagnostic" : error.description.UTF8String); + id function = [library newFunctionWithName:@"llm_reduction_probe"]; + ASSERT_NE(function, nil); + id pipeline = [device newComputePipelineStateWithFunction:function error:&error]; + ASSERT_NE(pipeline, nil); + id arguments = [function newArgumentEncoderWithBufferIndex:0]; + ASSERT_NE(arguments, nil); + id argument_buffer = [device newBufferWithLength:arguments.encodedLength + options:MTLResourceStorageModeShared]; + constexpr size_t kStatusWords = LlmMetalKernelContract::kTimedStatusWordCount; + id status = [device newBufferWithLength:kStatusWords * sizeof(uint32_t) + options:MTLResourceStorageModeShared]; + ASSERT_NE(argument_buffer, nil); + ASSERT_NE(status, nil); + [arguments setArgumentBuffer:argument_buffer offset:0]; + [arguments setBuffer:status offset:0 atIndex:LlmMetalKernelContract::kStatusChecksumResourceId]; + id queue = [device newCommandQueue]; + ASSERT_NE(queue, nil); + const size_t width = pipeline.threadExecutionWidth; + ASSERT_GT(width, 1U); + ASSERT_GE(pipeline.maxTotalThreadsPerThreadgroup, 2 * width + 1); + const std::array group_sizes = {1, width - 1, width, width + 1, + 2 * width - 1, 2 * width, 2 * width + 1}; + for (size_t group_size : group_sizes) { + for (size_t count : {group_size, 3 * group_size + 1}) { + SCOPED_TRACE("threads=" + std::to_string(count) + ", group=" + std::to_string(group_size)); + auto* words = static_cast(status.contents); + for (size_t i = 0; i < kStatusWords; ++i) { + words[i] = 0; + } + id command = [queue commandBuffer]; + id encoder = [command computeCommandEncoder]; + ASSERT_NE(encoder, nil); + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:argument_buffer offset:0 atIndex:0]; + [encoder useResource:status usage:MTLResourceUsageRead | MTLResourceUsageWrite]; + [encoder setThreadgroupMemoryLength:group_size * 6 * sizeof(uint32_t) atIndex:0]; + [encoder dispatchThreads:MTLSizeMake(count, 1, 1) + threadsPerThreadgroup:MTLSizeMake(group_size, 1, 1)]; + [encoder endEncoding]; + [command commit]; + [command waitUntilCompleted]; + ASSERT_EQ(command.status, MTLCommandBufferStatusCompleted) + << (command.error == nil ? "no command diagnostic" : command.error.description.UTF8String); + std::array expected{}; + for (uint32_t n = 1; n <= count; ++n) { + expected[0] += n; + expected[1] += UINT32_MAX; + expected[2] += n * 0x9e3779b1U; + expected[3] += n * n; + expected[4] += 0x80000000U + n; + expected[5] += 7U; + } + for (size_t component = 0; component < expected.size(); ++component) { + EXPECT_EQ(words[LlmMetalKernelContract::kWeightChecksumAIndex + component], expected[component]); + } + EXPECT_EQ(words[LlmMetalKernelContract::kLayoutMetadataLookupCountIndex], 0U); + } + } + } +} From 870b175927d747fb1d90fadefebe8728da38407a Mon Sep 17 00:00:00 2001 From: timoheimonen Date: Sun, 6 Sep 2026 08:05:34 +0300 Subject: [PATCH 10/16] Clarify LLM position balance and comparison conditions --- documents/API.md | 12 ++++++ documents/LLM_MEMORY_PROFILE_WHITEPAPER.md | 35 ++++++++++++++-- documents/MANUAL.md | 12 +++++- tests/test_llm_memory_runner.cpp | 46 ++++++++++++++++++++++ 4 files changed, 101 insertions(+), 4 deletions(-) diff --git a/documents/API.md b/documents/API.md index bad8f9b..34fac10 100644 --- a/documents/API.md +++ b/documents/API.md @@ -607,6 +607,13 @@ A correct count-one run can have both booleans true and `scenario_order_balance_ can leave the measured population complete while run status is failed and `run_accepted` is false. Consumers retain process exit status as well as the snapshot; an older preserved file cannot report a later failure. +`scenario_order_balance_complete` describes positions only: all planned rows must be measured, each loop complete, and +each scenario's first/middle/last counts equal and nonzero. Loop `i` rotates `weights_only`/`kv_only`/`mixed` by `i +mod 3`, independently of seed. This does not balance directed predecessor pairs, including loop and repeated-block +boundaries. Partial final blocks retain their realized order. Canonical frozen-plan warmups run once before loop zero, +not immediately before each measured scenario; backend task-local preparation and validation remain outside the +primary timer. + Each `aggregates.scenarios..accepted_measurement_ids` defines one population shared by all three metrics. Excluded calibration never enters it. Derive each rate before applying the shared linear percentile interpolation; for even n, median(work/duration) generally differs from work/median(duration). Exact median/MAD/percentiles are @@ -621,6 +628,11 @@ confidence levels. Default LLM console output shows n, median, min/max, CV and M A comparison may impose stricter quality criteria, but those are separate policy and do not delete or retry noisy measurements. Match phase/model geometry and, for paged results, block size, permutation, physical-resource geometry and component identities. Contiguous and paged results are distinct cohorts even with identical logical geometry. +Match backend/profile, frozen work, seeds, full component identities and run policy, including conditioning and +output/checkpoint cadence. Inspect accepted n, CV, duration and environment separately from acceptance. +`environment.start`/`end` thermal-state and Low Power Mode values are instantaneous OS observations, not continuous +temperature, cache-residency or CPU-affinity evidence; unavailable is not nominal. Accepted artifacts alone do not +establish a machine or methodology performance difference. `quality_warnings` merges and deduplicates runner tokens `weights_only-high-cv`, `kv_only-high-cv`, `mixed-high-cv`, and `scenario-order-not-balanced` with final-report tokens `environment-not-nominal`, diff --git a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md index 23f60cd..36f386e 100644 --- a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md +++ b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md @@ -892,8 +892,26 @@ Measured duration-quality values are `within-target-window`, `above-target-singl `not-run`. Every measured non-window value produces the corresponding `-duration-` warning. The base order is `weights_only`, `kv_only`, `mixed`; loop `i` rotates it by `i mod 3`. A complete block of three loops -gives every scenario one first, middle, and last position. Count need not be divisible by three. Balance is reported independently; a comparison policy may require it, but -producer correctness acceptance does not. +gives every scenario one first, middle, and last position. This is **position balance**, not directed predecessor-pair +balance. With W=weights_only, K=kv_only and M=mixed, the finite three-loop stream is `WKM KMW MWK`. +Including loop boundaries, its transition counts are W→K=2, K→M=2, M→W=2, M→K=1, W→M=1, K→W=0. +Repeating the block adds K→W at its boundary. Count need not be divisible by three; a partial final block retains its +actual positions and transitions. `scenario_order_balance_complete` checks complete measured loops and equal nonzero +first/middle/last counts per scenario, without asserting carryover balance. A comparison policy may require position +balance, but producer correctness acceptance does not. The order depends on loop index, not the resolved seed. + +The canonical W/K/M frozen-plan warmups occur once before loop zero. There is no runner-level same-scenario warmup +immediately before every measurement. Backend task-local reset, precondition, expected-witness construction and +post-validation still apply as documented above; they can affect the state seen by the next kernel. The initial warmup +prefix ends in M before measured W. Measured-order records do not enumerate those excluded tasks, calibration, or +host/output work. File checkpoints and stdout can produce different intertask gaps even with the same measured order. + +Using all six permutations balances within-loop directed pairs, but the order of those loops and repeated-block +boundaries still matters. For example, `WKM WMK KWM KMW MWK MKW` adds M→W, K→K, M→K, W→M and K→M once each +at its five loop boundaries, plus W→W when the block repeats. It does not balance the whole execution stream. +Per-measurement conditioning would also add a same-scenario predecessor and change the prepared-state phenomenon; +it requires an explicit run-policy/methodology review rather than treating a higher rate as equivalent work under the +current preparation policy. The production default remains three cyclic loops and canonical initial frozen warmups. Only `measured` records with accepted required timing, checksum and cold-check evidence enter aggregates. One retained authoritative duration/work/payload sample feeds all three metrics through one @@ -1142,7 +1160,18 @@ Correctness gates cover: For a performance comparison, keep the exact command/model geometry, layout, paged `G`/physical geometry/table and permutation identity when applicable, explicit-versus-automatic policy, frozen plan, seed, worker counts, software/methodology, hardware, macOS, power/thermal state, and background load matched. Prefer a count divisible by -three, inspect CV and warnings, and retain both stdout/file payload and stderr transcript. A separate +three, inspect each scenario's accepted n, CV, duration quality and warnings, and retain both stdout/file payload and +stderr transcript. Match the complete component identities and run policy, including ordering, conditioning and output +transport/checkpoint cadence. Freeze scenario-specific work across an A/B pair; separately calibrated work can differ. +Compare independent, alternating process pairs in repeated series; a single accepted artifact or lower median is not +sufficient evidence of a machine or methodology performance difference. If hardware or methodology is the experimental +variable, label that contrast and keep its populations separate instead of claiming matched-cohort reproducibility. + +The `environment.start` and `environment.end` thermal-state and Low Power Mode fields are instantaneous operating-system +observations. They are not continuous temperature monitoring, cache-residency evidence, or guaranteed CPU affinity; +equal nominal endpoints do not exclude an intervening state change. Unavailable observations are missing evidence. +Intertask host gaps include orchestration and optional output work and are distinct from authoritative CPU/GPU task +times. Added excluded conditioning costs wall time even when the following measured kernel gets faster. A separate real inference-engine run can be useful correlation evidence, but it is not part of this benchmark's correctness or acceptance predicate. diff --git a/documents/MANUAL.md b/documents/MANUAL.md index 3400fda..fb3cf0b 100644 --- a/documents/MANUAL.md +++ b/documents/MANUAL.md @@ -690,7 +690,11 @@ middle, and trailing items. one same-shape warmup; later candidates add a warmup only for the first irreducible one-work-unit confirmation. After all three candidates resolve, their plans freeze atomically and each frozen plan receives one canonical-order warmup before loop zero. Explicit iterations freeze all three exact plans first and use the same frozen-plan warmup boundary. - Cyclic measured order gives all scenarios each position once when count is three + The measured WKM/KMW/MWK rotation balances first/middle/last positions per complete three-loop block, independently + of seed. It does not balance directed predecessor pairs across loop/block boundaries. A partial final block keeps its + actual order. There is no runner-level same-scenario conditioning immediately before each measurement; backend + preparation and validation still occur outside its primary time. Changing conditioning or the six-permutation block + order would change comparison policy and requires a versioned methodology/run-policy decision. - Uses task-boundary completion-wins interruption. A started scenario is not polled in its hot kernel; a completed and checksum-valid current task stays measured, while no next task starts after the stop is observed - Output values retain the shared raw-target syntax: empty disables JSON, exact `-` emits one final LLM schema 2 document, @@ -2041,6 +2045,12 @@ Schema 2 ownership and interpretation: - `run_accepted` requires complete status, every planned measurement measured, accepted required runtime evidence and no known command/checkpoint error. `results_complete` only describes that measured population. Correct count one can be accepted while balance is false. Quality, CV, duration and environment remain separate comparison criteria. + Compare the same backend/profile, phase geometry, layout/permutation, frozen work, seed, component identities and + run policy, including conditioning and file/stdout checkpoint cadence. Inspect accepted sample count and repeat + alternating process comparisons; artifact acceptance alone does not establish a machine or methodology speed + difference. Thermal-state/Low Power Mode start/end snapshots are instantaneous OS observations, not continuous + temperatures, cache residency or guaranteed CPU affinity. Nominal endpoints cannot exclude intervening changes; + unavailable observations remain missing evidence. - One `accepted_measurement_ids` population feeds all three metrics. Derive each rate before statistics; exact median, MAD and linearly interpolated P90/P95/P99 are prepared at snapshots/terminal. Empty populations are null; n=1 stddev and positive-mean CV are zero. Classification is `insufficient-samples` for n<3, `undefined` for undefined CV, diff --git a/tests/test_llm_memory_runner.cpp b/tests/test_llm_memory_runner.cpp index 8af73a9..f9acf5d 100644 --- a/tests/test_llm_memory_runner.cpp +++ b/tests/test_llm_memory_runner.cpp @@ -819,6 +819,52 @@ TEST(LlmMemoryRunnerTest, } } +TEST(LlmMemoryRunnerTest, PositionBalanceRetainsDirectedCarryoverAndPartialBlocks) { + // Independent finite schedule: includes loop boundaries and the repeated three-loop block. + constexpr std::array expected = { + 0, 1, 2, 1, 2, 0, 2, 0, 1, 0, 1, 2, 1, 2, 0, 2, 0, 1, 0, 1, 2}; + for (size_t count : {1u, 2u, 3u, 4u, 6u, 7u}) { + SCOPED_TRACE(count); + const LlmMemoryConfig config = explicit_config(count); + const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); + ASSERT_TRUE(plan.valid) << plan.reason_code; + FakeLlmBackend backend; + LlmMemoryResult result; + ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_SUCCESS); + EXPECT_TRUE(result.run_accepted); + EXPECT_EQ(result.scenario_order_balance_complete, count % 3 == 0); + ASSERT_EQ(backend.calls.size(), 3 + 3 * count); + std::array, 3> transitions{}; + for (size_t index = 0; index < 3; ++index) { + EXPECT_EQ(backend.calls[index].context.kind, LlmRunnerTaskKind::Warmup); + EXPECT_EQ(backend.calls[index].context.scenario, static_cast(index)); + EXPECT_EQ(backend.calls[index].work_units, config.iterations); + } + for (size_t index = 0; index < count * 3; ++index) { + const TaskRecord& task = backend.calls[index + 3]; + EXPECT_EQ(task.context.kind, LlmRunnerTaskKind::Measurement); + EXPECT_EQ(task.context.scenario, static_cast(expected[index])); + EXPECT_EQ(task.context.loop_index, index / 3); + EXPECT_EQ(task.context.order_position, index % 3); + EXPECT_EQ(task.work_units, config.iterations); + if (index != 0) { + const size_t predecessor = static_cast(backend.calls[index + 2].context.scenario); + const size_t current = static_cast(task.context.scenario); + ASSERT_LT(predecessor, 3u); + ASSERT_LT(current, 3u); + ++transitions[predecessor][current]; + } + } + if (count == 3) { + // W->K=2, K->M=2, M->W=2; reverse directions occur only at loop boundaries. + EXPECT_EQ(transitions, (std::array, 3>{{{0, 2, 1}, {0, 0, 2}, {2, 1, 0}}})); + } + if (count == 6) { + EXPECT_EQ(transitions, (std::array, 3>{{{0, 4, 2}, {1, 0, 4}, {4, 2, 0}}})); + } + } +} + TEST(LlmMemoryRunnerTest, CompleteSingleLoopIsAcceptedWithSeparateBalanceAndSampleQuality) { const LlmMemoryConfig config = explicit_config(1); const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); From 1b3d2c273b6db655dee6c1e67d7b95aea5be6671 Mon Sep 17 00:00:00 2001 From: timoheimonen Date: Sun, 6 Sep 2026 10:06:22 +0300 Subject: [PATCH 11/16] Retain LLM build provenance and original CPU timing snapshots --- Makefile | 15 +++++++- README.md | 6 +++ build-support/generate_provenance.py | 45 ++++++++++++++++++++++ documents/API.md | 28 +++++++++++--- documents/LLM_MEMORY_PROFILE_WHITEPAPER.md | 6 +++ documents/MANUAL.md | 6 +++ src/core/timing/timer.cpp | 3 ++ src/core/timing/timer.h | 14 +++++++ src/llm_memory/llm_backend.h | 1 + src/llm_memory/llm_cpu_backend.cpp | 1 + src/llm_memory/llm_environment.h | 6 +++ src/llm_memory/llm_environment.mm | 27 +++++++++++++ src/llm_memory/llm_executor.cpp | 1 + src/llm_memory/llm_executor.h | 3 +- src/llm_memory/llm_json.cpp | 14 +++++-- src/llm_memory/llm_json.h | 4 ++ src/llm_memory/llm_memory.cpp | 8 ++++ src/llm_memory/llm_runner.cpp | 1 + src/llm_memory/llm_runner.h | 1 + tests/test_llm_memory_json.cpp | 22 +++++++++++ tests/test_timer.cpp | 37 ++++++++++++++++++ 21 files changed, 238 insertions(+), 11 deletions(-) create mode 100644 build-support/generate_provenance.py diff --git a/Makefile b/Makefile index a07870c..0f0f276 100644 --- a/Makefile +++ b/Makefile @@ -71,6 +71,19 @@ TARGET = memory_benchmark # Default target: build the executable all: $(TARGET) +# Recompute provenance before considering objects. A changed source revision, +# dirty state or build flags invalidates every object, avoiding mixed manifests. +export PROVENANCE_CXX = $(CXX) +export PROVENANCE_CXXFLAGS = $(CXXFLAGS) +export PROVENANCE_TEST_CXXFLAGS = $(TEST_CXXFLAGS) +export PROVENANCE_ASFLAGS = $(ASFLAGS) +export PROVENANCE_LDFLAGS = $(LDFLAGS) $(APPLE_FRAMEWORKS) +.build-provenance.h: FORCE + python3 build-support/generate_provenance.py $@ + +.PHONY: FORCE +FORCE: + # Rule for linking the executable from object files $(TARGET): $(OBJ_FILES) @echo "Linking $(TARGET)..." @@ -99,7 +112,7 @@ TEST_LIB_OBJS := $(filter-out main.o, $(OBJ_FILES)) # Recompile once after a Makefile change so an existing pre-dependency-file # workspace cannot keep stale objects. Subsequent header changes are tracked by # the generated .d files. -$(OBJ_FILES) $(TEST_OBJS): Makefile +$(OBJ_FILES) $(TEST_OBJS): Makefile .build-provenance.h # Rule for compiling test files (must come before generic %.o rule) $(TEST_DIR)/%.o: $(TEST_DIR)/%.cpp diff --git a/README.md b/README.md index 770a2e0..82924bb 100644 --- a/README.md +++ b/README.md @@ -419,3 +419,9 @@ The benchmark performs sustained, intensive memory operations. Use it at your ow Copyright 2025-2026 Timo Heimonen \ Licensed under the [GNU General Public License v3.0 or later](LICENSE). + +CPU LLM results retain the original Mach tick boundaries and timebase for duration reconstruction. +The build embeds compiler, flags, SDK/deployment target and Git provenance; the command hashes the +executable once before tasks. These fields bind available artifacts and do not constitute signed +execution attestation. See the [LLM process contract](documents/API.md) for availability and numeric rules. +Python 3 is required to generate build provenance. diff --git a/build-support/generate_provenance.py b/build-support/generate_provenance.py new file mode 100644 index 0000000..aa214e2 --- /dev/null +++ b/build-support/generate_provenance.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Freeze source/build inputs at make time; update the header only on change.""" +import json +import os +from pathlib import Path +import shlex +import subprocess +import sys + + +def command(args): + try: + return subprocess.check_output(args, stderr=subprocess.DEVNULL, text=True).strip() + except (OSError, subprocess.CalledProcessError): + return None + + +def main(): + commit = command(['git', 'rev-parse', '--verify', 'HEAD']) + status = command(['git', 'status', '--porcelain', '--untracked-files=normal']) if commit else None + compiler = shlex.split(os.environ['PROVENANCE_CXX']) + manifest = dict(manifest_version=1, status='available', reason_code='valid', binary_sha256=None, + git_commit=commit, git_dirty=bool(status) if status is not None else None, + compiler=command(compiler + ['--version']), + compile_flags={key: os.environ['PROVENANCE_' + key.upper()] + for key in ('cxxflags', 'test_cxxflags', 'asflags')}, + link_flags=os.environ['PROVENANCE_LDFLAGS'], + target_arch=command(compiler + ['-arch', 'arm64', '-dumpmachine']), + sdk=command(['xcrun', '--sdk', 'macosx', '--show-sdk-version']), + min_os=os.environ.get('MACOSX_DEPLOYMENT_TARGET')) + if any(manifest[key] is None for key in ('git_commit', 'git_dirty', 'compiler', 'target_arch', 'sdk', 'min_os')): + manifest.update(status='partial', reason_code='build-fields-unavailable') + payload = json.dumps(manifest, sort_keys=True) + if len(payload) > 16384: + raise SystemExit('build provenance exceeds 16 KiB budget') + header = '// Generated build inputs; do not edit.\n#define LLM_BUILD_MANIFEST_JSON ' + json.dumps(payload) + '\n' + path = Path(sys.argv[1]) + if not path.exists() or path.read_text() != header: + temporary = path.with_suffix('.tmp') + temporary.write_text(header) + temporary.replace(path) + + +if __name__ == '__main__': + main() diff --git a/documents/API.md b/documents/API.md index 34fac10..c8006f6 100644 --- a/documents/API.md +++ b/documents/API.md @@ -696,11 +696,29 @@ use `unavailable:arithmetic-overflow`. Actual byte/prefix/lookup/allocation over `h_q` (`--query-heads`) controls theoretical attention context; query tile `Q` (`--attention-query-tile-tokens`) controls executed prefix-read geometry. No Transformer or FMA computation is performed. -`build_manifest` has `manifest_version: 1`, `status: "unavailable"`, -`reason_code: "build-provenance-not-provided"`, and null `binary_sha256`, `git_commit`, `git_dirty`, `compiler`, -`compile_flags`, `link_flags`, `target_arch`, `sdk`, and `min_os`. This revision reserves the contract and does not -supply build attestation. Optional `execution.timing.cpu_raw` is not emitted here; absence cannot be promoted to -raw-tick verification. Software identity and MSL-source provenance alone are not independent build/runtime proof. +`build_manifest` version 1 records build-time `git_commit` and `git_dirty`, compiler version, +`compile_flags` (`cxxflags`, `test_cxxflags`, `asflags`), effective link flags including frameworks, +compiler target triple (`target_arch`), SDK version and `min_os`. Make regenerates the embedded manifest +when those inputs change and invalidates the object graph. Python 3 is required for this build step. +A source archive without Git metadata records null Git fields and status `partial` with +`build-fields-unavailable`; it never consults the run directory's Git state. The software version remains +separate provenance. A caller supplying no manifest retains the explicit `unavailable` reservation. +The command streams `binary_sha256` once before tasks, from the executable path. Read failure leaves +null with status `partial` and `binary-hash-unavailable`. This is a file binding, not signed attestation +of executed machine code; replacement of the executable during capture is outside the claim. + +CPU measurement `execution.timing.cpu_raw` and excluded-attempt `execution.timing.cpu_raw` contain +`start_ticks`, `stop_ticks`, `delta_ticks` as uint64 decimal strings and `timebase_numer` and +`timebase_denom` as uint32 JSON integers. Null means no captured snapshot; older schema-2 artifacts may +omit this optional field. The shared timer captures the original boundaries with one clock read each, +then computes `delta=(stop-start) mod 2^64` (at most one wrap assumed) and +`elapsed_seconds=(double(delta)*double(numer)/double(denom))/1e9`. Floating multiplication avoids +integer overflow. Zero delta or numerator produces zero and cannot admit an LLM measurement; a zero +denominator rejects timer creation, or defensively returns zero if externally corrupted afterwards. +A new start clears the prior snapshot. Invalid measurements keep diagnostic elapsed and any original +raw snapshot; they retain null headline/rate fields. Metal has null CPU raw evidence and retains its +GPU timestamps. CPU `completion_derivation=accepted-plan-derived` refers to accepted planned work; +the kernel's exact-byte checksum counter is programmed evidence, not measured hardware traffic. All byte, lookup, seed and checksum values follow canonical unsigned decimal-string encoding (`0` or `[1-9][0-9]*`); Metal lanes retain uint32 bounds. Indexes, work units and validated small inputs remain bounded JSON diff --git a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md index 36f386e..5141320 100644 --- a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md +++ b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md @@ -1189,3 +1189,9 @@ multiple weight passes, or KV replay factors other than one are separate methodo vocabulary does not activate them: each requires its own end-to-end implementation gate, exact selector-derived methodology and component identity, public CLI/documentation update, and compatibility review before it becomes a supported profile. + +CPU LLM results retain the original Mach tick boundaries and timebase for duration reconstruction. +The build embeds compiler, flags, SDK/deployment target and Git provenance; the command hashes the +executable once before tasks. These fields bind available artifacts and do not constitute signed +execution attestation. See the [LLM process contract](API.md) for availability and numeric rules. +Python 3 is required to generate build provenance. diff --git a/documents/MANUAL.md b/documents/MANUAL.md index fb3cf0b..7095753 100644 --- a/documents/MANUAL.md +++ b/documents/MANUAL.md @@ -2887,3 +2887,9 @@ Command help: ```bash memory_benchmark -h ``` + +CPU LLM results retain the original Mach tick boundaries and timebase for duration reconstruction. +The build embeds compiler, flags, SDK/deployment target and Git provenance; the command hashes the +executable once before tasks. These fields bind available artifacts and do not constitute signed +execution attestation. See the [LLM process contract](API.md) for availability and numeric rules. +Python 3 is required to generate build provenance. diff --git a/src/core/timing/timer.cpp b/src/core/timing/timer.cpp index ed48475..b6f512d 100644 --- a/src/core/timing/timer.cpp +++ b/src/core/timing/timer.cpp @@ -133,6 +133,7 @@ std::optional HighResTimer::create() { // timestamp read and the measured region, introducing run-to-run jitter. void HighResTimer::start() { asm volatile("dsb ish\n\tisb" ::: "memory"); + last_snapshot.reset(); start_ticks = active_timer_system_calls.absolute_time(); } @@ -147,6 +148,7 @@ double HighResTimer::stop() { uint64_t end = active_timer_system_calls.absolute_time(); // Calculate elapsed ticks. Unsigned arithmetic automatically handles wrap-around. uint64_t elapsed_ticks = end - start_ticks; + last_snapshot = MachTimingSnapshot{start_ticks, end, elapsed_ticks, timebase_info.numer, timebase_info.denom}; // Convert ticks to nanoseconds using the timebase info. // Defensive check: ensure denom is not zero (should never happen after constructor validation) if (timebase_info.denom == 0) { @@ -173,6 +175,7 @@ double HighResTimer::stop_ns() { uint64_t end = active_timer_system_calls.absolute_time(); // Calculate elapsed ticks. Unsigned arithmetic automatically handles wrap-around. uint64_t elapsed_ticks = end - start_ticks; + last_snapshot = MachTimingSnapshot{start_ticks, end, elapsed_ticks, timebase_info.numer, timebase_info.denom}; // Convert ticks to nanoseconds and return. // Defensive check: ensure denom is not zero (should never happen after constructor validation) if (timebase_info.denom == 0) { diff --git a/src/core/timing/timer.h b/src/core/timing/timer.h index a581a2e..55562a6 100644 --- a/src/core/timing/timer.h +++ b/src/core/timing/timer.h @@ -47,6 +47,19 @@ std::optional convert_mach_ticks_to_nanoseconds(uint64_t ticks, uint32_t numer, uint32_t denom); +/** Original timer boundaries, copied without clock reads. + * Unsigned delta permits one wrap. Floating multiplication avoids integer + * overflow. Zero delta/numerator yields zero, not a valid LLM duration. + * Access across threads requires external synchronization. + */ +struct MachTimingSnapshot { + uint64_t start_ticks = 0; + uint64_t stop_ticks = 0; + uint64_t delta_ticks = 0; + uint32_t numer = 0; + uint32_t denom = 0; +}; + // --- High-resolution timer helper --- /** * @struct HighResTimer @@ -56,6 +69,7 @@ std::optional convert_mach_ticks_to_nanoseconds(uint64_t ticks, * Automatically handles timebase conversion for accurate measurements. */ struct HighResTimer { + std::optional last_snapshot; ///< Reset at start, captured at stop. uint64_t start_ticks = 0; ///< Ticks at timer start mach_timebase_info_data_t timebase_info; ///< Timebase info for conversion diff --git a/src/llm_memory/llm_backend.h b/src/llm_memory/llm_backend.h index 31c74c5..da248b6 100644 --- a/src/llm_memory/llm_backend.h +++ b/src/llm_memory/llm_backend.h @@ -179,6 +179,7 @@ struct LlmAuthoritativeTiming { bool evaluated = false; bool valid = false; double elapsed_seconds = 0.0; + std::optional cpu_raw; }; /** Exact planned and completed logical work reported by one task. */ diff --git a/src/llm_memory/llm_cpu_backend.cpp b/src/llm_memory/llm_cpu_backend.cpp index 43371fd..76d82f0 100644 --- a/src/llm_memory/llm_cpu_backend.cpp +++ b/src/llm_memory/llm_cpu_backend.cpp @@ -199,6 +199,7 @@ LlmTaskExecutionResult adapt_llm_cpu_executor_result(const LlmMemoryWorkPlan& mo executor_result.post_validation_evaluated && write_evidence_complete; result.timing.evaluated = executor_result.timer_started && executor_result.timer_stopped; result.timing.elapsed_seconds = executor_result.elapsed_seconds; + result.timing.cpu_raw = executor_result.cpu_raw; result.timing.valid = result.timing.evaluated && std::isfinite(executor_result.elapsed_seconds) && executor_result.elapsed_seconds > 0.0; result.validation.evaluated = checksum_evidence_complete; diff --git a/src/llm_memory/llm_environment.h b/src/llm_memory/llm_environment.h index 4708c7f..466e917 100644 --- a/src/llm_memory/llm_environment.h +++ b/src/llm_memory/llm_environment.h @@ -42,4 +42,10 @@ struct LlmHostEnvironmentSnapshot { */ LlmHostEnvironmentSnapshot capture_llm_host_environment() noexcept; +/** Stream SHA-256 of the executable path once on the cold command path. + * Returns empty on lookup/read failure. This binds a file, not signed execution + * attestation; concurrent replacement of that file is outside this evidence. + */ +std::string capture_llm_binary_sha256() noexcept; + #endif // LLM_ENVIRONMENT_H diff --git a/src/llm_memory/llm_environment.mm b/src/llm_memory/llm_environment.mm index 9b55b5a..902d549 100644 --- a/src/llm_memory/llm_environment.mm +++ b/src/llm_memory/llm_environment.mm @@ -21,6 +21,11 @@ #import #include "llm_memory/llm_environment.h" +#include "utils/hash_utils.h" +#include +#include +#include +#include namespace { @@ -67,3 +72,25 @@ LlmHostEnvironmentSnapshot capture_llm_host_environment() noexcept { return LlmHostEnvironmentSnapshot{}; } } + +std::string capture_llm_binary_sha256() noexcept { + try { + uint32_t size = 0; + _NSGetExecutablePath(nullptr, &size); + if (size == 0 || size > 1024 * 1024) return {}; + std::vector path(size); + if (_NSGetExecutablePath(path.data(), &size) != 0) return {}; + std::ifstream input(path.data(), std::ios::binary); + if (!input) return {}; + HashUtils::Sha256Hasher hasher; + std::array buffer; + while (input) { + input.read(buffer.data(), buffer.size()); + hasher.update(buffer.data(), static_cast(input.gcount())); + } + if (!input.eof()) return {}; + return hasher.finalize_hex(); + } catch (...) { + return {}; + } +} diff --git a/src/llm_memory/llm_executor.cpp b/src/llm_memory/llm_executor.cpp index 345ade5..030e296 100644 --- a/src/llm_memory/llm_executor.cpp +++ b/src/llm_memory/llm_executor.cpp @@ -4350,6 +4350,7 @@ LlmExecutorResult execute_llm_scenario(const LlmMemoryWorkPlan& model_plan, cons result.completed_workers = completed_workers.load(std::memory_order_relaxed); result.qos_successful_workers = qos_successful_workers.load(std::memory_order_relaxed); result.qos_failed_workers = qos_failed_workers.load(std::memory_order_relaxed); + if (timer_stop_succeeded) result.cpu_raw = timer.last_snapshot; result.elapsed_seconds = measured_duration; result.timer_stopped = timer_stop_succeeded; diff --git a/src/llm_memory/llm_executor.h b/src/llm_memory/llm_executor.h index 04c7c06..5a34760 100644 --- a/src/llm_memory/llm_executor.h +++ b/src/llm_memory/llm_executor.h @@ -33,7 +33,7 @@ #include "core/memory/memory_manager.h" #include "llm_memory/llm_work_plan.h" -struct HighResTimer; +#include "core/timing/timer.h" /** Stable machine-readable reasons returned by the LLM executor boundary. */ namespace LlmExecutorReason { @@ -350,6 +350,7 @@ struct LlmCpuRuntimeEvidence { /** Transient synchronized executor result. Expected data is consumed by validation * and never retained once per measurement. */ struct LlmExecutorResult : LlmCpuRuntimeEvidence { + std::optional cpu_raw; double elapsed_seconds = 0.0; std::vector expected_checksums; LlmRunChecksum expected_run_checksum{0, 0}; diff --git a/src/llm_memory/llm_json.cpp b/src/llm_memory/llm_json.cpp index 28b3609..5059cc1 100644 --- a/src/llm_memory/llm_json.cpp +++ b/src/llm_memory/llm_json.cpp @@ -1394,6 +1394,13 @@ OrderedJson cold_checks_json(const LlmColdChecks& expected, const LlmColdChecks* return output; } +OrderedJson cpu_raw_json(const std::optional& raw) { + if (!raw) return nullptr; + return OrderedJson{{"start_ticks", decimal_string(raw->start_ticks)}, + {"stop_ticks", decimal_string(raw->stop_ticks)}, {"delta_ticks", decimal_string(raw->delta_ticks)}, + {"timebase_numer", raw->numer}, {"timebase_denom", raw->denom}}; +} + OrderedJson compact_execution_json(const LlmTaskExecutionEvidence& execution, std::string_view fallback_reason_code, std::string_view /*checksum_algorithm_version*/) { const bool available = execution.available; @@ -1416,6 +1423,7 @@ OrderedJson compact_execution_json(const LlmTaskExecutionEvidence& execution, st output["valid"] = available ? OrderedJson(execution.valid) : OrderedJson(nullptr); output["elapsed_seconds"] = positive_finite_or_null( execution.elapsed_seconds, available && execution.timing_evaluated && execution.timing_valid); + output["timing"]["cpu_raw"] = cpu_raw_json(execution.cpu_raw); const bool cpu_available = available && execution.cpu_evidence_available; output["requested_workers"] = number_or_null(execution.requested_workers, cpu_available); output["created_workers"] = number_or_null(execution.created_workers, cpu_available); @@ -1581,6 +1589,7 @@ OrderedJson measurement_execution_json(const LlmMeasurementState& measurement) { {"valid", available && execution.timing.evaluated ? OrderedJson(execution.timing.valid) : OrderedJson(nullptr)}, {"diagnostic_elapsed_seconds", measurement.status != LlmMeasurementStatus::Measured && available && execution.timing.evaluated ? finite_or_null(execution.timing.elapsed_seconds) : OrderedJson(nullptr)}}; + output["timing"]["cpu_raw"] = cpu_raw_json(execution.timing.cpu_raw); const bool cpu_available = available && cpu != nullptr; output["requested_workers"] = number_or_null(cpu_available ? cpu->requested_workers : 0, cpu_available); output["created_workers"] = number_or_null(cpu_available ? cpu->created_workers : 0, cpu_available); @@ -2292,10 +2301,7 @@ nlohmann::ordered_json build_llm_memory_json(const LlmMemoryConfig& config, cons output["kv_layout"] = llm_kv_layout_to_string(model_plan.kv_layout); output["methodology_version"] = model_plan.methodology_version; output["software"] = software_json(metadata); - output["build_manifest"] = OrderedJson{{"manifest_version", 1}, {"status", "unavailable"}, - {"reason_code", "build-provenance-not-provided"}, {"binary_sha256", nullptr}, {"git_commit", nullptr}, - {"git_dirty", nullptr}, {"compiler", nullptr}, {"compile_flags", nullptr}, {"link_flags", nullptr}, {"target_arch", nullptr}, - {"sdk", nullptr}, {"min_os", nullptr}}; + output["build_manifest"] = metadata.build_manifest; output["configuration"] = configuration_json(config); output["resolved_plan"] = resolved_plan_json(model_plan, result, backend_evidence); output["backend_evidence"] = diff --git a/src/llm_memory/llm_json.h b/src/llm_memory/llm_json.h index 59e86f2..910703b 100644 --- a/src/llm_memory/llm_json.h +++ b/src/llm_memory/llm_json.h @@ -51,6 +51,10 @@ struct LlmJsonPeakEstimate { /** Cold-path host and command metadata shared by every checkpoint snapshot. */ struct LlmResultMetadata { + nlohmann::ordered_json build_manifest = {{"manifest_version", 1}, {"status", "unavailable"}, + {"reason_code", "build-provenance-not-provided"}, {"binary_sha256", nullptr}, {"git_commit", nullptr}, + {"git_dirty", nullptr}, {"compiler", nullptr}, {"compile_flags", nullptr}, {"link_flags", nullptr}, + {"target_arch", nullptr}, {"sdk", nullptr}, {"min_os", nullptr}}; std::string timestamp; std::string processor_name; std::string macos_version; diff --git a/src/llm_memory/llm_memory.cpp b/src/llm_memory/llm_memory.cpp index 75f92bd..29decfa 100644 --- a/src/llm_memory/llm_memory.cpp +++ b/src/llm_memory/llm_memory.cpp @@ -18,6 +18,7 @@ * @brief Standalone parsing, command boundary, validation, and status tokens */ +#include "../../.build-provenance.h" #include "llm_memory/llm_memory.h" #include @@ -922,6 +923,13 @@ int run_llm_memory_mode(int argc, char* argv[]) { bool post_run_exception = false; try { print_runtime_banner(); + metadata.build_manifest = nlohmann::ordered_json::parse(LLM_BUILD_MANIFEST_JSON); + const std::string binary_hash = capture_llm_binary_sha256(); + if (!binary_hash.empty()) metadata.build_manifest["binary_sha256"] = binary_hash; + else { + metadata.build_manifest["status"] = "partial"; + metadata.build_manifest["reason_code"] = "binary-hash-unavailable"; + } metadata.timestamp = build_utc_timestamp(); metadata.main_thread_qos = prepare_main_thread_benchmark_qos(MainThreadQosSetter{}, false); BenchmarkSignalMaskGuard signal_guard; diff --git a/src/llm_memory/llm_runner.cpp b/src/llm_memory/llm_runner.cpp index 8174302..feaf268 100644 --- a/src/llm_memory/llm_runner.cpp +++ b/src/llm_memory/llm_runner.cpp @@ -526,6 +526,7 @@ LlmTaskExecutionEvidence compact_execution( evidence.valid = execution.status == LlmTaskExecutionStatus::Complete; evidence.reason_code = canonicalize_llm_result_reason_code(execution.reason_code); evidence.elapsed_seconds = execution.timing.elapsed_seconds; + evidence.cpu_raw = execution.timing.cpu_raw; evidence.timing_evaluated = execution.timing.evaluated; evidence.timing_valid = execution.timing.valid; evidence.completion = execution.completion; diff --git a/src/llm_memory/llm_runner.h b/src/llm_memory/llm_runner.h index cfe68b4..dc6f48a 100644 --- a/src/llm_memory/llm_runner.h +++ b/src/llm_memory/llm_runner.h @@ -105,6 +105,7 @@ inline const LlmMetalRuntimeEvidence* get_llm_metal_task_evidence(const LlmRetai /** Compact excluded-task evidence without retained per-worker vectors. */ struct LlmTaskExecutionEvidence { + std::optional cpu_raw; LlmColdChecks cpu_cold_checks; ///< Compact CPU copy; Metal owns its own array. bool available = false; ///< Complete generic task evidence was retained. bool valid = false; diff --git a/tests/test_llm_memory_json.cpp b/tests/test_llm_memory_json.cpp index 38326ab..f0d6f9e 100644 --- a/tests/test_llm_memory_json.cpp +++ b/tests/test_llm_memory_json.cpp @@ -3108,3 +3108,25 @@ TEST(LlmMemoryJsonTest, AllEightProfilesPublishExactSchemaTwoSelectorsAndNamedKi } } } + +TEST(LlmMemoryJsonTest, OriginalCpuTimingAndCommandManifestSurviveProjection) { + const auto config = explicit_config(1); + const auto plan = admitted_plan(config); + auto result = complete_result(config, plan); + auto metadata = fixed_metadata(config, plan); + metadata.build_manifest["binary_sha256"] = std::string(64, 'a'); + metadata.build_manifest["status"] = "partial"; + const MachTimingSnapshot raw{100, 103, 3, 125, 3}; + result.measurements[0].execution.timing.cpu_raw = raw; + result.calibration_attempts[0][0].execution.cpu_raw = raw; + const auto doc = build_llm_memory_json(config, plan, preparation_for(plan), metadata, result); + EXPECT_EQ(doc["build_manifest"], metadata.build_manifest); + const auto& captured = doc["measurements"][0]["execution"]["timing"]["cpu_raw"]; + EXPECT_EQ(captured["start_ticks"], "100"); + EXPECT_EQ(captured["stop_ticks"], "103"); + EXPECT_EQ(captured["delta_ticks"], "3"); + EXPECT_EQ(captured["timebase_numer"], 125); + EXPECT_EQ(captured["timebase_denom"], 3); + EXPECT_EQ(doc["calibration"]["attempts"]["weights_only"][0]["execution"]["timing"]["cpu_raw"], captured); + EXPECT_TRUE(doc["measurements"][1]["execution"]["timing"]["cpu_raw"].is_null()); +} diff --git a/tests/test_timer.cpp b/tests/test_timer.cpp index 99d3114..820fb11 100644 --- a/tests/test_timer.cpp +++ b/tests/test_timer.cpp @@ -130,3 +130,40 @@ TEST_F(HighResTimerTest, UnsignedTickSubtractionPreservesWraparound) { timer->start(); EXPECT_DOUBLE_EQ(timer->stop_ns(), 10.0); } + +TEST_F(HighResTimerTest, SnapshotRetainsOriginalBoundariesWithoutExtraClockReads) { + state.numer = 125; + state.denom = 3; + set_ticks({100, 103, 200, 200}); + auto timer = HighResTimer::create(); + ASSERT_TRUE(timer); + EXPECT_FALSE(timer->last_snapshot); + timer->start(); + EXPECT_FALSE(timer->last_snapshot); + EXPECT_DOUBLE_EQ(timer->stop(), 125.0 / 1e9); + ASSERT_TRUE(timer->last_snapshot); + EXPECT_EQ(timer->last_snapshot->start_ticks, 100u); + EXPECT_EQ(timer->last_snapshot->stop_ticks, 103u); + EXPECT_EQ(timer->last_snapshot->delta_ticks, 3u); + EXPECT_EQ(timer->last_snapshot->numer, 125u); + EXPECT_EQ(timer->last_snapshot->denom, 3u); + EXPECT_EQ(state.next_tick, 2u); + timer->start(); + EXPECT_FALSE(timer->last_snapshot); + EXPECT_DOUBLE_EQ(timer->stop_ns(), 0.0); + EXPECT_EQ(timer->last_snapshot->delta_ticks, 0u); + EXPECT_EQ(state.next_tick, 4u); +} + +TEST_F(HighResTimerTest, SnapshotWrapAndLargeConversionAvoidIntegerOverflow) { + state.numer = std::numeric_limits::max(); + state.denom = 1; + set_ticks({10, 9}); + auto timer = HighResTimer::create(); + ASSERT_TRUE(timer); + timer->start(); + const double ns = timer->stop_ns(); + ASSERT_TRUE(timer->last_snapshot); + EXPECT_EQ(timer->last_snapshot->delta_ticks, std::numeric_limits::max()); + EXPECT_DOUBLE_EQ(ns, static_cast(std::numeric_limits::max()) * state.numer); +} From e8e3cb5a3ddfa8880000765ddeaa4b929e264ff0 Mon Sep 17 00:00:00 2001 From: timoheimonen Date: Sun, 6 Sep 2026 10:37:16 +0300 Subject: [PATCH 12/16] Add bounded independent LLM artifact verification --- Makefile | 11 +- README.md | 27 +- build-support/generate_provenance.py | 54 +- documents/API.md | 49 + documents/LLM_MEMORY_PROFILE_WHITEPAPER.md | 12 +- documents/MANUAL.md | 27 +- documents/PROJECT_STRUCTURE.md | 10 +- script-examples/llm_verify_oracles.py | 552 ++++++ script-examples/llm_verify_profiles.py | 124 ++ script-examples/verify_llm_result.py | 1524 +++++++++++++++++ tests/fixtures/README.md | 16 + .../llm-schema-v2/cpu-decode-contiguous.json | 1 + .../llm-schema-v2/cpu-decode-paged.json | 1 + .../llm-schema-v2/cpu-prefill-contiguous.json | 1 + .../llm-schema-v2/cpu-prefill-paged.json | 1 + .../metal-decode-contiguous.json | 1 + .../llm-schema-v2/metal-decode-paged.json | 1 + .../metal-prefill-contiguous.json | 1 + .../llm-schema-v2/metal-prefill-paged.json | 1 + tests/fixtures/llm-schema-v2/unsupported.json | 1 + tests/test_llm_result_verifier.py | 489 ++++++ 21 files changed, 2858 insertions(+), 46 deletions(-) create mode 100644 script-examples/llm_verify_oracles.py create mode 100644 script-examples/llm_verify_profiles.py create mode 100644 script-examples/verify_llm_result.py create mode 100644 tests/fixtures/llm-schema-v2/cpu-decode-contiguous.json create mode 100644 tests/fixtures/llm-schema-v2/cpu-decode-paged.json create mode 100644 tests/fixtures/llm-schema-v2/cpu-prefill-contiguous.json create mode 100644 tests/fixtures/llm-schema-v2/cpu-prefill-paged.json create mode 100644 tests/fixtures/llm-schema-v2/metal-decode-contiguous.json create mode 100644 tests/fixtures/llm-schema-v2/metal-decode-paged.json create mode 100644 tests/fixtures/llm-schema-v2/metal-prefill-contiguous.json create mode 100644 tests/fixtures/llm-schema-v2/metal-prefill-paged.json create mode 100644 tests/fixtures/llm-schema-v2/unsupported.json create mode 100644 tests/test_llm_result_verifier.py diff --git a/Makefile b/Makefile index 0f0f276..bb455ee 100644 --- a/Makefile +++ b/Makefile @@ -154,11 +154,16 @@ test-integration: $(TARGET) $(TEST_TARGET) test-script-examples: python3 -m unittest -v tests/test_script_examples.py -# All tests (unit tests + integration tests + bundled script examples) +# Independent LLM artifact contract and mutation tests. +test-llm-verifier: + python3 -m unittest -v tests/test_llm_result_verifier.py + +# All tests (unit, integration, bundled examples, and independent LLM verifier) test-all: $(TARGET) $(TEST_TARGET) - @echo "Running all tests (unit + integration + bundled script examples)..." + @echo "Running all tests (unit + integration + bundled examples + LLM verifier)..." ./$(TEST_TARGET) $(MAKE) test-script-examples + $(MAKE) test-llm-verifier # Reproducible production C++ source coverage in an isolated /tmp build. coverage-unit: @@ -236,7 +241,7 @@ uninstall: @echo "$(TARGET) uninstalled successfully." # Define targets that don't correspond to files -.PHONY: all clean test test-integration test-script-examples test-all \ +.PHONY: all clean test test-integration test-script-examples test-llm-verifier test-all \ coverage-unit coverage-all clean-test docs clean-docs install uninstall # Missing dependency files are expected on a clean tree. Existing files carry diff --git a/README.md b/README.md index 82924bb..0872c78 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ See [Measurement Capabilities](documents/CAPABILITIES.md) for the full measureme - macOS 26 or later on Apple Silicon (ARM64) - Xcode Command Line Tools for source builds - GoogleTest from Homebrew for the test suite -- Python 3 for the script-example entry test included in the aggregate `make test-all` gate; `jq` is optional for JSON +- Python 3 for build provenance and the Python tests included in the aggregate `make test-all` gate; `jq` is optional for JSON inspection and the jq-backed latency-script path - Metal modes: a unified-memory device with `MTLGPUFamilyApple7` or compatible later-family capability; LLM Metal also requires Tier 2 argument buffers and `maxBufferLength >= 256 MiB` @@ -395,7 +395,7 @@ make test-integration make test-all ``` -`make test-all` requires Python 3; it runs all GTest cases followed by the focused script-example entry test. `jq` is +`make test-all` requires Python 3; it runs all GTest cases followed by the script-example and independent LLM verifier tests. `jq` is not required by the test gate. Generate isolated LLVM production-source coverage reports under `/tmp`: @@ -408,6 +408,23 @@ make coverage-all See [CONTRIBUTING.md](documents/CONTRIBUTING.md) for contribution guidance and [Project Structure](documents/PROJECT_STRUCTURE.md) for repository navigation and the current test-suite map. C++ reference documentation can be generated with `make docs`. +CPU LLM results retain the original Mach tick boundaries and timebase for duration reconstruction. +The build embeds compiler, flags, SDK/deployment target and Git provenance; the command hashes the +executable once before tasks. These fields bind available artifacts and do not constitute signed +execution attestation. See the [LLM process contract](documents/API.md) for availability and numeric rules. +Python 3 is required to generate build provenance. + +Verify a saved LLM schema-2 result independently (standard library only): + +```bash +python3 script-examples/verify_llm_result.py llm-result.json --binary ./memory_benchmark --require-raw-timing +make test-llm-verifier +``` + +The verifier reports artifact consistency separately from run acceptance and identifies missing timing/build +evidence. Its bounded arithmetic reconstructs logical work and checksums; it does not attest execution or measure +physical DRAM traffic. See [the verifier contract](documents/API.md#independent-llm-artifact-verifier). + ## Scope and Safety This project intentionally does not target Intel Macs or other operating systems, provide a GUI, or host a public leaderboard/backend. @@ -419,9 +436,3 @@ The benchmark performs sustained, intensive memory operations. Use it at your ow Copyright 2025-2026 Timo Heimonen \ Licensed under the [GNU General Public License v3.0 or later](LICENSE). - -CPU LLM results retain the original Mach tick boundaries and timebase for duration reconstruction. -The build embeds compiler, flags, SDK/deployment target and Git provenance; the command hashes the -executable once before tasks. These fields bind available artifacts and do not constitute signed -execution attestation. See the [LLM process contract](documents/API.md) for availability and numeric rules. -Python 3 is required to generate build provenance. diff --git a/build-support/generate_provenance.py b/build-support/generate_provenance.py index aa214e2..70c7cde 100644 --- a/build-support/generate_provenance.py +++ b/build-support/generate_provenance.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Freeze source/build inputs at make time; update the header only on change.""" + import json import os from pathlib import Path @@ -16,30 +17,47 @@ def command(args): def main(): - commit = command(['git', 'rev-parse', '--verify', 'HEAD']) - status = command(['git', 'status', '--porcelain', '--untracked-files=normal']) if commit else None - compiler = shlex.split(os.environ['PROVENANCE_CXX']) - manifest = dict(manifest_version=1, status='available', reason_code='valid', binary_sha256=None, - git_commit=commit, git_dirty=bool(status) if status is not None else None, - compiler=command(compiler + ['--version']), - compile_flags={key: os.environ['PROVENANCE_' + key.upper()] - for key in ('cxxflags', 'test_cxxflags', 'asflags')}, - link_flags=os.environ['PROVENANCE_LDFLAGS'], - target_arch=command(compiler + ['-arch', 'arm64', '-dumpmachine']), - sdk=command(['xcrun', '--sdk', 'macosx', '--show-sdk-version']), - min_os=os.environ.get('MACOSX_DEPLOYMENT_TARGET')) - if any(manifest[key] is None for key in ('git_commit', 'git_dirty', 'compiler', 'target_arch', 'sdk', 'min_os')): - manifest.update(status='partial', reason_code='build-fields-unavailable') + # A tarball placed inside another checkout must not inherit that repo's HEAD. + top = command(["git", "rev-parse", "--show-toplevel"]) + own_checkout = top is not None and Path(top).resolve() == Path.cwd().resolve() + commit = command(["git", "rev-parse", "--verify", "HEAD"]) if own_checkout else None + status = command(["git", "status", "--porcelain", "--untracked-files=normal"]) if commit else None + compiler = shlex.split(os.environ["PROVENANCE_CXX"]) + manifest = dict( + manifest_version=1, + status="available", + reason_code="valid", + binary_sha256=None, + git_commit=commit, + git_dirty=bool(status) if status is not None else None, + compiler=command(compiler + ["--version"]), + compile_flags={ + key: os.environ["PROVENANCE_" + key.upper()] for key in ("cxxflags", "test_cxxflags", "asflags") + }, + link_flags=os.environ["PROVENANCE_LDFLAGS"], + target_arch=command(compiler + ["-arch", "arm64", "-dumpmachine"]), + sdk=command(["xcrun", "--sdk", "macosx", "--show-sdk-version"]), + min_os=os.environ.get("MACOSX_DEPLOYMENT_TARGET"), + ) + if any( + manifest[key] is None + for key in ("git_commit", "git_dirty", "compiler", "target_arch", "sdk", "min_os") + ): + manifest.update(status="partial", reason_code="build-fields-unavailable") payload = json.dumps(manifest, sort_keys=True) if len(payload) > 16384: - raise SystemExit('build provenance exceeds 16 KiB budget') - header = '// Generated build inputs; do not edit.\n#define LLM_BUILD_MANIFEST_JSON ' + json.dumps(payload) + '\n' + raise SystemExit("build provenance exceeds 16 KiB budget") + header = ( + "// Generated build inputs; do not edit.\n#define LLM_BUILD_MANIFEST_JSON " + + json.dumps(payload) + + "\n" + ) path = Path(sys.argv[1]) if not path.exists() or path.read_text() != header: - temporary = path.with_suffix('.tmp') + temporary = path.with_suffix(".tmp") temporary.write_text(header) temporary.replace(path) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/documents/API.md b/documents/API.md index c8006f6..657cf13 100644 --- a/documents/API.md +++ b/documents/API.md @@ -727,6 +727,55 @@ Canonical plan/expected vector capacities, transient expected reconstruction, ac DOM and stdout serialized strings are charged together at their simultaneous peak; bounded snapshot count does not reduce the single-snapshot peak allowance. +## Independent LLM artifact verifier + +`python3 script-examples/verify_llm_result.py result.json` reads exactly one schema-2 `llm_memory` document. +It supports the eight current CPU/Metal × decode/prefill × contiguous/paged component sets. Other schemas, +methodologies and profiles are unsupported; there is no migration or historical field fallback. The independent +Python arithmetic never imports or invokes producer helpers or executes the optional binary. + +The JSON verdict separates `artifact_consistent` from `run_accepted`. Exit 0 means a consistent accepted artifact; +exit 1 means inconsistency or a consistently described unaccepted run; exit 2 means unsupported schema/profile, +a resource bound or a requested evidence level that is unavailable. `artifact_consistent` is null on unsupported +cases. An artifact alone cannot establish the original process exit status; the process acceptance procedure below +still applies to callers launching the producer. + +Checks reconstruct geometry, frozen work, payload/layout/lookup quantities, seed derivation and the canonical +Fisher–Yates table digest; bind component, model, scenario, layout and nested identity evidence; derive independent +CPU worker/run and Metal dual-mod32 expectations once per canonical plan; check original timing, rates, named +validation, ordered measurement prefix, accepted populations, exact statistics, counters and run acceptance. +The CPU oracle reconstructs owner ranges, including scenario-specific prefill prefix-cost partitions. Affine range +sums avoid materializing weight or KV pools. The verifier does not reproduce allocator capacity, host admission +samples or GPU traces; those remain reported implementation/environment evidence. Identity/hash agreement binds +reported content and never authenticates a build or a runtime observation. + +`checks.checksum` is `independent-oracle` when expectations were available, otherwise `unavailable`. +`checks.timing` is `cpu-raw-ticks`, `gpu-timestamps`, `elapsed-only`, or `unavailable`. Optional CPU raw evidence +may be absent from earlier schema-2 artifacts; rates are still checked but duration reconstruction is limited. +`--require-raw-timing` rejects missing CPU raw evidence with `unsupported-missing-raw-timing`. +`checks.build` is `manifest-only`, `unavailable`, or `binary-bound`. `--binary PATH` streams the supplied file's +SHA-256 and requires equality with the manifest. It does not run the file. A correctly described failed, +partial, interrupted or unsupported run can be consistent while `run_accepted` is false. + +Counts use exact integers and the producer's canonical decimal representation. Derived floats use relative +tolerance `1e-10` and absolute tolerance `1e-15`; integer equality never uses float tolerance. CPU conversion +uses the documented double evaluation order. Zero/nonfinite accepted durations and invalid rates are rejected. +Statistics use sample standard deviation (n−1; singleton zero), linearly interpolated quantiles at `(n−1)p`, +median absolute deviation and the accepted measurement IDs only. Empty populations require null statistics. + +Input limits are 64 MiB UTF-8 JSON, depth 64, one million JSON nodes, 100,000 measurements, 1,024 canonical +plans, 1 MiB per identity and one million charged oracle/partition steps across the artifact. Decimal uint64 +strings have at most 20 digits. Duplicate JSON keys, booleans used as integers and nonfinite JSON numbers +are rejected. A limit produces an explicit unsupported verdict, never successful verification; large calibrated +work can exceed this verifier's budget even when the benchmark itself supports it. Parsing errors do not become +partial success. Exact acceptance claims remain subject to the checksum collision boundaries documented in the +[LLM whitepaper](LLM_MEMORY_PROFILE_WHITEPAPER.md#checksum-fault-model-and-evidence-limits), and the verifier +cannot replace kernel qualification tests, prove executed instructions or establish physical GPU/DRAM traffic. + +`make test-llm-verifier` runs independent small arithmetic goldens, all eight captured current profile fixtures, +status fixtures, malformed input and specific artifact mutations. `make test-all` includes this separate gate +alongside the existing standard-memory script-example gate. + ## Consumer acceptance procedure A caller accepts a benchmark conclusion only after all of the following checks succeed: diff --git a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md index 5141320..92112ff 100644 --- a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md +++ b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md @@ -799,6 +799,12 @@ The maintained examples are in [CPU kernel tests](../tests/test_llm_memory_kerne (forced validation boolean), which test result handling. Pure independent checksum-oracle goldens test arithmetic; they are not GPU execution evidence. Algorithms and their existing identities remain unchanged by this fault model. +CPU LLM results retain the original Mach tick boundaries and timebase for duration reconstruction. +The build embeds compiler, flags, SDK/deployment target and Git provenance; the command hashes the +executable once before tasks. These fields bind available artifacts and do not constitute signed +execution attestation. See the [LLM process contract](API.md) for availability and numeric rules. +Python 3 is required to generate build provenance. + ## Timing boundary and backend lifecycle A CPU scenario task follows this boundary: @@ -1189,9 +1195,3 @@ multiple weight passes, or KV replay factors other than one are separate methodo vocabulary does not activate them: each requires its own end-to-end implementation gate, exact selector-derived methodology and component identity, public CLI/documentation update, and compatibility review before it becomes a supported profile. - -CPU LLM results retain the original Mach tick boundaries and timebase for duration reconstruction. -The build embeds compiler, flags, SDK/deployment target and Git provenance; the command hashes the -executable once before tasks. These fields bind available artifacts and do not constitute signed -execution attestation. See the [LLM process contract](API.md) for availability and numeric rules. -Python 3 is required to generate build provenance. diff --git a/documents/MANUAL.md b/documents/MANUAL.md index 7095753..6d65aa2 100644 --- a/documents/MANUAL.md +++ b/documents/MANUAL.md @@ -51,7 +51,7 @@ This manual focuses on practical usage and interpretation. For implementation de - Apple Silicon Mac running macOS 26 or later - Xcode Command Line Tools -- GoogleTest for C++ tests; Python 3 for the script-example entry test in the aggregate `make test-all` gate. `jq` is +- GoogleTest for C++ tests; Python 3 for build provenance and the Python tests in the aggregate `make test-all` gate. `jq` is optional for JSON inspection and the jq-backed latency-script path. - For `--gpu-bandwidth`: a unified-memory Metal device supporting `MTLGPUFamilyApple7` or a compatible later family. Capability support is distinct from a controlled performance-validation cohort. @@ -99,7 +99,7 @@ production C++ and Objective-C++ only and excludes tests, GoogleTest, the bundle assembly. The macOS 26.0 build links the system Metal and Foundation frameworks. GPU kernels are embedded MSL 2.3 source compiled at runtime; the optional offline Metal Toolchain is not required. -`make test-all` requires Python 3 for its script-example entry test. It does not require `jq`. +`make test-all` requires Python 3 for its script-example and LLM verifier tests. It does not require `jq`. ### First run @@ -2815,6 +2815,23 @@ positive power-of-two page-rounded weight, physical K/V, optional table, transient, and auxiliary peak fits the reported available-memory policy. +CPU LLM results retain the original Mach tick boundaries and timebase for duration reconstruction. +The build embeds compiler, flags, SDK/deployment target and Git provenance; the command hashes the +executable once before tasks. These fields bind available artifacts and do not constitute signed +execution attestation. See the [LLM process contract](API.md) for availability and numeric rules. +Python 3 is required to generate build provenance. + +A saved schema-2 artifact can be checked without running the benchmark: + +```bash +python3 script-examples/verify_llm_result.py llm-result.json --binary ./memory_benchmark --require-raw-timing +``` + +The optional binary is hashed, never executed. The verdict distinguishes artifact consistency, run acceptance, +independent checksum reconstruction, raw timing availability and binary binding. A consistent failed run exits 1; +an unsupported schema or exceeded verifier work budget exits 2. Only a consistent accepted artifact exits 0. +See the [bounded verifier contract](API.md#independent-llm-artifact-verifier) for limits and float tolerances. + ### LLM result is incomplete, invalid, or not comparable Check top-level backend/phase/layout/methodology, status/reason, `results_complete`, `run_accepted`, scenario-order @@ -2887,9 +2904,3 @@ Command help: ```bash memory_benchmark -h ``` - -CPU LLM results retain the original Mach tick boundaries and timebase for duration reconstruction. -The build embeds compiler, flags, SDK/deployment target and Git provenance; the command hashes the -executable once before tasks. These fields bind available artifacts and do not constitute signed -execution attestation. See the [LLM process contract](API.md) for availability and numeric rules. -Python 3 is required to generate build provenance. diff --git a/documents/PROJECT_STRUCTURE.md b/documents/PROJECT_STRUCTURE.md index b1093ff..a7777e2 100644 --- a/documents/PROJECT_STRUCTURE.md +++ b/documents/PROJECT_STRUCTURE.md @@ -362,9 +362,9 @@ bandwidth warm-up covers the full target buffer, and latency warm-up page-touche ## 3. tests/ — Test suite -GoogleTest-based unit and integration tests are supplemented by a focused Python script-example entry test. -`make test-script-examples` exercises the current JSON-reading examples directly, and `make test-all` runs it after all -GTest cases pass. Python 3 is required; `jq` is optional, and only its dedicated test branch is skipped when it is not +GoogleTest-based unit and integration tests are supplemented by Python example and LLM verifier tests. +`make test-script-examples` exercises the current JSON-reading examples directly. `make test-llm-verifier` checks the +independent schema-2 LLM reader, numeric goldens and semantic mutations. `make test-all` runs both after all GTest cases pass. Python 3 is required; `jq` is optional, and only its dedicated test branch is skipped when it is not installed. All `.cpp` files are picked up automatically by the Makefile. Tests named `*Integration*` are excluded from `make test` (unit-only) and run through the integration or all-test targets. @@ -396,6 +396,7 @@ installed. All `.cpp` files are picked up automatically by the Makefile. Tests n | `test_hash_utils.cpp` | `HashUtilsTest` | CommonCrypto SHA-256 standard vectors and source-provenance helper behavior | | `test_analysis.cpp` | `AnalysisTest` | Injected TLB coordination, counters/status, boundary detection, validation, and paired analysis | | `test_json_schema.cpp` | `JsonSchemaTest` | Current standard schema-3 output structure, completion fields, and other mode schema contracts | +| `test_llm_result_verifier.py` | Independent LLM verifier | Eight current profiles, arithmetic goldens, status/population and semantic mutations, bounded input and build provenance | | `test_script_examples.py` | Python script-example entry test | Compatible standard schema-3/methodology inputs, provenance-version handling, completion and field-shape checks, and metric extraction for the bundled standard-memory examples, including the optional jq-backed shell path when jq is installed | | `test_json_output_session.cpp` | `JsonOutputTargetTest`, `JsonOutputSessionTest` | Exact sentinel/path classification, lazy checkpoint dispatch, atomic-file parity, stdout routing/restoration, and stream failure containment | | `test_json_utils.cpp` | `JsonUtilsTest`, `JsonFileWriterTest` | JSON parse/statistics and atomic writer success/failure contracts | @@ -517,6 +518,9 @@ policy documented in their own row below. | File | Purpose | |---|---| +| `verify_llm_result.py` | Bounded standalone LLM schema-2 artifact verifier; separate consistency, acceptance and timing/build evidence levels | +| `llm_verify_oracles.py` | Independent CPU and Metal arithmetic/checksum reconstruction for the verifier | +| `llm_verify_profiles.py` | Supported LLM component identities for the eight verifier profiles | | `final_output.txt` | Small bundled sample of pooled latency statistics for `plot_cache_percentiles.py`; regenerated by `latency_test_script.sh` | | `latency_test_script.sh` | Sweeps custom cache size and configured latency-locality windows, writes per-run JSON, and extracts current pooled sample statistics into `final_output.txt` using jq when available or Python 3 otherwise | | `latency_test_script_stride_tlb.sh` | Sweeps cache size, configured locality, and latency stride; uses embedded Python 3 to read current pooled sample statistics, retains per-run JSON, and requires one complete CSV row per planned run | diff --git a/script-examples/llm_verify_oracles.py b/script-examples/llm_verify_oracles.py new file mode 100644 index 0000000..aa8a13f --- /dev/null +++ b/script-examples/llm_verify_oracles.py @@ -0,0 +1,552 @@ +"""Independent bounded arithmetic for LLM schema 2 (no producer imports). + +All modular arithmetic is explicit. Ranges are summed from affine words, not +materialized buffers; the caller supplies a shared operation budget. +""" + +from functools import lru_cache + +U64 = (1 << 64) - 1 +U32 = (1 << 32) - 1 +A = 0x9E3779B97F4A7C15 +B = 0xBF58476D1CE4E5B9 +C = 0x94D049BB133111EB +D = 0xD6E8FEB86659FD93 +DOMAINS = (0x5745494748545F31, 0x4B5F524541445F31, 0x565F524541445F31) +SEED_DOMAINS = ( + 0x4C4C4D5745494748, + 0x4C4C4D4B42554631, + 0x4C4C4D5642554631, + 0x4C4C4D5357454947, + 0x4C4C4D534B564F4E, + 0x4C4C4D534D495845, +) + + +def splitmix(value): + value = (value + A) & U64 + value = ((value ^ (value >> 30)) * B) & U64 + value = ((value ^ (value >> 27)) * C) & U64 + return value ^ (value >> 31) + + +def permutation(seed, count, budget): + budget(count) + values = list(range(count)) + state = splitmix(seed ^ 0x4C4C4D4B56504731) + for bound in range(count, 1, -1): + while True: + draw = splitmix(state) + state = (state + A) & U64 + if draw >= (1 << 64) % bound: + break + budget(1) + j = draw % bound + values[bound - 1], values[j] = values[j], values[bound - 1] + return values + + +def rot(value, shift): + value &= U64 + return ((value << shift) | (value >> (64 - shift))) & U64 + + +def partition(offset, size, workers): + """Nearest absolute 32-byte boundary, ties down, preserving nonempty owners.""" + active = min(size, workers) + if not active: + return [(0, 0)] * workers + boundaries = [0] + for rank in range(1, active): + target = size // active * rank + min(rank, size % active) + lo, hi = boundaries[-1] + 1, size - (active - rank) + aligned_lo, aligned_hi = ((offset + lo + 31) // 32) * 32, ((offset + hi) // 32) * 32 + if aligned_lo <= aligned_hi: + down = max(aligned_lo, min(aligned_hi, (offset + target) // 32 * 32)) + up = min(aligned_hi, down + 32) if down < offset + target else down + target = min((down, up), key=lambda x: (abs(x - offset - target), x)) - offset + boundaries.append(target) + boundaries.append(size) + return [(offset + a, b - a) for a, b in zip(boundaries, boundaries[1:])] + [(0, 0)] * (workers - active) + + +def affine_slice(base, step, start, count, bits=64, parity_origin=None): + """Sum clipped canonical words; parity_origin=None means absolute word parity. + + With parity_origin set, assemble little-endian words starting at that byte + boundary. Only unaligned spans need bounded scalar reads (caller budgets). + """ + word_bytes = bits // 8 + mask = (1 << bits) - 1 + sums = [0, 0] + end = start + count + origin = start if parity_origin is not None else start // word_bytes * word_bytes + if parity_origin is not None and start % word_bytes: + for pos in range(start, end, word_bytes): + value = 0 + for b in range(min(word_bytes, end - pos)): + absolute = pos + b + word = (base + step * (absolute // word_bytes + 1)) & mask + value |= ((word >> (8 * (absolute % word_bytes))) & 255) << (8 * b) + sums[((pos - start) // word_bytes) % 2] += value + return [v & mask for v in sums] + first, last = (start + word_bytes - 1) // word_bytes, end // word_bytes + for parity in range(2): + first_p = first + ((parity - first) % 2) + n = max(0, (last - first_p + 1) // 2) + total = n * base + step * (n * (first_p + 1) + n * (n - 1)) + destination = (parity - (origin // word_bytes if parity_origin is not None else 0)) % 2 + sums[destination] += total + for word in set((start // word_bytes, end // word_bytes)): + low, high = max(start, word * word_bytes), min(end, (word + 1) * word_bytes) + if low >= high or (low == word * word_bytes and high == (word + 1) * word_bytes): + continue + value = (base + step * (word + 1)) & mask + byte_mask = ((1 << ((high - low) * 8)) - 1) << ((low - word * word_bytes) * 8) + value &= byte_mask + destination = (word - (origin // word_bytes if parity_origin is not None else 0)) % 2 + sums[destination] += value + return [v & mask for v in sums] + + +def absorb(state, even, odd, count): + if count: + state[0] = rot(state[0] + even + A * (state[3] + 1), 17) + state[1] = rot(state[1] + odd + count + D * (state[3] + 1), 29) + state[2] += count + state[3] += 1 + + +def lookup(state, logical, physical, kind, work): + term = (logical + 1) * (physical + 1) + (kind + 1) * C + (work + 1) * B + state[0] = rot(state[0] + term + A, 13) + state[1] = rot(state[1] ^ ((term + D) & U64), 31) + + +def fold(workers): + a, b, ordinal = 0x6A09E667F3BCC909, 0xBB67AE8584CAA73B, 0 + for worker in workers: + for x, y, count, spans in worker: + a = rot(a + x + A * (ordinal + 1), 23) + b = rot(b + y + count + D * spans + C * (ordinal + 1), 41) + ordinal += 1 + return {"state_a_uint64_decimal": str(a), "state_b_uint64_decimal": str(b)} + + +def cpu_checksum(model, scenario, work, budget): + """Reconstruct worker checksums once per canonical scenario/T plan.""" + g = model + workers, layers, batch, record, tokens = ( + g[k] for k in ("workers", "layers", "batch", "record", "tokens") + ) + weights = [] + offset = 0 + for layer in range(layers): + size = g["weight"] // layers + (layer < g["weight"] % layers) + weights.append(partition(offset, size, workers)) + offset += size + budget( + work + * workers + * layers + * ( + 1 + + batch + * ( + (g["blocks"] * len(g["tile_ends"]) + g["blocks"]) + if g["prefill"] and g["paged"] + else tokens if g["prefill"] else g["blocks"] or 1 + ) + ) + ) + states = [ + [[0x243F6A8885A308D3 ^ d, (0x13198A2E03707344 + d) & U64, 0, 0] for d in DOMAINS] + for _ in range(workers) + ] + if g["prefill"]: + for state in states: + state[1:] = [[0, 0, 0, 0], [0, 0, 0, 0]] + seed = g["scenario_seeds"][scenario] + + @lru_cache(maxsize=4096) + def reference(base, step, first, size): + if first % 8: + budget((size + 7) // 8) + return affine_slice(base, step, first, size, parity_origin=first) + + def read_decode(state, base, step, first, size, append_start, append_base): + # Exact replacement of only the current record, preserving span parity. + even, odd = reference(base, step, first, size) + lo, hi = max(first, append_start), min(first + size, append_start + record) + if lo < hi: + budget((hi - lo + 7) // 8 + 2) + # Only intersecting span words need replacement, including unaligned tails. + for pos in range(first + ((lo - first) // 8) * 8, hi, 8): + old = new = 0 + for byte in range(min(8, first + size - pos)): + address = pos + byte + value = ((base + step * (address // 8 + 1)) & U64) >> (8 * (address % 8)) & 255 + replacement = value + if append_start <= address < append_start + record: + local = address - append_start + replacement = ((append_base + D * (local // 8 + 1)) & U64) >> (8 * (local % 8)) & 255 + old |= value << (8 * byte) + new |= replacement << (8 * byte) + if ((pos - first) // 8) % 2: + odd = (odd + new - old) & U64 + else: + even = (even + new - old) & U64 + absorb(state, even, odd, size) + + for w, state in enumerate(states): + for unit in range(work): + for layer in range(layers): + start, size = weights[layer][w] + if scenario != "kv_only" and size: + even, odd = reference(g["buffer_seeds"][0], A, start, size) + absorb(state[0], even, odd, size) + if scenario == "weights_only": + continue + for b in range(batch): + row = layer * batch + b + if g["prefill"]: + first, count = g["ownership"][scenario][row][w] + if not count: + continue + first_token = first * g["block_tokens"] if g["paged"] else first + last_token = ( + min(tokens, (first + count) * g["block_tokens"]) if g["paged"] else first + count + ) + if g["paged"]: + for block in range(first, first + count): + logical = b * g["blocks"] + block + lookup(state[1], logical, g["table"][logical], 0, unit) + for end in g["tile_ends"]: + if end <= first_token: + continue + for pool in (1, 2): + base = ( + seed + + 0x50524546494C4C31 + + A * (unit + 1) + + B * (layer + 1) + + C * (b + 1) + + (0x4B4B4B4B4B4B4B4B if pool == 1 else 0x5656565656565656) + ) + ranges = [ + (first_token * record, (min(end, last_token) - first_token) * record) + ] + if g["paged"]: + ranges = [] + for block in range( + first, + min( + first + count, (end + g["block_tokens"] - 1) // g["block_tokens"] + ), + ): + logical = b * g["blocks"] + block + ranges.append( + ( + block * g["block_bytes"], + min( + g["block_bytes"], end * record - block * g["block_bytes"] + ), + ) + ) + for pos, length in ranges: + if g["paged"]: + logical = b * g["blocks"] + pos // g["block_bytes"] + lookup(state[pool], logical, g["table"][logical], pool, unit) + even, odd = affine_slice(base, D, pos, length) + state[pool][0] = (state[pool][0] + even) & U64 + state[pool][1] = (state[pool][1] + odd) & U64 + state[pool][2] += length + state[pool][3] += 1 + elif g["paged"]: + first, count = g["ownership"][scenario][row][w] + if not count: + continue + if first + count == g["blocks"]: + logical = b * g["blocks"] + g["blocks"] - 1 + lookup(state[1], logical, g["table"][logical], 0, unit) + for pool in (1, 2): + for block in range(first, first + count): + logical = b * g["blocks"] + block + physical = g["table"][logical] + lookup(state[pool], logical, physical, pool, unit) + size = min(g["block_tokens"], tokens - block * g["block_tokens"]) * record + base = ( + g["buffer_seeds"][pool] + + 0xA24BAED4963EE407 * (layer + 1) + + 0x9FB21C651E98DF25 * (physical + 1) + ) + append_base = ( + seed + + A * (unit + 1) + + B * (layer + 1) + + C * (b + 1) + + (0x4B4B4B4B4B4B4B4B if pool == 1 else 0x5656565656565656) + ) + read_decode( + state[pool], + base, + 0xC13FA9A902A6328F, + 0, + size, + size - record if block == g["blocks"] - 1 else size, + append_base, + ) + else: + first, size = partition(row * tokens * record, tokens * record, workers)[w] + for pool in (1, 2): + append_base = ( + seed + + A * (unit + 1) + + B * (layer + 1) + + C * (b + 1) + + (0x4B4B4B4B4B4B4B4B if pool == 1 else 0x5656565656565656) + ) + read_decode( + state[pool], + g["buffer_seeds"][pool], + A, + first, + size, + (row * tokens + tokens - 1) * record, + append_base, + ) + keys = ( + "state_a_uint64_decimal", + "state_b_uint64_decimal", + "exact_bytes_read", + "span_count_uint64_decimal", + ) + output = [ + dict( + worker_index=i, + **{name: dict(zip(keys, map(str, values))) for name, values in zip(("weight", "k", "v"), worker)} + ) + for i, worker in enumerate(states) + ] + return output, fold(states) + + +def metal_checksum(g, scenario, work, budget): + """Scalar semantic visits with O(1) affine sums per range; no GPU helpers.""" + profile = { + (False, False): 0x4D444331, + (False, True): 0x4D445031, + (True, False): 0x4D504331, + (True, True): 0x4D505031, + }[g["prefill"], g["paged"]] + seed = g["scenario_seeds"][scenario] + states = [[0, 0] for _ in range(3)] + pools = (0x57474854, 0x4B455943, 0x56414C43) + budget(work * g["layers"] * (1 + g["batch"] * (g["tokens"] + len(g["tile_ends"]) * (g["blocks"] or 1)))) + + def domain(pool, visit, u, l, b, tile, mask, logical=None, physical=None, address=False): + v = ( + profile + + (seed & U32) + + 0xA24BAED5 * (seed >> 32) + + pools[pool] + + visit + + 0xC2B2AE3D * (u + 1) + + 0x27D4EB35 * (l + 1) + + 0x165667C5 * (b + 1) + + 0xD1B54A35 * tile + + 0xD3A2646D * mask + ) + if logical is not None: + v += ( + 0x7F4A7C15 * (logical + 1) + + 0x94D049BB * (physical + 1) + + 0x369DEA0F * (logical + 1) * (physical + 1) + ) + if address: + global_block = l * g["batch"] * g["blocks"] + physical + per_segment = (256 * 1024 * 1024) // g["block_bytes"] + segment = global_block // per_segment + local = (global_block % per_segment) * g["block_bytes"] + absolute = global_block * g["block_bytes"] + token = ( + 0x41444452 + + 0xDB4F0B91 * (global_block & U32) + + 0xBBE05633 * (global_block >> 32) + + 0xA0F2EC75 * segment + + 0x89E18285 * (local & U32) + + 0xC6D1D6C9 * (local >> 32) + + 0xB492B66F * (absolute & U32) + + 0x9AE16A3B * (absolute >> 32) + ) & U32 + v += token + 0xD6E8FEB9 * (logical + 1) * token + return v & U32 + + def mix(pool, value, indices, domains): + states[pool][0] = (states[pool][0] + value + domains) & U32 + states[pool][1] = ( + states[pool][1] + value * 0x9E3779B1 + indices * 0x85EBCA77 + domains * 0x7FEB352D + ) & U32 + + def ranged( + pool, base, step, start, length, u, l, b, visit, tile=0, logical=None, physical=None, index_base=0 + ): + if not length: + return + first, last = (start + 3) // 4, (start + length) // 4 + count = max(0, last - first) + indexsum = count * first + count * (count - 1) // 2 + values = count * base + step * (indexsum + count) + mix( + pool, + values, + indexsum + index_base * count, + count * domain(pool, visit, u, l, b, tile, 15, logical, physical), + ) + for i in set((start // 4, (start + length) // 4)): + lo, hi = max(start, 4 * i), min(start + length, 4 * i + 4) + if lo >= hi or (lo == 4 * i and hi == 4 * i + 4): + continue + mask = ((1 << (hi - lo)) - 1) << (lo - 4 * i) + byte_mask = sum(255 << (8 * j) for j in range(4) if mask & (1 << j)) + value = ((base + step * (i + 1)) & U32) & byte_mask + mix(pool, value, i + index_base, domain(pool, visit, u, l, b, tile, mask, logical, physical)) + + def table_visit(pool, kind, u, l, b, block, tile=0): + logical = b * g["blocks"] + block + physical = g["table"][logical] + mix(pool, physical, logical, domain(pool, kind, u, l, b, tile, 0, logical, physical, g["prefill"])) + + for u in range(work): + weight_start = 0 + for l in range(g["layers"]): + length = g["weight"] // g["layers"] + (l < g["weight"] % g["layers"]) + if scenario != "kv_only": + ranged(0, g["buffer_seeds"][0] & U32, 0x9E3779B9, weight_start, length, u, l, 0, 0x57524541) + weight_start += length + if scenario == "weights_only": + continue + for b in range(g["batch"]): + row = l * g["batch"] + b + for pool in (1, 2): + write_domain = ( + (0x504B5731 if pool == 1 else 0x50565731) + if g["prefill"] + else (0x4B455931 if pool == 1 else 0x56414C31) + ) + write_base = ( + (seed & U32) + + 0x85EBCA6B * (u + 1) + + 0xC2B2AE35 * (l + 1) + + 0x27D4EB2F * (b + 1) + + write_domain + ) + if g["prefill"]: + if g["paged"]: + # Paired population has one K-owned lookup per block. + for block in range(g["blocks"]): + logical = b * g["blocks"] + block + table_visit(pool, 0x5050574C, u, l, b, block) + start = (row * g["tokens"] + block * g["block_tokens"]) * g["record"] + size = ( + min(g["block_tokens"], g["tokens"] - block * g["block_tokens"]) + * g["record"] + ) + ranged(pool, write_base, 0x165667B1, start, size, u, l, b, 0x50575254) + for tile, end in enumerate(g["tile_ends"], 1): + for block in range((end + g["block_tokens"] - 1) // g["block_tokens"]): + table_visit( + pool, 0x50504B4C if pool == 1 else 0x5050564C, u, l, b, block, tile + ) + start = (row * g["tokens"] + block * g["block_tokens"]) * g["record"] + size = ( + min(g["block_tokens"], end - block * g["block_tokens"]) * g["record"] + ) + ranged( + pool, write_base, 0x165667B1, start, size, u, l, b, 0x4B565244, tile + ) + else: + for token in range(g["tokens"]): + ranged( + pool, + write_base, + 0x165667B1, + (row * g["tokens"] + token) * g["record"], + g["record"], + u, + l, + b, + 0x50575254, + ) + for tile, end in enumerate(g["tile_ends"], 1): + ranged( + pool, + write_base, + 0x165667B1, + row * g["tokens"] * g["record"], + end * g["record"], + u, + l, + b, + 0x4B565244, + tile, + ) + else: + block_range = range(g["blocks"]) if g["paged"] else range(1) + for block in block_range: + logical = b * g["blocks"] + block if g["paged"] else None + physical = g["table"][logical] if g["paged"] else None + start = 0 if g["paged"] else row * g["tokens"] * g["record"] + length = ( + min(g["block_tokens"], g["tokens"] - block * g["block_tokens"]) + if g["paged"] + else g["tokens"] + ) * g["record"] + last = not g["paged"] or block == g["blocks"] - 1 + index_base = logical * ((g["block_bytes"] + 3) // 4) if g["paged"] else 0 + initial = g["buffer_seeds"][pool] & U32 + step = 0x9E3779B9 + if g["paged"]: + initial += 0xA24BAED5 * (l + 1) + 0x9FB21C65 * (physical + 1) + step = 0xC13FA9A9 + if last: + table_visit(pool, 0x50414C55, u, l, b, block) + table_visit(pool, 0x504B4C55 if pool == 1 else 0x50564C55, u, l, b, block) + ranged( + pool, + initial, + step, + start, + length, + u, + l, + b, + 0x4B565244, + logical=logical, + physical=physical, + index_base=index_base, + ) + if last: + append = start + length - g["record"] + ranged( + pool, + write_base, + 0x165667B1, + append, + g["record"], + u, + l, + b, + 0x41505044, + logical=logical, + physical=physical, + index_base=index_base, + ) + # Replace just the appended bytes in the scan content sum. + old = sum(affine_slice(initial, step, append, g["record"], 32)) + new = sum(affine_slice(write_base, 0x165667B1, append, g["record"], 32)) + mix(pool, new - old, 0, 0) + return None, { + name: {"a_uint32_decimal": str(v[0]), "b_uint32_decimal": str(v[1])} + for name, v in zip(("weight", "k", "v"), states) + } diff --git a/script-examples/llm_verify_profiles.py b/script-examples/llm_verify_profiles.py new file mode 100644 index 0000000..15dd0a3 --- /dev/null +++ b/script-examples/llm_verify_profiles.py @@ -0,0 +1,124 @@ +"""Frozen schema-2 component selectors; changes require explicit verifier support.""" + +PROFILES = { + "cpu-decode-contiguous": { + "logical_profile_version": "decode_steady_fixed_context", + "kv_layout_version": "contiguous_layer_batch_token_head_dimension", + "permutation_version": None, + "backend_executor_version": "llm-cpu-executor-v1-arm64-decode-contiguous", + "resource_abi_version": "llm-memory-descriptor-abi-v1", + "schedule_version": "worker-local-layer-order-no-per-layer-global-barrier", + "timer_policy_version": "synchronized-start-to-last-worker-completion-per-scenario-task", + "buffer_pattern_version": "llm-buffer-pattern-v1", + "write_pattern_version": "llm-kv-append-affine64-v1", + "checksum_pattern_version": "llm-read-checksum-v1", + "msl_revision": None, + "msl_source_sha256": None, + "run_policy_version": "llm-run-policy-bounded-loop-snapshots-v1", + }, + "metal-decode-contiguous": { + "logical_profile_version": "decode_steady_fixed_context", + "kv_layout_version": "contiguous_layer_batch_token_head_dimension", + "permutation_version": None, + "backend_executor_version": "llm-metal-executor-v1-decode-contiguous", + "resource_abi_version": "llm-metal-argument-buffer-tier2-v1", + "schedule_version": "llm-metal-decode-contiguous-grid-stride-v1", + "timer_policy_version": "metal-command-buffer-gpu-start-end-v1", + "buffer_pattern_version": "llm-metal-contiguous-affine32-v1", + "write_pattern_version": "llm-metal-decode-append-affine32-v1", + "checksum_pattern_version": "llm-metal-dual-mod32-v1", + "msl_revision": "llm-metal-decode-prefill-contiguous-paged-msl23-v5", + "msl_source_sha256": "2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2", + "run_policy_version": "llm-run-policy-bounded-loop-snapshots-v1", + }, + "metal-decode-paged": { + "logical_profile_version": "decode_steady_fixed_context", + "kv_layout_version": "paged-uint32-block-table-full-blocks-v1", + "permutation_version": "splitmix64-fisher-yates-rejection-v1", + "backend_executor_version": "llm-metal-executor-v1-decode-paged", + "resource_abi_version": "llm-metal-argument-buffer-tier2-v1", + "schedule_version": "llm-metal-decode-paged-block-owner-grid-stride-v1", + "timer_policy_version": "metal-command-buffer-gpu-start-end-v1", + "buffer_pattern_version": "llm-paged-physical-buffer-pattern-v1", + "write_pattern_version": "llm-metal-decode-paged-append-affine32-v1", + "checksum_pattern_version": "llm-metal-paged-dual-mod32-lookup-mix-v1", + "msl_revision": "llm-metal-decode-prefill-contiguous-paged-msl23-v5", + "msl_source_sha256": "2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2", + "run_policy_version": "llm-run-policy-bounded-loop-snapshots-v1", + }, + "cpu-prefill-paged": { + "logical_profile_version": "full_prompt_causal_prefix_tiled", + "kv_layout_version": "paged-uint32-block-table-full-blocks-v1", + "permutation_version": "splitmix64-fisher-yates-rejection-v1", + "backend_executor_version": "llm-cpu-executor-v1-arm64-prefill-paged", + "resource_abi_version": "llm-memory-prefill-paged-descriptor-abi-v1", + "schedule_version": "llm-prefill-owner-local-write-then-tile-k-v-v1", + "timer_policy_version": "synchronized-start-to-last-worker-completion-per-scenario-task", + "buffer_pattern_version": "llm-paged-physical-buffer-pattern-v1", + "write_pattern_version": "llm-prefill-kv-affine64-v1", + "checksum_pattern_version": "llm-prefill-paged-affine64-lookup-v1", + "msl_revision": None, + "msl_source_sha256": None, + "run_policy_version": "llm-run-policy-bounded-loop-snapshots-v1", + }, + "cpu-prefill-contiguous": { + "logical_profile_version": "full_prompt_causal_prefix_tiled", + "kv_layout_version": "contiguous_layer_batch_token_head_dimension", + "permutation_version": None, + "backend_executor_version": "llm-cpu-executor-v1-arm64-prefill-contiguous", + "resource_abi_version": "llm-memory-prefill-contiguous-descriptor-abi-v1", + "schedule_version": "llm-prefill-owner-local-write-then-tile-k-v-v1", + "timer_policy_version": "synchronized-start-to-last-worker-completion-per-scenario-task", + "buffer_pattern_version": "llm-buffer-pattern-v1", + "write_pattern_version": "llm-prefill-kv-affine64-v1", + "checksum_pattern_version": "llm-prefill-affine64-parity-sum-v1", + "msl_revision": None, + "msl_source_sha256": None, + "run_policy_version": "llm-run-policy-bounded-loop-snapshots-v1", + }, + "metal-prefill-paged": { + "logical_profile_version": "full_prompt_causal_prefix_tiled", + "kv_layout_version": "paged-uint32-block-table-full-blocks-v1", + "permutation_version": "splitmix64-fisher-yates-rejection-v1", + "backend_executor_version": "llm-metal-executor-v1-prefill-paged", + "resource_abi_version": "llm-metal-argument-buffer-tier2-v1", + "schedule_version": "llm-metal-prefill-paged-cyclic-block-owner-grid-stride-v1", + "timer_policy_version": "metal-command-buffer-gpu-start-end-v1", + "buffer_pattern_version": "llm-paged-physical-buffer-pattern-v1", + "write_pattern_version": "llm-metal-prefill-paged-full-prompt-affine32-v1", + "checksum_pattern_version": "llm-metal-paged-prefill-dual-mod32-lookup-address-mix-v1", + "msl_revision": "llm-metal-decode-prefill-contiguous-paged-msl23-v5", + "msl_source_sha256": "2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2", + "run_policy_version": "llm-run-policy-bounded-loop-snapshots-v1", + }, + "metal-prefill-contiguous": { + "logical_profile_version": "full_prompt_causal_prefix_tiled", + "kv_layout_version": "contiguous_layer_batch_token_head_dimension", + "permutation_version": None, + "backend_executor_version": "llm-metal-executor-v1-prefill-contiguous", + "resource_abi_version": "llm-metal-argument-buffer-tier2-v1", + "schedule_version": "llm-metal-prefill-contiguous-lane-local-vector-stripe-v1", + "timer_policy_version": "metal-command-buffer-gpu-start-end-v1", + "buffer_pattern_version": "llm-metal-contiguous-affine32-v1", + "write_pattern_version": "llm-metal-prefill-contiguous-full-prompt-affine32-v1", + "checksum_pattern_version": "llm-metal-dual-mod32-v1", + "msl_revision": "llm-metal-decode-prefill-contiguous-paged-msl23-v5", + "msl_source_sha256": "2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2", + "run_policy_version": "llm-run-policy-bounded-loop-snapshots-v1", + }, + "cpu-decode-paged": { + "logical_profile_version": "decode_steady_fixed_context", + "kv_layout_version": "paged-uint32-block-table-full-blocks-v1", + "permutation_version": "splitmix64-fisher-yates-rejection-v1", + "backend_executor_version": "llm-cpu-executor-v1-arm64-decode-paged", + "resource_abi_version": "llm-memory-paged-descriptor-abi-v1", + "schedule_version": "decode-kv-accounted-prefix-balanced-rotating-v1", + "timer_policy_version": "synchronized-start-to-last-worker-completion-per-scenario-task", + "buffer_pattern_version": "llm-paged-physical-buffer-pattern-v1", + "write_pattern_version": "llm-kv-append-affine64-v1", + "checksum_pattern_version": "llm-paged-read-checksum-v1", + "msl_revision": None, + "msl_source_sha256": None, + "run_policy_version": "llm-run-policy-bounded-loop-snapshots-v1", + }, +} diff --git a/script-examples/verify_llm_result.py b/script-examples/verify_llm_result.py new file mode 100644 index 0000000..f906b4b --- /dev/null +++ b/script-examples/verify_llm_result.py @@ -0,0 +1,1524 @@ +#!/usr/bin/env python3 +"""Verify one LLM schema-2 artifact independently of the C++ producer. + +Usage: python3 script-examples/verify_llm_result.py result.json [--binary memory_benchmark] +Exit 0: consistent accepted artifact; 1: inconsistent or unaccepted; 2: unsupported. +See documents/API.md for evidence limits. No benchmark or executable is invoked. +""" + +import argparse +import hashlib +import json +import math +from pathlib import Path +import re +import statistics +import sys + +from llm_verify_oracles import U64, U32, SEED_DOMAINS, splitmix, permutation, partition, cpu_checksum +from llm_verify_profiles import PROFILES + +SCENARIOS = ("weights_only", "kv_only", "mixed") +METRICS = ( + "synthetic_work_unit_latency_seconds", + "synthetic_memory_work_units_per_second", + "effective_model_payload_gb_s", +) +MAX_INPUT = 64 * 1024 * 1024 +MAX_WORK = 1000000 + + +class Rejected(Exception): + def __init__(self, reason, unsupported=False): + self.reason, self.unsupported = reason, unsupported + + +def require(condition, reason): + if not condition: + raise Rejected(reason) + + +def equal(actual, expected, reason): + require(type(actual) is type(expected), reason) + if isinstance(expected, dict): + require(actual.keys() == expected.keys(), reason) + for key in expected: + equal(actual[key], expected[key], reason) + elif isinstance(expected, (list, tuple)): + require(len(actual) == len(expected), reason) + for a, b in zip(actual, expected): + equal(a, b, reason) + else: + require(actual == expected, reason) + + +def integer(value, decimal=False, maximum=U64): + if decimal: + require( + isinstance(value, str) and len(value) <= 20 and re.fullmatch(r"0|[1-9][0-9]*", value), + "integer-encoding", + ) + number = int(value) + else: + require(type(value) is int, "integer-encoding") + number = value + require(0 <= number <= maximum, "integer-range") + return number + + +def close(actual, expected, reason="rate-mismatch"): + require( + type(actual) in (int, float) + and math.isfinite(actual) + and math.isclose(actual, expected, rel_tol=1e-10, abs_tol=1e-15), + reason, + ) + + +class Budget: + def __init__(self): + self.remaining = MAX_WORK + + def __call__(self, amount): + self.remaining -= amount + if self.remaining < 0: + raise Rejected("unsupported-work-limit", True) + + +def identity_fields(text): + """Parse both documented length-prefix forms; retain repeated field names.""" + require(isinstance(text, str) and len(text) <= 1024 * 1024, "identity-encoding") + parts = [] + pos = text.find("|") + require(pos > 0, "identity-encoding") + prefix = text[:pos] + while pos < len(text): + match = re.match(r"\|([a-zA-Z0-9_]+)=", text[pos:]) + require(match is not None, "identity-encoding") + key = match[1] + pos += len(match[0]) + length = re.match(r"(0|[1-9][0-9]*):", text[pos:]) + if length: + size = int(length[1]) + pos += len(length[0]) + value = text[pos : pos + size] + pos += size + require(len(value) == size, "identity-length") + elif parts and parts[-1][0] == key + "_size": + size = integer(parts[-1][1], True) + value = text[pos : pos + size] + pos += size + require(len(value) == size, "identity-length") + else: + end = text.find("|", pos) + if end < 0: + end = len(text) + value = text[pos:end] + pos = end + parts.append((key, value)) + return prefix, parts + + +def fields(text): + prefix, pairs = identity_fields(text) + require(len(dict(pairs)) == len(pairs), "identity-duplicate-field") + return prefix, dict(pairs) + + +def pipe(prefix, pairs): + return prefix + "".join("|" + k + "=" + str(v) for k, v in pairs) + + +def check_geometry(doc, budget): + r = doc["resolved_plan"] + c = doc["configuration"] + g = r["geometry"] + layout = r["layout"] + backend, phase, kind = (doc[k] for k in ("backend", "phase", "kv_layout")) + profile = "-".join((backend, phase, kind)) + if profile not in PROFILES: + raise Rejected("unsupported-profile", True) + if doc["methodology_version"] != "llm-memory-v2-" + profile: + raise Rejected("unsupported-methodology", True) + components = r["component_identities"] + profile_components = dict(PROFILES[profile]) + if backend == "metal" and not r["resources"]["metal"]["valid"]: + require(not any(row["attempted"] for row in doc["measurements"]), "unresolved-execution") + profile_components.update(msl_revision=None, msl_source_sha256=None) + for key, value in profile_components.items(): + equal(components[key], value, "component-mismatch") + component_identity = "llm-memory-components-v1" + "".join( + "|" + k + "=" + ("null" if v is None else str(len(v)) + ":" + v) + for k, v in profile_components.items() + if k != "run_policy_version" + ) + equal(components["identity"], component_identity, "component-identity-mismatch") + for obj in (r, c, r["model_work_plan"]): + for key in ("backend", "phase", "kv_layout"): + equal(obj[key], doc[key], "profile-mismatch") + require(r["valid"] is True and g["valid"] is True, "model-invalid") + ints = { + k: integer(g[k], k == "kv_element_bytes", (1 << 53) - 1) + for k in ( + "layer_count", + "batch_size", + "query_head_count", + "kv_head_count", + "head_dimension", + "kv_element_bytes", + ) + } + for k, v in ints.items(): + require(v > 0, "geometry-range") + equal(c[k], g[k], "configuration-geometry-mismatch") + l, b, h, d, e = ( + ints[k] for k in ("layer_count", "batch_size", "kv_head_count", "head_dimension", "kv_element_bytes") + ) + require(e in (1, 2, 4) and ints["query_head_count"] % h == 0, "geometry-range") + prefill = phase == "prefill" + paged = kind == "paged" + n = integer( + g["prefill"]["prompt_tokens"] if prefill else g["decode"]["visible_context_tokens"], + maximum=(1 << 53) - 1, + ) + require(n > 0, "geometry-range") + equal(c["prompt_tokens" if prefill else "visible_context_tokens"], n, "configuration-geometry-mismatch") + q = integer(g["prefill"]["attention_query_tile_tokens"]) if prefill else n + require(0 < q <= n, "geometry-range") + tiles = (n + q - 1) // q + budget(tiles + l * b) + ends = [min(n, (i + 1) * q) for i in range(tiles)] if prefill else [n] + if prefill: + equal(c["attention_query_tile_tokens"], q, "configuration-geometry-mismatch") + equal(g["prefill"]["tile_count"], str(tiles), "geometry-mismatch") + equal(g["prefill"]["attention_prefix_token_visits_per_sequence"], str(sum(ends)), "geometry-mismatch") + block = integer(layout["kv_block_tokens"], maximum=U32) if paged else 0 + if paged: + require(block > 0 and block & (block - 1) == 0, "geometry-range") + blocks = (n + block - 1) // block if paged else 0 + record = h * d * e + weight = integer(g["active_weight_bytes_per_work_unit"], True) + require(weight > 0, "geometry-range") + equal(weight, integer(c["weight_size_mb"]) * 1048576, "configuration-geometry-mismatch") + logical = l * b * n * record + physical = l * b * blocks * block * record if paged else logical + read = 2 * l * b * record * sum(ends) + write = 2 * l * b * record * (n if prefill else 1) + derived = { + "kv_vector_bytes": d * e, + "k_or_v_record_bytes_per_layer": record, + "kv_record_bytes_per_layer": 2 * record, + "kv_bytes_per_visible_token": 2 * l * record, + "k_or_v_sequence_visible_bytes": n * record, + "k_mapping_bytes": physical, + "v_mapping_bytes": physical, + "kv_capacity_bytes": 2 * physical, + "weight_read_bytes_per_work_unit": weight, + "kv_read_bytes_per_work_unit": read, + "kv_write_bytes_per_work_unit": write, + "kv_only_effective_model_payload_bytes_per_work_unit": read + write, + "mixed_effective_model_payload_bytes_per_work_unit": weight + read + write, + "total_data_mapping_bytes": weight + 2 * physical, + } + for key, value in derived.items(): + equal(g[key], str(value), "geometry-mismatch") + equal(g["query_heads_per_kv_head"], ints["query_head_count"] // h, "geometry-mismatch") + equal( + g["attention_kind"], + ( + "mqa" + if h == 1 and ints["query_head_count"] > 1 + else "mha" if h == ints["query_head_count"] else "gqa" + ), + "geometry-mismatch", + ) + resource = r["resources"] + for key, value in { + "weight_logical_bytes": weight, + "k_logical_bytes": logical, + "v_logical_bytes": logical, + "k_physical_length_bytes": physical, + "v_physical_length_bytes": physical, + "k_layout_padding_bytes": physical - logical, + "v_layout_padding_bytes": physical - logical, + }.items(): + equal(resource[key], str(value), "resource-bytes-mismatch") + model = dict( + backend=backend, + prefill=prefill, + paged=paged, + layers=l, + batch=b, + record=record, + tokens=n, + q=q, + tile_ends=ends, + blocks=blocks, + block_tokens=block, + block_bytes=block * record, + weight=weight, + workers=integer(r["model_work_plan"]["effective_workers"]) if backend == "cpu" else 0, + ) + budget(model["workers"] * l * b) + if backend == "cpu": + mw = r["model_work_plan"] + requested = integer(c["requested_workers"]) + available = integer(c["available_workers"]) + max_kv = ( + blocks + if paged and prefill + else min(l * b * blocks, l * b + blocks - 1) if paged else n if prefill else n * record + ) + equal( + model["workers"], min(requested, available, (weight + l - 1) // l, max_kv), "worker-plan-mismatch" + ) + sequence_count = l * b * (3 if prefill else 1) + for key, value in dict( + layer_descriptors_per_worker=l, + sequence_descriptors_per_worker=sequence_count, + total_layer_descriptors=l * model["workers"], + total_sequence_descriptors=sequence_count * model["workers"], + worker_plan_count=model["workers"], + ).items(): + equal(mw[key], value, "descriptor-count-mismatch") + descriptor_bytes = model["workers"] * ( + l * 48 + sequence_count * (112 if paged and prefill else 96 if paged else 80) + ) + equal(mw["descriptor_bytes"], str(descriptor_bytes), "descriptor-bytes-mismatch") + seed = integer(doc["seeds"]["base_seed_uint64_decimal"], True) + equal(c["base_seed_uint64_decimal"], str(seed), "seed-mismatch") + seeds = [splitmix(seed ^ domain) for domain in SEED_DOMAINS] + for key, value in zip(("weight_uint64_decimal", "k_uint64_decimal", "v_uint64_decimal"), seeds): + equal(doc["seeds"]["buffer_domain_seeds"][key], str(value), "seed-mismatch") + for key, value in zip(SCENARIOS, seeds[3:]): + equal(doc["seeds"]["scenario_domain_seeds"][key], str(value), "seed-mismatch") + model.update(buffer_seeds=seeds[:3], scenario_seeds=dict(zip(SCENARIOS, seeds[3:])), base_seed=seed) + if paged: + for key, value in { + "blocks_per_sequence": blocks, + "physical_blocks_per_layer": b * blocks, + "total_physical_blocks": l * b * blocks, + "block_bytes": block * record, + "last_block_tokens": n - (blocks - 1) * block, + "last_block_valid_bytes": (n - (blocks - 1) * block) * record, + "block_table_entries": b * blocks, + "block_table_bytes": 4 * b * blocks, + }.items(): + equal(layout[key], str(value), "layout-mismatch") + table = permutation(seed, b * blocks, budget) + digest = hashlib.sha256(b"".join(v.to_bytes(4, "little") for v in table)).hexdigest() + equal( + layout["permutation_seed_uint64_decimal"], + str(splitmix(seed ^ 0x4C4C4D4B56504731)), + "seed-mismatch", + ) + equal(layout["permutation_sha256"], digest, "permutation-digest-mismatch") + equal(layout["permutation_entry_count"], str(len(table)), "layout-mismatch") + model["table"] = table + model["read"], model["write"] = read, write + return model + + +def ownership(model, doc, budget): + """Independently balance exact prefix costs with rational targets, ties down.""" + m = model + workers = m["workers"] + result = {s: [] for s in SCENARIOS} + if not workers: + return result + if not m["paged"] and not m["prefill"]: + return result + n = m["blocks"] if m["paged"] else m["tokens"] + active = min(n, workers) + budget(n * len(m["tile_ends"]) + workers * m["layers"] * m["batch"]) + payload = [0] + lookups = [0] + for unit in range(n): + start = unit * m["block_tokens"] if m["paged"] else unit + end = min(m["tokens"], start + (m["block_tokens"] if m["paged"] else 1)) + if m["prefill"]: + visits = sum(max(0, min(t, end) - start) for t in m["tile_ends"]) + payload.append(payload[-1] + 2 * m["record"] * (end - start + visits)) + lookups.append( + lookups[-1] + (1 + 2 * sum(t > start for t in m["tile_ends"]) if m["paged"] else 0) + ) + else: + payload.append(payload[-1] + 2 * m["record"] * (end - start + (unit == n - 1))) + lookups.append(lookups[-1] + 2 + (unit == n - 1)) + costs = [p + 4 * k for p, k in zip(payload, lookups)] + offset = 0 + for layer in range(m["layers"]): + size = m["weight"] // m["layers"] + (layer < m["weight"] % m["layers"]) + shards = [x[1] for x in partition(offset, size, workers)] + offset += size + for b in range(m["batch"]): + row = layer * m["batch"] + b + rotation = row % workers + for scenario in SCENARIOS: + weights = shards if m["prefill"] and scenario == "mixed" and b == 0 else [0] * workers + total = costs[-1] + sum(weights[(rank + rotation) % workers] for rank in range(active)) + boundaries = [0] + preceding = 0 + for rank in range(1, active): + preceding += weights[(rank - 1 + rotation) % workers] + target = max(0, rank * total - preceding * active) + lo = boundaries[-1] + 1 + hi = n - (active - rank) + # Monotone search avoids work proportional to virtual memory sizes. + left, right = lo, hi + while left < right: + mid = (left + right) // 2 + if costs[mid] * active >= target: + right = mid + else: + left = mid + 1 + candidates = (max(lo, left - 1), left) + boundaries.append(min(candidates, key=lambda k: (abs(costs[k] * active - target), k))) + boundaries.append(n) + ranges = [(0, 0)] * workers + for rank, (first, last) in enumerate(zip(boundaries, boundaries[1:])): + ranges[(rank + rotation) % workers] = (first, last - first) + result[scenario].append(ranges) + return result + + +def check_identities(doc, m, budget): + r = doc["resolved_plan"] + g = r["geometry"] + layout = r["layout"] + cpu = doc["backend_evidence"]["cpu"] + prefix, values = fields(r["plan_identity"]) + equal(prefix, "llm-memory-work-plan-v1", "model-identity-mismatch") + aliases = { + "weight": "active_weight_bytes_per_work_unit", + "layers": "layer_count", + "query_heads": "query_head_count", + "kv_heads": "kv_head_count", + "head_dim": "head_dimension", + "batch": "batch_size", + "kv_only_payload_bytes_per_work_unit": "kv_only_effective_model_payload_bytes_per_work_unit", + "mixed_payload_bytes_per_work_unit": "mixed_effective_model_payload_bytes_per_work_unit", + } + expected = {k: v for k, v in g.items() if type(v) in (str, int) and k not in ("reason_code",)} + expected.update({k: g[v] for k, v in aliases.items()}) + expected.update( + backend=m["backend"], + phase=doc["phase"], + kv_layout=doc["kv_layout"], + methodology=doc["methodology_version"], + component_identity=r["component_identities"]["identity"], + component_identity_size=len(r["component_identities"]["identity"]), + range_alignment=32, + weight_passes_per_work_unit=1, + kv_replay_factor=1, + base_seed=m["base_seed"], + weight_buffer_seed=m["buffer_seeds"][0], + k_buffer_seed=m["buffer_seeds"][1], + v_buffer_seed=m["buffer_seeds"][2], + ) + expected.update({s + "_scenario_seed": v for s, v in m["scenario_seeds"].items()}) + expected.update( + k_logical_bytes=r["resources"]["k_logical_bytes"], + v_logical_bytes=r["resources"]["v_logical_bytes"], + k_layout_padding_bytes=r["resources"]["k_layout_padding_bytes"], + v_layout_padding_bytes=r["resources"]["v_layout_padding_bytes"], + ) + layout_alias = {"kv_blocks_per_sequence": "blocks_per_sequence", "kv_block_bytes": "block_bytes"} + for key in ( + "kv_block_tokens", + "kv_blocks_per_sequence", + "physical_blocks_per_layer", + "total_physical_blocks", + "kv_block_bytes", + "last_block_tokens", + "last_block_valid_bytes", + "decode_append_offset_in_last_block", + "block_table_entries", + "block_table_bytes", + ): + expected[key] = layout[layout_alias.get(key, key)] or 0 + expected["layout_metadata_lookups_per_layer_sequence_per_work_unit"] = ( + ( + m["blocks"] + 2 * sum((t + m["block_tokens"] - 1) // m["block_tokens"] for t in m["tile_ends"]) + if m["prefill"] + else 1 + 2 * m["blocks"] + ) + if m["paged"] + else 0 + ) + expected["traffic_crossover_numerator"] = 0 if m["prefill"] else m["weight"] + expected["traffic_crossover_denominator"] = ( + 0 if m["prefill"] else 2 * m["layers"] * m["batch"] * m["record"] + ) + if m["prefill"]: + p = m["tokens"] + pairs = p * (p + 1) // 2 + attention = pairs * m["layers"] * m["batch"] * g["query_head_count"] + context = r["model_context"]["prefill"] + for key, value in { + "causal_token_pairs_per_sequence": pairs, + "logical_attention_pairs": attention, + "logical_attention_fma_terms": attention * g["head_dimension"], + }.items(): + equal(context[key], str(value) if value <= U64 else None, "model-context-mismatch") + equal( + context[key + "_reason_code"], + "valid" if value <= U64 else "arithmetic-overflow", + "model-context-mismatch", + ) + expected.update( + prefill_planner_version="llm-prefill-planner-v1", + prefill_cpu_partition_version="llm-prefill-cpu-accounted-prefix-balanced-v1", + prefill_prompt_tokens=p, + prefill_query_tile_tokens=m["q"], + prefill_tile_count=len(m["tile_ends"]), + prefill_prefix_token_visits_per_sequence=sum(m["tile_ends"]), + prefill_causal_token_pairs=pairs, + prefill_logical_attention_pairs=attention, + prefill_logical_attention_fma_terms=attention * g["head_dimension"], + prefill_prefix_block_visits_per_sequence=( + sum((t + m["block_tokens"] - 1) // m["block_tokens"] for t in m["tile_ends"]) + if m["paged"] + else 0 + ), + ) + else: + expected["decode_context"] = m["tokens"] + if m["backend"] == "cpu": + for key in ( + "requested_workers", + "effective_workers", + "layer_descriptors_per_worker", + "sequence_descriptors_per_worker", + "total_layer_descriptors", + "total_sequence_descriptors", + "descriptor_bytes", + ): + expected[key] = r["model_work_plan"][key] + if m["prefill"]: + expected["prefill_execution_identity"] = cpu["prefill"]["identity"] + expected["prefill_execution_identity_size"] = len(cpu["prefill"]["identity"]) + if m["paged"]: + expected.update( + paged_layout_identity=cpu["paged"]["layout_identity"], + paged_layout_identity_size=len(cpu["paged"]["layout_identity"]), + paged_execution_identity=cpu["paged"]["execution_identity"], + paged_execution_identity_size=len(cpu["paged"]["execution_identity"]), + ) + equal(cpu["paged"]["layout_identity"], layout["layout_identity"], "layout-identity-mismatch") + else: + identity = r["resources"]["metal"]["execution_identity"] + resolved = r["resources"]["metal"]["valid"] + expected.update( + metal_execution_resolved=int(resolved), + metal_execution_identity_size=len(identity) if resolved else 0, + metal_execution_identity_sha256=( + hashlib.sha256(identity.encode()).hexdigest() if resolved else "0" * 64 + ), + ) + for key, value in values.items(): + require(key in expected, "unknown-model-identity-field") + actual = expected[key] + if type(actual) is int and actual > U64: + actual = "unavailable:arithmetic-overflow" + equal(value, str(actual), "model-identity-mismatch") + # Mandatory scalar fields must not disappear even when every downstream ref is changed. + required = { + "backend", + "phase", + "kv_layout", + "work_unit_kind", + "methodology", + "component_identity", + "component_identity_size", + "weight", + "layers", + "query_heads", + "kv_heads", + "head_dim", + "kv_element_bytes", + "batch", + "base_seed", + "weight_buffer_seed", + "k_buffer_seed", + "v_buffer_seed", + "weights_only_scenario_seed", + "kv_only_scenario_seed", + "mixed_scenario_seed", + } + required.update( + "range_alignment weight_passes_per_work_unit kv_replay_factor query_heads_per_kv_head attention_kind kv_vector_bytes k_or_v_record_bytes_per_layer kv_record_bytes_per_layer kv_bytes_per_visible_token k_or_v_sequence_visible_bytes kv_block_tokens kv_blocks_per_sequence physical_blocks_per_layer total_physical_blocks kv_block_bytes last_block_tokens last_block_valid_bytes decode_append_offset_in_last_block k_logical_bytes v_logical_bytes k_layout_padding_bytes v_layout_padding_bytes block_table_entries block_table_bytes layout_metadata_lookups_per_layer_sequence_per_work_unit k_mapping_bytes v_mapping_bytes kv_capacity_bytes weight_read_bytes_per_work_unit kv_read_bytes_per_work_unit kv_write_bytes_per_work_unit kv_only_payload_bytes_per_work_unit mixed_payload_bytes_per_work_unit total_data_mapping_bytes traffic_crossover_numerator traffic_crossover_denominator".split() + ) + if m["prefill"]: + required.update( + "prefill_planner_version prefill_prompt_tokens prefill_query_tile_tokens prefill_tile_count prefill_prefix_token_visits_per_sequence prefill_causal_token_pairs prefill_logical_attention_pairs prefill_logical_attention_fma_terms prefill_prefix_block_visits_per_sequence".split() + ) + else: + required.add("decode_context") + if m["backend"] == "cpu": + required.update( + "requested_workers effective_workers layer_descriptors_per_worker sequence_descriptors_per_worker total_layer_descriptors total_sequence_descriptors descriptor_bytes".split() + ) + if m["prefill"]: + required.update( + "prefill_cpu_partition_version prefill_execution_identity_size prefill_execution_identity".split() + ) + if m["paged"]: + required.update( + "paged_layout_identity_size paged_layout_identity paged_execution_identity_size paged_execution_identity".split() + ) + else: + required.update( + "metal_execution_resolved metal_execution_identity_size metal_execution_identity_sha256".split() + ) + require(required == values.keys(), "model-identity-field-set") + if m["paged"]: + lp, lv = fields(layout["layout_geometry_identity"]) + equal(lp, "llm-kv-layout-geometry-v1", "layout-identity-mismatch") + mapping = dict( + expected, + sequence_tokens=m["tokens"], + layer_count=m["layers"], + batch_size=m["batch"], + blocks_per_sequence=m["blocks"], + block_bytes=m["block_bytes"], + k_physical_bytes=g["k_mapping_bytes"], + v_physical_bytes=g["v_mapping_bytes"], + kv_layout_version=r["component_identities"]["kv_layout_version"], + ) + entries = m["blocks"] * m["batch"] + table_bytes = 4 * entries + bits = (entries + 7) // 8 + resident = int(g["kv_capacity_bytes"]) + table_bytes + mapping.update( + validation_bitset_bytes=bits, + transient_peak_bytes=table_bytes + bits, + resident_layout_bytes=resident, + known_owned_peak_bytes=resident + bits, + permutation_iterations=max(0, entries - 1), + validation_entries=entries, + hash_entries=entries, + upload_bytes=table_bytes, + ) + for key, value in lv.items(): + require(key in mapping, "unknown-layout-identity-field") + equal(value, str(mapping[key]), "layout-identity-mismatch") + _, lv = fields(layout["layout_identity"]) + mapping = dict( + geometry_identity=layout["layout_geometry_identity"], + permutation_algorithm_version="splitmix64-fisher-yates-rejection-v1", + permutation_domain=0x4C4C4D4B56504731, + permutation_domain_uint64_hex="0x4c4c4d4b56504731", + permutation_seed=splitmix(m["base_seed"] ^ 0x4C4C4D4B56504731), + permutation_entry_count=entries, + permutation_sha256=layout["permutation_sha256"], + ) + equal(lv, {k: str(v) for k, v in mapping.items()}, "layout-identity-mismatch") + _, pv = fields(layout["permutation_identity"]) + equal( + pv, + dict( + domain=str(0x4C4C4D4B56504731), + domain_uint64_hex="0x4c4c4d4b56504731", + resolved_seed=str(mapping["permutation_seed"]), + entry_count=str(entries), + sha256=layout["permutation_sha256"], + ), + "permutation-identity-mismatch", + ) + # Validate nested identity hashes against reconstructed/published canonical content. + strings = set() + stack = [r, doc["backend_evidence"]] + while stack: + value = stack.pop() + if isinstance(value, dict): + stack.extend(value.values()) + elif isinstance(value, list): + stack.extend(value) + elif isinstance(value, str) and "|" in value: + strings.add(value) + by_hash = {hashlib.sha256(s.encode()).hexdigest(): s for s in strings} + for text in strings: + budget(1) + _, items = identity_fields(text) + for key, value in items: + if key.endswith("identity_sha256"): + if ( + key == "metal_execution_identity_sha256" + and value == "0" * 64 + and not r["resources"]["metal"]["valid"] + ): + continue + require(value in by_hash, "identity-digest-mismatch") + if m["backend"] == "cpu" and m["prefill"]: + for sc in cpu["prefill"]["scenarios"]: + scenario = sc["scenario"] + for row, text in enumerate(sc["scope_identities"]): + _, scope = fields(text) + for key, value in { + "scenario": scenario, + "worker_count": m["workers"], + "prompt_tokens": m["tokens"], + "query_tile_tokens": m["q"], + "layer_count": m["layers"], + "batch_size": m["batch"], + "k_or_v_record_bytes_per_layer": m["record"], + }.items(): + equal(scope[key], str(value), "ownership-identity-mismatch") + if scenario == "weights_only": + continue + observed = [(0, 0)] * m["workers"] + count = integer(scope["assignment_count"], True) + require(count <= m["workers"], "ownership-assignment-count") + for i in range(count): + name = "assignment_" + str(i) + "_" + w = integer(scope[name + "worker_index"], True) + require(w < m["workers"], "ownership-worker") + observed[w] = ( + integer(scope[name + "first_unit"], True), + integer(scope[name + "unit_count"], True), + ) + equal(observed, m["ownership"][scenario][row], "ownership-assignment-mismatch") + + +def check_prefill_scopes(doc, m, budget): + if m["backend"] != "cpu" or not m["prefill"]: + return + prefill = doc["backend_evidence"]["cpu"]["prefill"] + g = doc["resolved_plan"]["geometry"] + workers = m["workers"] + offsets = [] + offset = 0 + for layer in range(m["layers"]): + size = m["weight"] // m["layers"] + (layer < m["weight"] % m["layers"]) + offsets.append([v[1] for v in partition(offset, size, workers)]) + offset += size + + def cost(first, count): + start = first * m["block_tokens"] if m["paged"] else first + end = min(m["tokens"], (first + count) * m["block_tokens"]) if m["paged"] else first + count + reads = sum(max(0, min(t, end) - start) for t in m["tile_ends"]) + payload = 2 * m["record"] * (end - start + reads) + lookups = ( + count + + 2 + * sum( + max(0, min(first + count, (t + m["block_tokens"] - 1) // m["block_tokens"]) - first) + for t in m["tile_ends"] + ) + if m["paged"] + else 0 + ) + return payload, lookups, payload + 4 * lookups + + execution = [("sequence_descriptors_per_scenario_per_worker", m["layers"] * m["batch"])] + for index, sc in enumerate(prefill["scenarios"]): + scenario = SCENARIOS[index] + equal(sc["scenario"], scenario, "prefill-scenario-order") + scope_count = m["layers"] if index == 0 else m["layers"] * m["batch"] + equal(len(sc["scope_identities"]), scope_count, "scope-count") + totals = [0] * workers + for row, text in enumerate(sc["scope_identities"]): + layer = row if index == 0 else row // m["batch"] + batch = 0 if index == 0 else row % m["batch"] + rotation = row % workers + include = index == 0 or (index == 2 and batch == 0) + weights = offsets[layer] if include else [0] * workers + ranges = [(0, 0)] * workers if index == 0 else m["ownership"][scenario][row] + quantities = [cost(first, count) if count else (0, 0, 0) for first, count in ranges] + accounted = [w + x[2] for w, x in zip(weights, quantities)] + total_payload = sum(x[0] for x in quantities) + total_lookups = sum(x[1] for x in quantities) + active = sum(bool(count) for first, count in ranges) + scope = dict( + planner_version="llm-prefill-planner-v1", + schedule_version="llm-prefill-owner-local-write-then-tile-k-v-v1", + unit_kind="paged_block" if m["paged"] else "contiguous_token", + scenario=scenario, + active_weight_bytes=m["weight"], + prompt_tokens=m["tokens"], + query_tile_tokens=m["q"], + tile_count=len(m["tile_ends"]), + layer_count=m["layers"], + batch_size=m["batch"], + query_head_count=g["query_head_count"], + head_dimension=g["head_dimension"], + k_or_v_record_bytes_per_layer=m["record"], + paged=int(m["paged"]), + kv_block_tokens=m["block_tokens"], + blocks_per_sequence=m["blocks"], + worker_count=workers, + worker_rotation=rotation, + logical_unit_count=0 if index == 0 else m["blocks"] if m["paged"] else m["tokens"], + active_worker_count=active, + weight_shards_included=int(include), + total_weight_shard_bytes=sum(weights), + total_kv_model_payload_bytes=total_payload, + total_layout_metadata_lookup_count=total_lookups, + total_layout_metadata_read_bytes=4 * total_lookups, + total_kv_accounted_bytes=total_payload + 4 * total_lookups, + total_scenario_accounted_bytes=sum(accounted), + minimum_worker_accounted_bytes=min(accounted), + maximum_worker_accounted_bytes=max(accounted), + worker_accounted_imbalance_bytes=max(accounted) - min(accounted), + worker_vector_count=workers, + ) + for w, (weight, (payload, lookups, cost_value)) in enumerate(zip(weights, quantities)): + for k, v in dict( + weight_shard_bytes=weight, + kv_model_payload_bytes=payload, + layout_metadata_lookup_count=lookups, + layout_metadata_read_bytes=4 * lookups, + scenario_accounted_bytes=weight + cost_value, + ).items(): + scope[f"worker_{w}_{k}"] = v + totals[w] += weight + cost_value + scope["assignment_count"] = active + for rank in range(active): + w = (rotation + rank) % workers + first, count = ranges[w] + payload, lookups, cost_value = quantities[w] + for k, v in dict( + range_rank=rank, + worker_index=w, + first_unit=first, + unit_count=count, + kv_model_payload_bytes=payload, + layout_metadata_lookup_count=lookups, + layout_metadata_read_bytes=4 * lookups, + kv_accounted_bytes=cost_value, + ).items(): + scope[f"assignment_{rank}_{k}"] = v + equal( + text, + pipe("llm-prefill-cpu-accounted-prefix-balanced-v1", scope.items()), + "ownership-identity-mismatch", + ) + common = [("scenario", scenario), ("scope_count", scope_count)] + for text in sc["scope_identities"]: + common.extend( + [ + ("scope_identity_size", len(text)), + ("scope_identity_sha256", hashlib.sha256(text.encode()).hexdigest()), + ] + ) + tail = ( + [("worker_count", workers)] + + [("worker_accounted_bytes", v) for v in totals] + + [ + ("minimum_worker_accounted_bytes", min(totals)), + ("maximum_worker_accounted_bytes", max(totals)), + ("worker_accounted_imbalance_bytes", max(totals) - min(totals)), + ] + ) + equal( + sc["identity"], + pipe("llm-prefill-cpu-scenario-execution-v1", common + tail), + "scenario-execution-identity-mismatch", + ) + equal(sc["worker_accounted_bytes_per_work_unit"], list(map(str, totals)), "worker-cost-mismatch") + execution.extend( + [("scenario_index", index)] + + common + + [ + ("scenario_identity_size", len(sc["identity"])), + ("scenario_identity_sha256", hashlib.sha256(sc["identity"].encode()).hexdigest()), + ] + + tail + ) + equal( + prefill["identity"], + pipe("llm-prefill-cpu-execution-v1", execution), + "prefill-execution-identity-mismatch", + ) + + +def check_plans(doc, m, budget): + from llm_verify_oracles import metal_checksum + + r = doc["resolved_plan"] + plans = r["scenario_plans"] + if len(plans) > 1024: + raise Rejected("unsupported-plan-limit", True) + keys = [] + for p in plans: + scenario = p["scenario"] + require(scenario in SCENARIOS, "scenario-mismatch") + work = integer(p["work_units"], maximum=1000000000) + require(work > 0, "work-mismatch") + equal(p["model_ref"], "resolved_plan", "plan-reference") + equal(p["scenario_seed_uint64_decimal"], str(m["scenario_seeds"][scenario]), "seed-mismatch") + equal(p["work_unit_kind"], "prefill_operation" if m["prefill"] else "decode_step", "work-mismatch") + equal( + p["kv_write_kind"], + ( + "none" + if scenario == "weights_only" + else "full_prompt_population" if m["prefill"] else "current_token_append" + ), + "work-mismatch", + ) + require(type(p["explicit_iterations"]) is bool, "work-mismatch") + equal( + p["work_policy"], + "explicit_fixed_work" if p["explicit_iterations"] else "automatic_calibration", + "work-mismatch", + ) + weight = m["weight"] if scenario != "kv_only" else 0 + read = m["read"] if scenario != "weights_only" else 0 + write = m["write"] if scenario != "weights_only" else 0 + lookups = 0 + if m["paged"] and scenario != "weights_only": + lookups = ( + m["layers"] + * m["batch"] + * ( + m["blocks"] + + 2 * sum((t + m["block_tokens"] - 1) // m["block_tokens"] for t in m["tile_ends"]) + if m["prefill"] + else 1 + 2 * m["blocks"] + ) + ) + per = { + "weight_read_bytes": weight, + "kv_read_bytes": read, + "kv_write_bytes": write, + "effective_model_payload_bytes": weight + read + write, + "layout_metadata_lookup_count": lookups, + "layout_metadata_read_bytes": 4 * lookups, + } + for key, value in per.items(): + equal(p[key + "_per_work_unit"], str(value), "plan-bytes-mismatch") + equal(p[key], str(value * work), "plan-bytes-mismatch") + accounted = weight + read + write + 4 * lookups + equal(p["accounted_bytes_per_work_unit"], str(accounted), "plan-bytes-mismatch") + equal(p["task_accounted_bytes"], str(accounted * work), "plan-bytes-mismatch") + guard = 68719476736 // accounted + cap = 65536 if m["backend"] == "metal" else 1000000000 + if m["backend"] == "metal" and m["prefill"] and (not m["paged"] or scenario == "weights_only"): + visits = (m["layers"] if weight else 0) + ( + 2 * m["layers"] * m["batch"] * (m["tokens"] + len(m["tile_ends"])) if read else 0 + ) + cap = min(cap, 1048576 // visits) + if m["backend"] == "metal" and m["paged"] and lookups: + cap = min(cap, 268435456 // lookups) + equal(p["maximum_work_units_by_work_unit_cap"], cap, "work-cap-mismatch") + equal(p["effective_maximum_work_units"], min(cap, guard), "work-cap-mismatch") + equal(p["maximum_work_units_by_guardrail"], guard, "work-cap-mismatch") + # Metal prefill additionally constrains serial visits; published effective cap is checked below. + require(work <= min(guard, p["effective_maximum_work_units"], 1000000000), "work-cap-mismatch") + prefix, pairs = identity_fields(p["plan_identity"]) + equal(prefix, "llm-memory-work-plan-v1", "plan-identity-mismatch") + values = dict(pairs) + equal(len(values), len(pairs), "plan-identity-mismatch") + equal(values["model_plan_identity"], r["plan_identity"], "plan-identity-mismatch") + equal(values["model_plan_identity_size"], str(len(r["plan_identity"])), "plan-identity-mismatch") + mapped = { + k: str(v) + for k, v in p.items() + if type(v) in (int, str) + and k + not in ( + "model_ref", + "plan_identity", + "scenario_seed_uint64_decimal", + "work_policy", + "reason_code", + ) + } + mapped.update( + scenario_seed=p["scenario_seed_uint64_decimal"], + explicit=str(int(p["explicit_iterations"])), + model_plan_identity=r["plan_identity"], + model_plan_identity_size=str(len(r["plan_identity"])), + ) + equal(values, mapped, "plan-identity-mismatch") + expected = p["expected_checksum"] + if expected["status"] == "available": + oracle = cpu_checksum if m["backend"] == "cpu" else metal_checksum + workers, run = oracle(m, scenario, work, budget) + equal(expected["expected_worker_checksums"], workers, "expected-worker-checksum-mismatch") + equal(expected["expected_run_checksum"], run, "expected-run-checksum-mismatch") + else: + equal(expected["status"], "unavailable", "checksum-status") + require( + expected["expected_worker_checksums"] is None and expected["expected_run_checksum"] is None, + "checksum-null", + ) + keys.append((SCENARIOS.index(scenario), work, p["explicit_iterations"])) + equal(keys, sorted(set(keys)), "canonical-plan-order") + for scenario, ref in r["frozen_plan_refs"].items(): + if ref is not None: + integer(ref) + require(ref < len(plans) and plans[ref]["scenario"] == scenario, "plan-reference") + if doc["configuration"]["iterations"] is not None: + equal(plans[ref]["work_units"], doc["configuration"]["iterations"], "frozen-work-mismatch") + return plans + + +def timing(execution, elapsed, backend): + raw = execution.get("timing", {}).get("cpu_raw") + if backend == "metal": + require(raw is None, "timing-backend-mismatch") + metal = execution.get("metal") + if metal and metal["timing"]["valid"] is True: + t = metal["timing"] + start = t["gpu_start_seconds"] + stop = t["gpu_end_seconds"] + require( + type(start) in (float, int) and type(stop) in (float, int) and 0 < start < stop, + "gpu-timing-invalid", + ) + if elapsed is not None: + close(elapsed, stop - start, "elapsed-mismatch") + return "gpu-timestamps" + if raw is None: + return "elapsed-only" + start, stop, delta = (integer(raw[k], True) for k in ("start_ticks", "stop_ticks", "delta_ticks")) + numer, denom = (integer(raw[k], maximum=U32) for k in ("timebase_numer", "timebase_denom")) + equal(delta, (stop - start) & U64, "tick-delta-mismatch") + reconstructed = float(delta) * float(numer) / float(denom) / 1e9 if denom else 0.0 + if elapsed is not None: + close(elapsed, reconstructed, "elapsed-mismatch") + return "cpu-raw-ticks" + + +def check_checksum(observed, expected, backend, compact=False): + status = observed["status"] + if status == "unavailable" or status == "not_evaluated": + require( + observed["checksum_valid"] is None and observed["actual_run_checksum"] is None, "checksum-null" + ) + if not compact: + require(observed["actual_worker_checksums"] is None, "checksum-null") + return False + require(status in ("valid", "invalid"), "checksum-status") + require(expected["status"] == "available", "expected-checksum-unavailable") + matches = observed["actual_run_checksum"] == expected["expected_run_checksum"] + if not compact: + matches = matches and observed["actual_worker_checksums"] == expected["expected_worker_checksums"] + if backend == "cpu": + from llm_verify_oracles import fold + + states = [] + for i, worker in enumerate(observed["actual_worker_checksums"]): + equal(worker["worker_index"], i, "actual-worker-index") + states.append( + [ + [ + integer(worker[p][key], True) + for key in ( + "state_a_uint64_decimal", + "state_b_uint64_decimal", + "exact_bytes_read", + "span_count_uint64_decimal", + ) + ] + for p in ("weight", "k", "v") + ] + ) + equal(observed["actual_run_checksum"], fold(states), "actual-run-fold-mismatch") + equal(observed["checksum_valid"], matches, "checksum-valid-mismatch") + equal(status, "valid" if matches else "invalid", "checksum-status") + return matches + + +def check_measurements(doc, m, plans): + rows = doc["measurements"] + count = integer(doc["configuration"]["loop_count"]) + if len(rows) > 100000: + raise Rejected("unsupported-measurement-limit", True) + equal(len(rows), 3 * count, "measurement-count-mismatch") + require(len(doc["loop_records"]) == count, "loop-count-mismatch") + levels = [] + accepted = {s: [] for s in SCENARIOS} + stopped = False + for i, row in enumerate(rows): + integer(row["measurement_id"]) + equal(row["measurement_id"], i, "measurement-id-mismatch") + loop, pos = divmod(i, 3) + scenario = SCENARIOS[(loop + pos) % 3] + equal(row["loop_index"], loop, "measurement-order-mismatch") + equal(row["order_position"], pos, "measurement-order-mismatch") + equal(row["scenario"], scenario, "measurement-order-mismatch") + status = row["status"] + require(status in ("measured", "invalid", "failed", "interrupted", "not_run"), "measurement-status") + require(type(row["attempted"]) is bool, "measurement-attempted") + if row["attempted"]: + require(not stopped, "invalid-execution-prefix") + if status != "measured": + stopped = True + ref = row["plan_ref"] + p = None + if ref is not None: + integer(ref) + require(ref < len(plans) and plans[ref]["scenario"] == scenario, "plan-reference") + p = plans[ref] + equal(ref, doc["resolved_plan"]["frozen_plan_refs"][scenario], "frozen-plan-reference") + ex = row["execution"] + elapsed = ( + row["elapsed_seconds"] if status == "measured" else ex["timing"]["diagnostic_elapsed_seconds"] + ) + if row["attempted"]: + require(p is not None, "plan-reference") + checksum_ok = check_checksum(row["checksum"], p["expected_checksum"], m["backend"]) + levels.append(timing(ex, elapsed, m["backend"])) + else: + require(row["elapsed_seconds"] is None, "metric-null") + checksum_ok = False + named_checks(ex, m, scenario, status == "measured") + for check in ex["validation"]["checks"]: + if check["applicable"] is False: + require(check["evaluated"] is None and check["valid"] is None, "validation-null") + else: + require(check["applicable"] is True, "validation-applicability") + if status == "measured": + require( + check["evaluated"] is True and check["valid"] is True, "required-validation-failed" + ) + if status == "measured": + require( + row["attempted"] + and checksum_ok + and ex["valid"] is True + and ex["timing"]["evaluated"] is True + and ex["timing"]["valid"] is True, + "measurement-valid-mismatch", + ) + require( + type(elapsed) in (int, float) and math.isfinite(elapsed) and elapsed > 0, "elapsed-invalid" + ) + equal(row["completed_work_units"], p["work_units"], "completion-mismatch") + for name in ( + "effective_model_payload_bytes", + "layout_metadata_lookup_count", + "layout_metadata_read_bytes", + "task_accounted_bytes", + ): + equal(row["completed_" + name], p[name], "completion-mismatch") + t = p["work_units"] + payload = integer(p["effective_model_payload_bytes"], True) + for key, value in zip(METRICS, (elapsed / t, t / elapsed, payload / elapsed / 1e9)): + close(row[key], value) + if m["backend"] == "cpu": + equal(row["completion_derivation"], "accepted-plan-derived", "completion-derivation") + for key in ("requested_workers", "created_workers", "completed_workers"): + equal( + ex[key], + ( + m["workers"] + if key != "requested_workers" + else doc["configuration"]["requested_workers"] + ), + "worker-lifecycle", + ) + require( + ex["timer_started"] is True + and ex["timer_stopped"] is True + and ex["kernel_succeeded"] is True + and ex["worker_startup_failed"] is False, + "worker-lifecycle", + ) + accepted[scenario].append(i) + else: + require(all(row[k] is None for k in METRICS + ("elapsed_seconds",)), "metric-null") + for loop, record in enumerate(doc["loop_records"]): + equal(record["loop_index"], loop, "loop-order-mismatch") + order = [SCENARIOS[(loop + i) % 3] for i in range(3)] + equal(record["planned_order"], order, "loop-order-mismatch") + attempted = [rows[3 * loop + i]["scenario"] for i in range(3) if rows[3 * loop + i]["attempted"]] + equal(record["realized_order"], attempted, "loop-prefix-mismatch") + equal(record["realized_order_count"], len(attempted), "loop-prefix-mismatch") + equal( + record["measurement_indexes"], list(range(3 * loop, 3 * loop + 3)), "loop-measurement-reference" + ) + return accepted, levels + + +def stats(values): + n = len(values) + if not n: + return None + s = sorted(values) + mean = statistics.fmean(s) + median = statistics.median(s) + + def percentile(p): + x = (n - 1) * p + lo = int(x) + return s[lo] + (s[min(n - 1, lo + 1)] - s[lo]) * (x - lo) + + sd = statistics.stdev(s) if n > 1 else 0.0 + return dict( + sample_count=n, + average=mean, + min=s[0], + max=s[-1], + median=median, + p90=percentile(0.9), + p95=percentile(0.95), + p99=percentile(0.99), + stddev=sd, + coefficient_of_variation_pct=sd / mean * 100, + median_absolute_deviation=statistics.median(abs(v - median) for v in s), + ) + + +def check_aggregates(doc, accepted): + for scenario, ids in accepted.items(): + a = doc["aggregates"]["scenarios"][scenario] + equal(a["accepted_measurement_ids"], ids, "aggregate-population-mismatch") + n = len(ids) + equal( + a["status"], + "unavailable" if not n else "complete" if n == doc["configuration"]["loop_count"] else "partial", + "aggregate-status-mismatch", + ) + cv = stats([doc["measurements"][i]["effective_model_payload_gb_s"] for i in ids]) + classification = ( + "insufficient-samples" + if n < 3 + else "above-threshold" if cv["coefficient_of_variation_pct"] > 5 else "below-threshold" + ) + equal(a["observed_cv_classification"], classification, "aggregate-cv-mismatch") + close(a["cv_warning_threshold_pct"], 5.0, "aggregate-cv-mismatch") + for metric in METRICS: + values = [doc["measurements"][i][metric] for i in ids] + expected = stats(values) + got = a[metric] + equal(got["sample_count"], n, "aggregate-count-mismatch") + equal( + got["headline_semantics"], + "unavailable" if not n else "single_measurement" if n == 1 else "median_p50", + "aggregate-headline-semantics", + ) + if expected is None: + require(got["headline"] is None and got["statistics"] is None, "aggregate-null") + continue + close(got["headline"], expected["median"], "aggregate-headline-mismatch") + for key, value in expected.items(): + if key == "sample_count": + equal(got["statistics"][key], value, "aggregate-count-mismatch") + else: + close(got["statistics"][key], value, "aggregate-statistic-mismatch") + + +def check_run_state(doc, accepted, plans): + rows = doc["measurements"] + count = doc["configuration"]["loop_count"] + c = doc["counters"] + measured = sum(map(len, accepted.values())) + attempted = sum(r["attempted"] for r in rows) + expected = dict( + planned_loops=count, + attempted_loops=len({r["loop_index"] for r in rows if r["attempted"]}), + completed_loops=sum( + all(r["status"] == "measured" for r in rows[i : i + 3]) for i in range(0, len(rows), 3) + ), + planned_measurements=len(rows), + attempted_measurements=attempted, + terminal_measurements=sum(r["status"] != "not_run" for r in rows), + measured_measurements=measured, + ) + for key, value in expected.items(): + equal(c[key], value, "counter-mismatch") + for name in ( + "work_units", + "effective_model_payload_bytes", + "layout_metadata_lookup_count", + "layout_metadata_read_bytes", + "task_accounted_bytes", + ): + completed = sum(integer(r["completed_" + name], name != "work_units") for r in rows) + equal(c["completed_" + name], str(completed), "counter-mismatch") + refs = doc["resolved_plan"]["frozen_plan_refs"] + planned = ( + sum(integer(plans[refs[s]][name], name != "work_units") for s in SCENARIOS) * count + if all(refs[s] is not None for s in SCENARIOS) + else 0 + ) + equal(c["planned_" + name], str(planned), "counter-mismatch") + status = doc["status"] + require( + status in ("complete", "partial", "interrupted", "failed", "unsupported", "not_started"), "run-status" + ) + require(type(doc["interruption_requested"]) is bool, "interruption-flag") + if status == "complete": + require(measured == len(rows) and len(rows) > 0, "run-status-mismatch") + elif status == "partial": + require( + 0 < attempted and measured < len(rows) and not doc["interruption_requested"], + "run-status-mismatch", + ) + elif status == "interrupted": + require(doc["interruption_requested"] and measured < len(rows), "run-status-mismatch") + elif status in ("not_started", "unsupported"): + require(attempted == 0, "run-status-mismatch") + if any(r["status"] == "invalid" for r in rows): + require(status == "failed", "run-status-mismatch") + for row in rows: + if row["status"] == "interrupted": + require(doc["interruption_requested"], "interruption-flag") + if not row["attempted"]: + require( + row["checksum"]["checksum_valid"] is None + and row["checksum"]["actual_run_checksum"] is None + and row["checksum"]["actual_worker_checksums"] is None, + "checksum-null", + ) + require(row["execution"]["valid"] is None, "execution-null") + require( + row["completed_work_units"] == 0 + and all( + row["completed_" + k] == "0" + for k in ( + "effective_model_payload_bytes", + "layout_metadata_lookup_count", + "layout_metadata_read_bytes", + "task_accounted_bytes", + ) + ), + "completion-mismatch", + ) + life = doc["checkpoint_lifecycle"] + equal(life["snapshot_interval_loops"], max(1, (count + 7) // 8), "checkpoint-policy") + equal(life["checkpoint_policy"], "bounded-loop-snapshots", "checkpoint-policy") + require(life["current_persistence_success"] is None, "checkpoint-current-persistence") + require(type(life["checkpoint_failed"]) is bool, "checkpoint-failure") + if life["checkpoint_failed"]: + require(status == "failed", "run-status-mismatch") + + +def named_checks(ex, m, scenario, measured): + writes = scenario != "weights_only" + padding = m["paged"] and m["tokens"] % m["block_tokens"] != 0 + cpu = m["backend"] == "cpu" + kinds = [ + "post-validation-structure", + ( + "kv-prefill-final-samples" + if m["prefill"] + else "kv-append-final" if writes or not cpu else "kv-append-unchanged" + ), + "kv-padding-canary", + ] + applies = [ + m["paged"] or m["prefill"] or writes if cpu else writes, + writes or (m["paged"] and not m["prefill"]) if cpu else writes, + padding if cpu else writes and padding, + ] + checks = ex["validation"]["checks"] + equal(len(checks), 3, "validation-count") + for check, kind, applicable in zip(checks, kinds, applies): + equal(check["kind"], kind, "validation-kind") + equal(check["applicable"], applicable, "validation-applicability") + if not applicable: + require(check["evaluated"] is None and check["valid"] is None, "validation-null") + else: + require(type(check["evaluated"]) is bool, "validation-evaluated") + if check["evaluated"]: + require(type(check["valid"]) is bool, "validation-valid") + else: + require(check["valid"] is None, "validation-null") + if measured: + require(check["evaluated"] and check["valid"], "required-validation-failed") + + +def check_manifest(manifest): + equal(manifest["manifest_version"], 1, "manifest-version") + require(manifest["status"] in ("available", "partial", "unavailable"), "manifest-status") + if manifest["binary_sha256"] is not None: + require( + isinstance(manifest["binary_sha256"], str) + and re.fullmatch("[0-9a-f]{64}", manifest["binary_sha256"]) is not None, + "manifest-hash-encoding", + ) + if manifest["git_commit"] is not None: + require( + isinstance(manifest["git_commit"], str) + and re.fullmatch("[0-9a-f]{40,64}", manifest["git_commit"]) is not None, + "manifest-commit-encoding", + ) + require(manifest["git_dirty"] is None or type(manifest["git_dirty"]) is bool, "manifest-dirty-encoding") + if manifest["status"] == "available": + require( + all( + manifest[k] is not None + for k in ( + "binary_sha256", + "git_commit", + "git_dirty", + "compiler", + "compile_flags", + "link_flags", + "target_arch", + "sdk", + "min_os", + ) + ), + "manifest-availability", + ) + + +def verify(doc, binary=None, require_raw=False): + if type(doc.get("schema_version")) is not int or doc["schema_version"] != 2: + raise Rejected("unsupported-schema", True) + if doc.get("mode") != "llm_memory": + raise Rejected("unsupported-mode", True) + budget = Budget() + m = check_geometry(doc, budget) + m["ownership"] = ownership(m, doc, budget) + check_identities(doc, m, budget) + check_prefill_scopes(doc, m, budget) + plans = check_plans(doc, m, budget) + accepted, levels = check_measurements(doc, m, plans) + check_aggregates(doc, accepted) + check_run_state(doc, accepted, plans) + for scenario, attempts in doc["calibration"]["attempts"].items(): + for index, attempt in enumerate(attempts): + equal(attempt["attempt_index"], index, "calibration-index") + ref = attempt["plan_ref"] + if ref is None: + require( + not attempt["terminal"] + and attempt["valid"] is False + and attempt["execution"]["valid"] is None + and attempt["execution"]["elapsed_seconds"] is None, + "calibration-placeholder", + ) + continue + integer(ref) + require(ref < len(plans) and plans[ref]["scenario"] == scenario, "plan-reference") + ex = attempt["execution"] + ok = check_checksum(ex["checksum"], plans[ref]["expected_checksum"], m["backend"], True) + if attempt["valid"]: + require( + ok and ex["valid"] is True and ex["elapsed_seconds"] is not None, + "calibration-valid-mismatch", + ) + named_checks(ex, m, scenario, attempt["valid"]) + levels.append(timing(ex, ex["elapsed_seconds"], m["backend"])) + measured = sum(map(len, accepted.values())) + total = len(doc["measurements"]) + complete = total > 0 and measured == total + equal(doc["results_complete"], complete, "results-complete-mismatch") + accepted_run = ( + doc["status"] == "complete" and complete and not doc["checkpoint_lifecycle"]["checkpoint_failed"] + ) + equal(doc["run_accepted"], accepted_run, "run-acceptance-mismatch") + equal( + doc["scenario_order_balance_complete"], + complete and doc["configuration"]["loop_count"] % 3 == 0, + "order-balance-mismatch", + ) + if require_raw and m["backend"] == "cpu" and ("elapsed-only" in levels or not levels): + raise Rejected("unsupported-missing-raw-timing", True) + manifest = doc["build_manifest"] + build = "manifest-only" if doc["build_manifest"]["status"] != "unavailable" else "unavailable" + check_manifest(manifest) + if binary is not None: + require(manifest["binary_sha256"] is not None, "binary-hash-unavailable") + h = hashlib.sha256() + with open(binary, "rb") as f: + while chunk := f.read(65536): + h.update(chunk) + equal(h.hexdigest(), manifest["binary_sha256"], "binary-hash-mismatch") + build = "binary-bound" + return dict( + artifact_consistent=True, + run_accepted=accepted_run, + reason_code="valid" if accepted_run else "run-not-accepted", + checks=dict( + checksum=( + "independent-oracle" + if any(p["expected_checksum"]["status"] == "available" for p in plans) + else "unavailable" + ), + timing=( + "elapsed-only" + if "elapsed-only" in levels + else ( + ("cpu-raw-ticks" if m["backend"] == "cpu" else "gpu-timestamps") + if levels + else "unavailable" + ) + ), + build=build, + ), + oracle_work_used=MAX_WORK - budget.remaining, + ) + + +def load(path): + with open(path, "rb") as f: + data = f.read(MAX_INPUT + 1) + if len(data) > MAX_INPUT: + raise Rejected("unsupported-input-limit", True) + + def pairs(items): + result = {} + for k, v in items: + require(k not in result, "duplicate-json-key") + result[k] = v + return result + + doc = json.loads( + data.decode("utf-8"), + object_pairs_hook=pairs, + parse_constant=lambda _: (_ for _ in ()).throw(Rejected("nonfinite-json-number")), + ) + stack = [(doc, 0)] + count = 0 + while stack: + obj, depth = stack.pop() + count += 1 + if depth > 64 or count > 1000000: + raise Rejected("unsupported-structure-limit", True) + if isinstance(obj, dict): + stack.extend((v, depth + 1) for v in obj.values()) + elif isinstance(obj, list): + stack.extend((v, depth + 1) for v in obj) + return doc + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("result") + parser.add_argument("--binary", type=Path) + parser.add_argument( + "--require-raw-timing", action="store_true", help="Require original timing evidence for CPU tasks" + ) + args = parser.parse_args() + try: + verdict = verify(load(args.result), args.binary, args.require_raw_timing) + code = 0 if verdict["run_accepted"] else 1 + except Rejected as error: + verdict = dict( + artifact_consistent=None if error.unsupported else False, + run_accepted=False, + reason_code=error.reason, + ) + code = 2 if error.unsupported else 1 + except ( + OSError, + ValueError, + TypeError, + KeyError, + IndexError, + RecursionError, + ArithmeticError, + AttributeError, + ) as error: + verdict = dict( + artifact_consistent=False, + run_accepted=False, + reason_code="invalid-input", + detail=str(error)[:200], + ) + code = 1 + print(json.dumps(verdict, sort_keys=True)) + return code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 5a277c4..00f8e36 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -18,3 +18,19 @@ latency, and automatic-locality paths used by both plotters. The custom-cache ru and work metadata used by the shell examples. Refresh the fixtures, examples, and entry-path assertions together when the consumed schema, methodology, or result shape changes. A `SOFTVERSION`-only change does not require a fixture refresh. Run `make test-script-examples` after updating them. + +## LLM schema 2 verifier fixtures + +`llm-schema-v2/` contains exact producer captures from the Phase 7A binary (build Git provenance +`870b175927d747fb1d90fadefebe8728da38407a`, dirty tree; executable SHA-256 retained in each artifact). +The eight admitted profiles used W=1 MiB, layers=2, query heads=2, KV heads=1, head dimension=3, +element bytes=1, batch=2, iterations=2, count=3 and seed=424242. CPU requested two workers. +Decode used context=5; prefill used P=5/Q=2; paged layouts used block tokens=2. Explicit file output +preserves the terminal snapshot. The unsupported fixture used Metal decode contiguous, layers/heads/dimension/ +context/iterations/count=1, weight=1 MiB, seed=0 and default element width, under seatbelt without device access. +These captured timestamps/build values are immutable provenance data, not documentation release dates. + +`test_llm_result_verifier.py` separately authors partial/interrupted/failed/invalid states with exact populations. +Its independent one-byte goldens cover every profile without treating these producer captures as the oracle. +The multi-byte affine arithmetic is also checked against direct byte enumeration, including unaligned boundaries. +The fixture gate has no device or producer executable dependency; real-device readback remains a separate gate. diff --git a/tests/fixtures/llm-schema-v2/cpu-decode-contiguous.json b/tests/fixtures/llm-schema-v2/cpu-decode-contiguous.json new file mode 100644 index 0000000..cd9985a --- /dev/null +++ b/tests/fixtures/llm-schema-v2/cpu-decode-contiguous.json @@ -0,0 +1 @@ +{"schema_version":2,"mode":"llm_memory","backend":"cpu","phase":"decode","kv_layout":"contiguous","methodology_version":"llm-memory-v2-cpu-decode-contiguous","software":{"version":"0.63.1","timestamp":"2026-09-06T07:07:07Z"},"build_manifest":{"binary_sha256":"a9c0aa68c61fccf12bda222a0485a3eba1b642a54b9b1caf8292348f608a1100","compile_flags":{"asflags":"-arch arm64","cxxflags":"-Wall -O3 -std=c++17 -arch arm64 -pthread -Isrc","test_cxxflags":"-Wall -g -std=c++17 -arch arm64 -pthread -Isrc -I/opt/homebrew/opt/googletest/include"},"compiler":"Apple clang version 21.0.0 (clang-2100.1.1.101)\nTarget: arm64-apple-darwin25.6.0\nThread model: posix\nInstalledDir: /Library/Developer/CommandLineTools/usr/bin","git_commit":"870b175927d747fb1d90fadefebe8728da38407a","git_dirty":true,"link_flags":"-pthread -framework Metal -framework Foundation","manifest_version":1,"min_os":"26.0","reason_code":"valid","sdk":"26.5","status":"available","target_arch":"arm64-apple-darwin25.6.0"},"configuration":{"backend":"cpu","phase":"decode","kv_layout":"contiguous","kv_block_tokens":null,"weight_size_mb":1,"layer_count":2,"query_head_count":2,"kv_head_count":1,"head_dimension":3,"kv_element_bytes":"1","visible_context_tokens":5,"prompt_tokens":null,"attention_query_tile_tokens":null,"batch_size":2,"requested_workers":2,"available_workers":10,"worker_source":"user","iterations":2,"work_policy":"explicit_fixed_work","loop_count":3,"base_seed_uint64_decimal":"424242","seed_source":"user","output_file":"plans/llm-benchmark-remediation-evidence/phase7/profiles/cpu-decode-contiguous.json","argv":["./memory_benchmark","--llm-memory","--llm-memory-backend","cpu","--phase","decode","--kv-layout","contiguous","--weight-size-mb","1","--layers","2","--query-heads","2","--kv-heads","1","--head-dim","3","--kv-element-bytes","1","--batch-size","2","--iterations","2","--count","3","--seed","424242","-o","plans/llm-benchmark-remediation-evidence/phase7/profiles/cpu-decode-contiguous.json","--context-tokens","5","--threads","2"],"resolved_sources":{"backend":"explicit","phase":"explicit","kv_layout":"explicit","kv_block_tokens":null,"visible_context_tokens":"explicit","prompt_tokens":null,"attention_query_tile_tokens":null,"workers":"explicit","iterations":"explicit","seed":"explicit"}},"resolved_plan":{"valid":true,"reason_code":"valid","plan_identity":"llm-memory-work-plan-v1|backend=cpu|phase=decode|kv_layout=contiguous|work_unit_kind=decode_step|methodology=llm-memory-v2-cpu-decode-contiguous|component_identity_size=643|component_identity=llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=43:llm-cpu-executor-v1-arm64-decode-contiguous|resource_abi_version=28:llm-memory-descriptor-abi-v1|schedule_version=52:worker-local-layer-order-no-per-layer-global-barrier|timer_policy_version=62:synchronized-start-to-last-worker-completion-per-scenario-task|buffer_pattern_version=21:llm-buffer-pattern-v1|write_pattern_version=25:llm-kv-append-affine64-v1|checksum_pattern_version=20:llm-read-checksum-v1|msl_revision=null|msl_source_sha256=null|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|decode_context=5|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=0|kv_blocks_per_sequence=0|physical_blocks_per_layer=0|total_physical_blocks=0|kv_block_bytes=0|last_block_tokens=0|last_block_valid_bytes=0|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=0|v_layout_padding_bytes=0|block_table_entries=0|block_table_bytes=0|layout_metadata_lookups_per_layer_sequence_per_work_unit=0|k_mapping_bytes=60|v_mapping_bytes=60|kv_capacity_bytes=120|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|kv_only_payload_bytes_per_work_unit=144|mixed_payload_bytes_per_work_unit=1048720|total_data_mapping_bytes=1048696|traffic_crossover_numerator=1048576|traffic_crossover_denominator=24|requested_workers=2|effective_workers=2|layer_descriptors_per_worker=2|sequence_descriptors_per_worker=4|total_layer_descriptors=4|total_sequence_descriptors=8|descriptor_bytes=832|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391","backend":"cpu","phase":"decode","kv_layout":"contiguous","work_unit_kind":"decode_step","geometry":{"valid":true,"reason_code":"valid","phase":"decode","work_unit_kind":"decode_step","decode":{"visible_context_tokens":5},"prefill":null,"attention_kind":"mqa","active_weight_bytes_per_work_unit":"1048576","layer_count":2,"query_head_count":2,"kv_head_count":1,"query_heads_per_kv_head":2,"head_dimension":3,"kv_element_bytes":"1","batch_size":2,"kv_vector_bytes":"3","k_or_v_record_bytes_per_layer":"3","kv_record_bytes_per_layer":"6","kv_bytes_per_visible_token":"12","k_or_v_sequence_visible_bytes":"15","k_mapping_bytes":"60","v_mapping_bytes":"60","kv_capacity_bytes":"120","weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"120","kv_write_bytes_per_work_unit":"24","kv_only_effective_model_payload_bytes_per_work_unit":"144","mixed_effective_model_payload_bytes_per_work_unit":"1048720","total_data_mapping_bytes":"1048696","traffic_crossover_numerator":"1048576","traffic_crossover_denominator":"24","traffic_crossover_context_tokens":43690.666666666664},"model_context":{"prefill":null},"layout":{"kv_layout":"contiguous","kv_block_tokens":null,"blocks_per_sequence":null,"physical_blocks_per_layer":null,"total_physical_blocks":null,"block_bytes":null,"last_block_tokens":null,"last_block_valid_bytes":null,"decode_append_offset_in_last_block":null,"block_table_entries":null,"block_table_bytes":null,"layout_geometry_identity":null,"layout_identity":null,"permutation_domain_uint64_hex":null,"permutation_seed_uint64_decimal":null,"permutation_algorithm_version":null,"permutation_entry_count":null,"permutation_sha256":null,"permutation_identity":null},"resources":{"weight_logical_bytes":"1048576","k_logical_bytes":"60","v_logical_bytes":"60","k_physical_length_bytes":"60","v_physical_length_bytes":"60","k_layout_padding_bytes":"0","v_layout_padding_bytes":"0","block_table_bytes":null},"component_identities":{"logical_profile_version":"decode_steady_fixed_context","kv_layout_version":"contiguous_layer_batch_token_head_dimension","permutation_version":null,"backend_executor_version":"llm-cpu-executor-v1-arm64-decode-contiguous","resource_abi_version":"llm-memory-descriptor-abi-v1","schedule_version":"worker-local-layer-order-no-per-layer-global-barrier","timer_policy_version":"synchronized-start-to-last-worker-completion-per-scenario-task","buffer_pattern_version":"llm-buffer-pattern-v1","write_pattern_version":"llm-kv-append-affine64-v1","checksum_pattern_version":"llm-read-checksum-v1","msl_revision":null,"msl_source_sha256":null,"identity":"llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=43:llm-cpu-executor-v1-arm64-decode-contiguous|resource_abi_version=28:llm-memory-descriptor-abi-v1|schedule_version=52:worker-local-layer-order-no-per-layer-global-barrier|timer_policy_version=62:synchronized-start-to-last-worker-completion-per-scenario-task|buffer_pattern_version=21:llm-buffer-pattern-v1|write_pattern_version=25:llm-kv-append-affine64-v1|checksum_pattern_version=20:llm-read-checksum-v1|msl_revision=null|msl_source_sha256=null","run_policy_version":"llm-run-policy-bounded-loop-snapshots-v1"},"methodology":{"backend":"cpu","phase":"decode","kv_layout":"contiguous","work_unit_kind":"decode_step","weight_passes_per_work_unit":1,"kv_replay_factor":1,"warmup_policy":"same-shape-excluded-steady-state-warm-memory","context_policy":"fixed-visible-context-including-current-token-slot","scenario_order_policy":"cyclic-rotation-across-count-loops","timing_policy":"synchronized-start-to-last-worker-completion-per-scenario-task","cache_policy":"regular-cacheable-no-explicit-flush-between-scenarios","calibration_policy":"per-scenario-automatic-or-explicit-frozen-before-loop-zero","calibration_target_seconds":0.15,"calibration_min_seconds":0.1,"calibration_max_seconds":0.25,"calibration_max_corrections":2,"calibration_min_pilot_accounted_bytes":"8388608","maximum_work_units_per_measurement":1000000000,"maximum_serial_range_visits_per_lane_per_task":null,"maximum_accounted_bytes_per_task":"68719476736","repeatability_cv_warning_threshold_pct":5.0,"calibration_excluded_from_results":true,"snapshot_policy":"bounded-loop-snapshots","timed_region_exclusions":["mapping-allocation","buffer-initialization-and-pre-touch","descriptor-materialization-and-validation","worker-thread-creation-and-qos","same-shape-warmup","calibration-attempts","checksum-reference-generation-and-validation","worker-join-and-json-serialization"]},"model_work_plan":{"valid":true,"reason_code":"valid","backend":"cpu","phase":"decode","kv_layout":"contiguous","work_unit_kind":"decode_step","weight_passes_per_work_unit":1,"kv_replay_factor":1,"requested_workers":2,"available_workers":10,"effective_workers":2,"worker_plan_count":2,"weight_layer_count":2,"layer_descriptors_per_worker":2,"sequence_descriptors_per_worker":4,"total_layer_descriptors":4,"total_sequence_descriptors":8,"descriptor_bytes":"832","planner_storage_bytes":"1152"},"scenario_plans":[{"valid":true,"reason_code":"valid","scenario":"weights_only","work_unit_kind":"decode_step","kv_write_kind":"none","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"17410844953366850226","work_units":2,"weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"0","kv_write_bytes_per_work_unit":"0","effective_model_payload_bytes_per_work_unit":"1048576","layout_metadata_lookup_count_per_work_unit":"0","layout_metadata_read_bytes_per_work_unit":"0","accounted_bytes_per_work_unit":"1048576","weight_read_bytes":"2097152","kv_read_bytes":"0","kv_write_bytes":"0","effective_model_payload_bytes":"2097152","layout_metadata_lookup_count":"0","layout_metadata_read_bytes":"0","task_accounted_bytes":"2097152","maximum_work_units_by_work_unit_cap":1000000000,"maximum_work_units_by_guardrail":65536,"effective_maximum_work_units":65536,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=2360|model_plan_identity=llm-memory-work-plan-v1|backend=cpu|phase=decode|kv_layout=contiguous|work_unit_kind=decode_step|methodology=llm-memory-v2-cpu-decode-contiguous|component_identity_size=643|component_identity=llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=43:llm-cpu-executor-v1-arm64-decode-contiguous|resource_abi_version=28:llm-memory-descriptor-abi-v1|schedule_version=52:worker-local-layer-order-no-per-layer-global-barrier|timer_policy_version=62:synchronized-start-to-last-worker-completion-per-scenario-task|buffer_pattern_version=21:llm-buffer-pattern-v1|write_pattern_version=25:llm-kv-append-affine64-v1|checksum_pattern_version=20:llm-read-checksum-v1|msl_revision=null|msl_source_sha256=null|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|decode_context=5|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=0|kv_blocks_per_sequence=0|physical_blocks_per_layer=0|total_physical_blocks=0|kv_block_bytes=0|last_block_tokens=0|last_block_valid_bytes=0|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=0|v_layout_padding_bytes=0|block_table_entries=0|block_table_bytes=0|layout_metadata_lookups_per_layer_sequence_per_work_unit=0|k_mapping_bytes=60|v_mapping_bytes=60|kv_capacity_bytes=120|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|kv_only_payload_bytes_per_work_unit=144|mixed_payload_bytes_per_work_unit=1048720|total_data_mapping_bytes=1048696|traffic_crossover_numerator=1048576|traffic_crossover_denominator=24|requested_workers=2|effective_workers=2|layer_descriptors_per_worker=2|sequence_descriptors_per_worker=4|total_layer_descriptors=4|total_sequence_descriptors=8|descriptor_bytes=832|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=weights_only|work_unit_kind=decode_step|kv_write_kind=none|scenario_seed=17410844953366850226|explicit=1|work_units=2|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=0|kv_write_bytes_per_work_unit=0|effective_model_payload_bytes_per_work_unit=1048576|layout_metadata_lookup_count_per_work_unit=0|layout_metadata_read_bytes_per_work_unit=0|accounted_bytes_per_work_unit=1048576|weight_read_bytes=2097152|kv_read_bytes=0|kv_write_bytes=0|effective_model_payload_bytes=2097152|layout_metadata_lookup_count=0|layout_metadata_read_bytes=0|task_accounted_bytes=2097152|maximum_work_units_by_work_unit_cap=1000000000|maximum_work_units_by_guardrail=65536|effective_maximum_work_units=65536","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"8025476992397170658","state_b_uint64_decimal":"6807433224402555509","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"8241649774510954466","state_b_uint64_decimal":"7600066758819762805","exact_bytes_read":"0","span_count_uint64_decimal":"0"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"8025476992397170658","state_b_uint64_decimal":"6807433224402555509","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"8241649774510954466","state_b_uint64_decimal":"7600066758819762805","exact_bytes_read":"0","span_count_uint64_decimal":"0"}}],"expected_run_checksum":{"state_a_uint64_decimal":"1826493790751740180","state_b_uint64_decimal":"2326349888306066833"}}},{"valid":true,"reason_code":"valid","scenario":"kv_only","work_unit_kind":"decode_step","kv_write_kind":"current_token_append","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"16590208891222127307","work_units":2,"weight_read_bytes_per_work_unit":"0","kv_read_bytes_per_work_unit":"120","kv_write_bytes_per_work_unit":"24","effective_model_payload_bytes_per_work_unit":"144","layout_metadata_lookup_count_per_work_unit":"0","layout_metadata_read_bytes_per_work_unit":"0","accounted_bytes_per_work_unit":"144","weight_read_bytes":"0","kv_read_bytes":"240","kv_write_bytes":"48","effective_model_payload_bytes":"288","layout_metadata_lookup_count":"0","layout_metadata_read_bytes":"0","task_accounted_bytes":"288","maximum_work_units_by_work_unit_cap":1000000000,"maximum_work_units_by_guardrail":477218588,"effective_maximum_work_units":477218588,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=2360|model_plan_identity=llm-memory-work-plan-v1|backend=cpu|phase=decode|kv_layout=contiguous|work_unit_kind=decode_step|methodology=llm-memory-v2-cpu-decode-contiguous|component_identity_size=643|component_identity=llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=43:llm-cpu-executor-v1-arm64-decode-contiguous|resource_abi_version=28:llm-memory-descriptor-abi-v1|schedule_version=52:worker-local-layer-order-no-per-layer-global-barrier|timer_policy_version=62:synchronized-start-to-last-worker-completion-per-scenario-task|buffer_pattern_version=21:llm-buffer-pattern-v1|write_pattern_version=25:llm-kv-append-affine64-v1|checksum_pattern_version=20:llm-read-checksum-v1|msl_revision=null|msl_source_sha256=null|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|decode_context=5|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=0|kv_blocks_per_sequence=0|physical_blocks_per_layer=0|total_physical_blocks=0|kv_block_bytes=0|last_block_tokens=0|last_block_valid_bytes=0|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=0|v_layout_padding_bytes=0|block_table_entries=0|block_table_bytes=0|layout_metadata_lookups_per_layer_sequence_per_work_unit=0|k_mapping_bytes=60|v_mapping_bytes=60|kv_capacity_bytes=120|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|kv_only_payload_bytes_per_work_unit=144|mixed_payload_bytes_per_work_unit=1048720|total_data_mapping_bytes=1048696|traffic_crossover_numerator=1048576|traffic_crossover_denominator=24|requested_workers=2|effective_workers=2|layer_descriptors_per_worker=2|sequence_descriptors_per_worker=4|total_layer_descriptors=4|total_sequence_descriptors=8|descriptor_bytes=832|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=kv_only|work_unit_kind=decode_step|kv_write_kind=current_token_append|scenario_seed=16590208891222127307|explicit=1|work_units=2|weight_read_bytes_per_work_unit=0|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|effective_model_payload_bytes_per_work_unit=144|layout_metadata_lookup_count_per_work_unit=0|layout_metadata_read_bytes_per_work_unit=0|accounted_bytes_per_work_unit=144|weight_read_bytes=0|kv_read_bytes=240|kv_write_bytes=48|effective_model_payload_bytes=288|layout_metadata_lookup_count=0|layout_metadata_read_bytes=0|task_accounted_bytes=288|maximum_work_units_by_work_unit_cap=1000000000|maximum_work_units_by_guardrail=477218588|effective_maximum_work_units=477218588","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"11898181619645713650","state_b_uint64_decimal":"18167339280465603252","exact_bytes_read":"52","span_count_uint64_decimal":"8"},"v":{"state_a_uint64_decimal":"14337786283661910650","state_b_uint64_decimal":"18167339327710243508","exact_bytes_read":"52","span_count_uint64_decimal":"8"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"11335208175666694355","state_b_uint64_decimal":"17960946654246414605","exact_bytes_read":"68","span_count_uint64_decimal":"8"},"v":{"state_a_uint64_decimal":"14982831505298971441","state_b_uint64_decimal":"5623828104260097821","exact_bytes_read":"68","span_count_uint64_decimal":"8"}}],"expected_run_checksum":{"state_a_uint64_decimal":"1377760989781549385","state_b_uint64_decimal":"10363833834494835208"}}},{"valid":true,"reason_code":"valid","scenario":"mixed","work_unit_kind":"decode_step","kv_write_kind":"current_token_append","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"1546516732201688391","work_units":2,"weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"120","kv_write_bytes_per_work_unit":"24","effective_model_payload_bytes_per_work_unit":"1048720","layout_metadata_lookup_count_per_work_unit":"0","layout_metadata_read_bytes_per_work_unit":"0","accounted_bytes_per_work_unit":"1048720","weight_read_bytes":"2097152","kv_read_bytes":"240","kv_write_bytes":"48","effective_model_payload_bytes":"2097440","layout_metadata_lookup_count":"0","layout_metadata_read_bytes":"0","task_accounted_bytes":"2097440","maximum_work_units_by_work_unit_cap":1000000000,"maximum_work_units_by_guardrail":65527,"effective_maximum_work_units":65527,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=2360|model_plan_identity=llm-memory-work-plan-v1|backend=cpu|phase=decode|kv_layout=contiguous|work_unit_kind=decode_step|methodology=llm-memory-v2-cpu-decode-contiguous|component_identity_size=643|component_identity=llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=43:llm-cpu-executor-v1-arm64-decode-contiguous|resource_abi_version=28:llm-memory-descriptor-abi-v1|schedule_version=52:worker-local-layer-order-no-per-layer-global-barrier|timer_policy_version=62:synchronized-start-to-last-worker-completion-per-scenario-task|buffer_pattern_version=21:llm-buffer-pattern-v1|write_pattern_version=25:llm-kv-append-affine64-v1|checksum_pattern_version=20:llm-read-checksum-v1|msl_revision=null|msl_source_sha256=null|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|decode_context=5|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=0|kv_blocks_per_sequence=0|physical_blocks_per_layer=0|total_physical_blocks=0|kv_block_bytes=0|last_block_tokens=0|last_block_valid_bytes=0|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=0|v_layout_padding_bytes=0|block_table_entries=0|block_table_bytes=0|layout_metadata_lookups_per_layer_sequence_per_work_unit=0|k_mapping_bytes=60|v_mapping_bytes=60|kv_capacity_bytes=120|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|kv_only_payload_bytes_per_work_unit=144|mixed_payload_bytes_per_work_unit=1048720|total_data_mapping_bytes=1048696|traffic_crossover_numerator=1048576|traffic_crossover_denominator=24|requested_workers=2|effective_workers=2|layer_descriptors_per_worker=2|sequence_descriptors_per_worker=4|total_layer_descriptors=4|total_sequence_descriptors=8|descriptor_bytes=832|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=mixed|work_unit_kind=decode_step|kv_write_kind=current_token_append|scenario_seed=1546516732201688391|explicit=1|work_units=2|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|effective_model_payload_bytes_per_work_unit=1048720|layout_metadata_lookup_count_per_work_unit=0|layout_metadata_read_bytes_per_work_unit=0|accounted_bytes_per_work_unit=1048720|weight_read_bytes=2097152|kv_read_bytes=240|kv_write_bytes=48|effective_model_payload_bytes=2097440|layout_metadata_lookup_count=0|layout_metadata_read_bytes=0|task_accounted_bytes=2097440|maximum_work_units_by_work_unit_cap=1000000000|maximum_work_units_by_guardrail=65527|effective_maximum_work_units=65527","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"11898181619645713650","state_b_uint64_decimal":"18167339280465603252","exact_bytes_read":"52","span_count_uint64_decimal":"8"},"v":{"state_a_uint64_decimal":"14337786283661910650","state_b_uint64_decimal":"18167339327710243508","exact_bytes_read":"52","span_count_uint64_decimal":"8"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"598985949067047316","state_b_uint64_decimal":"17960946658671039404","exact_bytes_read":"68","span_count_uint64_decimal":"8"},"v":{"state_a_uint64_decimal":"4246609295879193602","state_b_uint64_decimal":"5623828109217268156","exact_bytes_read":"68","span_count_uint64_decimal":"8"}}],"expected_run_checksum":{"state_a_uint64_decimal":"7557809546606715305","state_b_uint64_decimal":"12589281716502849552"}}}],"frozen_plan_refs":{"weights_only":0,"kv_only":1,"mixed":2}},"backend_evidence":{"cpu":{"requested_workers":2,"available_workers":10,"effective_workers":2,"resource_abi_version":"llm-memory-descriptor-abi-v1","schedule_version":"worker-local-layer-order-no-per-layer-global-barrier","timer_policy_version":"synchronized-start-to-last-worker-completion-per-scenario-task","prefill":null,"paged":null,"resources":{"valid":true,"reason_code":"valid","model_ref":"resolved_plan","mappings":{"policy":"private_anonymous_regular_cacheable","full_size_physical_mappings":true,"weight":{"requested_bytes":"1048576","committed_bytes":"1048576"},"k":{"requested_bytes":"60","committed_bytes":"16384"},"v":{"requested_bytes":"60","committed_bytes":"16384"},"block_table":null,"requested_data_bytes":"1048696","committed_data_bytes":"1081344"},"descriptors":{"abi_version":"llm-memory-descriptor-abi-v1","layer_descriptors_per_worker":2,"sequence_descriptors_per_worker":4,"total_layer_descriptors":4,"total_sequence_descriptors":8,"descriptor_bytes":"832"},"executor_auxiliary":{"valid":true,"reason_code":"valid","static_reference_bytes":"480","expected_checksum_bytes":"192","actual_checksum_bytes":"192","run_checksum_bytes":"32","worker_status_bytes":"2","thread_handle_bytes":"16","checksum_auxiliary_bytes":"896","orchestration_auxiliary_bytes":"18","total_auxiliary_bytes":"914"},"json_output_peak_estimate":{"enabled":true,"valid":true,"reason_code":"valid","policy":"conservative-live-dom-plus-serialized-transport","fixed_schema_bytes":"2097152","input_string_bytes":"863360","measurement_record_bytes":"1179648","worker_checksum_bytes":"1966080","total_bytes":"6106240"},"allocation_memory_budget":{"resource_rounding_bytes":"32648","transient_peak_bytes":"6277972","layout_transient_bytes":null,"setup_peak_bytes":"1152","runtime_peak_bytes":"7359316","known_owned_peak_bytes":"7359316","admitted_budget_bytes":"9270250700","valid":true,"reason_code":"within-memory-budget","request":{"valid":true,"reason_code":"valid","mapping_granularity_bytes":"16384","requested_weight_mapping_bytes":"1048576","requested_k_mapping_bytes":"60","requested_v_mapping_bytes":"60","committed_weight_mapping_bytes":"1048576","committed_k_mapping_bytes":"16384","committed_v_mapping_bytes":"16384","requested_block_table_mapping_bytes":null,"committed_block_table_mapping_bytes":null,"requested_data_bytes":"1048696","committed_data_bytes":"1081344","layout_transient_bytes":null,"setup_peak_bytes":"1152","runtime_peak_bytes":"7359316","descriptor_bytes":"832","planner_storage_bytes":"1152","checksum_auxiliary_bytes":"4160","orchestration_auxiliary_bytes":"6271828","auxiliary_bytes":"6277972","required_total_bytes":"7359316"},"available_memory_bytes":"11587813376","allowed_memory_bytes":"9270250700","used_fallback":false},"initialization":{"complete":true,"pattern_version":"llm-buffer-pattern-v1","pre_touch_policy":"write-every-requested-mapping-byte-once","separate_reference_read_pass":false,"static_references_accumulated_during_initialization":true,"weight_bytes":"1048576","k_bytes":"60","v_bytes":"60","total_bytes":"1048696","non_empty_weight_spans":4,"non_empty_k_spans":8,"non_empty_v_spans":8,"block_table_logical_bytes":null,"block_table_page_rounded_bytes":null,"block_table_read_only":null,"k_layout_padding_bytes":null,"v_layout_padding_bytes":null}}},"metal":null},"memory_budget":{"resource_rounding_bytes":"32648","transient_peak_bytes":"6277972","layout_transient_bytes":null,"setup_peak_bytes":"1152","runtime_peak_bytes":"7359316","known_owned_peak_bytes":"7359316","admitted_budget_bytes":"9270250700","valid":true,"reason_code":"within-memory-budget","request":{"valid":true,"reason_code":"valid","mapping_granularity_bytes":"16384","requested_weight_mapping_bytes":"1048576","requested_k_mapping_bytes":"60","requested_v_mapping_bytes":"60","committed_weight_mapping_bytes":"1048576","committed_k_mapping_bytes":"16384","committed_v_mapping_bytes":"16384","requested_block_table_mapping_bytes":null,"committed_block_table_mapping_bytes":null,"requested_data_bytes":"1048696","committed_data_bytes":"1081344","layout_transient_bytes":null,"setup_peak_bytes":"1152","runtime_peak_bytes":"7359316","descriptor_bytes":"832","planner_storage_bytes":"1152","checksum_auxiliary_bytes":"4160","orchestration_auxiliary_bytes":"6271828","auxiliary_bytes":"6277972","required_total_bytes":"7359316"},"available_memory_bytes":"11587813376","allowed_memory_bytes":"9270250700","used_fallback":false},"calibration":{"excluded_from_results":true,"attempts":{"weights_only":[{"attempt_index":0,"scenario":"weights_only","plan_ref":0,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":1.9583333333333333e-05,"timing":{"cpu_raw":{"start_ticks":"36788195046445","stop_ticks":"36788195046915","delta_ticks":"470","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"state_a_uint64_decimal":"1826493790751740180","state_b_uint64_decimal":"2326349888306066833"}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-append-unchanged","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}}}],"kv_only":[{"attempt_index":0,"scenario":"kv_only","plan_ref":1,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":4.833333333333333e-06,"timing":{"cpu_raw":{"start_ticks":"36788195047880","stop_ticks":"36788195047996","delta_ticks":"116","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"state_a_uint64_decimal":"1377760989781549385","state_b_uint64_decimal":"10363833834494835208"}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}}}],"mixed":[{"attempt_index":0,"scenario":"mixed","plan_ref":2,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":1.6208333333333335e-05,"timing":{"cpu_raw":{"start_ticks":"36788195048919","stop_ticks":"36788195049308","delta_ticks":"389","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"state_a_uint64_decimal":"7557809546606715305","state_b_uint64_decimal":"12589281716502849552"}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}}}]}},"measurements":[{"measurement_id":0,"plan_ref":0,"scenario":"weights_only","loop_index":0,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":"accepted-plan-derived","elapsed_seconds":1.8875e-05,"synthetic_work_unit_latency_seconds":9.4375e-06,"synthetic_memory_work_units_per_second":105960.26490066224,"effective_model_payload_gb_s":111.1073907284768,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195050349","stop_ticks":"36788195050802","delta_ticks":"453","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-append-unchanged","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"8025476992397170658","state_b_uint64_decimal":"6807433224402555509","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"8241649774510954466","state_b_uint64_decimal":"7600066758819762805","exact_bytes_read":"0","span_count_uint64_decimal":"0"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"8025476992397170658","state_b_uint64_decimal":"6807433224402555509","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"8241649774510954466","state_b_uint64_decimal":"7600066758819762805","exact_bytes_read":"0","span_count_uint64_decimal":"0"}}],"actual_run_checksum":{"state_a_uint64_decimal":"1826493790751740180","state_b_uint64_decimal":"2326349888306066833"}}},{"measurement_id":1,"plan_ref":1,"scenario":"kv_only","loop_index":0,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"288","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"288","completion_derivation":"accepted-plan-derived","elapsed_seconds":4.041666666666666e-06,"synthetic_work_unit_latency_seconds":2.020833333333333e-06,"synthetic_memory_work_units_per_second":494845.3608247423,"effective_model_payload_gb_s":0.0712577319587629,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195051551","stop_ticks":"36788195051648","delta_ticks":"97","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"11898181619645713650","state_b_uint64_decimal":"18167339280465603252","exact_bytes_read":"52","span_count_uint64_decimal":"8"},"v":{"state_a_uint64_decimal":"14337786283661910650","state_b_uint64_decimal":"18167339327710243508","exact_bytes_read":"52","span_count_uint64_decimal":"8"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"11335208175666694355","state_b_uint64_decimal":"17960946654246414605","exact_bytes_read":"68","span_count_uint64_decimal":"8"},"v":{"state_a_uint64_decimal":"14982831505298971441","state_b_uint64_decimal":"5623828104260097821","exact_bytes_read":"68","span_count_uint64_decimal":"8"}}],"actual_run_checksum":{"state_a_uint64_decimal":"1377760989781549385","state_b_uint64_decimal":"10363833834494835208"}}},{"measurement_id":2,"plan_ref":2,"scenario":"mixed","loop_index":0,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":0.99986268975513,"kv_read_payload_fraction":0.00011442520405828057,"kv_write_payload_fraction":2.2885040811656113e-05,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097440","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097440","completion_derivation":"accepted-plan-derived","elapsed_seconds":2.020833333333333e-05,"synthetic_work_unit_latency_seconds":1.0104166666666665e-05,"synthetic_memory_work_units_per_second":98969.07216494846,"effective_model_payload_gb_s":103.79084536082475,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195052358","stop_ticks":"36788195052843","delta_ticks":"485","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"11898181619645713650","state_b_uint64_decimal":"18167339280465603252","exact_bytes_read":"52","span_count_uint64_decimal":"8"},"v":{"state_a_uint64_decimal":"14337786283661910650","state_b_uint64_decimal":"18167339327710243508","exact_bytes_read":"52","span_count_uint64_decimal":"8"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"598985949067047316","state_b_uint64_decimal":"17960946658671039404","exact_bytes_read":"68","span_count_uint64_decimal":"8"},"v":{"state_a_uint64_decimal":"4246609295879193602","state_b_uint64_decimal":"5623828109217268156","exact_bytes_read":"68","span_count_uint64_decimal":"8"}}],"actual_run_checksum":{"state_a_uint64_decimal":"7557809546606715305","state_b_uint64_decimal":"12589281716502849552"}}},{"measurement_id":3,"plan_ref":1,"scenario":"kv_only","loop_index":1,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"288","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"288","completion_derivation":"accepted-plan-derived","elapsed_seconds":6.125e-06,"synthetic_work_unit_latency_seconds":3.0625e-06,"synthetic_memory_work_units_per_second":326530.612244898,"effective_model_payload_gb_s":0.04702040816326531,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195076839","stop_ticks":"36788195076986","delta_ticks":"147","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"11898181619645713650","state_b_uint64_decimal":"18167339280465603252","exact_bytes_read":"52","span_count_uint64_decimal":"8"},"v":{"state_a_uint64_decimal":"14337786283661910650","state_b_uint64_decimal":"18167339327710243508","exact_bytes_read":"52","span_count_uint64_decimal":"8"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"11335208175666694355","state_b_uint64_decimal":"17960946654246414605","exact_bytes_read":"68","span_count_uint64_decimal":"8"},"v":{"state_a_uint64_decimal":"14982831505298971441","state_b_uint64_decimal":"5623828104260097821","exact_bytes_read":"68","span_count_uint64_decimal":"8"}}],"actual_run_checksum":{"state_a_uint64_decimal":"1377760989781549385","state_b_uint64_decimal":"10363833834494835208"}}},{"measurement_id":4,"plan_ref":2,"scenario":"mixed","loop_index":1,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":0.99986268975513,"kv_read_payload_fraction":0.00011442520405828057,"kv_write_payload_fraction":2.2885040811656113e-05,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097440","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097440","completion_derivation":"accepted-plan-derived","elapsed_seconds":1.9833333333333332e-05,"synthetic_work_unit_latency_seconds":9.916666666666666e-06,"synthetic_memory_work_units_per_second":100840.33613445378,"effective_model_payload_gb_s":105.75327731092437,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195077897","stop_ticks":"36788195078373","delta_ticks":"476","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"11898181619645713650","state_b_uint64_decimal":"18167339280465603252","exact_bytes_read":"52","span_count_uint64_decimal":"8"},"v":{"state_a_uint64_decimal":"14337786283661910650","state_b_uint64_decimal":"18167339327710243508","exact_bytes_read":"52","span_count_uint64_decimal":"8"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"598985949067047316","state_b_uint64_decimal":"17960946658671039404","exact_bytes_read":"68","span_count_uint64_decimal":"8"},"v":{"state_a_uint64_decimal":"4246609295879193602","state_b_uint64_decimal":"5623828109217268156","exact_bytes_read":"68","span_count_uint64_decimal":"8"}}],"actual_run_checksum":{"state_a_uint64_decimal":"7557809546606715305","state_b_uint64_decimal":"12589281716502849552"}}},{"measurement_id":5,"plan_ref":0,"scenario":"weights_only","loop_index":1,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":"accepted-plan-derived","elapsed_seconds":1.9166666666666667e-05,"synthetic_work_unit_latency_seconds":9.583333333333334e-06,"synthetic_memory_work_units_per_second":104347.82608695651,"effective_model_payload_gb_s":109.41662608695651,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195079117","stop_ticks":"36788195079577","delta_ticks":"460","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-append-unchanged","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"8025476992397170658","state_b_uint64_decimal":"6807433224402555509","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"8241649774510954466","state_b_uint64_decimal":"7600066758819762805","exact_bytes_read":"0","span_count_uint64_decimal":"0"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"8025476992397170658","state_b_uint64_decimal":"6807433224402555509","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"8241649774510954466","state_b_uint64_decimal":"7600066758819762805","exact_bytes_read":"0","span_count_uint64_decimal":"0"}}],"actual_run_checksum":{"state_a_uint64_decimal":"1826493790751740180","state_b_uint64_decimal":"2326349888306066833"}}},{"measurement_id":6,"plan_ref":2,"scenario":"mixed","loop_index":2,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":0.99986268975513,"kv_read_payload_fraction":0.00011442520405828057,"kv_write_payload_fraction":2.2885040811656113e-05,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097440","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097440","completion_derivation":"accepted-plan-derived","elapsed_seconds":1.6416666666666668e-05,"synthetic_work_unit_latency_seconds":8.208333333333334e-06,"synthetic_memory_work_units_per_second":121827.41116751268,"effective_model_payload_gb_s":127.7628426395939,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195103549","stop_ticks":"36788195103943","delta_ticks":"394","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"11898181619645713650","state_b_uint64_decimal":"18167339280465603252","exact_bytes_read":"52","span_count_uint64_decimal":"8"},"v":{"state_a_uint64_decimal":"14337786283661910650","state_b_uint64_decimal":"18167339327710243508","exact_bytes_read":"52","span_count_uint64_decimal":"8"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"598985949067047316","state_b_uint64_decimal":"17960946658671039404","exact_bytes_read":"68","span_count_uint64_decimal":"8"},"v":{"state_a_uint64_decimal":"4246609295879193602","state_b_uint64_decimal":"5623828109217268156","exact_bytes_read":"68","span_count_uint64_decimal":"8"}}],"actual_run_checksum":{"state_a_uint64_decimal":"7557809546606715305","state_b_uint64_decimal":"12589281716502849552"}}},{"measurement_id":7,"plan_ref":0,"scenario":"weights_only","loop_index":2,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":"accepted-plan-derived","elapsed_seconds":1.991666666666667e-05,"synthetic_work_unit_latency_seconds":9.958333333333334e-06,"synthetic_memory_work_units_per_second":100418.410041841,"effective_model_payload_gb_s":105.29633472803346,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195104630","stop_ticks":"36788195105108","delta_ticks":"478","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-append-unchanged","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"8025476992397170658","state_b_uint64_decimal":"6807433224402555509","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"8241649774510954466","state_b_uint64_decimal":"7600066758819762805","exact_bytes_read":"0","span_count_uint64_decimal":"0"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"8025476992397170658","state_b_uint64_decimal":"6807433224402555509","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"8241649774510954466","state_b_uint64_decimal":"7600066758819762805","exact_bytes_read":"0","span_count_uint64_decimal":"0"}}],"actual_run_checksum":{"state_a_uint64_decimal":"1826493790751740180","state_b_uint64_decimal":"2326349888306066833"}}},{"measurement_id":8,"plan_ref":1,"scenario":"kv_only","loop_index":2,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"288","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"288","completion_derivation":"accepted-plan-derived","elapsed_seconds":4.958333333333333e-06,"synthetic_work_unit_latency_seconds":2.4791666666666665e-06,"synthetic_memory_work_units_per_second":403361.34453781514,"effective_model_payload_gb_s":0.05808403361344538,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195105861","stop_ticks":"36788195105980","delta_ticks":"119","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"11898181619645713650","state_b_uint64_decimal":"18167339280465603252","exact_bytes_read":"52","span_count_uint64_decimal":"8"},"v":{"state_a_uint64_decimal":"14337786283661910650","state_b_uint64_decimal":"18167339327710243508","exact_bytes_read":"52","span_count_uint64_decimal":"8"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"11335208175666694355","state_b_uint64_decimal":"17960946654246414605","exact_bytes_read":"68","span_count_uint64_decimal":"8"},"v":{"state_a_uint64_decimal":"14982831505298971441","state_b_uint64_decimal":"5623828104260097821","exact_bytes_read":"68","span_count_uint64_decimal":"8"}}],"actual_run_checksum":{"state_a_uint64_decimal":"1377760989781549385","state_b_uint64_decimal":"10363833834494835208"}}}],"aggregates":{"scenarios":{"weights_only":{"scenario":"weights_only","status":"complete","observed_cv_classification":"below-threshold","accepted_measurement_ids":[0,5,7],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":9.583333333333334e-06,"statistics":{"sample_count":3,"average":9.659722222222223e-06,"min":9.4375e-06,"max":9.958333333333334e-06,"median":9.583333333333334e-06,"p90":9.883333333333334e-06,"p95":9.920833333333333e-06,"p99":9.950833333333335e-06,"stddev":2.686880850299044e-07,"coefficient_of_variation_pct":2.7815301397775865,"median_absolute_deviation":1.4583333333333293e-07}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":104347.82608695651,"statistics":{"sample_count":3,"average":103575.50034315325,"min":100418.410041841,"max":105960.26490066224,"median":104347.82608695651,"p90":105637.77713792109,"p95":105799.02101929167,"p99":105928.01612438813,"stddev":2850.5094474437556,"coefficient_of_variation_pct":2.7521078227957463,"median_absolute_deviation":1612.438813705725}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":109.41662608695651,"statistics":{"sample_count":3,"average":108.60678384782226,"min":105.29633472803346,"max":111.1073907284768,"median":109.41662608695651,"p90":110.76923780017275,"p95":110.93831426432477,"p99":111.07357543564639,"stddev":2.988975794362784,"coefficient_of_variation_pct":2.752107822795747,"median_absolute_deviation":1.690764641520289}}},"kv_only":{"scenario":"kv_only","status":"complete","observed_cv_classification":"above-threshold","accepted_measurement_ids":[1,3,8],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":2.4791666666666665e-06,"statistics":{"sample_count":3,"average":2.520833333333333e-06,"min":2.020833333333333e-06,"max":3.0625e-06,"median":2.4791666666666665e-06,"p90":2.9458333333333336e-06,"p95":3.0041666666666666e-06,"p99":3.050833333333333e-06,"stddev":5.220818369225696e-07,"coefficient_of_variation_pct":20.710684439903588,"median_absolute_deviation":4.583333333333333e-07}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":403361.34453781514,"statistics":{"sample_count":3,"average":408245.77253581845,"min":326530.612244898,"max":494845.3608247423,"median":403361.34453781514,"p90":476548.55756735685,"p95":485696.95919604955,"p99":493015.68049900373,"stddev":84263.61536882221,"coefficient_of_variation_pct":20.64041345619302,"median_absolute_deviation":76830.73229291715}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":0.05808403361344538,"statistics":{"sample_count":3,"average":0.05878739124515786,"min":0.04702040816326531,"max":0.0712577319587629,"median":0.05808403361344538,"p90":0.06862299228969938,"p95":0.06994036212423115,"p99":0.07099425799185655,"stddev":0.012133960613110398,"coefficient_of_variation_pct":20.640413456193016,"median_absolute_deviation":0.011063625450180067}}},"mixed":{"scenario":"mixed","status":"complete","observed_cv_classification":"above-threshold","accepted_measurement_ids":[2,4,6],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":9.916666666666666e-06,"statistics":{"sample_count":3,"average":9.409722222222222e-06,"min":8.208333333333334e-06,"max":1.0104166666666665e-05,"median":9.916666666666666e-06,"p90":1.0066666666666666e-05,"p95":1.0085416666666665e-05,"p99":1.0100416666666664e-05,"stddev":1.0446485099109926e-06,"coefficient_of_variation_pct":11.101799662522726,"median_absolute_deviation":1.8749999999999948e-07}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":100840.33613445378,"statistics":{"sample_count":3,"average":107212.27315563831,"min":98969.07216494846,"max":121827.41116751268,"median":100840.33613445378,"p90":117629.9961609009,"p95":119728.70366420678,"p99":121407.66966685149,"stddev":12691.61540311136,"coefficient_of_variation_pct":11.837838177992131,"median_absolute_deviation":1871.2639695053222}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":105.75327731092437,"statistics":{"sample_count":3,"average":112.435655103781,"min":103.79084536082475,"max":127.7628426395939,"median":105.75327731092437,"p90":123.36092957385999,"p95":125.56188610672694,"p99":127.3226513330205,"stddev":13.309950905550949,"coefficient_of_variation_pct":11.837838177992134,"median_absolute_deviation":1.962431950099628}}}},"traffic_diagnostics":{"classification_version":"llm-exact-weight-vs-kv-read-payload-v1","traffic_crossover_numerator":"1048576","traffic_crossover_denominator":"24","traffic_crossover_context_tokens":43690.666666666664,"current_visible_context_tokens":5,"current_weight_read_payload_bytes_per_work_unit":"1048576","current_kv_read_payload_bytes_per_work_unit":"120","current_weight_to_kv_read_payload_ratio":8738.133333333333,"current_context_classification":"weight_payload_dominant","classification_is_payload_only":true,"scenario_headlines":{"weights_only":{"synthetic_work_unit_latency_seconds":9.583333333333334e-06,"synthetic_memory_work_units_per_second":104347.82608695651,"effective_model_payload_gb_s":109.41662608695651},"kv_only":{"synthetic_work_unit_latency_seconds":2.4791666666666665e-06,"synthetic_memory_work_units_per_second":403361.34453781514,"effective_model_payload_gb_s":0.05808403361344538},"mixed":{"synthetic_work_unit_latency_seconds":9.916666666666666e-06,"synthetic_memory_work_units_per_second":100840.33613445378,"effective_model_payload_gb_s":105.75327731092437}}}},"status":"complete","reason_code":"complete","results_complete":true,"run_accepted":true,"interpretation":{"result_scope":"synthetic-memory-only-work-unit-not-real-inference-token-or-prefill-latency","reported_rate":"synthetic_memory_work_units_per_second","backend":"cpu","phase":"decode","work_unit_kind":"decode_step","transformer_math_included":false,"framework_scheduler_and_dispatch_included":false,"compute_memory_overlap_included":false,"physical_dram_traffic_measured":false,"dram_residency":"unverified","cache_residency":"unverified","fixed_context_includes_current_token_slot":true,"kv_layout":"contiguous","payload_semantics":"exact-logical-weight-plus-kv-read-plus-kv-write-bytes-divided-by-elapsed-time","layout_metadata_timed_but_excluded_from_effective_model_payload_gb_s":true,"prefill_transformer_compute_or_ttft_prediction_included":false,"private_metal_storage_implies_separate_vram":false,"cross_backend_performance_distributions_combined":false,"traffic_classification_semantics":"exact-weight-vs-kv-read-logical-payload-only-not-hardware-bottleneck","full_size_working_set_reduces_but_does_not_prove_dram_residency":true,"comparability_requires":["same-backend","same-methodology-version","same-frozen-work-plan-and-model-geometry","same-software-version","sufficiently-similar-hardware","sufficiently-similar-thermal-and-power-state"]},"diagnostic":null,"interruption_requested":false,"scenario_order_balance_complete":true,"seeds":{"base_seed_uint64_decimal":"424242","source":"user","buffer_domain_seeds":{"weight_uint64_decimal":"904846021374001452","k_uint64_decimal":"10718278915871169538","v_uint64_decimal":"2111147304547004201"},"scenario_domain_seeds":{"weights_only":"17410844953366850226","kv_only":"16590208891222127307","mixed":"1546516732201688391"}},"counters":{"planned_loops":3,"attempted_loops":3,"completed_loops":3,"planned_measurements":9,"attempted_measurements":9,"terminal_measurements":9,"measured_measurements":9,"planned_work_units":"18","completed_work_units":"18","planned_effective_model_payload_bytes":"12584640","completed_effective_model_payload_bytes":"12584640","planned_layout_metadata_lookup_count":"0","completed_layout_metadata_lookup_count":"0","planned_layout_metadata_read_bytes":"0","completed_layout_metadata_read_bytes":"0","planned_task_accounted_bytes":"12584640","completed_task_accounted_bytes":"12584640","runner_auxiliary":{"valid":true,"reason_code":"valid","measurement_record_bytes":"7056","loop_record_bytes":"144","calibration_record_bytes":"2784","calibration_identity_bytes":"66468","aggregate_value_bytes":"72","statistics_workspace_bytes":"72","warning_record_bytes":"64","fixed_metadata_bytes":"88910","retained_checksum_bytes":"3264","checksum_auxiliary_bytes":"3264","orchestration_auxiliary_bytes":"165570","total_auxiliary_bytes":"168834"}},"checkpoint_lifecycle":{"checkpoint_failed":false,"observation_point":"before-current-snapshot-preparation","prior_file_writer_attempts":3,"prior_successful_file_writes":3,"current_request":"terminal","current_persistence_success":null,"snapshot_interval_loops":1,"checkpoint_policy":"bounded-loop-snapshots","file_checkpoint_failure_is_terminal_and_not_retried":true,"stdout_intermediate_checkpoints_are_lazy":true},"loop_records":[{"loop_index":0,"planned_order":["weights_only","kv_only","mixed"],"realized_order":["weights_only","kv_only","mixed"],"realized_order_count":3,"measurement_indexes":[0,1,2]},{"loop_index":1,"planned_order":["kv_only","mixed","weights_only"],"realized_order":["kv_only","mixed","weights_only"],"realized_order_count":3,"measurement_indexes":[3,4,5]},{"loop_index":2,"planned_order":["mixed","weights_only","kv_only"],"realized_order":["mixed","weights_only","kv_only"],"realized_order_count":3,"measurement_indexes":[6,7,8]}],"environment":{"processor_name":"Apple M4","macos_version":"26.6.2","performance_core_count":4,"efficiency_core_count":6,"logical_core_count":10,"page_size_bytes":"16384","l1_data_cache_bytes":"131072","l2_data_cache_bytes":"16777216","available_memory_bytes":"11587813376","available_memory_source":"mach-free-plus-inactive-rounded-mib","main_thread_qos":{"requested":true,"applied":true,"code":0},"start":{"thermal_state":"nominal","low_power_mode_available":true,"low_power_mode_enabled":false,"physical_memory_bytes":"25769803776"},"end":{"thermal_state":"nominal","low_power_mode_available":true,"low_power_mode_enabled":false,"physical_memory_bytes":"25769803776"}},"quality_warnings":["kv_only-high-cv","mixed-high-cv","weight-working-set-cache-dominant","kv-working-set-cache-dominant","weights_only-duration-below-target-window","kv_only-duration-below-target-window","mixed-duration-below-target-window"]} diff --git a/tests/fixtures/llm-schema-v2/cpu-decode-paged.json b/tests/fixtures/llm-schema-v2/cpu-decode-paged.json new file mode 100644 index 0000000..1d41c20 --- /dev/null +++ b/tests/fixtures/llm-schema-v2/cpu-decode-paged.json @@ -0,0 +1 @@ +{"schema_version":2,"mode":"llm_memory","backend":"cpu","phase":"decode","kv_layout":"paged","methodology_version":"llm-memory-v2-cpu-decode-paged","software":{"version":"0.63.1","timestamp":"2026-09-06T07:07:07Z"},"build_manifest":{"binary_sha256":"a9c0aa68c61fccf12bda222a0485a3eba1b642a54b9b1caf8292348f608a1100","compile_flags":{"asflags":"-arch arm64","cxxflags":"-Wall -O3 -std=c++17 -arch arm64 -pthread -Isrc","test_cxxflags":"-Wall -g -std=c++17 -arch arm64 -pthread -Isrc -I/opt/homebrew/opt/googletest/include"},"compiler":"Apple clang version 21.0.0 (clang-2100.1.1.101)\nTarget: arm64-apple-darwin25.6.0\nThread model: posix\nInstalledDir: /Library/Developer/CommandLineTools/usr/bin","git_commit":"870b175927d747fb1d90fadefebe8728da38407a","git_dirty":true,"link_flags":"-pthread -framework Metal -framework Foundation","manifest_version":1,"min_os":"26.0","reason_code":"valid","sdk":"26.5","status":"available","target_arch":"arm64-apple-darwin25.6.0"},"configuration":{"backend":"cpu","phase":"decode","kv_layout":"paged","kv_block_tokens":2,"weight_size_mb":1,"layer_count":2,"query_head_count":2,"kv_head_count":1,"head_dimension":3,"kv_element_bytes":"1","visible_context_tokens":5,"prompt_tokens":null,"attention_query_tile_tokens":null,"batch_size":2,"requested_workers":2,"available_workers":10,"worker_source":"user","iterations":2,"work_policy":"explicit_fixed_work","loop_count":3,"base_seed_uint64_decimal":"424242","seed_source":"user","output_file":"plans/llm-benchmark-remediation-evidence/phase7/profiles/cpu-decode-paged.json","argv":["./memory_benchmark","--llm-memory","--llm-memory-backend","cpu","--phase","decode","--kv-layout","paged","--weight-size-mb","1","--layers","2","--query-heads","2","--kv-heads","1","--head-dim","3","--kv-element-bytes","1","--batch-size","2","--iterations","2","--count","3","--seed","424242","-o","plans/llm-benchmark-remediation-evidence/phase7/profiles/cpu-decode-paged.json","--context-tokens","5","--threads","2","--kv-block-tokens","2"],"resolved_sources":{"backend":"explicit","phase":"explicit","kv_layout":"explicit","kv_block_tokens":"explicit","visible_context_tokens":"explicit","prompt_tokens":null,"attention_query_tile_tokens":null,"workers":"explicit","iterations":"explicit","seed":"explicit"}},"resolved_plan":{"valid":true,"reason_code":"valid","plan_identity":"llm-memory-work-plan-v1|backend=cpu|phase=decode|kv_layout=paged|work_unit_kind=decode_step|methodology=llm-memory-v2-cpu-decode-paged|component_identity_size=691|component_identity=llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|permutation_version=36:splitmix64-fisher-yates-rejection-v1|backend_executor_version=38:llm-cpu-executor-v1-arm64-decode-paged|resource_abi_version=34:llm-memory-paged-descriptor-abi-v1|schedule_version=47:decode-kv-accounted-prefix-balanced-rotating-v1|timer_policy_version=62:synchronized-start-to-last-worker-completion-per-scenario-task|buffer_pattern_version=36:llm-paged-physical-buffer-pattern-v1|write_pattern_version=25:llm-kv-append-affine64-v1|checksum_pattern_version=26:llm-paged-read-checksum-v1|msl_revision=null|msl_source_sha256=null|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|decode_context=5|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=2|kv_blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|kv_block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|layout_metadata_lookups_per_layer_sequence_per_work_unit=7|k_mapping_bytes=72|v_mapping_bytes=72|kv_capacity_bytes=144|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|kv_only_payload_bytes_per_work_unit=144|mixed_payload_bytes_per_work_unit=1048720|total_data_mapping_bytes=1048720|traffic_crossover_numerator=1048576|traffic_crossover_denominator=24|requested_workers=2|effective_workers=2|layer_descriptors_per_worker=2|sequence_descriptors_per_worker=4|total_layer_descriptors=4|total_sequence_descriptors=8|descriptor_bytes=960|paged_layout_identity_size=1060|paged_layout_identity=llm-kv-layout-plan-v1|geometry_identity=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24|permutation_algorithm_version=36:splitmix64-fisher-yates-rejection-v1|permutation_domain=5497854231078520625|permutation_domain_uint64_hex=18:0x4c4c4d4b56504731|permutation_seed=1732835419980629872|permutation_entry_count=6|permutation_sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c|paged_execution_identity_size=1242|paged_execution_identity=llm-paged-cpu-execution-v1|layout_geometry_identity=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24|schedule_version=47:decode-kv-accounted-prefix-balanced-rotating-v1|worker_count=2|layer_sequence_count=4|total_owned_blocks=12|assignment_count=8|total_model_payload_bytes_per_work_unit=144|total_layout_metadata_lookup_count_per_work_unit=28|total_layout_metadata_read_bytes_per_work_unit=112|total_accounted_bytes_per_work_unit=256|minimum_worker_accounted_bytes_per_work_unit=128|maximum_worker_accounted_bytes_per_work_unit=128|worker_accounted_imbalance_bytes_per_work_unit=0|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391","backend":"cpu","phase":"decode","kv_layout":"paged","work_unit_kind":"decode_step","geometry":{"valid":true,"reason_code":"valid","phase":"decode","work_unit_kind":"decode_step","decode":{"visible_context_tokens":5},"prefill":null,"attention_kind":"mqa","active_weight_bytes_per_work_unit":"1048576","layer_count":2,"query_head_count":2,"kv_head_count":1,"query_heads_per_kv_head":2,"head_dimension":3,"kv_element_bytes":"1","batch_size":2,"kv_vector_bytes":"3","k_or_v_record_bytes_per_layer":"3","kv_record_bytes_per_layer":"6","kv_bytes_per_visible_token":"12","k_or_v_sequence_visible_bytes":"15","k_mapping_bytes":"72","v_mapping_bytes":"72","kv_capacity_bytes":"144","weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"120","kv_write_bytes_per_work_unit":"24","kv_only_effective_model_payload_bytes_per_work_unit":"144","mixed_effective_model_payload_bytes_per_work_unit":"1048720","total_data_mapping_bytes":"1048720","traffic_crossover_numerator":"1048576","traffic_crossover_denominator":"24","traffic_crossover_context_tokens":43690.666666666664},"model_context":{"prefill":null},"layout":{"kv_layout":"paged","kv_block_tokens":2,"blocks_per_sequence":"3","physical_blocks_per_layer":"6","total_physical_blocks":"12","block_bytes":"6","last_block_tokens":"1","last_block_valid_bytes":"3","decode_append_offset_in_last_block":"0","block_table_entries":"6","block_table_bytes":"24","layout_geometry_identity":"llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24","layout_identity":"llm-kv-layout-plan-v1|geometry_identity=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24|permutation_algorithm_version=36:splitmix64-fisher-yates-rejection-v1|permutation_domain=5497854231078520625|permutation_domain_uint64_hex=18:0x4c4c4d4b56504731|permutation_seed=1732835419980629872|permutation_entry_count=6|permutation_sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c","permutation_domain_uint64_hex":"0x4c4c4d4b56504731","permutation_seed_uint64_decimal":"1732835419980629872","permutation_algorithm_version":"splitmix64-fisher-yates-rejection-v1","permutation_entry_count":"6","permutation_sha256":"14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c","permutation_identity":"splitmix64-fisher-yates-rejection-v1|domain=5497854231078520625|domain_uint64_hex=18:0x4c4c4d4b56504731|resolved_seed=1732835419980629872|entry_count=6|sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c"},"resources":{"weight_logical_bytes":"1048576","k_logical_bytes":"60","v_logical_bytes":"60","k_physical_length_bytes":"72","v_physical_length_bytes":"72","k_layout_padding_bytes":"12","v_layout_padding_bytes":"12","block_table_bytes":"24"},"component_identities":{"logical_profile_version":"decode_steady_fixed_context","kv_layout_version":"paged-uint32-block-table-full-blocks-v1","permutation_version":"splitmix64-fisher-yates-rejection-v1","backend_executor_version":"llm-cpu-executor-v1-arm64-decode-paged","resource_abi_version":"llm-memory-paged-descriptor-abi-v1","schedule_version":"decode-kv-accounted-prefix-balanced-rotating-v1","timer_policy_version":"synchronized-start-to-last-worker-completion-per-scenario-task","buffer_pattern_version":"llm-paged-physical-buffer-pattern-v1","write_pattern_version":"llm-kv-append-affine64-v1","checksum_pattern_version":"llm-paged-read-checksum-v1","msl_revision":null,"msl_source_sha256":null,"identity":"llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|permutation_version=36:splitmix64-fisher-yates-rejection-v1|backend_executor_version=38:llm-cpu-executor-v1-arm64-decode-paged|resource_abi_version=34:llm-memory-paged-descriptor-abi-v1|schedule_version=47:decode-kv-accounted-prefix-balanced-rotating-v1|timer_policy_version=62:synchronized-start-to-last-worker-completion-per-scenario-task|buffer_pattern_version=36:llm-paged-physical-buffer-pattern-v1|write_pattern_version=25:llm-kv-append-affine64-v1|checksum_pattern_version=26:llm-paged-read-checksum-v1|msl_revision=null|msl_source_sha256=null","run_policy_version":"llm-run-policy-bounded-loop-snapshots-v1"},"methodology":{"backend":"cpu","phase":"decode","kv_layout":"paged","work_unit_kind":"decode_step","weight_passes_per_work_unit":1,"kv_replay_factor":1,"warmup_policy":"same-shape-excluded-steady-state-warm-memory","context_policy":"fixed-visible-context-including-current-token-slot","scenario_order_policy":"cyclic-rotation-across-count-loops","timing_policy":"synchronized-start-to-last-worker-completion-per-scenario-task","cache_policy":"regular-cacheable-no-explicit-flush-between-scenarios","calibration_policy":"per-scenario-automatic-or-explicit-frozen-before-loop-zero","calibration_target_seconds":0.15,"calibration_min_seconds":0.1,"calibration_max_seconds":0.25,"calibration_max_corrections":2,"calibration_min_pilot_accounted_bytes":"8388608","maximum_work_units_per_measurement":1000000000,"maximum_serial_range_visits_per_lane_per_task":null,"maximum_accounted_bytes_per_task":"68719476736","repeatability_cv_warning_threshold_pct":5.0,"calibration_excluded_from_results":true,"snapshot_policy":"bounded-loop-snapshots","timed_region_exclusions":["mapping-allocation","buffer-initialization-and-pre-touch","descriptor-materialization-and-validation","worker-thread-creation-and-qos","same-shape-warmup","calibration-attempts","checksum-reference-generation-and-validation","worker-join-and-json-serialization"]},"model_work_plan":{"valid":true,"reason_code":"valid","backend":"cpu","phase":"decode","kv_layout":"paged","work_unit_kind":"decode_step","weight_passes_per_work_unit":1,"kv_replay_factor":1,"requested_workers":2,"available_workers":10,"effective_workers":2,"worker_plan_count":2,"weight_layer_count":2,"layer_descriptors_per_worker":2,"sequence_descriptors_per_worker":4,"total_layer_descriptors":4,"total_sequence_descriptors":8,"descriptor_bytes":"960","planner_storage_bytes":"1360"},"scenario_plans":[{"valid":true,"reason_code":"valid","scenario":"weights_only","work_unit_kind":"decode_step","kv_write_kind":"none","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"17410844953366850226","work_units":2,"weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"0","kv_write_bytes_per_work_unit":"0","effective_model_payload_bytes_per_work_unit":"1048576","layout_metadata_lookup_count_per_work_unit":"0","layout_metadata_read_bytes_per_work_unit":"0","accounted_bytes_per_work_unit":"1048576","weight_read_bytes":"2097152","kv_read_bytes":"0","kv_write_bytes":"0","effective_model_payload_bytes":"2097152","layout_metadata_lookup_count":"0","layout_metadata_read_bytes":"0","task_accounted_bytes":"2097152","maximum_work_units_by_work_unit_cap":1000000000,"maximum_work_units_by_guardrail":65536,"effective_maximum_work_units":65536,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=4820|model_plan_identity=llm-memory-work-plan-v1|backend=cpu|phase=decode|kv_layout=paged|work_unit_kind=decode_step|methodology=llm-memory-v2-cpu-decode-paged|component_identity_size=691|component_identity=llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|permutation_version=36:splitmix64-fisher-yates-rejection-v1|backend_executor_version=38:llm-cpu-executor-v1-arm64-decode-paged|resource_abi_version=34:llm-memory-paged-descriptor-abi-v1|schedule_version=47:decode-kv-accounted-prefix-balanced-rotating-v1|timer_policy_version=62:synchronized-start-to-last-worker-completion-per-scenario-task|buffer_pattern_version=36:llm-paged-physical-buffer-pattern-v1|write_pattern_version=25:llm-kv-append-affine64-v1|checksum_pattern_version=26:llm-paged-read-checksum-v1|msl_revision=null|msl_source_sha256=null|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|decode_context=5|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=2|kv_blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|kv_block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|layout_metadata_lookups_per_layer_sequence_per_work_unit=7|k_mapping_bytes=72|v_mapping_bytes=72|kv_capacity_bytes=144|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|kv_only_payload_bytes_per_work_unit=144|mixed_payload_bytes_per_work_unit=1048720|total_data_mapping_bytes=1048720|traffic_crossover_numerator=1048576|traffic_crossover_denominator=24|requested_workers=2|effective_workers=2|layer_descriptors_per_worker=2|sequence_descriptors_per_worker=4|total_layer_descriptors=4|total_sequence_descriptors=8|descriptor_bytes=960|paged_layout_identity_size=1060|paged_layout_identity=llm-kv-layout-plan-v1|geometry_identity=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24|permutation_algorithm_version=36:splitmix64-fisher-yates-rejection-v1|permutation_domain=5497854231078520625|permutation_domain_uint64_hex=18:0x4c4c4d4b56504731|permutation_seed=1732835419980629872|permutation_entry_count=6|permutation_sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c|paged_execution_identity_size=1242|paged_execution_identity=llm-paged-cpu-execution-v1|layout_geometry_identity=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24|schedule_version=47:decode-kv-accounted-prefix-balanced-rotating-v1|worker_count=2|layer_sequence_count=4|total_owned_blocks=12|assignment_count=8|total_model_payload_bytes_per_work_unit=144|total_layout_metadata_lookup_count_per_work_unit=28|total_layout_metadata_read_bytes_per_work_unit=112|total_accounted_bytes_per_work_unit=256|minimum_worker_accounted_bytes_per_work_unit=128|maximum_worker_accounted_bytes_per_work_unit=128|worker_accounted_imbalance_bytes_per_work_unit=0|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=weights_only|work_unit_kind=decode_step|kv_write_kind=none|scenario_seed=17410844953366850226|explicit=1|work_units=2|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=0|kv_write_bytes_per_work_unit=0|effective_model_payload_bytes_per_work_unit=1048576|layout_metadata_lookup_count_per_work_unit=0|layout_metadata_read_bytes_per_work_unit=0|accounted_bytes_per_work_unit=1048576|weight_read_bytes=2097152|kv_read_bytes=0|kv_write_bytes=0|effective_model_payload_bytes=2097152|layout_metadata_lookup_count=0|layout_metadata_read_bytes=0|task_accounted_bytes=2097152|maximum_work_units_by_work_unit_cap=1000000000|maximum_work_units_by_guardrail=65536|effective_maximum_work_units=65536","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"8025476992397170658","state_b_uint64_decimal":"6807433224402555509","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"8241649774510954466","state_b_uint64_decimal":"7600066758819762805","exact_bytes_read":"0","span_count_uint64_decimal":"0"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"8025476992397170658","state_b_uint64_decimal":"6807433224402555509","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"8241649774510954466","state_b_uint64_decimal":"7600066758819762805","exact_bytes_read":"0","span_count_uint64_decimal":"0"}}],"expected_run_checksum":{"state_a_uint64_decimal":"1826493790751740180","state_b_uint64_decimal":"2326349888306066833"}}},{"valid":true,"reason_code":"valid","scenario":"kv_only","work_unit_kind":"decode_step","kv_write_kind":"current_token_append","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"16590208891222127307","work_units":2,"weight_read_bytes_per_work_unit":"0","kv_read_bytes_per_work_unit":"120","kv_write_bytes_per_work_unit":"24","effective_model_payload_bytes_per_work_unit":"144","layout_metadata_lookup_count_per_work_unit":"28","layout_metadata_read_bytes_per_work_unit":"112","accounted_bytes_per_work_unit":"256","weight_read_bytes":"0","kv_read_bytes":"240","kv_write_bytes":"48","effective_model_payload_bytes":"288","layout_metadata_lookup_count":"56","layout_metadata_read_bytes":"224","task_accounted_bytes":"512","maximum_work_units_by_work_unit_cap":1000000000,"maximum_work_units_by_guardrail":268435456,"effective_maximum_work_units":268435456,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=4820|model_plan_identity=llm-memory-work-plan-v1|backend=cpu|phase=decode|kv_layout=paged|work_unit_kind=decode_step|methodology=llm-memory-v2-cpu-decode-paged|component_identity_size=691|component_identity=llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|permutation_version=36:splitmix64-fisher-yates-rejection-v1|backend_executor_version=38:llm-cpu-executor-v1-arm64-decode-paged|resource_abi_version=34:llm-memory-paged-descriptor-abi-v1|schedule_version=47:decode-kv-accounted-prefix-balanced-rotating-v1|timer_policy_version=62:synchronized-start-to-last-worker-completion-per-scenario-task|buffer_pattern_version=36:llm-paged-physical-buffer-pattern-v1|write_pattern_version=25:llm-kv-append-affine64-v1|checksum_pattern_version=26:llm-paged-read-checksum-v1|msl_revision=null|msl_source_sha256=null|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|decode_context=5|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=2|kv_blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|kv_block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|layout_metadata_lookups_per_layer_sequence_per_work_unit=7|k_mapping_bytes=72|v_mapping_bytes=72|kv_capacity_bytes=144|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|kv_only_payload_bytes_per_work_unit=144|mixed_payload_bytes_per_work_unit=1048720|total_data_mapping_bytes=1048720|traffic_crossover_numerator=1048576|traffic_crossover_denominator=24|requested_workers=2|effective_workers=2|layer_descriptors_per_worker=2|sequence_descriptors_per_worker=4|total_layer_descriptors=4|total_sequence_descriptors=8|descriptor_bytes=960|paged_layout_identity_size=1060|paged_layout_identity=llm-kv-layout-plan-v1|geometry_identity=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24|permutation_algorithm_version=36:splitmix64-fisher-yates-rejection-v1|permutation_domain=5497854231078520625|permutation_domain_uint64_hex=18:0x4c4c4d4b56504731|permutation_seed=1732835419980629872|permutation_entry_count=6|permutation_sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c|paged_execution_identity_size=1242|paged_execution_identity=llm-paged-cpu-execution-v1|layout_geometry_identity=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24|schedule_version=47:decode-kv-accounted-prefix-balanced-rotating-v1|worker_count=2|layer_sequence_count=4|total_owned_blocks=12|assignment_count=8|total_model_payload_bytes_per_work_unit=144|total_layout_metadata_lookup_count_per_work_unit=28|total_layout_metadata_read_bytes_per_work_unit=112|total_accounted_bytes_per_work_unit=256|minimum_worker_accounted_bytes_per_work_unit=128|maximum_worker_accounted_bytes_per_work_unit=128|worker_accounted_imbalance_bytes_per_work_unit=0|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=kv_only|work_unit_kind=decode_step|kv_write_kind=current_token_append|scenario_seed=16590208891222127307|explicit=1|work_units=2|weight_read_bytes_per_work_unit=0|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|effective_model_payload_bytes_per_work_unit=144|layout_metadata_lookup_count_per_work_unit=28|layout_metadata_read_bytes_per_work_unit=112|accounted_bytes_per_work_unit=256|weight_read_bytes=0|kv_read_bytes=240|kv_write_bytes=48|effective_model_payload_bytes=288|layout_metadata_lookup_count=56|layout_metadata_read_bytes=224|task_accounted_bytes=512|maximum_work_units_by_work_unit_cap=1000000000|maximum_work_units_by_guardrail=268435456|effective_maximum_work_units=268435456","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"15394552837414810619","state_b_uint64_decimal":"2747549447208547797","exact_bytes_read":"60","span_count_uint64_decimal":"12"},"v":{"state_a_uint64_decimal":"12374392096140509556","state_b_uint64_decimal":"11678308986910037800","exact_bytes_read":"60","span_count_uint64_decimal":"12"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"11071130346649771451","state_b_uint64_decimal":"8100033999639209314","exact_bytes_read":"60","span_count_uint64_decimal":"12"},"v":{"state_a_uint64_decimal":"6410772862550282490","state_b_uint64_decimal":"1834243767302171144","exact_bytes_read":"60","span_count_uint64_decimal":"12"}}],"expected_run_checksum":{"state_a_uint64_decimal":"9522903600940082822","state_b_uint64_decimal":"1645033434810956561"}}},{"valid":true,"reason_code":"valid","scenario":"mixed","work_unit_kind":"decode_step","kv_write_kind":"current_token_append","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"1546516732201688391","work_units":2,"weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"120","kv_write_bytes_per_work_unit":"24","effective_model_payload_bytes_per_work_unit":"1048720","layout_metadata_lookup_count_per_work_unit":"28","layout_metadata_read_bytes_per_work_unit":"112","accounted_bytes_per_work_unit":"1048832","weight_read_bytes":"2097152","kv_read_bytes":"240","kv_write_bytes":"48","effective_model_payload_bytes":"2097440","layout_metadata_lookup_count":"56","layout_metadata_read_bytes":"224","task_accounted_bytes":"2097664","maximum_work_units_by_work_unit_cap":1000000000,"maximum_work_units_by_guardrail":65520,"effective_maximum_work_units":65520,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=4820|model_plan_identity=llm-memory-work-plan-v1|backend=cpu|phase=decode|kv_layout=paged|work_unit_kind=decode_step|methodology=llm-memory-v2-cpu-decode-paged|component_identity_size=691|component_identity=llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|permutation_version=36:splitmix64-fisher-yates-rejection-v1|backend_executor_version=38:llm-cpu-executor-v1-arm64-decode-paged|resource_abi_version=34:llm-memory-paged-descriptor-abi-v1|schedule_version=47:decode-kv-accounted-prefix-balanced-rotating-v1|timer_policy_version=62:synchronized-start-to-last-worker-completion-per-scenario-task|buffer_pattern_version=36:llm-paged-physical-buffer-pattern-v1|write_pattern_version=25:llm-kv-append-affine64-v1|checksum_pattern_version=26:llm-paged-read-checksum-v1|msl_revision=null|msl_source_sha256=null|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|decode_context=5|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=2|kv_blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|kv_block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|layout_metadata_lookups_per_layer_sequence_per_work_unit=7|k_mapping_bytes=72|v_mapping_bytes=72|kv_capacity_bytes=144|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|kv_only_payload_bytes_per_work_unit=144|mixed_payload_bytes_per_work_unit=1048720|total_data_mapping_bytes=1048720|traffic_crossover_numerator=1048576|traffic_crossover_denominator=24|requested_workers=2|effective_workers=2|layer_descriptors_per_worker=2|sequence_descriptors_per_worker=4|total_layer_descriptors=4|total_sequence_descriptors=8|descriptor_bytes=960|paged_layout_identity_size=1060|paged_layout_identity=llm-kv-layout-plan-v1|geometry_identity=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24|permutation_algorithm_version=36:splitmix64-fisher-yates-rejection-v1|permutation_domain=5497854231078520625|permutation_domain_uint64_hex=18:0x4c4c4d4b56504731|permutation_seed=1732835419980629872|permutation_entry_count=6|permutation_sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c|paged_execution_identity_size=1242|paged_execution_identity=llm-paged-cpu-execution-v1|layout_geometry_identity=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24|schedule_version=47:decode-kv-accounted-prefix-balanced-rotating-v1|worker_count=2|layer_sequence_count=4|total_owned_blocks=12|assignment_count=8|total_model_payload_bytes_per_work_unit=144|total_layout_metadata_lookup_count_per_work_unit=28|total_layout_metadata_read_bytes_per_work_unit=112|total_accounted_bytes_per_work_unit=256|minimum_worker_accounted_bytes_per_work_unit=128|maximum_worker_accounted_bytes_per_work_unit=128|worker_accounted_imbalance_bytes_per_work_unit=0|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=mixed|work_unit_kind=decode_step|kv_write_kind=current_token_append|scenario_seed=1546516732201688391|explicit=1|work_units=2|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|effective_model_payload_bytes_per_work_unit=1048720|layout_metadata_lookup_count_per_work_unit=28|layout_metadata_read_bytes_per_work_unit=112|accounted_bytes_per_work_unit=1048832|weight_read_bytes=2097152|kv_read_bytes=240|kv_write_bytes=48|effective_model_payload_bytes=2097440|layout_metadata_lookup_count=56|layout_metadata_read_bytes=224|task_accounted_bytes=2097664|maximum_work_units_by_work_unit_cap=1000000000|maximum_work_units_by_guardrail=65520|effective_maximum_work_units=65520","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"5856190242604209382","state_b_uint64_decimal":"2747549447208547797","exact_bytes_read":"60","span_count_uint64_decimal":"12"},"v":{"state_a_uint64_decimal":"13488150379565584630","state_b_uint64_decimal":"11678308986910037800","exact_bytes_read":"60","span_count_uint64_decimal":"12"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"5793351235053588522","state_b_uint64_decimal":"8100033999639209314","exact_bytes_read":"60","span_count_uint64_decimal":"12"},"v":{"state_a_uint64_decimal":"8779328562325579250","state_b_uint64_decimal":"1834243767302171144","exact_bytes_read":"60","span_count_uint64_decimal":"12"}}],"expected_run_checksum":{"state_a_uint64_decimal":"6740753326854537986","state_b_uint64_decimal":"4962222860565210826"}}}],"frozen_plan_refs":{"weights_only":0,"kv_only":1,"mixed":2}},"backend_evidence":{"cpu":{"requested_workers":2,"available_workers":10,"effective_workers":2,"resource_abi_version":"llm-memory-paged-descriptor-abi-v1","schedule_version":"decode-kv-accounted-prefix-balanced-rotating-v1","timer_policy_version":"synchronized-start-to-last-worker-completion-per-scenario-task","prefill":null,"paged":{"layout_identity":"llm-kv-layout-plan-v1|geometry_identity=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24|permutation_algorithm_version=36:splitmix64-fisher-yates-rejection-v1|permutation_domain=5497854231078520625|permutation_domain_uint64_hex=18:0x4c4c4d4b56504731|permutation_seed=1732835419980629872|permutation_entry_count=6|permutation_sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c","execution_identity":"llm-paged-cpu-execution-v1|layout_geometry_identity=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24|schedule_version=47:decode-kv-accounted-prefix-balanced-rotating-v1|worker_count=2|layer_sequence_count=4|total_owned_blocks=12|assignment_count=8|total_model_payload_bytes_per_work_unit=144|total_layout_metadata_lookup_count_per_work_unit=28|total_layout_metadata_read_bytes_per_work_unit=112|total_accounted_bytes_per_work_unit=256|minimum_worker_accounted_bytes_per_work_unit=128|maximum_worker_accounted_bytes_per_work_unit=128|worker_accounted_imbalance_bytes_per_work_unit=0","block_table_logical_bytes":"24","block_table_page_rounded_bytes":"16384","block_table_read_only":true,"table_validation":{"valid":true,"interrupted":false,"reason_code":"valid","expected_entries":"6","examined_entries":"6","validation_bitset_bytes":"1"},"permutation":{"algorithm_version":"splitmix64-fisher-yates-rejection-v1","domain_uint64_hex":"0x4c4c4d4b56504731","resolved_seed_uint64_decimal":"1732835419980629872","entry_count":"6","sha256":"14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c","identity":"splitmix64-fisher-yates-rejection-v1|domain=5497854231078520625|domain_uint64_hex=18:0x4c4c4d4b56504731|resolved_seed=1732835419980629872|entry_count=6|sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c"},"ownership":{"valid":true,"reason_code":"valid","worker_count":2,"layout_geometry_identity":"llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24","layer_sequence_count":"4","total_owned_blocks":"12","total_model_payload_bytes_per_work_unit":"144","total_layout_metadata_lookup_count_per_work_unit":"28","total_layout_metadata_read_bytes_per_work_unit":"112","total_accounted_bytes_per_work_unit":"256","minimum_worker_accounted_bytes_per_work_unit":"128","maximum_worker_accounted_bytes_per_work_unit":"128","worker_accounted_imbalance_bytes_per_work_unit":"0","assignment_count":"8","identity":"llm-paged-cpu-execution-v1|layout_geometry_identity=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24|schedule_version=47:decode-kv-accounted-prefix-balanced-rotating-v1|worker_count=2|layer_sequence_count=4|total_owned_blocks=12|assignment_count=8|total_model_payload_bytes_per_work_unit=144|total_layout_metadata_lookup_count_per_work_unit=28|total_layout_metadata_read_bytes_per_work_unit=112|total_accounted_bytes_per_work_unit=256|minimum_worker_accounted_bytes_per_work_unit=128|maximum_worker_accounted_bytes_per_work_unit=128|worker_accounted_imbalance_bytes_per_work_unit=0"}},"resources":{"valid":true,"reason_code":"valid","model_ref":"resolved_plan","mappings":{"policy":"private_anonymous_regular_cacheable","full_size_physical_mappings":true,"weight":{"requested_bytes":"1048576","committed_bytes":"1048576"},"k":{"requested_bytes":"72","committed_bytes":"16384"},"v":{"requested_bytes":"72","committed_bytes":"16384"},"block_table":{"requested_bytes":"24","committed_bytes":"16384"},"requested_data_bytes":"1048720","committed_data_bytes":"1081344"},"descriptors":{"abi_version":"llm-memory-paged-descriptor-abi-v1","layer_descriptors_per_worker":2,"sequence_descriptors_per_worker":4,"total_layer_descriptors":4,"total_sequence_descriptors":8,"descriptor_bytes":"960"},"executor_auxiliary":{"valid":true,"reason_code":"valid","static_reference_bytes":"672","expected_checksum_bytes":"192","actual_checksum_bytes":"192","run_checksum_bytes":"32","worker_status_bytes":"2","thread_handle_bytes":"16","checksum_auxiliary_bytes":"1088","orchestration_auxiliary_bytes":"18","total_auxiliary_bytes":"1106"},"json_output_peak_estimate":{"enabled":true,"valid":true,"reason_code":"valid","policy":"conservative-live-dom-plus-serialized-transport","fixed_schema_bytes":"2097152","input_string_bytes":"1660592","measurement_record_bytes":"1179648","worker_checksum_bytes":"1966080","total_bytes":"6903472"},"allocation_memory_budget":{"resource_rounding_bytes":"48984","transient_peak_bytes":"7208940","layout_transient_bytes":"1","setup_peak_bytes":"17745","runtime_peak_bytes":"8306668","known_owned_peak_bytes":"8306668","admitted_budget_bytes":"9270250700","valid":true,"reason_code":"within-memory-budget","request":{"valid":true,"reason_code":"valid","mapping_granularity_bytes":"16384","requested_weight_mapping_bytes":"1048576","requested_k_mapping_bytes":"72","requested_v_mapping_bytes":"72","committed_weight_mapping_bytes":"1048576","committed_k_mapping_bytes":"16384","committed_v_mapping_bytes":"16384","requested_block_table_mapping_bytes":"24","committed_block_table_mapping_bytes":"16384","requested_data_bytes":"1048720","committed_data_bytes":"1081344","layout_transient_bytes":"1","setup_peak_bytes":"17745","runtime_peak_bytes":"8306668","descriptor_bytes":"960","planner_storage_bytes":"1360","checksum_auxiliary_bytes":"4352","orchestration_auxiliary_bytes":"7202268","auxiliary_bytes":"7208940","required_total_bytes":"8306668"},"available_memory_bytes":"11587813376","allowed_memory_bytes":"9270250700","used_fallback":false},"initialization":{"complete":true,"pattern_version":"llm-paged-physical-buffer-pattern-v1","pre_touch_policy":"write-every-requested-mapping-byte-once","separate_reference_read_pass":false,"static_references_accumulated_during_initialization":true,"weight_bytes":"1048576","k_bytes":"72","v_bytes":"72","total_bytes":"1048720","non_empty_weight_spans":4,"non_empty_k_spans":12,"non_empty_v_spans":12,"block_table_logical_bytes":"24","block_table_page_rounded_bytes":"16384","block_table_read_only":true,"k_layout_padding_bytes":"12","v_layout_padding_bytes":"12"}}},"metal":null},"memory_budget":{"resource_rounding_bytes":"48984","transient_peak_bytes":"7208940","layout_transient_bytes":"1","setup_peak_bytes":"17745","runtime_peak_bytes":"8306668","known_owned_peak_bytes":"8306668","admitted_budget_bytes":"9270250700","valid":true,"reason_code":"within-memory-budget","request":{"valid":true,"reason_code":"valid","mapping_granularity_bytes":"16384","requested_weight_mapping_bytes":"1048576","requested_k_mapping_bytes":"72","requested_v_mapping_bytes":"72","committed_weight_mapping_bytes":"1048576","committed_k_mapping_bytes":"16384","committed_v_mapping_bytes":"16384","requested_block_table_mapping_bytes":"24","committed_block_table_mapping_bytes":"16384","requested_data_bytes":"1048720","committed_data_bytes":"1081344","layout_transient_bytes":"1","setup_peak_bytes":"17745","runtime_peak_bytes":"8306668","descriptor_bytes":"960","planner_storage_bytes":"1360","checksum_auxiliary_bytes":"4352","orchestration_auxiliary_bytes":"7202268","auxiliary_bytes":"7208940","required_total_bytes":"8306668"},"available_memory_bytes":"11587813376","allowed_memory_bytes":"9270250700","used_fallback":false},"calibration":{"excluded_from_results":true,"attempts":{"weights_only":[{"attempt_index":0,"scenario":"weights_only","plan_ref":0,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":1.9125e-05,"timing":{"cpu_raw":{"start_ticks":"36788195291584","stop_ticks":"36788195292043","delta_ticks":"459","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"state_a_uint64_decimal":"1826493790751740180","state_b_uint64_decimal":"2326349888306066833"}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-unchanged","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}}}],"kv_only":[{"attempt_index":0,"scenario":"kv_only","plan_ref":1,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":2.9166666666666666e-06,"timing":{"cpu_raw":{"start_ticks":"36788195292875","stop_ticks":"36788195292945","delta_ticks":"70","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"state_a_uint64_decimal":"9522903600940082822","state_b_uint64_decimal":"1645033434810956561"}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}}}],"mixed":[{"attempt_index":0,"scenario":"mixed","plan_ref":2,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":1.6166666666666665e-05,"timing":{"cpu_raw":{"start_ticks":"36788195293648","stop_ticks":"36788195294036","delta_ticks":"388","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"state_a_uint64_decimal":"6740753326854537986","state_b_uint64_decimal":"4962222860565210826"}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}}}]}},"measurements":[{"measurement_id":0,"plan_ref":0,"scenario":"weights_only","loop_index":0,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":"accepted-plan-derived","elapsed_seconds":1.6041666666666666e-05,"synthetic_work_unit_latency_seconds":8.020833333333333e-06,"synthetic_memory_work_units_per_second":124675.32467532468,"effective_model_payload_gb_s":130.73155324675324,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195294772","stop_ticks":"36788195295157","delta_ticks":"385","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-unchanged","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"8025476992397170658","state_b_uint64_decimal":"6807433224402555509","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"8241649774510954466","state_b_uint64_decimal":"7600066758819762805","exact_bytes_read":"0","span_count_uint64_decimal":"0"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"8025476992397170658","state_b_uint64_decimal":"6807433224402555509","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"8241649774510954466","state_b_uint64_decimal":"7600066758819762805","exact_bytes_read":"0","span_count_uint64_decimal":"0"}}],"actual_run_checksum":{"state_a_uint64_decimal":"1826493790751740180","state_b_uint64_decimal":"2326349888306066833"}}},{"measurement_id":1,"plan_ref":1,"scenario":"kv_only","loop_index":0,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"288","completed_layout_metadata_lookup_count":"56","completed_layout_metadata_read_bytes":"224","completed_task_accounted_bytes":"512","completion_derivation":"accepted-plan-derived","elapsed_seconds":2.6666666666666664e-06,"synthetic_work_unit_latency_seconds":1.3333333333333332e-06,"synthetic_memory_work_units_per_second":750000.0000000001,"effective_model_payload_gb_s":0.10800000000000001,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195295859","stop_ticks":"36788195295923","delta_ticks":"64","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"15394552837414810619","state_b_uint64_decimal":"2747549447208547797","exact_bytes_read":"60","span_count_uint64_decimal":"12"},"v":{"state_a_uint64_decimal":"12374392096140509556","state_b_uint64_decimal":"11678308986910037800","exact_bytes_read":"60","span_count_uint64_decimal":"12"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"11071130346649771451","state_b_uint64_decimal":"8100033999639209314","exact_bytes_read":"60","span_count_uint64_decimal":"12"},"v":{"state_a_uint64_decimal":"6410772862550282490","state_b_uint64_decimal":"1834243767302171144","exact_bytes_read":"60","span_count_uint64_decimal":"12"}}],"actual_run_checksum":{"state_a_uint64_decimal":"9522903600940082822","state_b_uint64_decimal":"1645033434810956561"}}},{"measurement_id":2,"plan_ref":2,"scenario":"mixed","loop_index":0,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":0.99986268975513,"kv_read_payload_fraction":0.00011442520405828057,"kv_write_payload_fraction":2.2885040811656113e-05,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097440","completed_layout_metadata_lookup_count":"56","completed_layout_metadata_read_bytes":"224","completed_task_accounted_bytes":"2097664","completion_derivation":"accepted-plan-derived","elapsed_seconds":1.5666666666666667e-05,"synthetic_work_unit_latency_seconds":7.833333333333333e-06,"synthetic_memory_work_units_per_second":127659.5744680851,"effective_model_payload_gb_s":133.87914893617022,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195296688","stop_ticks":"36788195297064","delta_ticks":"376","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"5856190242604209382","state_b_uint64_decimal":"2747549447208547797","exact_bytes_read":"60","span_count_uint64_decimal":"12"},"v":{"state_a_uint64_decimal":"13488150379565584630","state_b_uint64_decimal":"11678308986910037800","exact_bytes_read":"60","span_count_uint64_decimal":"12"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"5793351235053588522","state_b_uint64_decimal":"8100033999639209314","exact_bytes_read":"60","span_count_uint64_decimal":"12"},"v":{"state_a_uint64_decimal":"8779328562325579250","state_b_uint64_decimal":"1834243767302171144","exact_bytes_read":"60","span_count_uint64_decimal":"12"}}],"actual_run_checksum":{"state_a_uint64_decimal":"6740753326854537986","state_b_uint64_decimal":"4962222860565210826"}}},{"measurement_id":3,"plan_ref":1,"scenario":"kv_only","loop_index":1,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"288","completed_layout_metadata_lookup_count":"56","completed_layout_metadata_read_bytes":"224","completed_task_accounted_bytes":"512","completion_derivation":"accepted-plan-derived","elapsed_seconds":3.3333333333333333e-06,"synthetic_work_unit_latency_seconds":1.6666666666666667e-06,"synthetic_memory_work_units_per_second":600000.0,"effective_model_payload_gb_s":0.0864,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195323985","stop_ticks":"36788195324065","delta_ticks":"80","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"15394552837414810619","state_b_uint64_decimal":"2747549447208547797","exact_bytes_read":"60","span_count_uint64_decimal":"12"},"v":{"state_a_uint64_decimal":"12374392096140509556","state_b_uint64_decimal":"11678308986910037800","exact_bytes_read":"60","span_count_uint64_decimal":"12"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"11071130346649771451","state_b_uint64_decimal":"8100033999639209314","exact_bytes_read":"60","span_count_uint64_decimal":"12"},"v":{"state_a_uint64_decimal":"6410772862550282490","state_b_uint64_decimal":"1834243767302171144","exact_bytes_read":"60","span_count_uint64_decimal":"12"}}],"actual_run_checksum":{"state_a_uint64_decimal":"9522903600940082822","state_b_uint64_decimal":"1645033434810956561"}}},{"measurement_id":4,"plan_ref":2,"scenario":"mixed","loop_index":1,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":0.99986268975513,"kv_read_payload_fraction":0.00011442520405828057,"kv_write_payload_fraction":2.2885040811656113e-05,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097440","completed_layout_metadata_lookup_count":"56","completed_layout_metadata_read_bytes":"224","completed_task_accounted_bytes":"2097664","completion_derivation":"accepted-plan-derived","elapsed_seconds":1.658333333333333e-05,"synthetic_work_unit_latency_seconds":8.291666666666665e-06,"synthetic_memory_work_units_per_second":120603.0150753769,"effective_model_payload_gb_s":126.47879396984926,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195324974","stop_ticks":"36788195325372","delta_ticks":"398","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"5856190242604209382","state_b_uint64_decimal":"2747549447208547797","exact_bytes_read":"60","span_count_uint64_decimal":"12"},"v":{"state_a_uint64_decimal":"13488150379565584630","state_b_uint64_decimal":"11678308986910037800","exact_bytes_read":"60","span_count_uint64_decimal":"12"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"5793351235053588522","state_b_uint64_decimal":"8100033999639209314","exact_bytes_read":"60","span_count_uint64_decimal":"12"},"v":{"state_a_uint64_decimal":"8779328562325579250","state_b_uint64_decimal":"1834243767302171144","exact_bytes_read":"60","span_count_uint64_decimal":"12"}}],"actual_run_checksum":{"state_a_uint64_decimal":"6740753326854537986","state_b_uint64_decimal":"4962222860565210826"}}},{"measurement_id":5,"plan_ref":0,"scenario":"weights_only","loop_index":1,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":"accepted-plan-derived","elapsed_seconds":1.6083333333333332e-05,"synthetic_work_unit_latency_seconds":8.041666666666666e-06,"synthetic_memory_work_units_per_second":124352.33160621763,"effective_model_payload_gb_s":130.39287046632126,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195326042","stop_ticks":"36788195326428","delta_ticks":"386","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-unchanged","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"8025476992397170658","state_b_uint64_decimal":"6807433224402555509","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"8241649774510954466","state_b_uint64_decimal":"7600066758819762805","exact_bytes_read":"0","span_count_uint64_decimal":"0"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"8025476992397170658","state_b_uint64_decimal":"6807433224402555509","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"8241649774510954466","state_b_uint64_decimal":"7600066758819762805","exact_bytes_read":"0","span_count_uint64_decimal":"0"}}],"actual_run_checksum":{"state_a_uint64_decimal":"1826493790751740180","state_b_uint64_decimal":"2326349888306066833"}}},{"measurement_id":6,"plan_ref":2,"scenario":"mixed","loop_index":2,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":0.99986268975513,"kv_read_payload_fraction":0.00011442520405828057,"kv_write_payload_fraction":2.2885040811656113e-05,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097440","completed_layout_metadata_lookup_count":"56","completed_layout_metadata_read_bytes":"224","completed_task_accounted_bytes":"2097664","completion_derivation":"accepted-plan-derived","elapsed_seconds":2.1291666666666668e-05,"synthetic_work_unit_latency_seconds":1.0645833333333334e-05,"synthetic_memory_work_units_per_second":93933.46379647749,"effective_model_payload_gb_s":98.50990215264187,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195348914","stop_ticks":"36788195349425","delta_ticks":"511","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"5856190242604209382","state_b_uint64_decimal":"2747549447208547797","exact_bytes_read":"60","span_count_uint64_decimal":"12"},"v":{"state_a_uint64_decimal":"13488150379565584630","state_b_uint64_decimal":"11678308986910037800","exact_bytes_read":"60","span_count_uint64_decimal":"12"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"5793351235053588522","state_b_uint64_decimal":"8100033999639209314","exact_bytes_read":"60","span_count_uint64_decimal":"12"},"v":{"state_a_uint64_decimal":"8779328562325579250","state_b_uint64_decimal":"1834243767302171144","exact_bytes_read":"60","span_count_uint64_decimal":"12"}}],"actual_run_checksum":{"state_a_uint64_decimal":"6740753326854537986","state_b_uint64_decimal":"4962222860565210826"}}},{"measurement_id":7,"plan_ref":0,"scenario":"weights_only","loop_index":2,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":"accepted-plan-derived","elapsed_seconds":1.5833333333333333e-05,"synthetic_work_unit_latency_seconds":7.916666666666667e-06,"synthetic_memory_work_units_per_second":126315.78947368421,"effective_model_payload_gb_s":132.4517052631579,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195350071","stop_ticks":"36788195350451","delta_ticks":"380","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-unchanged","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"8025476992397170658","state_b_uint64_decimal":"6807433224402555509","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"8241649774510954466","state_b_uint64_decimal":"7600066758819762805","exact_bytes_read":"0","span_count_uint64_decimal":"0"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"8025476992397170658","state_b_uint64_decimal":"6807433224402555509","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"8241649774510954466","state_b_uint64_decimal":"7600066758819762805","exact_bytes_read":"0","span_count_uint64_decimal":"0"}}],"actual_run_checksum":{"state_a_uint64_decimal":"1826493790751740180","state_b_uint64_decimal":"2326349888306066833"}}},{"measurement_id":8,"plan_ref":1,"scenario":"kv_only","loop_index":2,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"288","completed_layout_metadata_lookup_count":"56","completed_layout_metadata_read_bytes":"224","completed_task_accounted_bytes":"512","completion_derivation":"accepted-plan-derived","elapsed_seconds":2.6666666666666664e-06,"synthetic_work_unit_latency_seconds":1.3333333333333332e-06,"synthetic_memory_work_units_per_second":750000.0000000001,"effective_model_payload_gb_s":0.10800000000000001,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195351104","stop_ticks":"36788195351168","delta_ticks":"64","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"15394552837414810619","state_b_uint64_decimal":"2747549447208547797","exact_bytes_read":"60","span_count_uint64_decimal":"12"},"v":{"state_a_uint64_decimal":"12374392096140509556","state_b_uint64_decimal":"11678308986910037800","exact_bytes_read":"60","span_count_uint64_decimal":"12"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"11071130346649771451","state_b_uint64_decimal":"8100033999639209314","exact_bytes_read":"60","span_count_uint64_decimal":"12"},"v":{"state_a_uint64_decimal":"6410772862550282490","state_b_uint64_decimal":"1834243767302171144","exact_bytes_read":"60","span_count_uint64_decimal":"12"}}],"actual_run_checksum":{"state_a_uint64_decimal":"9522903600940082822","state_b_uint64_decimal":"1645033434810956561"}}}],"aggregates":{"scenarios":{"weights_only":{"scenario":"weights_only","status":"complete","observed_cv_classification":"below-threshold","accepted_measurement_ids":[0,5,7],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":8.020833333333333e-06,"statistics":{"sample_count":3,"average":7.993055555555554e-06,"min":7.916666666666667e-06,"max":8.041666666666666e-06,"median":8.020833333333333e-06,"p90":8.037499999999999e-06,"p95":8.039583333333333e-06,"p99":8.041249999999999e-06,"stddev":6.696979695133979e-08,"coefficient_of_variation_pct":0.8378497620324006,"median_absolute_deviation":2.0833333333333275e-08}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":124675.32467532468,"statistics":{"sample_count":3,"average":125114.48191840884,"min":124352.33160621763,"max":126315.78947368421,"median":124675.32467532468,"p90":125987.6965140123,"p95":126151.74299384827,"p99":126282.98017771704,"stddev":1052.822878001835,"coefficient_of_variation_pct":0.8414876214636885,"median_absolute_deviation":322.99306910704763}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":130.73155324675324,"statistics":{"sample_count":3,"average":131.1920429920775,"min":130.39287046632126,"max":132.4517052631579,"median":130.73155324675324,"p90":132.10767485987697,"p95":132.2796900615174,"p99":132.4173022228298,"stddev":1.103964802123652,"coefficient_of_variation_pct":0.8414876214636882,"median_absolute_deviation":0.33868278043198075}}},"kv_only":{"scenario":"kv_only","status":"complete","observed_cv_classification":"above-threshold","accepted_measurement_ids":[1,3,8],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":1.3333333333333332e-06,"statistics":{"sample_count":3,"average":1.4444444444444443e-06,"min":1.3333333333333332e-06,"max":1.6666666666666667e-06,"median":1.3333333333333332e-06,"p90":1.6e-06,"p95":1.6333333333333333e-06,"p99":1.66e-06,"stddev":1.924500897298753e-07,"coefficient_of_variation_pct":13.32346775052983,"median_absolute_deviation":0.0}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":750000.0000000001,"statistics":{"sample_count":3,"average":700000.0,"min":600000.0,"max":750000.0000000001,"median":750000.0000000001,"p90":750000.0000000001,"p95":750000.0000000001,"p99":750000.0000000001,"stddev":86602.54037844393,"coefficient_of_variation_pct":12.371791482634846,"median_absolute_deviation":0.0}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":0.10800000000000001,"statistics":{"sample_count":3,"average":0.1008,"min":0.0864,"max":0.10800000000000001,"median":0.10800000000000001,"p90":0.10800000000000001,"p95":0.10800000000000001,"p99":0.10800000000000001,"stddev":0.01247076581449592,"coefficient_of_variation_pct":12.371791482634842,"median_absolute_deviation":0.0}}},"mixed":{"scenario":"mixed","status":"complete","observed_cv_classification":"above-threshold","accepted_measurement_ids":[2,4,6],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":8.291666666666665e-06,"statistics":{"sample_count":3,"average":8.923611111111111e-06,"min":7.833333333333333e-06,"max":1.0645833333333334e-05,"median":8.291666666666665e-06,"p90":1.0175e-05,"p95":1.0410416666666666e-05,"p99":1.0598749999999999e-05,"stddev":1.508991185576692e-06,"coefficient_of_variation_pct":16.910095776112346,"median_absolute_deviation":4.5833333333333205e-07}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":120603.0150753769,"statistics":{"sample_count":3,"average":114065.35111331317,"min":93933.46379647749,"max":127659.5744680851,"median":120603.0150753769,"p90":126248.26258954348,"p95":126953.91852881429,"p99":127518.44328023095,"stddev":17788.15400256675,"coefficient_of_variation_pct":15.594704113868808,"median_absolute_deviation":7056.559392708208}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":126.47879396984926,"statistics":{"sample_count":3,"average":119.62261501955379,"min":98.50990215264187,"max":133.87914893617022,"median":126.47879396984926,"p90":132.39907794290602,"p95":133.13911343953814,"p99":133.73114183684382,"stddev":18.654792865571807,"coefficient_of_variation_pct":15.594704113868813,"median_absolute_deviation":7.400354966320961}}}},"traffic_diagnostics":{"classification_version":"llm-exact-weight-vs-kv-read-payload-v1","traffic_crossover_numerator":"1048576","traffic_crossover_denominator":"24","traffic_crossover_context_tokens":43690.666666666664,"current_visible_context_tokens":5,"current_weight_read_payload_bytes_per_work_unit":"1048576","current_kv_read_payload_bytes_per_work_unit":"120","current_weight_to_kv_read_payload_ratio":8738.133333333333,"current_context_classification":"weight_payload_dominant","classification_is_payload_only":true,"scenario_headlines":{"weights_only":{"synthetic_work_unit_latency_seconds":8.020833333333333e-06,"synthetic_memory_work_units_per_second":124675.32467532468,"effective_model_payload_gb_s":130.73155324675324},"kv_only":{"synthetic_work_unit_latency_seconds":1.3333333333333332e-06,"synthetic_memory_work_units_per_second":750000.0000000001,"effective_model_payload_gb_s":0.10800000000000001},"mixed":{"synthetic_work_unit_latency_seconds":8.291666666666665e-06,"synthetic_memory_work_units_per_second":120603.0150753769,"effective_model_payload_gb_s":126.47879396984926}}}},"status":"complete","reason_code":"complete","results_complete":true,"run_accepted":true,"interpretation":{"result_scope":"synthetic-memory-only-work-unit-not-real-inference-token-or-prefill-latency","reported_rate":"synthetic_memory_work_units_per_second","backend":"cpu","phase":"decode","work_unit_kind":"decode_step","transformer_math_included":false,"framework_scheduler_and_dispatch_included":false,"compute_memory_overlap_included":false,"physical_dram_traffic_measured":false,"dram_residency":"unverified","cache_residency":"unverified","fixed_context_includes_current_token_slot":true,"kv_layout":"paged","payload_semantics":"exact-logical-weight-plus-kv-read-plus-kv-write-bytes-divided-by-elapsed-time","layout_metadata_timed_but_excluded_from_effective_model_payload_gb_s":true,"prefill_transformer_compute_or_ttft_prediction_included":false,"private_metal_storage_implies_separate_vram":false,"cross_backend_performance_distributions_combined":false,"traffic_classification_semantics":"exact-weight-vs-kv-read-logical-payload-only-not-hardware-bottleneck","full_size_working_set_reduces_but_does_not_prove_dram_residency":true,"comparability_requires":["same-backend","same-methodology-version","same-frozen-work-plan-and-model-geometry","same-software-version","sufficiently-similar-hardware","sufficiently-similar-thermal-and-power-state"]},"diagnostic":null,"interruption_requested":false,"scenario_order_balance_complete":true,"seeds":{"base_seed_uint64_decimal":"424242","source":"user","buffer_domain_seeds":{"weight_uint64_decimal":"904846021374001452","k_uint64_decimal":"10718278915871169538","v_uint64_decimal":"2111147304547004201"},"scenario_domain_seeds":{"weights_only":"17410844953366850226","kv_only":"16590208891222127307","mixed":"1546516732201688391"}},"counters":{"planned_loops":3,"attempted_loops":3,"completed_loops":3,"planned_measurements":9,"attempted_measurements":9,"terminal_measurements":9,"measured_measurements":9,"planned_work_units":"18","completed_work_units":"18","planned_effective_model_payload_bytes":"12584640","completed_effective_model_payload_bytes":"12584640","planned_layout_metadata_lookup_count":"336","completed_layout_metadata_lookup_count":"336","planned_layout_metadata_read_bytes":"1344","completed_layout_metadata_read_bytes":"1344","planned_task_accounted_bytes":"12585984","completed_task_accounted_bytes":"12585984","runner_auxiliary":{"valid":true,"reason_code":"valid","measurement_record_bytes":"7056","loop_record_bytes":"144","calibration_record_bytes":"2784","calibration_identity_bytes":"125652","aggregate_value_bytes":"72","statistics_workspace_bytes":"72","warning_record_bytes":"64","fixed_metadata_bytes":"162934","retained_checksum_bytes":"3264","checksum_auxiliary_bytes":"3264","orchestration_auxiliary_bytes":"298778","total_auxiliary_bytes":"302042"}},"checkpoint_lifecycle":{"checkpoint_failed":false,"observation_point":"before-current-snapshot-preparation","prior_file_writer_attempts":3,"prior_successful_file_writes":3,"current_request":"terminal","current_persistence_success":null,"snapshot_interval_loops":1,"checkpoint_policy":"bounded-loop-snapshots","file_checkpoint_failure_is_terminal_and_not_retried":true,"stdout_intermediate_checkpoints_are_lazy":true},"loop_records":[{"loop_index":0,"planned_order":["weights_only","kv_only","mixed"],"realized_order":["weights_only","kv_only","mixed"],"realized_order_count":3,"measurement_indexes":[0,1,2]},{"loop_index":1,"planned_order":["kv_only","mixed","weights_only"],"realized_order":["kv_only","mixed","weights_only"],"realized_order_count":3,"measurement_indexes":[3,4,5]},{"loop_index":2,"planned_order":["mixed","weights_only","kv_only"],"realized_order":["mixed","weights_only","kv_only"],"realized_order_count":3,"measurement_indexes":[6,7,8]}],"environment":{"processor_name":"Apple M4","macos_version":"26.6.2","performance_core_count":4,"efficiency_core_count":6,"logical_core_count":10,"page_size_bytes":"16384","l1_data_cache_bytes":"131072","l2_data_cache_bytes":"16777216","available_memory_bytes":"11587813376","available_memory_source":"mach-free-plus-inactive-rounded-mib","main_thread_qos":{"requested":true,"applied":true,"code":0},"start":{"thermal_state":"nominal","low_power_mode_available":true,"low_power_mode_enabled":false,"physical_memory_bytes":"25769803776"},"end":{"thermal_state":"nominal","low_power_mode_available":true,"low_power_mode_enabled":false,"physical_memory_bytes":"25769803776"}},"quality_warnings":["kv_only-high-cv","mixed-high-cv","weight-working-set-cache-dominant","kv-working-set-cache-dominant","weights_only-duration-below-target-window","kv_only-duration-below-target-window","mixed-duration-below-target-window"]} diff --git a/tests/fixtures/llm-schema-v2/cpu-prefill-contiguous.json b/tests/fixtures/llm-schema-v2/cpu-prefill-contiguous.json new file mode 100644 index 0000000..841790b --- /dev/null +++ b/tests/fixtures/llm-schema-v2/cpu-prefill-contiguous.json @@ -0,0 +1 @@ +{"schema_version":2,"mode":"llm_memory","backend":"cpu","phase":"prefill","kv_layout":"contiguous","methodology_version":"llm-memory-v2-cpu-prefill-contiguous","software":{"version":"0.63.1","timestamp":"2026-09-06T07:07:07Z"},"build_manifest":{"binary_sha256":"a9c0aa68c61fccf12bda222a0485a3eba1b642a54b9b1caf8292348f608a1100","compile_flags":{"asflags":"-arch arm64","cxxflags":"-Wall -O3 -std=c++17 -arch arm64 -pthread -Isrc","test_cxxflags":"-Wall -g -std=c++17 -arch arm64 -pthread -Isrc -I/opt/homebrew/opt/googletest/include"},"compiler":"Apple clang version 21.0.0 (clang-2100.1.1.101)\nTarget: arm64-apple-darwin25.6.0\nThread model: posix\nInstalledDir: /Library/Developer/CommandLineTools/usr/bin","git_commit":"870b175927d747fb1d90fadefebe8728da38407a","git_dirty":true,"link_flags":"-pthread -framework Metal -framework Foundation","manifest_version":1,"min_os":"26.0","reason_code":"valid","sdk":"26.5","status":"available","target_arch":"arm64-apple-darwin25.6.0"},"configuration":{"backend":"cpu","phase":"prefill","kv_layout":"contiguous","kv_block_tokens":null,"weight_size_mb":1,"layer_count":2,"query_head_count":2,"kv_head_count":1,"head_dimension":3,"kv_element_bytes":"1","visible_context_tokens":null,"prompt_tokens":5,"attention_query_tile_tokens":2,"batch_size":2,"requested_workers":2,"available_workers":10,"worker_source":"user","iterations":2,"work_policy":"explicit_fixed_work","loop_count":3,"base_seed_uint64_decimal":"424242","seed_source":"user","output_file":"plans/llm-benchmark-remediation-evidence/phase7/profiles/cpu-prefill-contiguous.json","argv":["./memory_benchmark","--llm-memory","--llm-memory-backend","cpu","--phase","prefill","--kv-layout","contiguous","--weight-size-mb","1","--layers","2","--query-heads","2","--kv-heads","1","--head-dim","3","--kv-element-bytes","1","--batch-size","2","--iterations","2","--count","3","--seed","424242","-o","plans/llm-benchmark-remediation-evidence/phase7/profiles/cpu-prefill-contiguous.json","--prompt-tokens","5","--attention-query-tile-tokens","2","--threads","2"],"resolved_sources":{"backend":"explicit","phase":"explicit","kv_layout":"explicit","kv_block_tokens":null,"visible_context_tokens":null,"prompt_tokens":"explicit","attention_query_tile_tokens":"explicit","workers":"explicit","iterations":"explicit","seed":"explicit"}},"resolved_plan":{"valid":true,"reason_code":"valid","plan_identity":"llm-memory-work-plan-v1|backend=cpu|phase=prefill|kv_layout=contiguous|work_unit_kind=prefill_operation|methodology=llm-memory-v2-cpu-prefill-contiguous|component_identity_size=676|component_identity=llm-memory-components-v1|logical_profile_version=31:full_prompt_causal_prefix_tiled|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=44:llm-cpu-executor-v1-arm64-prefill-contiguous|resource_abi_version=47:llm-memory-prefill-contiguous-descriptor-abi-v1|schedule_version=46:llm-prefill-owner-local-write-then-tile-k-v-v1|timer_policy_version=62:synchronized-start-to-last-worker-completion-per-scenario-task|buffer_pattern_version=21:llm-buffer-pattern-v1|write_pattern_version=26:llm-prefill-kv-affine64-v1|checksum_pattern_version=34:llm-prefill-affine64-parity-sum-v1|msl_revision=null|msl_source_sha256=null|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|prefill_planner_version=llm-prefill-planner-v1|prefill_cpu_partition_version=llm-prefill-cpu-accounted-prefix-balanced-v1|prefill_prompt_tokens=5|prefill_query_tile_tokens=2|prefill_tile_count=3|prefill_prefix_token_visits_per_sequence=11|prefill_causal_token_pairs=15|prefill_logical_attention_pairs=120|prefill_logical_attention_fma_terms=360|prefill_prefix_block_visits_per_sequence=0|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=0|kv_blocks_per_sequence=0|physical_blocks_per_layer=0|total_physical_blocks=0|kv_block_bytes=0|last_block_tokens=0|last_block_valid_bytes=0|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=0|v_layout_padding_bytes=0|block_table_entries=0|block_table_bytes=0|layout_metadata_lookups_per_layer_sequence_per_work_unit=0|k_mapping_bytes=60|v_mapping_bytes=60|kv_capacity_bytes=120|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|kv_only_payload_bytes_per_work_unit=384|mixed_payload_bytes_per_work_unit=1048960|total_data_mapping_bytes=1048696|traffic_crossover_numerator=0|traffic_crossover_denominator=0|requested_workers=2|effective_workers=2|layer_descriptors_per_worker=2|sequence_descriptors_per_worker=12|total_layer_descriptors=4|total_sequence_descriptors=24|descriptor_bytes=2112|prefill_execution_identity_size=2239|prefill_execution_identity=llm-prefill-cpu-execution-v1|sequence_descriptors_per_scenario_per_worker=4|scenario_index=0|scenario=weights_only|scope_count=2|scope_identity_size=1245|scope_identity_sha256=03ade36a2b8f2182bf053169fe8957c8128f5089848ca8aaa0c29ac106dc918f|scope_identity_size=1245|scope_identity_sha256=04cd5be22a7347524667e5415f8c0b3a2bb796f07de38d46716bfd909dffb87d|scenario_identity_size=483|scenario_identity_sha256=90914b27a2604468b5ad6946a204c0f8f930c440513984bd05b0b120548ea026|worker_count=2|worker_accounted_bytes=524288|worker_accounted_bytes=524288|minimum_worker_accounted_bytes=524288|maximum_worker_accounted_bytes=524288|worker_accounted_imbalance_bytes=0|scenario_index=1|scenario=kv_only|scope_count=4|scope_identity_size=1741|scope_identity_sha256=8c90eca75f04288ce0f56ef7f413d7a9c96a44cfc2b896280c5333351d3ffce3|scope_identity_size=1741|scope_identity_sha256=4e32ba02276fca3e29174e195611597f3365394eb823b5859bce31c31e703f86|scope_identity_size=1741|scope_identity_sha256=8c90eca75f04288ce0f56ef7f413d7a9c96a44cfc2b896280c5333351d3ffce3|scope_identity_size=1741|scope_identity_sha256=4e32ba02276fca3e29174e195611597f3365394eb823b5859bce31c31e703f86|scenario_identity_size=690|scenario_identity_sha256=a02ce00fac4f553b08b94cfb91136b14ac22d6fc7d702268cf1cdbc889a99ed1|worker_count=2|worker_accounted_bytes=192|worker_accounted_bytes=192|minimum_worker_accounted_bytes=192|maximum_worker_accounted_bytes=192|worker_accounted_imbalance_bytes=0|scenario_index=2|scenario=mixed|scope_count=4|scope_identity_size=1774|scope_identity_sha256=38fb1cb42a709c2470a5298db02a3680636c49343cb3e01ec5d0b5909ffa17e0|scope_identity_size=1739|scope_identity_sha256=e606419234c3462da7116af96861c2caa4dbec72b6cabd295778706d57f4f663|scope_identity_size=1774|scope_identity_sha256=38fb1cb42a709c2470a5298db02a3680636c49343cb3e01ec5d0b5909ffa17e0|scope_identity_size=1739|scope_identity_sha256=e606419234c3462da7116af96861c2caa4dbec72b6cabd295778706d57f4f663|scenario_identity_size=700|scenario_identity_sha256=abd76d2b793705f102d99fda27d8c18dde8f64bcf7da021281c522e585a17f69|worker_count=2|worker_accounted_bytes=524480|worker_accounted_bytes=524480|minimum_worker_accounted_bytes=524480|maximum_worker_accounted_bytes=524480|worker_accounted_imbalance_bytes=0|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391","backend":"cpu","phase":"prefill","kv_layout":"contiguous","work_unit_kind":"prefill_operation","geometry":{"valid":true,"reason_code":"valid","phase":"prefill","work_unit_kind":"prefill_operation","decode":null,"prefill":{"prompt_tokens":5,"attention_query_tile_tokens":2,"tile_count":"3","attention_prefix_token_visits_per_sequence":"11"},"attention_kind":"mqa","active_weight_bytes_per_work_unit":"1048576","layer_count":2,"query_head_count":2,"kv_head_count":1,"query_heads_per_kv_head":2,"head_dimension":3,"kv_element_bytes":"1","batch_size":2,"kv_vector_bytes":"3","k_or_v_record_bytes_per_layer":"3","kv_record_bytes_per_layer":"6","kv_bytes_per_visible_token":"12","k_or_v_sequence_visible_bytes":"15","k_mapping_bytes":"60","v_mapping_bytes":"60","kv_capacity_bytes":"120","weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"264","kv_write_bytes_per_work_unit":"120","kv_only_effective_model_payload_bytes_per_work_unit":"384","mixed_effective_model_payload_bytes_per_work_unit":"1048960","total_data_mapping_bytes":"1048696","traffic_crossover_numerator":null,"traffic_crossover_denominator":null,"traffic_crossover_context_tokens":null},"model_context":{"prefill":{"causal_token_pairs_per_sequence":"15","causal_token_pairs_per_sequence_reason_code":"valid","logical_attention_pairs":"120","logical_attention_pairs_reason_code":"valid","logical_attention_fma_terms":"360","logical_attention_fma_terms_reason_code":"valid"}},"layout":{"kv_layout":"contiguous","kv_block_tokens":null,"blocks_per_sequence":null,"physical_blocks_per_layer":null,"total_physical_blocks":null,"block_bytes":null,"last_block_tokens":null,"last_block_valid_bytes":null,"decode_append_offset_in_last_block":null,"block_table_entries":null,"block_table_bytes":null,"layout_geometry_identity":null,"layout_identity":null,"permutation_domain_uint64_hex":null,"permutation_seed_uint64_decimal":null,"permutation_algorithm_version":null,"permutation_entry_count":null,"permutation_sha256":null,"permutation_identity":null},"resources":{"weight_logical_bytes":"1048576","k_logical_bytes":"60","v_logical_bytes":"60","k_physical_length_bytes":"60","v_physical_length_bytes":"60","k_layout_padding_bytes":"0","v_layout_padding_bytes":"0","block_table_bytes":null},"component_identities":{"logical_profile_version":"full_prompt_causal_prefix_tiled","kv_layout_version":"contiguous_layer_batch_token_head_dimension","permutation_version":null,"backend_executor_version":"llm-cpu-executor-v1-arm64-prefill-contiguous","resource_abi_version":"llm-memory-prefill-contiguous-descriptor-abi-v1","schedule_version":"llm-prefill-owner-local-write-then-tile-k-v-v1","timer_policy_version":"synchronized-start-to-last-worker-completion-per-scenario-task","buffer_pattern_version":"llm-buffer-pattern-v1","write_pattern_version":"llm-prefill-kv-affine64-v1","checksum_pattern_version":"llm-prefill-affine64-parity-sum-v1","msl_revision":null,"msl_source_sha256":null,"identity":"llm-memory-components-v1|logical_profile_version=31:full_prompt_causal_prefix_tiled|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=44:llm-cpu-executor-v1-arm64-prefill-contiguous|resource_abi_version=47:llm-memory-prefill-contiguous-descriptor-abi-v1|schedule_version=46:llm-prefill-owner-local-write-then-tile-k-v-v1|timer_policy_version=62:synchronized-start-to-last-worker-completion-per-scenario-task|buffer_pattern_version=21:llm-buffer-pattern-v1|write_pattern_version=26:llm-prefill-kv-affine64-v1|checksum_pattern_version=34:llm-prefill-affine64-parity-sum-v1|msl_revision=null|msl_source_sha256=null","run_policy_version":"llm-run-policy-bounded-loop-snapshots-v1"},"methodology":{"backend":"cpu","phase":"prefill","kv_layout":"contiguous","work_unit_kind":"prefill_operation","weight_passes_per_work_unit":1,"kv_replay_factor":1,"warmup_policy":"same-shape-excluded-steady-state-warm-memory","context_policy":"full-prompt-population-with-tiled-causal-prefix-scans","scenario_order_policy":"cyclic-rotation-across-count-loops","timing_policy":"synchronized-start-to-last-worker-completion-per-scenario-task","cache_policy":"regular-cacheable-no-explicit-flush-between-scenarios","calibration_policy":"per-scenario-automatic-or-explicit-frozen-before-loop-zero","calibration_target_seconds":0.15,"calibration_min_seconds":0.1,"calibration_max_seconds":0.25,"calibration_max_corrections":2,"calibration_min_pilot_accounted_bytes":"8388608","maximum_work_units_per_measurement":1000000000,"maximum_serial_range_visits_per_lane_per_task":null,"maximum_accounted_bytes_per_task":"68719476736","repeatability_cv_warning_threshold_pct":5.0,"calibration_excluded_from_results":true,"snapshot_policy":"bounded-loop-snapshots","timed_region_exclusions":["mapping-allocation","buffer-initialization-and-pre-touch","descriptor-materialization-and-validation","worker-thread-creation-and-qos","same-shape-warmup","calibration-attempts","checksum-reference-generation-and-validation","worker-join-and-json-serialization"]},"model_work_plan":{"valid":true,"reason_code":"valid","backend":"cpu","phase":"prefill","kv_layout":"contiguous","work_unit_kind":"prefill_operation","weight_passes_per_work_unit":1,"kv_replay_factor":1,"requested_workers":2,"available_workers":10,"effective_workers":2,"worker_plan_count":2,"weight_layer_count":2,"layer_descriptors_per_worker":2,"sequence_descriptors_per_worker":12,"total_layer_descriptors":4,"total_sequence_descriptors":24,"descriptor_bytes":"2112","planner_storage_bytes":"41616"},"scenario_plans":[{"valid":true,"reason_code":"valid","scenario":"weights_only","work_unit_kind":"prefill_operation","kv_write_kind":"none","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"17410844953366850226","work_units":2,"weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"0","kv_write_bytes_per_work_unit":"0","effective_model_payload_bytes_per_work_unit":"1048576","layout_metadata_lookup_count_per_work_unit":"0","layout_metadata_read_bytes_per_work_unit":"0","accounted_bytes_per_work_unit":"1048576","weight_read_bytes":"2097152","kv_read_bytes":"0","kv_write_bytes":"0","effective_model_payload_bytes":"2097152","layout_metadata_lookup_count":"0","layout_metadata_read_bytes":"0","task_accounted_bytes":"2097152","maximum_work_units_by_work_unit_cap":1000000000,"maximum_work_units_by_guardrail":65536,"effective_maximum_work_units":65536,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=5073|model_plan_identity=llm-memory-work-plan-v1|backend=cpu|phase=prefill|kv_layout=contiguous|work_unit_kind=prefill_operation|methodology=llm-memory-v2-cpu-prefill-contiguous|component_identity_size=676|component_identity=llm-memory-components-v1|logical_profile_version=31:full_prompt_causal_prefix_tiled|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=44:llm-cpu-executor-v1-arm64-prefill-contiguous|resource_abi_version=47:llm-memory-prefill-contiguous-descriptor-abi-v1|schedule_version=46:llm-prefill-owner-local-write-then-tile-k-v-v1|timer_policy_version=62:synchronized-start-to-last-worker-completion-per-scenario-task|buffer_pattern_version=21:llm-buffer-pattern-v1|write_pattern_version=26:llm-prefill-kv-affine64-v1|checksum_pattern_version=34:llm-prefill-affine64-parity-sum-v1|msl_revision=null|msl_source_sha256=null|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|prefill_planner_version=llm-prefill-planner-v1|prefill_cpu_partition_version=llm-prefill-cpu-accounted-prefix-balanced-v1|prefill_prompt_tokens=5|prefill_query_tile_tokens=2|prefill_tile_count=3|prefill_prefix_token_visits_per_sequence=11|prefill_causal_token_pairs=15|prefill_logical_attention_pairs=120|prefill_logical_attention_fma_terms=360|prefill_prefix_block_visits_per_sequence=0|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=0|kv_blocks_per_sequence=0|physical_blocks_per_layer=0|total_physical_blocks=0|kv_block_bytes=0|last_block_tokens=0|last_block_valid_bytes=0|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=0|v_layout_padding_bytes=0|block_table_entries=0|block_table_bytes=0|layout_metadata_lookups_per_layer_sequence_per_work_unit=0|k_mapping_bytes=60|v_mapping_bytes=60|kv_capacity_bytes=120|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|kv_only_payload_bytes_per_work_unit=384|mixed_payload_bytes_per_work_unit=1048960|total_data_mapping_bytes=1048696|traffic_crossover_numerator=0|traffic_crossover_denominator=0|requested_workers=2|effective_workers=2|layer_descriptors_per_worker=2|sequence_descriptors_per_worker=12|total_layer_descriptors=4|total_sequence_descriptors=24|descriptor_bytes=2112|prefill_execution_identity_size=2239|prefill_execution_identity=llm-prefill-cpu-execution-v1|sequence_descriptors_per_scenario_per_worker=4|scenario_index=0|scenario=weights_only|scope_count=2|scope_identity_size=1245|scope_identity_sha256=03ade36a2b8f2182bf053169fe8957c8128f5089848ca8aaa0c29ac106dc918f|scope_identity_size=1245|scope_identity_sha256=04cd5be22a7347524667e5415f8c0b3a2bb796f07de38d46716bfd909dffb87d|scenario_identity_size=483|scenario_identity_sha256=90914b27a2604468b5ad6946a204c0f8f930c440513984bd05b0b120548ea026|worker_count=2|worker_accounted_bytes=524288|worker_accounted_bytes=524288|minimum_worker_accounted_bytes=524288|maximum_worker_accounted_bytes=524288|worker_accounted_imbalance_bytes=0|scenario_index=1|scenario=kv_only|scope_count=4|scope_identity_size=1741|scope_identity_sha256=8c90eca75f04288ce0f56ef7f413d7a9c96a44cfc2b896280c5333351d3ffce3|scope_identity_size=1741|scope_identity_sha256=4e32ba02276fca3e29174e195611597f3365394eb823b5859bce31c31e703f86|scope_identity_size=1741|scope_identity_sha256=8c90eca75f04288ce0f56ef7f413d7a9c96a44cfc2b896280c5333351d3ffce3|scope_identity_size=1741|scope_identity_sha256=4e32ba02276fca3e29174e195611597f3365394eb823b5859bce31c31e703f86|scenario_identity_size=690|scenario_identity_sha256=a02ce00fac4f553b08b94cfb91136b14ac22d6fc7d702268cf1cdbc889a99ed1|worker_count=2|worker_accounted_bytes=192|worker_accounted_bytes=192|minimum_worker_accounted_bytes=192|maximum_worker_accounted_bytes=192|worker_accounted_imbalance_bytes=0|scenario_index=2|scenario=mixed|scope_count=4|scope_identity_size=1774|scope_identity_sha256=38fb1cb42a709c2470a5298db02a3680636c49343cb3e01ec5d0b5909ffa17e0|scope_identity_size=1739|scope_identity_sha256=e606419234c3462da7116af96861c2caa4dbec72b6cabd295778706d57f4f663|scope_identity_size=1774|scope_identity_sha256=38fb1cb42a709c2470a5298db02a3680636c49343cb3e01ec5d0b5909ffa17e0|scope_identity_size=1739|scope_identity_sha256=e606419234c3462da7116af96861c2caa4dbec72b6cabd295778706d57f4f663|scenario_identity_size=700|scenario_identity_sha256=abd76d2b793705f102d99fda27d8c18dde8f64bcf7da021281c522e585a17f69|worker_count=2|worker_accounted_bytes=524480|worker_accounted_bytes=524480|minimum_worker_accounted_bytes=524480|maximum_worker_accounted_bytes=524480|worker_accounted_imbalance_bytes=0|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=weights_only|work_unit_kind=prefill_operation|kv_write_kind=none|scenario_seed=17410844953366850226|explicit=1|work_units=2|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=0|kv_write_bytes_per_work_unit=0|effective_model_payload_bytes_per_work_unit=1048576|layout_metadata_lookup_count_per_work_unit=0|layout_metadata_read_bytes_per_work_unit=0|accounted_bytes_per_work_unit=1048576|weight_read_bytes=2097152|kv_read_bytes=0|kv_write_bytes=0|effective_model_payload_bytes=2097152|layout_metadata_lookup_count=0|layout_metadata_read_bytes=0|task_accounted_bytes=2097152|maximum_work_units_by_work_unit_cap=1000000000|maximum_work_units_by_guardrail=65536|effective_maximum_work_units=65536","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"}}],"expected_run_checksum":{"state_a_uint64_decimal":"4693879361642038518","state_b_uint64_decimal":"15995465716993483296"}}},{"valid":true,"reason_code":"valid","scenario":"kv_only","work_unit_kind":"prefill_operation","kv_write_kind":"full_prompt_population","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"16590208891222127307","work_units":2,"weight_read_bytes_per_work_unit":"0","kv_read_bytes_per_work_unit":"264","kv_write_bytes_per_work_unit":"120","effective_model_payload_bytes_per_work_unit":"384","layout_metadata_lookup_count_per_work_unit":"0","layout_metadata_read_bytes_per_work_unit":"0","accounted_bytes_per_work_unit":"384","weight_read_bytes":"0","kv_read_bytes":"528","kv_write_bytes":"240","effective_model_payload_bytes":"768","layout_metadata_lookup_count":"0","layout_metadata_read_bytes":"0","task_accounted_bytes":"768","maximum_work_units_by_work_unit_cap":1000000000,"maximum_work_units_by_guardrail":178956970,"effective_maximum_work_units":178956970,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=5073|model_plan_identity=llm-memory-work-plan-v1|backend=cpu|phase=prefill|kv_layout=contiguous|work_unit_kind=prefill_operation|methodology=llm-memory-v2-cpu-prefill-contiguous|component_identity_size=676|component_identity=llm-memory-components-v1|logical_profile_version=31:full_prompt_causal_prefix_tiled|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=44:llm-cpu-executor-v1-arm64-prefill-contiguous|resource_abi_version=47:llm-memory-prefill-contiguous-descriptor-abi-v1|schedule_version=46:llm-prefill-owner-local-write-then-tile-k-v-v1|timer_policy_version=62:synchronized-start-to-last-worker-completion-per-scenario-task|buffer_pattern_version=21:llm-buffer-pattern-v1|write_pattern_version=26:llm-prefill-kv-affine64-v1|checksum_pattern_version=34:llm-prefill-affine64-parity-sum-v1|msl_revision=null|msl_source_sha256=null|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|prefill_planner_version=llm-prefill-planner-v1|prefill_cpu_partition_version=llm-prefill-cpu-accounted-prefix-balanced-v1|prefill_prompt_tokens=5|prefill_query_tile_tokens=2|prefill_tile_count=3|prefill_prefix_token_visits_per_sequence=11|prefill_causal_token_pairs=15|prefill_logical_attention_pairs=120|prefill_logical_attention_fma_terms=360|prefill_prefix_block_visits_per_sequence=0|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=0|kv_blocks_per_sequence=0|physical_blocks_per_layer=0|total_physical_blocks=0|kv_block_bytes=0|last_block_tokens=0|last_block_valid_bytes=0|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=0|v_layout_padding_bytes=0|block_table_entries=0|block_table_bytes=0|layout_metadata_lookups_per_layer_sequence_per_work_unit=0|k_mapping_bytes=60|v_mapping_bytes=60|kv_capacity_bytes=120|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|kv_only_payload_bytes_per_work_unit=384|mixed_payload_bytes_per_work_unit=1048960|total_data_mapping_bytes=1048696|traffic_crossover_numerator=0|traffic_crossover_denominator=0|requested_workers=2|effective_workers=2|layer_descriptors_per_worker=2|sequence_descriptors_per_worker=12|total_layer_descriptors=4|total_sequence_descriptors=24|descriptor_bytes=2112|prefill_execution_identity_size=2239|prefill_execution_identity=llm-prefill-cpu-execution-v1|sequence_descriptors_per_scenario_per_worker=4|scenario_index=0|scenario=weights_only|scope_count=2|scope_identity_size=1245|scope_identity_sha256=03ade36a2b8f2182bf053169fe8957c8128f5089848ca8aaa0c29ac106dc918f|scope_identity_size=1245|scope_identity_sha256=04cd5be22a7347524667e5415f8c0b3a2bb796f07de38d46716bfd909dffb87d|scenario_identity_size=483|scenario_identity_sha256=90914b27a2604468b5ad6946a204c0f8f930c440513984bd05b0b120548ea026|worker_count=2|worker_accounted_bytes=524288|worker_accounted_bytes=524288|minimum_worker_accounted_bytes=524288|maximum_worker_accounted_bytes=524288|worker_accounted_imbalance_bytes=0|scenario_index=1|scenario=kv_only|scope_count=4|scope_identity_size=1741|scope_identity_sha256=8c90eca75f04288ce0f56ef7f413d7a9c96a44cfc2b896280c5333351d3ffce3|scope_identity_size=1741|scope_identity_sha256=4e32ba02276fca3e29174e195611597f3365394eb823b5859bce31c31e703f86|scope_identity_size=1741|scope_identity_sha256=8c90eca75f04288ce0f56ef7f413d7a9c96a44cfc2b896280c5333351d3ffce3|scope_identity_size=1741|scope_identity_sha256=4e32ba02276fca3e29174e195611597f3365394eb823b5859bce31c31e703f86|scenario_identity_size=690|scenario_identity_sha256=a02ce00fac4f553b08b94cfb91136b14ac22d6fc7d702268cf1cdbc889a99ed1|worker_count=2|worker_accounted_bytes=192|worker_accounted_bytes=192|minimum_worker_accounted_bytes=192|maximum_worker_accounted_bytes=192|worker_accounted_imbalance_bytes=0|scenario_index=2|scenario=mixed|scope_count=4|scope_identity_size=1774|scope_identity_sha256=38fb1cb42a709c2470a5298db02a3680636c49343cb3e01ec5d0b5909ffa17e0|scope_identity_size=1739|scope_identity_sha256=e606419234c3462da7116af96861c2caa4dbec72b6cabd295778706d57f4f663|scope_identity_size=1774|scope_identity_sha256=38fb1cb42a709c2470a5298db02a3680636c49343cb3e01ec5d0b5909ffa17e0|scope_identity_size=1739|scope_identity_sha256=e606419234c3462da7116af96861c2caa4dbec72b6cabd295778706d57f4f663|scenario_identity_size=700|scenario_identity_sha256=abd76d2b793705f102d99fda27d8c18dde8f64bcf7da021281c522e585a17f69|worker_count=2|worker_accounted_bytes=524480|worker_accounted_bytes=524480|minimum_worker_accounted_bytes=524480|maximum_worker_accounted_bytes=524480|worker_accounted_imbalance_bytes=0|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=kv_only|work_unit_kind=prefill_operation|kv_write_kind=full_prompt_population|scenario_seed=16590208891222127307|explicit=1|work_units=2|weight_read_bytes_per_work_unit=0|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|effective_model_payload_bytes_per_work_unit=384|layout_metadata_lookup_count_per_work_unit=0|layout_metadata_read_bytes_per_work_unit=0|accounted_bytes_per_work_unit=384|weight_read_bytes=0|kv_read_bytes=528|kv_write_bytes=240|effective_model_payload_bytes=768|layout_metadata_lookup_count=0|layout_metadata_read_bytes=0|task_accounted_bytes=768|maximum_work_units_by_work_unit_cap=1000000000|maximum_work_units_by_guardrail=178956970|effective_maximum_work_units=178956970","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"8492071466883487160","state_b_uint64_decimal":"112617639640686528","exact_bytes_read":"132","span_count_uint64_decimal":"20"},"v":{"state_a_uint64_decimal":"14858055244865509948","state_b_uint64_decimal":"125051107588323352","exact_bytes_read":"132","span_count_uint64_decimal":"20"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"14940791572350823612","state_b_uint64_decimal":"94278968524492904","exact_bytes_read":"132","span_count_uint64_decimal":"20"},"v":{"state_a_uint64_decimal":"2860031276623294784","state_b_uint64_decimal":"106712436472129728","exact_bytes_read":"132","span_count_uint64_decimal":"20"}}],"expected_run_checksum":{"state_a_uint64_decimal":"13401016718287388321","state_b_uint64_decimal":"9725993772142933009"}}},{"valid":true,"reason_code":"valid","scenario":"mixed","work_unit_kind":"prefill_operation","kv_write_kind":"full_prompt_population","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"1546516732201688391","work_units":2,"weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"264","kv_write_bytes_per_work_unit":"120","effective_model_payload_bytes_per_work_unit":"1048960","layout_metadata_lookup_count_per_work_unit":"0","layout_metadata_read_bytes_per_work_unit":"0","accounted_bytes_per_work_unit":"1048960","weight_read_bytes":"2097152","kv_read_bytes":"528","kv_write_bytes":"240","effective_model_payload_bytes":"2097920","layout_metadata_lookup_count":"0","layout_metadata_read_bytes":"0","task_accounted_bytes":"2097920","maximum_work_units_by_work_unit_cap":1000000000,"maximum_work_units_by_guardrail":65512,"effective_maximum_work_units":65512,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=5073|model_plan_identity=llm-memory-work-plan-v1|backend=cpu|phase=prefill|kv_layout=contiguous|work_unit_kind=prefill_operation|methodology=llm-memory-v2-cpu-prefill-contiguous|component_identity_size=676|component_identity=llm-memory-components-v1|logical_profile_version=31:full_prompt_causal_prefix_tiled|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=44:llm-cpu-executor-v1-arm64-prefill-contiguous|resource_abi_version=47:llm-memory-prefill-contiguous-descriptor-abi-v1|schedule_version=46:llm-prefill-owner-local-write-then-tile-k-v-v1|timer_policy_version=62:synchronized-start-to-last-worker-completion-per-scenario-task|buffer_pattern_version=21:llm-buffer-pattern-v1|write_pattern_version=26:llm-prefill-kv-affine64-v1|checksum_pattern_version=34:llm-prefill-affine64-parity-sum-v1|msl_revision=null|msl_source_sha256=null|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|prefill_planner_version=llm-prefill-planner-v1|prefill_cpu_partition_version=llm-prefill-cpu-accounted-prefix-balanced-v1|prefill_prompt_tokens=5|prefill_query_tile_tokens=2|prefill_tile_count=3|prefill_prefix_token_visits_per_sequence=11|prefill_causal_token_pairs=15|prefill_logical_attention_pairs=120|prefill_logical_attention_fma_terms=360|prefill_prefix_block_visits_per_sequence=0|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=0|kv_blocks_per_sequence=0|physical_blocks_per_layer=0|total_physical_blocks=0|kv_block_bytes=0|last_block_tokens=0|last_block_valid_bytes=0|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=0|v_layout_padding_bytes=0|block_table_entries=0|block_table_bytes=0|layout_metadata_lookups_per_layer_sequence_per_work_unit=0|k_mapping_bytes=60|v_mapping_bytes=60|kv_capacity_bytes=120|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|kv_only_payload_bytes_per_work_unit=384|mixed_payload_bytes_per_work_unit=1048960|total_data_mapping_bytes=1048696|traffic_crossover_numerator=0|traffic_crossover_denominator=0|requested_workers=2|effective_workers=2|layer_descriptors_per_worker=2|sequence_descriptors_per_worker=12|total_layer_descriptors=4|total_sequence_descriptors=24|descriptor_bytes=2112|prefill_execution_identity_size=2239|prefill_execution_identity=llm-prefill-cpu-execution-v1|sequence_descriptors_per_scenario_per_worker=4|scenario_index=0|scenario=weights_only|scope_count=2|scope_identity_size=1245|scope_identity_sha256=03ade36a2b8f2182bf053169fe8957c8128f5089848ca8aaa0c29ac106dc918f|scope_identity_size=1245|scope_identity_sha256=04cd5be22a7347524667e5415f8c0b3a2bb796f07de38d46716bfd909dffb87d|scenario_identity_size=483|scenario_identity_sha256=90914b27a2604468b5ad6946a204c0f8f930c440513984bd05b0b120548ea026|worker_count=2|worker_accounted_bytes=524288|worker_accounted_bytes=524288|minimum_worker_accounted_bytes=524288|maximum_worker_accounted_bytes=524288|worker_accounted_imbalance_bytes=0|scenario_index=1|scenario=kv_only|scope_count=4|scope_identity_size=1741|scope_identity_sha256=8c90eca75f04288ce0f56ef7f413d7a9c96a44cfc2b896280c5333351d3ffce3|scope_identity_size=1741|scope_identity_sha256=4e32ba02276fca3e29174e195611597f3365394eb823b5859bce31c31e703f86|scope_identity_size=1741|scope_identity_sha256=8c90eca75f04288ce0f56ef7f413d7a9c96a44cfc2b896280c5333351d3ffce3|scope_identity_size=1741|scope_identity_sha256=4e32ba02276fca3e29174e195611597f3365394eb823b5859bce31c31e703f86|scenario_identity_size=690|scenario_identity_sha256=a02ce00fac4f553b08b94cfb91136b14ac22d6fc7d702268cf1cdbc889a99ed1|worker_count=2|worker_accounted_bytes=192|worker_accounted_bytes=192|minimum_worker_accounted_bytes=192|maximum_worker_accounted_bytes=192|worker_accounted_imbalance_bytes=0|scenario_index=2|scenario=mixed|scope_count=4|scope_identity_size=1774|scope_identity_sha256=38fb1cb42a709c2470a5298db02a3680636c49343cb3e01ec5d0b5909ffa17e0|scope_identity_size=1739|scope_identity_sha256=e606419234c3462da7116af96861c2caa4dbec72b6cabd295778706d57f4f663|scope_identity_size=1774|scope_identity_sha256=38fb1cb42a709c2470a5298db02a3680636c49343cb3e01ec5d0b5909ffa17e0|scope_identity_size=1739|scope_identity_sha256=e606419234c3462da7116af96861c2caa4dbec72b6cabd295778706d57f4f663|scenario_identity_size=700|scenario_identity_sha256=abd76d2b793705f102d99fda27d8c18dde8f64bcf7da021281c522e585a17f69|worker_count=2|worker_accounted_bytes=524480|worker_accounted_bytes=524480|minimum_worker_accounted_bytes=524480|maximum_worker_accounted_bytes=524480|worker_accounted_imbalance_bytes=0|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=mixed|work_unit_kind=prefill_operation|kv_write_kind=full_prompt_population|scenario_seed=1546516732201688391|explicit=1|work_units=2|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|effective_model_payload_bytes_per_work_unit=1048960|layout_metadata_lookup_count_per_work_unit=0|layout_metadata_read_bytes_per_work_unit=0|accounted_bytes_per_work_unit=1048960|weight_read_bytes=2097152|kv_read_bytes=528|kv_write_bytes=240|effective_model_payload_bytes=2097920|layout_metadata_lookup_count=0|layout_metadata_read_bytes=0|task_accounted_bytes=2097920|maximum_work_units_by_work_unit_cap=1000000000|maximum_work_units_by_guardrail=65512|effective_maximum_work_units=65512","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"17269820495715963784","state_b_uint64_decimal":"105940028400153504","exact_bytes_read":"132","span_count_uint64_decimal":"20"},"v":{"state_a_uint64_decimal":"5189623149941856268","state_b_uint64_decimal":"118373496347790328","exact_bytes_read":"132","span_count_uint64_decimal":"20"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"5271796527473748620","state_b_uint64_decimal":"159658951321887816","exact_bytes_read":"132","span_count_uint64_decimal":"20"},"v":{"state_a_uint64_decimal":"11636935880525639440","state_b_uint64_decimal":"172092419269524640","exact_bytes_read":"132","span_count_uint64_decimal":"20"}}],"expected_run_checksum":{"state_a_uint64_decimal":"4668412853063398514","state_b_uint64_decimal":"10985833669298179668"}}}],"frozen_plan_refs":{"weights_only":0,"kv_only":1,"mixed":2}},"backend_evidence":{"cpu":{"requested_workers":2,"available_workers":10,"effective_workers":2,"resource_abi_version":"llm-memory-prefill-contiguous-descriptor-abi-v1","schedule_version":"llm-prefill-owner-local-write-then-tile-k-v-v1","timer_policy_version":"synchronized-start-to-last-worker-completion-per-scenario-task","prefill":{"cost_unit":"worker-cost","sequence_descriptors_per_scenario_per_worker":4,"scenarios":[{"scenario":"weights_only","cost_unit":"worker-cost","scope_count":"2","scope_identities":["llm-prefill-cpu-accounted-prefix-balanced-v1|planner_version=llm-prefill-planner-v1|schedule_version=llm-prefill-owner-local-write-then-tile-k-v-v1|unit_kind=contiguous_token|scenario=weights_only|active_weight_bytes=1048576|prompt_tokens=5|query_tile_tokens=2|tile_count=3|layer_count=2|batch_size=2|query_head_count=2|head_dimension=3|k_or_v_record_bytes_per_layer=3|paged=0|kv_block_tokens=0|blocks_per_sequence=0|worker_count=2|worker_rotation=0|logical_unit_count=0|active_worker_count=0|weight_shards_included=1|total_weight_shard_bytes=524288|total_kv_model_payload_bytes=0|total_layout_metadata_lookup_count=0|total_layout_metadata_read_bytes=0|total_kv_accounted_bytes=0|total_scenario_accounted_bytes=524288|minimum_worker_accounted_bytes=262144|maximum_worker_accounted_bytes=262144|worker_accounted_imbalance_bytes=0|worker_vector_count=2|worker_0_weight_shard_bytes=262144|worker_0_kv_model_payload_bytes=0|worker_0_layout_metadata_lookup_count=0|worker_0_layout_metadata_read_bytes=0|worker_0_scenario_accounted_bytes=262144|worker_1_weight_shard_bytes=262144|worker_1_kv_model_payload_bytes=0|worker_1_layout_metadata_lookup_count=0|worker_1_layout_metadata_read_bytes=0|worker_1_scenario_accounted_bytes=262144|assignment_count=0","llm-prefill-cpu-accounted-prefix-balanced-v1|planner_version=llm-prefill-planner-v1|schedule_version=llm-prefill-owner-local-write-then-tile-k-v-v1|unit_kind=contiguous_token|scenario=weights_only|active_weight_bytes=1048576|prompt_tokens=5|query_tile_tokens=2|tile_count=3|layer_count=2|batch_size=2|query_head_count=2|head_dimension=3|k_or_v_record_bytes_per_layer=3|paged=0|kv_block_tokens=0|blocks_per_sequence=0|worker_count=2|worker_rotation=1|logical_unit_count=0|active_worker_count=0|weight_shards_included=1|total_weight_shard_bytes=524288|total_kv_model_payload_bytes=0|total_layout_metadata_lookup_count=0|total_layout_metadata_read_bytes=0|total_kv_accounted_bytes=0|total_scenario_accounted_bytes=524288|minimum_worker_accounted_bytes=262144|maximum_worker_accounted_bytes=262144|worker_accounted_imbalance_bytes=0|worker_vector_count=2|worker_0_weight_shard_bytes=262144|worker_0_kv_model_payload_bytes=0|worker_0_layout_metadata_lookup_count=0|worker_0_layout_metadata_read_bytes=0|worker_0_scenario_accounted_bytes=262144|worker_1_weight_shard_bytes=262144|worker_1_kv_model_payload_bytes=0|worker_1_layout_metadata_lookup_count=0|worker_1_layout_metadata_read_bytes=0|worker_1_scenario_accounted_bytes=262144|assignment_count=0"],"worker_accounted_bytes_per_work_unit":["524288","524288"],"minimum_worker_accounted_bytes_per_work_unit":"524288","maximum_worker_accounted_bytes_per_work_unit":"524288","worker_accounted_imbalance_bytes_per_work_unit":"0","identity":"llm-prefill-cpu-scenario-execution-v1|scenario=weights_only|scope_count=2|scope_identity_size=1245|scope_identity_sha256=03ade36a2b8f2182bf053169fe8957c8128f5089848ca8aaa0c29ac106dc918f|scope_identity_size=1245|scope_identity_sha256=04cd5be22a7347524667e5415f8c0b3a2bb796f07de38d46716bfd909dffb87d|worker_count=2|worker_accounted_bytes=524288|worker_accounted_bytes=524288|minimum_worker_accounted_bytes=524288|maximum_worker_accounted_bytes=524288|worker_accounted_imbalance_bytes=0"},{"scenario":"kv_only","cost_unit":"worker-cost","scope_count":"4","scope_identities":["llm-prefill-cpu-accounted-prefix-balanced-v1|planner_version=llm-prefill-planner-v1|schedule_version=llm-prefill-owner-local-write-then-tile-k-v-v1|unit_kind=contiguous_token|scenario=kv_only|active_weight_bytes=1048576|prompt_tokens=5|query_tile_tokens=2|tile_count=3|layer_count=2|batch_size=2|query_head_count=2|head_dimension=3|k_or_v_record_bytes_per_layer=3|paged=0|kv_block_tokens=0|blocks_per_sequence=0|worker_count=2|worker_rotation=0|logical_unit_count=5|active_worker_count=2|weight_shards_included=0|total_weight_shard_bytes=0|total_kv_model_payload_bytes=96|total_layout_metadata_lookup_count=0|total_layout_metadata_read_bytes=0|total_kv_accounted_bytes=96|total_scenario_accounted_bytes=96|minimum_worker_accounted_bytes=48|maximum_worker_accounted_bytes=48|worker_accounted_imbalance_bytes=0|worker_vector_count=2|worker_0_weight_shard_bytes=0|worker_0_kv_model_payload_bytes=48|worker_0_layout_metadata_lookup_count=0|worker_0_layout_metadata_read_bytes=0|worker_0_scenario_accounted_bytes=48|worker_1_weight_shard_bytes=0|worker_1_kv_model_payload_bytes=48|worker_1_layout_metadata_lookup_count=0|worker_1_layout_metadata_read_bytes=0|worker_1_scenario_accounted_bytes=48|assignment_count=2|assignment_0_range_rank=0|assignment_0_worker_index=0|assignment_0_first_unit=0|assignment_0_unit_count=2|assignment_0_kv_model_payload_bytes=48|assignment_0_layout_metadata_lookup_count=0|assignment_0_layout_metadata_read_bytes=0|assignment_0_kv_accounted_bytes=48|assignment_1_range_rank=1|assignment_1_worker_index=1|assignment_1_first_unit=2|assignment_1_unit_count=3|assignment_1_kv_model_payload_bytes=48|assignment_1_layout_metadata_lookup_count=0|assignment_1_layout_metadata_read_bytes=0|assignment_1_kv_accounted_bytes=48","llm-prefill-cpu-accounted-prefix-balanced-v1|planner_version=llm-prefill-planner-v1|schedule_version=llm-prefill-owner-local-write-then-tile-k-v-v1|unit_kind=contiguous_token|scenario=kv_only|active_weight_bytes=1048576|prompt_tokens=5|query_tile_tokens=2|tile_count=3|layer_count=2|batch_size=2|query_head_count=2|head_dimension=3|k_or_v_record_bytes_per_layer=3|paged=0|kv_block_tokens=0|blocks_per_sequence=0|worker_count=2|worker_rotation=1|logical_unit_count=5|active_worker_count=2|weight_shards_included=0|total_weight_shard_bytes=0|total_kv_model_payload_bytes=96|total_layout_metadata_lookup_count=0|total_layout_metadata_read_bytes=0|total_kv_accounted_bytes=96|total_scenario_accounted_bytes=96|minimum_worker_accounted_bytes=48|maximum_worker_accounted_bytes=48|worker_accounted_imbalance_bytes=0|worker_vector_count=2|worker_0_weight_shard_bytes=0|worker_0_kv_model_payload_bytes=48|worker_0_layout_metadata_lookup_count=0|worker_0_layout_metadata_read_bytes=0|worker_0_scenario_accounted_bytes=48|worker_1_weight_shard_bytes=0|worker_1_kv_model_payload_bytes=48|worker_1_layout_metadata_lookup_count=0|worker_1_layout_metadata_read_bytes=0|worker_1_scenario_accounted_bytes=48|assignment_count=2|assignment_0_range_rank=0|assignment_0_worker_index=1|assignment_0_first_unit=0|assignment_0_unit_count=2|assignment_0_kv_model_payload_bytes=48|assignment_0_layout_metadata_lookup_count=0|assignment_0_layout_metadata_read_bytes=0|assignment_0_kv_accounted_bytes=48|assignment_1_range_rank=1|assignment_1_worker_index=0|assignment_1_first_unit=2|assignment_1_unit_count=3|assignment_1_kv_model_payload_bytes=48|assignment_1_layout_metadata_lookup_count=0|assignment_1_layout_metadata_read_bytes=0|assignment_1_kv_accounted_bytes=48","llm-prefill-cpu-accounted-prefix-balanced-v1|planner_version=llm-prefill-planner-v1|schedule_version=llm-prefill-owner-local-write-then-tile-k-v-v1|unit_kind=contiguous_token|scenario=kv_only|active_weight_bytes=1048576|prompt_tokens=5|query_tile_tokens=2|tile_count=3|layer_count=2|batch_size=2|query_head_count=2|head_dimension=3|k_or_v_record_bytes_per_layer=3|paged=0|kv_block_tokens=0|blocks_per_sequence=0|worker_count=2|worker_rotation=0|logical_unit_count=5|active_worker_count=2|weight_shards_included=0|total_weight_shard_bytes=0|total_kv_model_payload_bytes=96|total_layout_metadata_lookup_count=0|total_layout_metadata_read_bytes=0|total_kv_accounted_bytes=96|total_scenario_accounted_bytes=96|minimum_worker_accounted_bytes=48|maximum_worker_accounted_bytes=48|worker_accounted_imbalance_bytes=0|worker_vector_count=2|worker_0_weight_shard_bytes=0|worker_0_kv_model_payload_bytes=48|worker_0_layout_metadata_lookup_count=0|worker_0_layout_metadata_read_bytes=0|worker_0_scenario_accounted_bytes=48|worker_1_weight_shard_bytes=0|worker_1_kv_model_payload_bytes=48|worker_1_layout_metadata_lookup_count=0|worker_1_layout_metadata_read_bytes=0|worker_1_scenario_accounted_bytes=48|assignment_count=2|assignment_0_range_rank=0|assignment_0_worker_index=0|assignment_0_first_unit=0|assignment_0_unit_count=2|assignment_0_kv_model_payload_bytes=48|assignment_0_layout_metadata_lookup_count=0|assignment_0_layout_metadata_read_bytes=0|assignment_0_kv_accounted_bytes=48|assignment_1_range_rank=1|assignment_1_worker_index=1|assignment_1_first_unit=2|assignment_1_unit_count=3|assignment_1_kv_model_payload_bytes=48|assignment_1_layout_metadata_lookup_count=0|assignment_1_layout_metadata_read_bytes=0|assignment_1_kv_accounted_bytes=48","llm-prefill-cpu-accounted-prefix-balanced-v1|planner_version=llm-prefill-planner-v1|schedule_version=llm-prefill-owner-local-write-then-tile-k-v-v1|unit_kind=contiguous_token|scenario=kv_only|active_weight_bytes=1048576|prompt_tokens=5|query_tile_tokens=2|tile_count=3|layer_count=2|batch_size=2|query_head_count=2|head_dimension=3|k_or_v_record_bytes_per_layer=3|paged=0|kv_block_tokens=0|blocks_per_sequence=0|worker_count=2|worker_rotation=1|logical_unit_count=5|active_worker_count=2|weight_shards_included=0|total_weight_shard_bytes=0|total_kv_model_payload_bytes=96|total_layout_metadata_lookup_count=0|total_layout_metadata_read_bytes=0|total_kv_accounted_bytes=96|total_scenario_accounted_bytes=96|minimum_worker_accounted_bytes=48|maximum_worker_accounted_bytes=48|worker_accounted_imbalance_bytes=0|worker_vector_count=2|worker_0_weight_shard_bytes=0|worker_0_kv_model_payload_bytes=48|worker_0_layout_metadata_lookup_count=0|worker_0_layout_metadata_read_bytes=0|worker_0_scenario_accounted_bytes=48|worker_1_weight_shard_bytes=0|worker_1_kv_model_payload_bytes=48|worker_1_layout_metadata_lookup_count=0|worker_1_layout_metadata_read_bytes=0|worker_1_scenario_accounted_bytes=48|assignment_count=2|assignment_0_range_rank=0|assignment_0_worker_index=1|assignment_0_first_unit=0|assignment_0_unit_count=2|assignment_0_kv_model_payload_bytes=48|assignment_0_layout_metadata_lookup_count=0|assignment_0_layout_metadata_read_bytes=0|assignment_0_kv_accounted_bytes=48|assignment_1_range_rank=1|assignment_1_worker_index=0|assignment_1_first_unit=2|assignment_1_unit_count=3|assignment_1_kv_model_payload_bytes=48|assignment_1_layout_metadata_lookup_count=0|assignment_1_layout_metadata_read_bytes=0|assignment_1_kv_accounted_bytes=48"],"worker_accounted_bytes_per_work_unit":["192","192"],"minimum_worker_accounted_bytes_per_work_unit":"192","maximum_worker_accounted_bytes_per_work_unit":"192","worker_accounted_imbalance_bytes_per_work_unit":"0","identity":"llm-prefill-cpu-scenario-execution-v1|scenario=kv_only|scope_count=4|scope_identity_size=1741|scope_identity_sha256=8c90eca75f04288ce0f56ef7f413d7a9c96a44cfc2b896280c5333351d3ffce3|scope_identity_size=1741|scope_identity_sha256=4e32ba02276fca3e29174e195611597f3365394eb823b5859bce31c31e703f86|scope_identity_size=1741|scope_identity_sha256=8c90eca75f04288ce0f56ef7f413d7a9c96a44cfc2b896280c5333351d3ffce3|scope_identity_size=1741|scope_identity_sha256=4e32ba02276fca3e29174e195611597f3365394eb823b5859bce31c31e703f86|worker_count=2|worker_accounted_bytes=192|worker_accounted_bytes=192|minimum_worker_accounted_bytes=192|maximum_worker_accounted_bytes=192|worker_accounted_imbalance_bytes=0"},{"scenario":"mixed","cost_unit":"worker-cost","scope_count":"4","scope_identities":["llm-prefill-cpu-accounted-prefix-balanced-v1|planner_version=llm-prefill-planner-v1|schedule_version=llm-prefill-owner-local-write-then-tile-k-v-v1|unit_kind=contiguous_token|scenario=mixed|active_weight_bytes=1048576|prompt_tokens=5|query_tile_tokens=2|tile_count=3|layer_count=2|batch_size=2|query_head_count=2|head_dimension=3|k_or_v_record_bytes_per_layer=3|paged=0|kv_block_tokens=0|blocks_per_sequence=0|worker_count=2|worker_rotation=0|logical_unit_count=5|active_worker_count=2|weight_shards_included=1|total_weight_shard_bytes=524288|total_kv_model_payload_bytes=96|total_layout_metadata_lookup_count=0|total_layout_metadata_read_bytes=0|total_kv_accounted_bytes=96|total_scenario_accounted_bytes=524384|minimum_worker_accounted_bytes=262192|maximum_worker_accounted_bytes=262192|worker_accounted_imbalance_bytes=0|worker_vector_count=2|worker_0_weight_shard_bytes=262144|worker_0_kv_model_payload_bytes=48|worker_0_layout_metadata_lookup_count=0|worker_0_layout_metadata_read_bytes=0|worker_0_scenario_accounted_bytes=262192|worker_1_weight_shard_bytes=262144|worker_1_kv_model_payload_bytes=48|worker_1_layout_metadata_lookup_count=0|worker_1_layout_metadata_read_bytes=0|worker_1_scenario_accounted_bytes=262192|assignment_count=2|assignment_0_range_rank=0|assignment_0_worker_index=0|assignment_0_first_unit=0|assignment_0_unit_count=2|assignment_0_kv_model_payload_bytes=48|assignment_0_layout_metadata_lookup_count=0|assignment_0_layout_metadata_read_bytes=0|assignment_0_kv_accounted_bytes=48|assignment_1_range_rank=1|assignment_1_worker_index=1|assignment_1_first_unit=2|assignment_1_unit_count=3|assignment_1_kv_model_payload_bytes=48|assignment_1_layout_metadata_lookup_count=0|assignment_1_layout_metadata_read_bytes=0|assignment_1_kv_accounted_bytes=48","llm-prefill-cpu-accounted-prefix-balanced-v1|planner_version=llm-prefill-planner-v1|schedule_version=llm-prefill-owner-local-write-then-tile-k-v-v1|unit_kind=contiguous_token|scenario=mixed|active_weight_bytes=1048576|prompt_tokens=5|query_tile_tokens=2|tile_count=3|layer_count=2|batch_size=2|query_head_count=2|head_dimension=3|k_or_v_record_bytes_per_layer=3|paged=0|kv_block_tokens=0|blocks_per_sequence=0|worker_count=2|worker_rotation=1|logical_unit_count=5|active_worker_count=2|weight_shards_included=0|total_weight_shard_bytes=0|total_kv_model_payload_bytes=96|total_layout_metadata_lookup_count=0|total_layout_metadata_read_bytes=0|total_kv_accounted_bytes=96|total_scenario_accounted_bytes=96|minimum_worker_accounted_bytes=48|maximum_worker_accounted_bytes=48|worker_accounted_imbalance_bytes=0|worker_vector_count=2|worker_0_weight_shard_bytes=0|worker_0_kv_model_payload_bytes=48|worker_0_layout_metadata_lookup_count=0|worker_0_layout_metadata_read_bytes=0|worker_0_scenario_accounted_bytes=48|worker_1_weight_shard_bytes=0|worker_1_kv_model_payload_bytes=48|worker_1_layout_metadata_lookup_count=0|worker_1_layout_metadata_read_bytes=0|worker_1_scenario_accounted_bytes=48|assignment_count=2|assignment_0_range_rank=0|assignment_0_worker_index=1|assignment_0_first_unit=0|assignment_0_unit_count=2|assignment_0_kv_model_payload_bytes=48|assignment_0_layout_metadata_lookup_count=0|assignment_0_layout_metadata_read_bytes=0|assignment_0_kv_accounted_bytes=48|assignment_1_range_rank=1|assignment_1_worker_index=0|assignment_1_first_unit=2|assignment_1_unit_count=3|assignment_1_kv_model_payload_bytes=48|assignment_1_layout_metadata_lookup_count=0|assignment_1_layout_metadata_read_bytes=0|assignment_1_kv_accounted_bytes=48","llm-prefill-cpu-accounted-prefix-balanced-v1|planner_version=llm-prefill-planner-v1|schedule_version=llm-prefill-owner-local-write-then-tile-k-v-v1|unit_kind=contiguous_token|scenario=mixed|active_weight_bytes=1048576|prompt_tokens=5|query_tile_tokens=2|tile_count=3|layer_count=2|batch_size=2|query_head_count=2|head_dimension=3|k_or_v_record_bytes_per_layer=3|paged=0|kv_block_tokens=0|blocks_per_sequence=0|worker_count=2|worker_rotation=0|logical_unit_count=5|active_worker_count=2|weight_shards_included=1|total_weight_shard_bytes=524288|total_kv_model_payload_bytes=96|total_layout_metadata_lookup_count=0|total_layout_metadata_read_bytes=0|total_kv_accounted_bytes=96|total_scenario_accounted_bytes=524384|minimum_worker_accounted_bytes=262192|maximum_worker_accounted_bytes=262192|worker_accounted_imbalance_bytes=0|worker_vector_count=2|worker_0_weight_shard_bytes=262144|worker_0_kv_model_payload_bytes=48|worker_0_layout_metadata_lookup_count=0|worker_0_layout_metadata_read_bytes=0|worker_0_scenario_accounted_bytes=262192|worker_1_weight_shard_bytes=262144|worker_1_kv_model_payload_bytes=48|worker_1_layout_metadata_lookup_count=0|worker_1_layout_metadata_read_bytes=0|worker_1_scenario_accounted_bytes=262192|assignment_count=2|assignment_0_range_rank=0|assignment_0_worker_index=0|assignment_0_first_unit=0|assignment_0_unit_count=2|assignment_0_kv_model_payload_bytes=48|assignment_0_layout_metadata_lookup_count=0|assignment_0_layout_metadata_read_bytes=0|assignment_0_kv_accounted_bytes=48|assignment_1_range_rank=1|assignment_1_worker_index=1|assignment_1_first_unit=2|assignment_1_unit_count=3|assignment_1_kv_model_payload_bytes=48|assignment_1_layout_metadata_lookup_count=0|assignment_1_layout_metadata_read_bytes=0|assignment_1_kv_accounted_bytes=48","llm-prefill-cpu-accounted-prefix-balanced-v1|planner_version=llm-prefill-planner-v1|schedule_version=llm-prefill-owner-local-write-then-tile-k-v-v1|unit_kind=contiguous_token|scenario=mixed|active_weight_bytes=1048576|prompt_tokens=5|query_tile_tokens=2|tile_count=3|layer_count=2|batch_size=2|query_head_count=2|head_dimension=3|k_or_v_record_bytes_per_layer=3|paged=0|kv_block_tokens=0|blocks_per_sequence=0|worker_count=2|worker_rotation=1|logical_unit_count=5|active_worker_count=2|weight_shards_included=0|total_weight_shard_bytes=0|total_kv_model_payload_bytes=96|total_layout_metadata_lookup_count=0|total_layout_metadata_read_bytes=0|total_kv_accounted_bytes=96|total_scenario_accounted_bytes=96|minimum_worker_accounted_bytes=48|maximum_worker_accounted_bytes=48|worker_accounted_imbalance_bytes=0|worker_vector_count=2|worker_0_weight_shard_bytes=0|worker_0_kv_model_payload_bytes=48|worker_0_layout_metadata_lookup_count=0|worker_0_layout_metadata_read_bytes=0|worker_0_scenario_accounted_bytes=48|worker_1_weight_shard_bytes=0|worker_1_kv_model_payload_bytes=48|worker_1_layout_metadata_lookup_count=0|worker_1_layout_metadata_read_bytes=0|worker_1_scenario_accounted_bytes=48|assignment_count=2|assignment_0_range_rank=0|assignment_0_worker_index=1|assignment_0_first_unit=0|assignment_0_unit_count=2|assignment_0_kv_model_payload_bytes=48|assignment_0_layout_metadata_lookup_count=0|assignment_0_layout_metadata_read_bytes=0|assignment_0_kv_accounted_bytes=48|assignment_1_range_rank=1|assignment_1_worker_index=0|assignment_1_first_unit=2|assignment_1_unit_count=3|assignment_1_kv_model_payload_bytes=48|assignment_1_layout_metadata_lookup_count=0|assignment_1_layout_metadata_read_bytes=0|assignment_1_kv_accounted_bytes=48"],"worker_accounted_bytes_per_work_unit":["524480","524480"],"minimum_worker_accounted_bytes_per_work_unit":"524480","maximum_worker_accounted_bytes_per_work_unit":"524480","worker_accounted_imbalance_bytes_per_work_unit":"0","identity":"llm-prefill-cpu-scenario-execution-v1|scenario=mixed|scope_count=4|scope_identity_size=1774|scope_identity_sha256=38fb1cb42a709c2470a5298db02a3680636c49343cb3e01ec5d0b5909ffa17e0|scope_identity_size=1739|scope_identity_sha256=e606419234c3462da7116af96861c2caa4dbec72b6cabd295778706d57f4f663|scope_identity_size=1774|scope_identity_sha256=38fb1cb42a709c2470a5298db02a3680636c49343cb3e01ec5d0b5909ffa17e0|scope_identity_size=1739|scope_identity_sha256=e606419234c3462da7116af96861c2caa4dbec72b6cabd295778706d57f4f663|worker_count=2|worker_accounted_bytes=524480|worker_accounted_bytes=524480|minimum_worker_accounted_bytes=524480|maximum_worker_accounted_bytes=524480|worker_accounted_imbalance_bytes=0"}],"identity":"llm-prefill-cpu-execution-v1|sequence_descriptors_per_scenario_per_worker=4|scenario_index=0|scenario=weights_only|scope_count=2|scope_identity_size=1245|scope_identity_sha256=03ade36a2b8f2182bf053169fe8957c8128f5089848ca8aaa0c29ac106dc918f|scope_identity_size=1245|scope_identity_sha256=04cd5be22a7347524667e5415f8c0b3a2bb796f07de38d46716bfd909dffb87d|scenario_identity_size=483|scenario_identity_sha256=90914b27a2604468b5ad6946a204c0f8f930c440513984bd05b0b120548ea026|worker_count=2|worker_accounted_bytes=524288|worker_accounted_bytes=524288|minimum_worker_accounted_bytes=524288|maximum_worker_accounted_bytes=524288|worker_accounted_imbalance_bytes=0|scenario_index=1|scenario=kv_only|scope_count=4|scope_identity_size=1741|scope_identity_sha256=8c90eca75f04288ce0f56ef7f413d7a9c96a44cfc2b896280c5333351d3ffce3|scope_identity_size=1741|scope_identity_sha256=4e32ba02276fca3e29174e195611597f3365394eb823b5859bce31c31e703f86|scope_identity_size=1741|scope_identity_sha256=8c90eca75f04288ce0f56ef7f413d7a9c96a44cfc2b896280c5333351d3ffce3|scope_identity_size=1741|scope_identity_sha256=4e32ba02276fca3e29174e195611597f3365394eb823b5859bce31c31e703f86|scenario_identity_size=690|scenario_identity_sha256=a02ce00fac4f553b08b94cfb91136b14ac22d6fc7d702268cf1cdbc889a99ed1|worker_count=2|worker_accounted_bytes=192|worker_accounted_bytes=192|minimum_worker_accounted_bytes=192|maximum_worker_accounted_bytes=192|worker_accounted_imbalance_bytes=0|scenario_index=2|scenario=mixed|scope_count=4|scope_identity_size=1774|scope_identity_sha256=38fb1cb42a709c2470a5298db02a3680636c49343cb3e01ec5d0b5909ffa17e0|scope_identity_size=1739|scope_identity_sha256=e606419234c3462da7116af96861c2caa4dbec72b6cabd295778706d57f4f663|scope_identity_size=1774|scope_identity_sha256=38fb1cb42a709c2470a5298db02a3680636c49343cb3e01ec5d0b5909ffa17e0|scope_identity_size=1739|scope_identity_sha256=e606419234c3462da7116af96861c2caa4dbec72b6cabd295778706d57f4f663|scenario_identity_size=700|scenario_identity_sha256=abd76d2b793705f102d99fda27d8c18dde8f64bcf7da021281c522e585a17f69|worker_count=2|worker_accounted_bytes=524480|worker_accounted_bytes=524480|minimum_worker_accounted_bytes=524480|maximum_worker_accounted_bytes=524480|worker_accounted_imbalance_bytes=0"},"paged":null,"resources":{"valid":true,"reason_code":"valid","model_ref":"resolved_plan","mappings":{"policy":"private_anonymous_regular_cacheable","full_size_physical_mappings":true,"weight":{"requested_bytes":"1048576","committed_bytes":"1048576"},"k":{"requested_bytes":"60","committed_bytes":"16384"},"v":{"requested_bytes":"60","committed_bytes":"16384"},"block_table":null,"requested_data_bytes":"1048696","committed_data_bytes":"1081344"},"descriptors":{"abi_version":"llm-memory-prefill-contiguous-descriptor-abi-v1","layer_descriptors_per_worker":2,"sequence_descriptors_per_worker":12,"total_layer_descriptors":4,"total_sequence_descriptors":24,"descriptor_bytes":"2112"},"executor_auxiliary":{"valid":true,"reason_code":"valid","static_reference_bytes":"96","expected_checksum_bytes":"192","actual_checksum_bytes":"192","run_checksum_bytes":"32","worker_status_bytes":"2","thread_handle_bytes":"16","checksum_auxiliary_bytes":"512","orchestration_auxiliary_bytes":"18","total_auxiliary_bytes":"530"},"json_output_peak_estimate":{"enabled":true,"valid":true,"reason_code":"valid","policy":"conservative-live-dom-plus-serialized-transport","fixed_schema_bytes":"2097152","input_string_bytes":"1978160","measurement_record_bytes":"1179648","worker_checksum_bytes":"1966080","total_bytes":"7221040"},"allocation_memory_budget":{"resource_rounding_bytes":"32648","transient_peak_bytes":"7580874","layout_transient_bytes":null,"setup_peak_bytes":"41616","runtime_peak_bytes":"8662218","known_owned_peak_bytes":"8662218","admitted_budget_bytes":"9266895257","valid":true,"reason_code":"within-memory-budget","request":{"valid":true,"reason_code":"valid","mapping_granularity_bytes":"16384","requested_weight_mapping_bytes":"1048576","requested_k_mapping_bytes":"60","requested_v_mapping_bytes":"60","committed_weight_mapping_bytes":"1048576","committed_k_mapping_bytes":"16384","committed_v_mapping_bytes":"16384","requested_block_table_mapping_bytes":null,"committed_block_table_mapping_bytes":null,"requested_data_bytes":"1048696","committed_data_bytes":"1081344","layout_transient_bytes":null,"setup_peak_bytes":"41616","runtime_peak_bytes":"8662218","descriptor_bytes":"2112","planner_storage_bytes":"41616","checksum_auxiliary_bytes":"3776","orchestration_auxiliary_bytes":"7533370","auxiliary_bytes":"7580874","required_total_bytes":"8662218"},"available_memory_bytes":"11583619072","allowed_memory_bytes":"9266895257","used_fallback":false},"initialization":{"complete":true,"pattern_version":"llm-buffer-pattern-v1","pre_touch_policy":"write-every-requested-mapping-byte-once","separate_reference_read_pass":false,"static_references_accumulated_during_initialization":true,"weight_bytes":"1048576","k_bytes":"60","v_bytes":"60","total_bytes":"1048696","non_empty_weight_spans":4,"non_empty_k_spans":1,"non_empty_v_spans":1,"block_table_logical_bytes":null,"block_table_page_rounded_bytes":null,"block_table_read_only":null,"k_layout_padding_bytes":null,"v_layout_padding_bytes":null}}},"metal":null},"memory_budget":{"resource_rounding_bytes":"32648","transient_peak_bytes":"7580874","layout_transient_bytes":null,"setup_peak_bytes":"41616","runtime_peak_bytes":"8662218","known_owned_peak_bytes":"8662218","admitted_budget_bytes":"9266895257","valid":true,"reason_code":"within-memory-budget","request":{"valid":true,"reason_code":"valid","mapping_granularity_bytes":"16384","requested_weight_mapping_bytes":"1048576","requested_k_mapping_bytes":"60","requested_v_mapping_bytes":"60","committed_weight_mapping_bytes":"1048576","committed_k_mapping_bytes":"16384","committed_v_mapping_bytes":"16384","requested_block_table_mapping_bytes":null,"committed_block_table_mapping_bytes":null,"requested_data_bytes":"1048696","committed_data_bytes":"1081344","layout_transient_bytes":null,"setup_peak_bytes":"41616","runtime_peak_bytes":"8662218","descriptor_bytes":"2112","planner_storage_bytes":"41616","checksum_auxiliary_bytes":"3776","orchestration_auxiliary_bytes":"7533370","auxiliary_bytes":"7580874","required_total_bytes":"8662218"},"available_memory_bytes":"11583619072","allowed_memory_bytes":"9266895257","used_fallback":false},"calibration":{"excluded_from_results":true,"attempts":{"weights_only":[{"attempt_index":0,"scenario":"weights_only","plan_ref":0,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":2.125e-05,"timing":{"cpu_raw":{"start_ticks":"36788195529377","stop_ticks":"36788195529887","delta_ticks":"510","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"state_a_uint64_decimal":"4693879361642038518","state_b_uint64_decimal":"15995465716993483296"}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}}}],"kv_only":[{"attempt_index":0,"scenario":"kv_only","plan_ref":1,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":3.4583333333333334e-06,"timing":{"cpu_raw":{"start_ticks":"36788195531360","stop_ticks":"36788195531443","delta_ticks":"83","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"state_a_uint64_decimal":"13401016718287388321","state_b_uint64_decimal":"9725993772142933009"}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}}}],"mixed":[{"attempt_index":0,"scenario":"mixed","plan_ref":2,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":1.6791666666666667e-05,"timing":{"cpu_raw":{"start_ticks":"36788195532980","stop_ticks":"36788195533383","delta_ticks":"403","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"state_a_uint64_decimal":"4668412853063398514","state_b_uint64_decimal":"10985833669298179668"}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}}}]}},"measurements":[{"measurement_id":0,"plan_ref":0,"scenario":"weights_only","loop_index":0,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":"accepted-plan-derived","elapsed_seconds":1.5875e-05,"synthetic_work_unit_latency_seconds":7.9375e-06,"synthetic_memory_work_units_per_second":125984.25196850394,"effective_model_payload_gb_s":132.104062992126,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195534563","stop_ticks":"36788195534944","delta_ticks":"381","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"}}],"actual_run_checksum":{"state_a_uint64_decimal":"4693879361642038518","state_b_uint64_decimal":"15995465716993483296"}}},{"measurement_id":1,"plan_ref":1,"scenario":"kv_only","loop_index":0,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"768","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"768","completion_derivation":"accepted-plan-derived","elapsed_seconds":3.0416666666666666e-06,"synthetic_work_unit_latency_seconds":1.5208333333333333e-06,"synthetic_memory_work_units_per_second":657534.2465753425,"effective_model_payload_gb_s":0.2524931506849315,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195536187","stop_ticks":"36788195536260","delta_ticks":"73","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"8492071466883487160","state_b_uint64_decimal":"112617639640686528","exact_bytes_read":"132","span_count_uint64_decimal":"20"},"v":{"state_a_uint64_decimal":"14858055244865509948","state_b_uint64_decimal":"125051107588323352","exact_bytes_read":"132","span_count_uint64_decimal":"20"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"14940791572350823612","state_b_uint64_decimal":"94278968524492904","exact_bytes_read":"132","span_count_uint64_decimal":"20"},"v":{"state_a_uint64_decimal":"2860031276623294784","state_b_uint64_decimal":"106712436472129728","exact_bytes_read":"132","span_count_uint64_decimal":"20"}}],"actual_run_checksum":{"state_a_uint64_decimal":"13401016718287388321","state_b_uint64_decimal":"9725993772142933009"}}},{"measurement_id":2,"plan_ref":2,"scenario":"mixed","loop_index":0,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":0.999633923123856,"kv_read_payload_fraction":0.0002516778523489933,"kv_write_payload_fraction":0.00011439902379499695,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097920","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097920","completion_derivation":"accepted-plan-derived","elapsed_seconds":1.5875e-05,"synthetic_work_unit_latency_seconds":7.9375e-06,"synthetic_memory_work_units_per_second":125984.25196850394,"effective_model_payload_gb_s":132.1524409448819,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195537674","stop_ticks":"36788195538055","delta_ticks":"381","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"17269820495715963784","state_b_uint64_decimal":"105940028400153504","exact_bytes_read":"132","span_count_uint64_decimal":"20"},"v":{"state_a_uint64_decimal":"5189623149941856268","state_b_uint64_decimal":"118373496347790328","exact_bytes_read":"132","span_count_uint64_decimal":"20"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"5271796527473748620","state_b_uint64_decimal":"159658951321887816","exact_bytes_read":"132","span_count_uint64_decimal":"20"},"v":{"state_a_uint64_decimal":"11636935880525639440","state_b_uint64_decimal":"172092419269524640","exact_bytes_read":"132","span_count_uint64_decimal":"20"}}],"actual_run_checksum":{"state_a_uint64_decimal":"4668412853063398514","state_b_uint64_decimal":"10985833669298179668"}}},{"measurement_id":3,"plan_ref":1,"scenario":"kv_only","loop_index":1,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"768","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"768","completion_derivation":"accepted-plan-derived","elapsed_seconds":3.125e-06,"synthetic_work_unit_latency_seconds":1.5625e-06,"synthetic_memory_work_units_per_second":640000.0,"effective_model_payload_gb_s":0.24576,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195563648","stop_ticks":"36788195563723","delta_ticks":"75","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"8492071466883487160","state_b_uint64_decimal":"112617639640686528","exact_bytes_read":"132","span_count_uint64_decimal":"20"},"v":{"state_a_uint64_decimal":"14858055244865509948","state_b_uint64_decimal":"125051107588323352","exact_bytes_read":"132","span_count_uint64_decimal":"20"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"14940791572350823612","state_b_uint64_decimal":"94278968524492904","exact_bytes_read":"132","span_count_uint64_decimal":"20"},"v":{"state_a_uint64_decimal":"2860031276623294784","state_b_uint64_decimal":"106712436472129728","exact_bytes_read":"132","span_count_uint64_decimal":"20"}}],"actual_run_checksum":{"state_a_uint64_decimal":"13401016718287388321","state_b_uint64_decimal":"9725993772142933009"}}},{"measurement_id":4,"plan_ref":2,"scenario":"mixed","loop_index":1,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":0.999633923123856,"kv_read_payload_fraction":0.0002516778523489933,"kv_write_payload_fraction":0.00011439902379499695,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097920","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097920","completion_derivation":"accepted-plan-derived","elapsed_seconds":1.6208333333333335e-05,"synthetic_work_unit_latency_seconds":8.104166666666668e-06,"synthetic_memory_work_units_per_second":123393.31619537274,"effective_model_payload_gb_s":129.43465295629818,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195565123","stop_ticks":"36788195565512","delta_ticks":"389","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"17269820495715963784","state_b_uint64_decimal":"105940028400153504","exact_bytes_read":"132","span_count_uint64_decimal":"20"},"v":{"state_a_uint64_decimal":"5189623149941856268","state_b_uint64_decimal":"118373496347790328","exact_bytes_read":"132","span_count_uint64_decimal":"20"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"5271796527473748620","state_b_uint64_decimal":"159658951321887816","exact_bytes_read":"132","span_count_uint64_decimal":"20"},"v":{"state_a_uint64_decimal":"11636935880525639440","state_b_uint64_decimal":"172092419269524640","exact_bytes_read":"132","span_count_uint64_decimal":"20"}}],"actual_run_checksum":{"state_a_uint64_decimal":"4668412853063398514","state_b_uint64_decimal":"10985833669298179668"}}},{"measurement_id":5,"plan_ref":0,"scenario":"weights_only","loop_index":1,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":"accepted-plan-derived","elapsed_seconds":1.6041666666666666e-05,"synthetic_work_unit_latency_seconds":8.020833333333333e-06,"synthetic_memory_work_units_per_second":124675.32467532468,"effective_model_payload_gb_s":130.73155324675324,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195566665","stop_ticks":"36788195567050","delta_ticks":"385","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"}}],"actual_run_checksum":{"state_a_uint64_decimal":"4693879361642038518","state_b_uint64_decimal":"15995465716993483296"}}},{"measurement_id":6,"plan_ref":2,"scenario":"mixed","loop_index":2,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":0.999633923123856,"kv_read_payload_fraction":0.0002516778523489933,"kv_write_payload_fraction":0.00011439902379499695,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097920","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097920","completion_derivation":"accepted-plan-derived","elapsed_seconds":2.091666666666667e-05,"synthetic_work_unit_latency_seconds":1.0458333333333335e-05,"synthetic_memory_work_units_per_second":95617.52988047808,"effective_model_payload_gb_s":100.29896414342629,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195590837","stop_ticks":"36788195591339","delta_ticks":"502","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"17269820495715963784","state_b_uint64_decimal":"105940028400153504","exact_bytes_read":"132","span_count_uint64_decimal":"20"},"v":{"state_a_uint64_decimal":"5189623149941856268","state_b_uint64_decimal":"118373496347790328","exact_bytes_read":"132","span_count_uint64_decimal":"20"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"5271796527473748620","state_b_uint64_decimal":"159658951321887816","exact_bytes_read":"132","span_count_uint64_decimal":"20"},"v":{"state_a_uint64_decimal":"11636935880525639440","state_b_uint64_decimal":"172092419269524640","exact_bytes_read":"132","span_count_uint64_decimal":"20"}}],"actual_run_checksum":{"state_a_uint64_decimal":"4668412853063398514","state_b_uint64_decimal":"10985833669298179668"}}},{"measurement_id":7,"plan_ref":0,"scenario":"weights_only","loop_index":2,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":"accepted-plan-derived","elapsed_seconds":1.5791666666666667e-05,"synthetic_work_unit_latency_seconds":7.895833333333333e-06,"synthetic_memory_work_units_per_second":126649.0765171504,"effective_model_payload_gb_s":132.8011820580475,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195592996","stop_ticks":"36788195593375","delta_ticks":"379","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"}}],"actual_run_checksum":{"state_a_uint64_decimal":"4693879361642038518","state_b_uint64_decimal":"15995465716993483296"}}},{"measurement_id":8,"plan_ref":1,"scenario":"kv_only","loop_index":2,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"768","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"768","completion_derivation":"accepted-plan-derived","elapsed_seconds":3.0416666666666666e-06,"synthetic_work_unit_latency_seconds":1.5208333333333333e-06,"synthetic_memory_work_units_per_second":657534.2465753425,"effective_model_payload_gb_s":0.2524931506849315,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195594618","stop_ticks":"36788195594691","delta_ticks":"73","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"8492071466883487160","state_b_uint64_decimal":"112617639640686528","exact_bytes_read":"132","span_count_uint64_decimal":"20"},"v":{"state_a_uint64_decimal":"14858055244865509948","state_b_uint64_decimal":"125051107588323352","exact_bytes_read":"132","span_count_uint64_decimal":"20"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"14940791572350823612","state_b_uint64_decimal":"94278968524492904","exact_bytes_read":"132","span_count_uint64_decimal":"20"},"v":{"state_a_uint64_decimal":"2860031276623294784","state_b_uint64_decimal":"106712436472129728","exact_bytes_read":"132","span_count_uint64_decimal":"20"}}],"actual_run_checksum":{"state_a_uint64_decimal":"13401016718287388321","state_b_uint64_decimal":"9725993772142933009"}}}],"aggregates":{"scenarios":{"weights_only":{"scenario":"weights_only","status":"complete","observed_cv_classification":"below-threshold","accepted_measurement_ids":[0,5,7],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":7.9375e-06,"statistics":{"sample_count":3,"average":7.95138888888889e-06,"min":7.895833333333333e-06,"max":8.020833333333333e-06,"median":7.9375e-06,"p90":8.004166666666665e-06,"p95":8.0125e-06,"p99":8.019166666666668e-06,"stddev":6.364688465216426e-08,"coefficient_of_variation_pct":0.8004499030490527,"median_absolute_deviation":4.166666666666655e-08}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":125984.25196850394,"statistics":{"sample_count":3,"average":125769.55105365968,"min":124675.32467532468,"max":126649.0765171504,"median":125984.25196850394,"p90":126516.1116074211,"p95":126582.59406228576,"p99":126635.78002617748,"stddev":1004.2392371361688,"coefficient_of_variation_pct":0.7984756475020804,"median_absolute_deviation":664.8245486464584}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":132.104062992126,"statistics":{"sample_count":3,"average":131.87893276564225,"min":130.73155324675324,"max":132.8011820580475,"median":132.104062992126,"p90":132.6617582448632,"p95":132.73147015145534,"p99":132.78723967672906,"stddev":1.0530211623192967,"coefficient_of_variation_pct":0.7984756475020816,"median_absolute_deviation":0.6971190659214983}}},"kv_only":{"scenario":"kv_only","status":"complete","observed_cv_classification":"below-threshold","accepted_measurement_ids":[1,3,8],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":1.5208333333333333e-06,"statistics":{"sample_count":3,"average":1.534722222222222e-06,"min":1.5208333333333333e-06,"max":1.5625e-06,"median":1.5208333333333333e-06,"p90":1.5541666666666667e-06,"p95":1.5583333333333334e-06,"p99":1.5616666666666666e-06,"stddev":2.4056261216234463e-08,"coefficient_of_variation_pct":1.5674667941799831,"median_absolute_deviation":0.0}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":657534.2465753425,"statistics":{"sample_count":3,"average":651689.497716895,"min":640000.0,"max":657534.2465753425,"median":657534.2465753425,"p90":657534.2465753425,"p95":657534.2465753425,"p99":657534.2465753425,"stddev":10123.40198031126,"coefficient_of_variation_pct":1.553408796025901,"median_absolute_deviation":0.0}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":0.2524931506849315,"statistics":{"sample_count":3,"average":0.25024876712328764,"min":0.24576,"max":0.2524931506849315,"median":0.2524931506849315,"p90":0.2524931506849315,"p95":0.2524931506849315,"p99":0.2524931506849315,"stddev":0.0038873863604395108,"coefficient_of_variation_pct":1.553408796025896,"median_absolute_deviation":0.0}}},"mixed":{"scenario":"mixed","status":"complete","observed_cv_classification":"above-threshold","accepted_measurement_ids":[2,4,6],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":8.104166666666668e-06,"statistics":{"sample_count":3,"average":8.833333333333334e-06,"min":7.9375e-06,"max":1.0458333333333335e-05,"median":8.104166666666668e-06,"p90":9.9875e-06,"p95":1.0222916666666668e-05,"p99":1.0411250000000001e-05,"stddev":1.409756430893098e-06,"coefficient_of_variation_pct":15.959506764827522,"median_absolute_deviation":1.666666666666679e-07}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":123393.31619537274,"statistics":{"sample_count":3,"average":114998.36601478492,"min":95617.52988047808,"max":125984.25196850394,"median":123393.31619537274,"p90":125466.0648138777,"p95":125725.15839119082,"p99":125932.43325304131,"stddev":16834.2164650976,"coefficient_of_variation_pct":14.638657094426271,"median_absolute_deviation":2590.9357731312048}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":129.43465295629818,"statistics":{"sample_count":3,"average":120.62868601486879,"min":100.29896414342629,"max":132.1524409448819,"median":129.43465295629818,"p90":131.60888334716518,"p95":131.88066214602352,"p99":132.09808518511022,"stddev":17.658419703228773,"coefficient_of_variation_pct":14.638657094426263,"median_absolute_deviation":2.717787988583723}}}},"traffic_diagnostics":{"classification_version":"llm-exact-weight-vs-kv-read-payload-v1","traffic_crossover_numerator":null,"traffic_crossover_denominator":null,"traffic_crossover_context_tokens":null,"current_visible_context_tokens":null,"current_weight_read_payload_bytes_per_work_unit":"1048576","current_kv_read_payload_bytes_per_work_unit":"264","current_weight_to_kv_read_payload_ratio":null,"current_context_classification":null,"classification_is_payload_only":true,"scenario_headlines":{"weights_only":{"synthetic_work_unit_latency_seconds":7.9375e-06,"synthetic_memory_work_units_per_second":125984.25196850394,"effective_model_payload_gb_s":132.104062992126},"kv_only":{"synthetic_work_unit_latency_seconds":1.5208333333333333e-06,"synthetic_memory_work_units_per_second":657534.2465753425,"effective_model_payload_gb_s":0.2524931506849315},"mixed":{"synthetic_work_unit_latency_seconds":8.104166666666668e-06,"synthetic_memory_work_units_per_second":123393.31619537274,"effective_model_payload_gb_s":129.43465295629818}}}},"status":"complete","reason_code":"complete","results_complete":true,"run_accepted":true,"interpretation":{"result_scope":"synthetic-memory-only-work-unit-not-real-inference-token-or-prefill-latency","reported_rate":"synthetic_memory_work_units_per_second","backend":"cpu","phase":"prefill","work_unit_kind":"prefill_operation","transformer_math_included":false,"framework_scheduler_and_dispatch_included":false,"compute_memory_overlap_included":false,"physical_dram_traffic_measured":false,"dram_residency":"unverified","cache_residency":"unverified","fixed_context_includes_current_token_slot":null,"kv_layout":"contiguous","payload_semantics":"exact-logical-weight-plus-kv-read-plus-kv-write-bytes-divided-by-elapsed-time","layout_metadata_timed_but_excluded_from_effective_model_payload_gb_s":true,"prefill_transformer_compute_or_ttft_prediction_included":false,"private_metal_storage_implies_separate_vram":false,"cross_backend_performance_distributions_combined":false,"traffic_classification_semantics":"exact-weight-vs-kv-read-logical-payload-only-not-hardware-bottleneck","full_size_working_set_reduces_but_does_not_prove_dram_residency":true,"comparability_requires":["same-backend","same-methodology-version","same-frozen-work-plan-and-model-geometry","same-software-version","sufficiently-similar-hardware","sufficiently-similar-thermal-and-power-state"]},"diagnostic":null,"interruption_requested":false,"scenario_order_balance_complete":true,"seeds":{"base_seed_uint64_decimal":"424242","source":"user","buffer_domain_seeds":{"weight_uint64_decimal":"904846021374001452","k_uint64_decimal":"10718278915871169538","v_uint64_decimal":"2111147304547004201"},"scenario_domain_seeds":{"weights_only":"17410844953366850226","kv_only":"16590208891222127307","mixed":"1546516732201688391"}},"counters":{"planned_loops":3,"attempted_loops":3,"completed_loops":3,"planned_measurements":9,"attempted_measurements":9,"terminal_measurements":9,"measured_measurements":9,"planned_work_units":"18","completed_work_units":"18","planned_effective_model_payload_bytes":"12587520","completed_effective_model_payload_bytes":"12587520","planned_layout_metadata_lookup_count":"0","completed_layout_metadata_lookup_count":"0","planned_layout_metadata_read_bytes":"0","completed_layout_metadata_read_bytes":"0","planned_task_accounted_bytes":"12587520","completed_task_accounted_bytes":"12587520","runner_auxiliary":{"valid":true,"reason_code":"valid","measurement_record_bytes":"7056","loop_record_bytes":"144","calibration_record_bytes":"2784","calibration_identity_bytes":"131680","aggregate_value_bytes":"72","statistics_workspace_bytes":"72","warning_record_bytes":"64","fixed_metadata_bytes":"170440","retained_checksum_bytes":"3264","checksum_auxiliary_bytes":"3264","orchestration_auxiliary_bytes":"312312","total_auxiliary_bytes":"315576"}},"checkpoint_lifecycle":{"checkpoint_failed":false,"observation_point":"before-current-snapshot-preparation","prior_file_writer_attempts":3,"prior_successful_file_writes":3,"current_request":"terminal","current_persistence_success":null,"snapshot_interval_loops":1,"checkpoint_policy":"bounded-loop-snapshots","file_checkpoint_failure_is_terminal_and_not_retried":true,"stdout_intermediate_checkpoints_are_lazy":true},"loop_records":[{"loop_index":0,"planned_order":["weights_only","kv_only","mixed"],"realized_order":["weights_only","kv_only","mixed"],"realized_order_count":3,"measurement_indexes":[0,1,2]},{"loop_index":1,"planned_order":["kv_only","mixed","weights_only"],"realized_order":["kv_only","mixed","weights_only"],"realized_order_count":3,"measurement_indexes":[3,4,5]},{"loop_index":2,"planned_order":["mixed","weights_only","kv_only"],"realized_order":["mixed","weights_only","kv_only"],"realized_order_count":3,"measurement_indexes":[6,7,8]}],"environment":{"processor_name":"Apple M4","macos_version":"26.6.2","performance_core_count":4,"efficiency_core_count":6,"logical_core_count":10,"page_size_bytes":"16384","l1_data_cache_bytes":"131072","l2_data_cache_bytes":"16777216","available_memory_bytes":"11583619072","available_memory_source":"mach-free-plus-inactive-rounded-mib","main_thread_qos":{"requested":true,"applied":true,"code":0},"start":{"thermal_state":"nominal","low_power_mode_available":true,"low_power_mode_enabled":false,"physical_memory_bytes":"25769803776"},"end":{"thermal_state":"nominal","low_power_mode_available":true,"low_power_mode_enabled":false,"physical_memory_bytes":"25769803776"}},"quality_warnings":["mixed-high-cv","weight-working-set-cache-dominant","kv-working-set-cache-dominant","weights_only-duration-below-target-window","kv_only-duration-below-target-window","mixed-duration-below-target-window"]} diff --git a/tests/fixtures/llm-schema-v2/cpu-prefill-paged.json b/tests/fixtures/llm-schema-v2/cpu-prefill-paged.json new file mode 100644 index 0000000..db69a34 --- /dev/null +++ b/tests/fixtures/llm-schema-v2/cpu-prefill-paged.json @@ -0,0 +1 @@ +{"schema_version":2,"mode":"llm_memory","backend":"cpu","phase":"prefill","kv_layout":"paged","methodology_version":"llm-memory-v2-cpu-prefill-paged","software":{"version":"0.63.1","timestamp":"2026-09-06T07:07:07Z"},"build_manifest":{"binary_sha256":"a9c0aa68c61fccf12bda222a0485a3eba1b642a54b9b1caf8292348f608a1100","compile_flags":{"asflags":"-arch arm64","cxxflags":"-Wall -O3 -std=c++17 -arch arm64 -pthread -Isrc","test_cxxflags":"-Wall -g -std=c++17 -arch arm64 -pthread -Isrc -I/opt/homebrew/opt/googletest/include"},"compiler":"Apple clang version 21.0.0 (clang-2100.1.1.101)\nTarget: arm64-apple-darwin25.6.0\nThread model: posix\nInstalledDir: /Library/Developer/CommandLineTools/usr/bin","git_commit":"870b175927d747fb1d90fadefebe8728da38407a","git_dirty":true,"link_flags":"-pthread -framework Metal -framework Foundation","manifest_version":1,"min_os":"26.0","reason_code":"valid","sdk":"26.5","status":"available","target_arch":"arm64-apple-darwin25.6.0"},"configuration":{"backend":"cpu","phase":"prefill","kv_layout":"paged","kv_block_tokens":2,"weight_size_mb":1,"layer_count":2,"query_head_count":2,"kv_head_count":1,"head_dimension":3,"kv_element_bytes":"1","visible_context_tokens":null,"prompt_tokens":5,"attention_query_tile_tokens":2,"batch_size":2,"requested_workers":2,"available_workers":10,"worker_source":"user","iterations":2,"work_policy":"explicit_fixed_work","loop_count":3,"base_seed_uint64_decimal":"424242","seed_source":"user","output_file":"plans/llm-benchmark-remediation-evidence/phase7/profiles/cpu-prefill-paged.json","argv":["./memory_benchmark","--llm-memory","--llm-memory-backend","cpu","--phase","prefill","--kv-layout","paged","--weight-size-mb","1","--layers","2","--query-heads","2","--kv-heads","1","--head-dim","3","--kv-element-bytes","1","--batch-size","2","--iterations","2","--count","3","--seed","424242","-o","plans/llm-benchmark-remediation-evidence/phase7/profiles/cpu-prefill-paged.json","--prompt-tokens","5","--attention-query-tile-tokens","2","--threads","2","--kv-block-tokens","2"],"resolved_sources":{"backend":"explicit","phase":"explicit","kv_layout":"explicit","kv_block_tokens":"explicit","visible_context_tokens":null,"prompt_tokens":"explicit","attention_query_tile_tokens":"explicit","workers":"explicit","iterations":"explicit","seed":"explicit"}},"resolved_plan":{"valid":true,"reason_code":"valid","plan_identity":"llm-memory-work-plan-v1|backend=cpu|phase=prefill|kv_layout=paged|work_unit_kind=prefill_operation|methodology=llm-memory-v2-cpu-prefill-paged|component_identity_size=714|component_identity=llm-memory-components-v1|logical_profile_version=31:full_prompt_causal_prefix_tiled|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|permutation_version=36:splitmix64-fisher-yates-rejection-v1|backend_executor_version=39:llm-cpu-executor-v1-arm64-prefill-paged|resource_abi_version=42:llm-memory-prefill-paged-descriptor-abi-v1|schedule_version=46:llm-prefill-owner-local-write-then-tile-k-v-v1|timer_policy_version=62:synchronized-start-to-last-worker-completion-per-scenario-task|buffer_pattern_version=36:llm-paged-physical-buffer-pattern-v1|write_pattern_version=26:llm-prefill-kv-affine64-v1|checksum_pattern_version=36:llm-prefill-paged-affine64-lookup-v1|msl_revision=null|msl_source_sha256=null|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|prefill_planner_version=llm-prefill-planner-v1|prefill_cpu_partition_version=llm-prefill-cpu-accounted-prefix-balanced-v1|prefill_prompt_tokens=5|prefill_query_tile_tokens=2|prefill_tile_count=3|prefill_prefix_token_visits_per_sequence=11|prefill_causal_token_pairs=15|prefill_logical_attention_pairs=120|prefill_logical_attention_fma_terms=360|prefill_prefix_block_visits_per_sequence=6|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=2|kv_blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|kv_block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|layout_metadata_lookups_per_layer_sequence_per_work_unit=15|k_mapping_bytes=72|v_mapping_bytes=72|kv_capacity_bytes=144|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|kv_only_payload_bytes_per_work_unit=384|mixed_payload_bytes_per_work_unit=1048960|total_data_mapping_bytes=1048720|traffic_crossover_numerator=0|traffic_crossover_denominator=0|requested_workers=2|effective_workers=2|layer_descriptors_per_worker=2|sequence_descriptors_per_worker=12|total_layer_descriptors=4|total_sequence_descriptors=24|descriptor_bytes=2880|prefill_execution_identity_size=2239|prefill_execution_identity=llm-prefill-cpu-execution-v1|sequence_descriptors_per_scenario_per_worker=4|scenario_index=0|scenario=weights_only|scope_count=2|scope_identity_size=1240|scope_identity_sha256=ec65388b21acf11d327e80e42eec02024a5e4e713880a07d5ddc23ecce969e86|scope_identity_size=1240|scope_identity_sha256=576390a7fd86b32a77e0623f06256f3db8c05ebb58fc8e66c4f0edaa45931aa5|scenario_identity_size=483|scenario_identity_sha256=bdea7d1380a216d61b118c5100ae14508441714b38871a894449ff7382ac324e|worker_count=2|worker_accounted_bytes=524288|worker_accounted_bytes=524288|minimum_worker_accounted_bytes=524288|maximum_worker_accounted_bytes=524288|worker_accounted_imbalance_bytes=0|scenario_index=1|scenario=kv_only|scope_count=4|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|scenario_identity_size=690|scenario_identity_sha256=2d1bbb68fcf68dfd8ec91c87e6637be4cb17fd48ba9448dd7832fdbae001bea2|worker_count=2|worker_accounted_bytes=312|worker_accounted_bytes=312|minimum_worker_accounted_bytes=312|maximum_worker_accounted_bytes=312|worker_accounted_imbalance_bytes=0|scenario_index=2|scenario=mixed|scope_count=4|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|scenario_identity_size=700|scenario_identity_sha256=c6559156693601203bec3632ab45e8d8ce2358333fd24379902964ce03030ec1|worker_count=2|worker_accounted_bytes=524600|worker_accounted_bytes=524600|minimum_worker_accounted_bytes=524600|maximum_worker_accounted_bytes=524600|worker_accounted_imbalance_bytes=0|paged_layout_identity_size=1060|paged_layout_identity=llm-kv-layout-plan-v1|geometry_identity=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24|permutation_algorithm_version=36:splitmix64-fisher-yates-rejection-v1|permutation_domain=5497854231078520625|permutation_domain_uint64_hex=18:0x4c4c4d4b56504731|permutation_seed=1732835419980629872|permutation_entry_count=6|permutation_sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c|paged_execution_identity_size=2239|paged_execution_identity=llm-prefill-cpu-execution-v1|sequence_descriptors_per_scenario_per_worker=4|scenario_index=0|scenario=weights_only|scope_count=2|scope_identity_size=1240|scope_identity_sha256=ec65388b21acf11d327e80e42eec02024a5e4e713880a07d5ddc23ecce969e86|scope_identity_size=1240|scope_identity_sha256=576390a7fd86b32a77e0623f06256f3db8c05ebb58fc8e66c4f0edaa45931aa5|scenario_identity_size=483|scenario_identity_sha256=bdea7d1380a216d61b118c5100ae14508441714b38871a894449ff7382ac324e|worker_count=2|worker_accounted_bytes=524288|worker_accounted_bytes=524288|minimum_worker_accounted_bytes=524288|maximum_worker_accounted_bytes=524288|worker_accounted_imbalance_bytes=0|scenario_index=1|scenario=kv_only|scope_count=4|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|scenario_identity_size=690|scenario_identity_sha256=2d1bbb68fcf68dfd8ec91c87e6637be4cb17fd48ba9448dd7832fdbae001bea2|worker_count=2|worker_accounted_bytes=312|worker_accounted_bytes=312|minimum_worker_accounted_bytes=312|maximum_worker_accounted_bytes=312|worker_accounted_imbalance_bytes=0|scenario_index=2|scenario=mixed|scope_count=4|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|scenario_identity_size=700|scenario_identity_sha256=c6559156693601203bec3632ab45e8d8ce2358333fd24379902964ce03030ec1|worker_count=2|worker_accounted_bytes=524600|worker_accounted_bytes=524600|minimum_worker_accounted_bytes=524600|maximum_worker_accounted_bytes=524600|worker_accounted_imbalance_bytes=0|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391","backend":"cpu","phase":"prefill","kv_layout":"paged","work_unit_kind":"prefill_operation","geometry":{"valid":true,"reason_code":"valid","phase":"prefill","work_unit_kind":"prefill_operation","decode":null,"prefill":{"prompt_tokens":5,"attention_query_tile_tokens":2,"tile_count":"3","attention_prefix_token_visits_per_sequence":"11"},"attention_kind":"mqa","active_weight_bytes_per_work_unit":"1048576","layer_count":2,"query_head_count":2,"kv_head_count":1,"query_heads_per_kv_head":2,"head_dimension":3,"kv_element_bytes":"1","batch_size":2,"kv_vector_bytes":"3","k_or_v_record_bytes_per_layer":"3","kv_record_bytes_per_layer":"6","kv_bytes_per_visible_token":"12","k_or_v_sequence_visible_bytes":"15","k_mapping_bytes":"72","v_mapping_bytes":"72","kv_capacity_bytes":"144","weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"264","kv_write_bytes_per_work_unit":"120","kv_only_effective_model_payload_bytes_per_work_unit":"384","mixed_effective_model_payload_bytes_per_work_unit":"1048960","total_data_mapping_bytes":"1048720","traffic_crossover_numerator":null,"traffic_crossover_denominator":null,"traffic_crossover_context_tokens":null},"model_context":{"prefill":{"causal_token_pairs_per_sequence":"15","causal_token_pairs_per_sequence_reason_code":"valid","logical_attention_pairs":"120","logical_attention_pairs_reason_code":"valid","logical_attention_fma_terms":"360","logical_attention_fma_terms_reason_code":"valid"}},"layout":{"kv_layout":"paged","kv_block_tokens":2,"blocks_per_sequence":"3","physical_blocks_per_layer":"6","total_physical_blocks":"12","block_bytes":"6","last_block_tokens":"1","last_block_valid_bytes":"3","decode_append_offset_in_last_block":null,"block_table_entries":"6","block_table_bytes":"24","layout_geometry_identity":"llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24","layout_identity":"llm-kv-layout-plan-v1|geometry_identity=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24|permutation_algorithm_version=36:splitmix64-fisher-yates-rejection-v1|permutation_domain=5497854231078520625|permutation_domain_uint64_hex=18:0x4c4c4d4b56504731|permutation_seed=1732835419980629872|permutation_entry_count=6|permutation_sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c","permutation_domain_uint64_hex":"0x4c4c4d4b56504731","permutation_seed_uint64_decimal":"1732835419980629872","permutation_algorithm_version":"splitmix64-fisher-yates-rejection-v1","permutation_entry_count":"6","permutation_sha256":"14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c","permutation_identity":"splitmix64-fisher-yates-rejection-v1|domain=5497854231078520625|domain_uint64_hex=18:0x4c4c4d4b56504731|resolved_seed=1732835419980629872|entry_count=6|sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c"},"resources":{"weight_logical_bytes":"1048576","k_logical_bytes":"60","v_logical_bytes":"60","k_physical_length_bytes":"72","v_physical_length_bytes":"72","k_layout_padding_bytes":"12","v_layout_padding_bytes":"12","block_table_bytes":"24"},"component_identities":{"logical_profile_version":"full_prompt_causal_prefix_tiled","kv_layout_version":"paged-uint32-block-table-full-blocks-v1","permutation_version":"splitmix64-fisher-yates-rejection-v1","backend_executor_version":"llm-cpu-executor-v1-arm64-prefill-paged","resource_abi_version":"llm-memory-prefill-paged-descriptor-abi-v1","schedule_version":"llm-prefill-owner-local-write-then-tile-k-v-v1","timer_policy_version":"synchronized-start-to-last-worker-completion-per-scenario-task","buffer_pattern_version":"llm-paged-physical-buffer-pattern-v1","write_pattern_version":"llm-prefill-kv-affine64-v1","checksum_pattern_version":"llm-prefill-paged-affine64-lookup-v1","msl_revision":null,"msl_source_sha256":null,"identity":"llm-memory-components-v1|logical_profile_version=31:full_prompt_causal_prefix_tiled|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|permutation_version=36:splitmix64-fisher-yates-rejection-v1|backend_executor_version=39:llm-cpu-executor-v1-arm64-prefill-paged|resource_abi_version=42:llm-memory-prefill-paged-descriptor-abi-v1|schedule_version=46:llm-prefill-owner-local-write-then-tile-k-v-v1|timer_policy_version=62:synchronized-start-to-last-worker-completion-per-scenario-task|buffer_pattern_version=36:llm-paged-physical-buffer-pattern-v1|write_pattern_version=26:llm-prefill-kv-affine64-v1|checksum_pattern_version=36:llm-prefill-paged-affine64-lookup-v1|msl_revision=null|msl_source_sha256=null","run_policy_version":"llm-run-policy-bounded-loop-snapshots-v1"},"methodology":{"backend":"cpu","phase":"prefill","kv_layout":"paged","work_unit_kind":"prefill_operation","weight_passes_per_work_unit":1,"kv_replay_factor":1,"warmup_policy":"same-shape-excluded-steady-state-warm-memory","context_policy":"full-prompt-population-with-tiled-causal-prefix-scans","scenario_order_policy":"cyclic-rotation-across-count-loops","timing_policy":"synchronized-start-to-last-worker-completion-per-scenario-task","cache_policy":"regular-cacheable-no-explicit-flush-between-scenarios","calibration_policy":"per-scenario-automatic-or-explicit-frozen-before-loop-zero","calibration_target_seconds":0.15,"calibration_min_seconds":0.1,"calibration_max_seconds":0.25,"calibration_max_corrections":2,"calibration_min_pilot_accounted_bytes":"8388608","maximum_work_units_per_measurement":1000000000,"maximum_serial_range_visits_per_lane_per_task":null,"maximum_accounted_bytes_per_task":"68719476736","repeatability_cv_warning_threshold_pct":5.0,"calibration_excluded_from_results":true,"snapshot_policy":"bounded-loop-snapshots","timed_region_exclusions":["mapping-allocation","buffer-initialization-and-pre-touch","descriptor-materialization-and-validation","worker-thread-creation-and-qos","same-shape-warmup","calibration-attempts","checksum-reference-generation-and-validation","worker-join-and-json-serialization"]},"model_work_plan":{"valid":true,"reason_code":"valid","backend":"cpu","phase":"prefill","kv_layout":"paged","work_unit_kind":"prefill_operation","weight_passes_per_work_unit":1,"kv_replay_factor":1,"requested_workers":2,"available_workers":10,"effective_workers":2,"worker_plan_count":2,"weight_layer_count":2,"layer_descriptors_per_worker":2,"sequence_descriptors_per_worker":12,"total_layer_descriptors":4,"total_sequence_descriptors":24,"descriptor_bytes":"2880","planner_storage_bytes":"40848"},"scenario_plans":[{"valid":true,"reason_code":"valid","scenario":"weights_only","work_unit_kind":"prefill_operation","kv_write_kind":"none","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"17410844953366850226","work_units":2,"weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"0","kv_write_bytes_per_work_unit":"0","effective_model_payload_bytes_per_work_unit":"1048576","layout_metadata_lookup_count_per_work_unit":"0","layout_metadata_read_bytes_per_work_unit":"0","accounted_bytes_per_work_unit":"1048576","weight_read_bytes":"2097152","kv_read_bytes":"0","kv_write_bytes":"0","effective_model_payload_bytes":"2097152","layout_metadata_lookup_count":"0","layout_metadata_read_bytes":"0","task_accounted_bytes":"2097152","maximum_work_units_by_work_unit_cap":1000000000,"maximum_work_units_by_guardrail":65536,"effective_maximum_work_units":65536,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=8521|model_plan_identity=llm-memory-work-plan-v1|backend=cpu|phase=prefill|kv_layout=paged|work_unit_kind=prefill_operation|methodology=llm-memory-v2-cpu-prefill-paged|component_identity_size=714|component_identity=llm-memory-components-v1|logical_profile_version=31:full_prompt_causal_prefix_tiled|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|permutation_version=36:splitmix64-fisher-yates-rejection-v1|backend_executor_version=39:llm-cpu-executor-v1-arm64-prefill-paged|resource_abi_version=42:llm-memory-prefill-paged-descriptor-abi-v1|schedule_version=46:llm-prefill-owner-local-write-then-tile-k-v-v1|timer_policy_version=62:synchronized-start-to-last-worker-completion-per-scenario-task|buffer_pattern_version=36:llm-paged-physical-buffer-pattern-v1|write_pattern_version=26:llm-prefill-kv-affine64-v1|checksum_pattern_version=36:llm-prefill-paged-affine64-lookup-v1|msl_revision=null|msl_source_sha256=null|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|prefill_planner_version=llm-prefill-planner-v1|prefill_cpu_partition_version=llm-prefill-cpu-accounted-prefix-balanced-v1|prefill_prompt_tokens=5|prefill_query_tile_tokens=2|prefill_tile_count=3|prefill_prefix_token_visits_per_sequence=11|prefill_causal_token_pairs=15|prefill_logical_attention_pairs=120|prefill_logical_attention_fma_terms=360|prefill_prefix_block_visits_per_sequence=6|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=2|kv_blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|kv_block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|layout_metadata_lookups_per_layer_sequence_per_work_unit=15|k_mapping_bytes=72|v_mapping_bytes=72|kv_capacity_bytes=144|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|kv_only_payload_bytes_per_work_unit=384|mixed_payload_bytes_per_work_unit=1048960|total_data_mapping_bytes=1048720|traffic_crossover_numerator=0|traffic_crossover_denominator=0|requested_workers=2|effective_workers=2|layer_descriptors_per_worker=2|sequence_descriptors_per_worker=12|total_layer_descriptors=4|total_sequence_descriptors=24|descriptor_bytes=2880|prefill_execution_identity_size=2239|prefill_execution_identity=llm-prefill-cpu-execution-v1|sequence_descriptors_per_scenario_per_worker=4|scenario_index=0|scenario=weights_only|scope_count=2|scope_identity_size=1240|scope_identity_sha256=ec65388b21acf11d327e80e42eec02024a5e4e713880a07d5ddc23ecce969e86|scope_identity_size=1240|scope_identity_sha256=576390a7fd86b32a77e0623f06256f3db8c05ebb58fc8e66c4f0edaa45931aa5|scenario_identity_size=483|scenario_identity_sha256=bdea7d1380a216d61b118c5100ae14508441714b38871a894449ff7382ac324e|worker_count=2|worker_accounted_bytes=524288|worker_accounted_bytes=524288|minimum_worker_accounted_bytes=524288|maximum_worker_accounted_bytes=524288|worker_accounted_imbalance_bytes=0|scenario_index=1|scenario=kv_only|scope_count=4|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|scenario_identity_size=690|scenario_identity_sha256=2d1bbb68fcf68dfd8ec91c87e6637be4cb17fd48ba9448dd7832fdbae001bea2|worker_count=2|worker_accounted_bytes=312|worker_accounted_bytes=312|minimum_worker_accounted_bytes=312|maximum_worker_accounted_bytes=312|worker_accounted_imbalance_bytes=0|scenario_index=2|scenario=mixed|scope_count=4|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|scenario_identity_size=700|scenario_identity_sha256=c6559156693601203bec3632ab45e8d8ce2358333fd24379902964ce03030ec1|worker_count=2|worker_accounted_bytes=524600|worker_accounted_bytes=524600|minimum_worker_accounted_bytes=524600|maximum_worker_accounted_bytes=524600|worker_accounted_imbalance_bytes=0|paged_layout_identity_size=1060|paged_layout_identity=llm-kv-layout-plan-v1|geometry_identity=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24|permutation_algorithm_version=36:splitmix64-fisher-yates-rejection-v1|permutation_domain=5497854231078520625|permutation_domain_uint64_hex=18:0x4c4c4d4b56504731|permutation_seed=1732835419980629872|permutation_entry_count=6|permutation_sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c|paged_execution_identity_size=2239|paged_execution_identity=llm-prefill-cpu-execution-v1|sequence_descriptors_per_scenario_per_worker=4|scenario_index=0|scenario=weights_only|scope_count=2|scope_identity_size=1240|scope_identity_sha256=ec65388b21acf11d327e80e42eec02024a5e4e713880a07d5ddc23ecce969e86|scope_identity_size=1240|scope_identity_sha256=576390a7fd86b32a77e0623f06256f3db8c05ebb58fc8e66c4f0edaa45931aa5|scenario_identity_size=483|scenario_identity_sha256=bdea7d1380a216d61b118c5100ae14508441714b38871a894449ff7382ac324e|worker_count=2|worker_accounted_bytes=524288|worker_accounted_bytes=524288|minimum_worker_accounted_bytes=524288|maximum_worker_accounted_bytes=524288|worker_accounted_imbalance_bytes=0|scenario_index=1|scenario=kv_only|scope_count=4|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|scenario_identity_size=690|scenario_identity_sha256=2d1bbb68fcf68dfd8ec91c87e6637be4cb17fd48ba9448dd7832fdbae001bea2|worker_count=2|worker_accounted_bytes=312|worker_accounted_bytes=312|minimum_worker_accounted_bytes=312|maximum_worker_accounted_bytes=312|worker_accounted_imbalance_bytes=0|scenario_index=2|scenario=mixed|scope_count=4|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|scenario_identity_size=700|scenario_identity_sha256=c6559156693601203bec3632ab45e8d8ce2358333fd24379902964ce03030ec1|worker_count=2|worker_accounted_bytes=524600|worker_accounted_bytes=524600|minimum_worker_accounted_bytes=524600|maximum_worker_accounted_bytes=524600|worker_accounted_imbalance_bytes=0|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=weights_only|work_unit_kind=prefill_operation|kv_write_kind=none|scenario_seed=17410844953366850226|explicit=1|work_units=2|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=0|kv_write_bytes_per_work_unit=0|effective_model_payload_bytes_per_work_unit=1048576|layout_metadata_lookup_count_per_work_unit=0|layout_metadata_read_bytes_per_work_unit=0|accounted_bytes_per_work_unit=1048576|weight_read_bytes=2097152|kv_read_bytes=0|kv_write_bytes=0|effective_model_payload_bytes=2097152|layout_metadata_lookup_count=0|layout_metadata_read_bytes=0|task_accounted_bytes=2097152|maximum_work_units_by_work_unit_cap=1000000000|maximum_work_units_by_guardrail=65536|effective_maximum_work_units=65536","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"}}],"expected_run_checksum":{"state_a_uint64_decimal":"4693879361642038518","state_b_uint64_decimal":"15995465716993483296"}}},{"valid":true,"reason_code":"valid","scenario":"kv_only","work_unit_kind":"prefill_operation","kv_write_kind":"full_prompt_population","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"16590208891222127307","work_units":2,"weight_read_bytes_per_work_unit":"0","kv_read_bytes_per_work_unit":"264","kv_write_bytes_per_work_unit":"120","effective_model_payload_bytes_per_work_unit":"384","layout_metadata_lookup_count_per_work_unit":"60","layout_metadata_read_bytes_per_work_unit":"240","accounted_bytes_per_work_unit":"624","weight_read_bytes":"0","kv_read_bytes":"528","kv_write_bytes":"240","effective_model_payload_bytes":"768","layout_metadata_lookup_count":"120","layout_metadata_read_bytes":"480","task_accounted_bytes":"1248","maximum_work_units_by_work_unit_cap":1000000000,"maximum_work_units_by_guardrail":110127366,"effective_maximum_work_units":110127366,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=8521|model_plan_identity=llm-memory-work-plan-v1|backend=cpu|phase=prefill|kv_layout=paged|work_unit_kind=prefill_operation|methodology=llm-memory-v2-cpu-prefill-paged|component_identity_size=714|component_identity=llm-memory-components-v1|logical_profile_version=31:full_prompt_causal_prefix_tiled|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|permutation_version=36:splitmix64-fisher-yates-rejection-v1|backend_executor_version=39:llm-cpu-executor-v1-arm64-prefill-paged|resource_abi_version=42:llm-memory-prefill-paged-descriptor-abi-v1|schedule_version=46:llm-prefill-owner-local-write-then-tile-k-v-v1|timer_policy_version=62:synchronized-start-to-last-worker-completion-per-scenario-task|buffer_pattern_version=36:llm-paged-physical-buffer-pattern-v1|write_pattern_version=26:llm-prefill-kv-affine64-v1|checksum_pattern_version=36:llm-prefill-paged-affine64-lookup-v1|msl_revision=null|msl_source_sha256=null|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|prefill_planner_version=llm-prefill-planner-v1|prefill_cpu_partition_version=llm-prefill-cpu-accounted-prefix-balanced-v1|prefill_prompt_tokens=5|prefill_query_tile_tokens=2|prefill_tile_count=3|prefill_prefix_token_visits_per_sequence=11|prefill_causal_token_pairs=15|prefill_logical_attention_pairs=120|prefill_logical_attention_fma_terms=360|prefill_prefix_block_visits_per_sequence=6|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=2|kv_blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|kv_block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|layout_metadata_lookups_per_layer_sequence_per_work_unit=15|k_mapping_bytes=72|v_mapping_bytes=72|kv_capacity_bytes=144|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|kv_only_payload_bytes_per_work_unit=384|mixed_payload_bytes_per_work_unit=1048960|total_data_mapping_bytes=1048720|traffic_crossover_numerator=0|traffic_crossover_denominator=0|requested_workers=2|effective_workers=2|layer_descriptors_per_worker=2|sequence_descriptors_per_worker=12|total_layer_descriptors=4|total_sequence_descriptors=24|descriptor_bytes=2880|prefill_execution_identity_size=2239|prefill_execution_identity=llm-prefill-cpu-execution-v1|sequence_descriptors_per_scenario_per_worker=4|scenario_index=0|scenario=weights_only|scope_count=2|scope_identity_size=1240|scope_identity_sha256=ec65388b21acf11d327e80e42eec02024a5e4e713880a07d5ddc23ecce969e86|scope_identity_size=1240|scope_identity_sha256=576390a7fd86b32a77e0623f06256f3db8c05ebb58fc8e66c4f0edaa45931aa5|scenario_identity_size=483|scenario_identity_sha256=bdea7d1380a216d61b118c5100ae14508441714b38871a894449ff7382ac324e|worker_count=2|worker_accounted_bytes=524288|worker_accounted_bytes=524288|minimum_worker_accounted_bytes=524288|maximum_worker_accounted_bytes=524288|worker_accounted_imbalance_bytes=0|scenario_index=1|scenario=kv_only|scope_count=4|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|scenario_identity_size=690|scenario_identity_sha256=2d1bbb68fcf68dfd8ec91c87e6637be4cb17fd48ba9448dd7832fdbae001bea2|worker_count=2|worker_accounted_bytes=312|worker_accounted_bytes=312|minimum_worker_accounted_bytes=312|maximum_worker_accounted_bytes=312|worker_accounted_imbalance_bytes=0|scenario_index=2|scenario=mixed|scope_count=4|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|scenario_identity_size=700|scenario_identity_sha256=c6559156693601203bec3632ab45e8d8ce2358333fd24379902964ce03030ec1|worker_count=2|worker_accounted_bytes=524600|worker_accounted_bytes=524600|minimum_worker_accounted_bytes=524600|maximum_worker_accounted_bytes=524600|worker_accounted_imbalance_bytes=0|paged_layout_identity_size=1060|paged_layout_identity=llm-kv-layout-plan-v1|geometry_identity=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24|permutation_algorithm_version=36:splitmix64-fisher-yates-rejection-v1|permutation_domain=5497854231078520625|permutation_domain_uint64_hex=18:0x4c4c4d4b56504731|permutation_seed=1732835419980629872|permutation_entry_count=6|permutation_sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c|paged_execution_identity_size=2239|paged_execution_identity=llm-prefill-cpu-execution-v1|sequence_descriptors_per_scenario_per_worker=4|scenario_index=0|scenario=weights_only|scope_count=2|scope_identity_size=1240|scope_identity_sha256=ec65388b21acf11d327e80e42eec02024a5e4e713880a07d5ddc23ecce969e86|scope_identity_size=1240|scope_identity_sha256=576390a7fd86b32a77e0623f06256f3db8c05ebb58fc8e66c4f0edaa45931aa5|scenario_identity_size=483|scenario_identity_sha256=bdea7d1380a216d61b118c5100ae14508441714b38871a894449ff7382ac324e|worker_count=2|worker_accounted_bytes=524288|worker_accounted_bytes=524288|minimum_worker_accounted_bytes=524288|maximum_worker_accounted_bytes=524288|worker_accounted_imbalance_bytes=0|scenario_index=1|scenario=kv_only|scope_count=4|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|scenario_identity_size=690|scenario_identity_sha256=2d1bbb68fcf68dfd8ec91c87e6637be4cb17fd48ba9448dd7832fdbae001bea2|worker_count=2|worker_accounted_bytes=312|worker_accounted_bytes=312|minimum_worker_accounted_bytes=312|maximum_worker_accounted_bytes=312|worker_accounted_imbalance_bytes=0|scenario_index=2|scenario=mixed|scope_count=4|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|scenario_identity_size=700|scenario_identity_sha256=c6559156693601203bec3632ab45e8d8ce2358333fd24379902964ce03030ec1|worker_count=2|worker_accounted_bytes=524600|worker_accounted_bytes=524600|minimum_worker_accounted_bytes=524600|maximum_worker_accounted_bytes=524600|worker_accounted_imbalance_bytes=0|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=kv_only|work_unit_kind=prefill_operation|kv_write_kind=full_prompt_population|scenario_seed=16590208891222127307|explicit=1|work_units=2|weight_read_bytes_per_work_unit=0|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|effective_model_payload_bytes_per_work_unit=384|layout_metadata_lookup_count_per_work_unit=60|layout_metadata_read_bytes_per_work_unit=240|accounted_bytes_per_work_unit=624|weight_read_bytes=0|kv_read_bytes=528|kv_write_bytes=240|effective_model_payload_bytes=768|layout_metadata_lookup_count=120|layout_metadata_read_bytes=480|task_accounted_bytes=1248|maximum_work_units_by_work_unit_cap=1000000000|maximum_work_units_by_guardrail=110127366|effective_maximum_work_units=110127366","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"1140714041612029950","state_b_uint64_decimal":"8498857480141418907","exact_bytes_read":"132","span_count_uint64_decimal":"24"},"v":{"state_a_uint64_decimal":"16681237628322075608","state_b_uint64_decimal":"414276627016090771","exact_bytes_read":"132","span_count_uint64_decimal":"24"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"10639026457221265723","state_b_uint64_decimal":"11898353891714788459","exact_bytes_read":"132","span_count_uint64_decimal":"24"},"v":{"state_a_uint64_decimal":"18400180927322680104","state_b_uint64_decimal":"4392117935559101237","exact_bytes_read":"132","span_count_uint64_decimal":"24"}}],"expected_run_checksum":{"state_a_uint64_decimal":"3424005326438551271","state_b_uint64_decimal":"6237725668856807365"}}},{"valid":true,"reason_code":"valid","scenario":"mixed","work_unit_kind":"prefill_operation","kv_write_kind":"full_prompt_population","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"1546516732201688391","work_units":2,"weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"264","kv_write_bytes_per_work_unit":"120","effective_model_payload_bytes_per_work_unit":"1048960","layout_metadata_lookup_count_per_work_unit":"60","layout_metadata_read_bytes_per_work_unit":"240","accounted_bytes_per_work_unit":"1049200","weight_read_bytes":"2097152","kv_read_bytes":"528","kv_write_bytes":"240","effective_model_payload_bytes":"2097920","layout_metadata_lookup_count":"120","layout_metadata_read_bytes":"480","task_accounted_bytes":"2098400","maximum_work_units_by_work_unit_cap":1000000000,"maximum_work_units_by_guardrail":65497,"effective_maximum_work_units":65497,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=8521|model_plan_identity=llm-memory-work-plan-v1|backend=cpu|phase=prefill|kv_layout=paged|work_unit_kind=prefill_operation|methodology=llm-memory-v2-cpu-prefill-paged|component_identity_size=714|component_identity=llm-memory-components-v1|logical_profile_version=31:full_prompt_causal_prefix_tiled|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|permutation_version=36:splitmix64-fisher-yates-rejection-v1|backend_executor_version=39:llm-cpu-executor-v1-arm64-prefill-paged|resource_abi_version=42:llm-memory-prefill-paged-descriptor-abi-v1|schedule_version=46:llm-prefill-owner-local-write-then-tile-k-v-v1|timer_policy_version=62:synchronized-start-to-last-worker-completion-per-scenario-task|buffer_pattern_version=36:llm-paged-physical-buffer-pattern-v1|write_pattern_version=26:llm-prefill-kv-affine64-v1|checksum_pattern_version=36:llm-prefill-paged-affine64-lookup-v1|msl_revision=null|msl_source_sha256=null|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|prefill_planner_version=llm-prefill-planner-v1|prefill_cpu_partition_version=llm-prefill-cpu-accounted-prefix-balanced-v1|prefill_prompt_tokens=5|prefill_query_tile_tokens=2|prefill_tile_count=3|prefill_prefix_token_visits_per_sequence=11|prefill_causal_token_pairs=15|prefill_logical_attention_pairs=120|prefill_logical_attention_fma_terms=360|prefill_prefix_block_visits_per_sequence=6|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=2|kv_blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|kv_block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|layout_metadata_lookups_per_layer_sequence_per_work_unit=15|k_mapping_bytes=72|v_mapping_bytes=72|kv_capacity_bytes=144|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|kv_only_payload_bytes_per_work_unit=384|mixed_payload_bytes_per_work_unit=1048960|total_data_mapping_bytes=1048720|traffic_crossover_numerator=0|traffic_crossover_denominator=0|requested_workers=2|effective_workers=2|layer_descriptors_per_worker=2|sequence_descriptors_per_worker=12|total_layer_descriptors=4|total_sequence_descriptors=24|descriptor_bytes=2880|prefill_execution_identity_size=2239|prefill_execution_identity=llm-prefill-cpu-execution-v1|sequence_descriptors_per_scenario_per_worker=4|scenario_index=0|scenario=weights_only|scope_count=2|scope_identity_size=1240|scope_identity_sha256=ec65388b21acf11d327e80e42eec02024a5e4e713880a07d5ddc23ecce969e86|scope_identity_size=1240|scope_identity_sha256=576390a7fd86b32a77e0623f06256f3db8c05ebb58fc8e66c4f0edaa45931aa5|scenario_identity_size=483|scenario_identity_sha256=bdea7d1380a216d61b118c5100ae14508441714b38871a894449ff7382ac324e|worker_count=2|worker_accounted_bytes=524288|worker_accounted_bytes=524288|minimum_worker_accounted_bytes=524288|maximum_worker_accounted_bytes=524288|worker_accounted_imbalance_bytes=0|scenario_index=1|scenario=kv_only|scope_count=4|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|scenario_identity_size=690|scenario_identity_sha256=2d1bbb68fcf68dfd8ec91c87e6637be4cb17fd48ba9448dd7832fdbae001bea2|worker_count=2|worker_accounted_bytes=312|worker_accounted_bytes=312|minimum_worker_accounted_bytes=312|maximum_worker_accounted_bytes=312|worker_accounted_imbalance_bytes=0|scenario_index=2|scenario=mixed|scope_count=4|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|scenario_identity_size=700|scenario_identity_sha256=c6559156693601203bec3632ab45e8d8ce2358333fd24379902964ce03030ec1|worker_count=2|worker_accounted_bytes=524600|worker_accounted_bytes=524600|minimum_worker_accounted_bytes=524600|maximum_worker_accounted_bytes=524600|worker_accounted_imbalance_bytes=0|paged_layout_identity_size=1060|paged_layout_identity=llm-kv-layout-plan-v1|geometry_identity=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24|permutation_algorithm_version=36:splitmix64-fisher-yates-rejection-v1|permutation_domain=5497854231078520625|permutation_domain_uint64_hex=18:0x4c4c4d4b56504731|permutation_seed=1732835419980629872|permutation_entry_count=6|permutation_sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c|paged_execution_identity_size=2239|paged_execution_identity=llm-prefill-cpu-execution-v1|sequence_descriptors_per_scenario_per_worker=4|scenario_index=0|scenario=weights_only|scope_count=2|scope_identity_size=1240|scope_identity_sha256=ec65388b21acf11d327e80e42eec02024a5e4e713880a07d5ddc23ecce969e86|scope_identity_size=1240|scope_identity_sha256=576390a7fd86b32a77e0623f06256f3db8c05ebb58fc8e66c4f0edaa45931aa5|scenario_identity_size=483|scenario_identity_sha256=bdea7d1380a216d61b118c5100ae14508441714b38871a894449ff7382ac324e|worker_count=2|worker_accounted_bytes=524288|worker_accounted_bytes=524288|minimum_worker_accounted_bytes=524288|maximum_worker_accounted_bytes=524288|worker_accounted_imbalance_bytes=0|scenario_index=1|scenario=kv_only|scope_count=4|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|scenario_identity_size=690|scenario_identity_sha256=2d1bbb68fcf68dfd8ec91c87e6637be4cb17fd48ba9448dd7832fdbae001bea2|worker_count=2|worker_accounted_bytes=312|worker_accounted_bytes=312|minimum_worker_accounted_bytes=312|maximum_worker_accounted_bytes=312|worker_accounted_imbalance_bytes=0|scenario_index=2|scenario=mixed|scope_count=4|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|scenario_identity_size=700|scenario_identity_sha256=c6559156693601203bec3632ab45e8d8ce2358333fd24379902964ce03030ec1|worker_count=2|worker_accounted_bytes=524600|worker_accounted_bytes=524600|minimum_worker_accounted_bytes=524600|maximum_worker_accounted_bytes=524600|worker_accounted_imbalance_bytes=0|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=mixed|work_unit_kind=prefill_operation|kv_write_kind=full_prompt_population|scenario_seed=1546516732201688391|explicit=1|work_units=2|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|effective_model_payload_bytes_per_work_unit=1048960|layout_metadata_lookup_count_per_work_unit=60|layout_metadata_read_bytes_per_work_unit=240|accounted_bytes_per_work_unit=1049200|weight_read_bytes=2097152|kv_read_bytes=528|kv_write_bytes=240|effective_model_payload_bytes=2097920|layout_metadata_lookup_count=120|layout_metadata_read_bytes=480|task_accounted_bytes=2098400|maximum_work_units_by_work_unit_cap=1000000000|maximum_work_units_by_guardrail=65497|effective_maximum_work_units=65497","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"11155577244390976892","state_b_uint64_decimal":"11131867310101203795","exact_bytes_read":"132","span_count_uint64_decimal":"24"},"v":{"state_a_uint64_decimal":"183736363622793439","state_b_uint64_decimal":"11344593588544046653","exact_bytes_read":"132","span_count_uint64_decimal":"24"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"4072218455961768215","state_b_uint64_decimal":"2950715978437760927","exact_bytes_read":"132","span_count_uint64_decimal":"24"},"v":{"state_a_uint64_decimal":"16361630500378501476","state_b_uint64_decimal":"10061267757372228855","exact_bytes_read":"132","span_count_uint64_decimal":"24"}}],"expected_run_checksum":{"state_a_uint64_decimal":"14877702997802518770","state_b_uint64_decimal":"7409280873164736813"}}}],"frozen_plan_refs":{"weights_only":0,"kv_only":1,"mixed":2}},"backend_evidence":{"cpu":{"requested_workers":2,"available_workers":10,"effective_workers":2,"resource_abi_version":"llm-memory-prefill-paged-descriptor-abi-v1","schedule_version":"llm-prefill-owner-local-write-then-tile-k-v-v1","timer_policy_version":"synchronized-start-to-last-worker-completion-per-scenario-task","prefill":{"cost_unit":"worker-cost","sequence_descriptors_per_scenario_per_worker":4,"scenarios":[{"scenario":"weights_only","cost_unit":"worker-cost","scope_count":"2","scope_identities":["llm-prefill-cpu-accounted-prefix-balanced-v1|planner_version=llm-prefill-planner-v1|schedule_version=llm-prefill-owner-local-write-then-tile-k-v-v1|unit_kind=paged_block|scenario=weights_only|active_weight_bytes=1048576|prompt_tokens=5|query_tile_tokens=2|tile_count=3|layer_count=2|batch_size=2|query_head_count=2|head_dimension=3|k_or_v_record_bytes_per_layer=3|paged=1|kv_block_tokens=2|blocks_per_sequence=3|worker_count=2|worker_rotation=0|logical_unit_count=0|active_worker_count=0|weight_shards_included=1|total_weight_shard_bytes=524288|total_kv_model_payload_bytes=0|total_layout_metadata_lookup_count=0|total_layout_metadata_read_bytes=0|total_kv_accounted_bytes=0|total_scenario_accounted_bytes=524288|minimum_worker_accounted_bytes=262144|maximum_worker_accounted_bytes=262144|worker_accounted_imbalance_bytes=0|worker_vector_count=2|worker_0_weight_shard_bytes=262144|worker_0_kv_model_payload_bytes=0|worker_0_layout_metadata_lookup_count=0|worker_0_layout_metadata_read_bytes=0|worker_0_scenario_accounted_bytes=262144|worker_1_weight_shard_bytes=262144|worker_1_kv_model_payload_bytes=0|worker_1_layout_metadata_lookup_count=0|worker_1_layout_metadata_read_bytes=0|worker_1_scenario_accounted_bytes=262144|assignment_count=0","llm-prefill-cpu-accounted-prefix-balanced-v1|planner_version=llm-prefill-planner-v1|schedule_version=llm-prefill-owner-local-write-then-tile-k-v-v1|unit_kind=paged_block|scenario=weights_only|active_weight_bytes=1048576|prompt_tokens=5|query_tile_tokens=2|tile_count=3|layer_count=2|batch_size=2|query_head_count=2|head_dimension=3|k_or_v_record_bytes_per_layer=3|paged=1|kv_block_tokens=2|blocks_per_sequence=3|worker_count=2|worker_rotation=1|logical_unit_count=0|active_worker_count=0|weight_shards_included=1|total_weight_shard_bytes=524288|total_kv_model_payload_bytes=0|total_layout_metadata_lookup_count=0|total_layout_metadata_read_bytes=0|total_kv_accounted_bytes=0|total_scenario_accounted_bytes=524288|minimum_worker_accounted_bytes=262144|maximum_worker_accounted_bytes=262144|worker_accounted_imbalance_bytes=0|worker_vector_count=2|worker_0_weight_shard_bytes=262144|worker_0_kv_model_payload_bytes=0|worker_0_layout_metadata_lookup_count=0|worker_0_layout_metadata_read_bytes=0|worker_0_scenario_accounted_bytes=262144|worker_1_weight_shard_bytes=262144|worker_1_kv_model_payload_bytes=0|worker_1_layout_metadata_lookup_count=0|worker_1_layout_metadata_read_bytes=0|worker_1_scenario_accounted_bytes=262144|assignment_count=0"],"worker_accounted_bytes_per_work_unit":["524288","524288"],"minimum_worker_accounted_bytes_per_work_unit":"524288","maximum_worker_accounted_bytes_per_work_unit":"524288","worker_accounted_imbalance_bytes_per_work_unit":"0","identity":"llm-prefill-cpu-scenario-execution-v1|scenario=weights_only|scope_count=2|scope_identity_size=1240|scope_identity_sha256=ec65388b21acf11d327e80e42eec02024a5e4e713880a07d5ddc23ecce969e86|scope_identity_size=1240|scope_identity_sha256=576390a7fd86b32a77e0623f06256f3db8c05ebb58fc8e66c4f0edaa45931aa5|worker_count=2|worker_accounted_bytes=524288|worker_accounted_bytes=524288|minimum_worker_accounted_bytes=524288|maximum_worker_accounted_bytes=524288|worker_accounted_imbalance_bytes=0"},{"scenario":"kv_only","cost_unit":"worker-cost","scope_count":"4","scope_identities":["llm-prefill-cpu-accounted-prefix-balanced-v1|planner_version=llm-prefill-planner-v1|schedule_version=llm-prefill-owner-local-write-then-tile-k-v-v1|unit_kind=paged_block|scenario=kv_only|active_weight_bytes=1048576|prompt_tokens=5|query_tile_tokens=2|tile_count=3|layer_count=2|batch_size=2|query_head_count=2|head_dimension=3|k_or_v_record_bytes_per_layer=3|paged=1|kv_block_tokens=2|blocks_per_sequence=3|worker_count=2|worker_rotation=0|logical_unit_count=3|active_worker_count=2|weight_shards_included=0|total_weight_shard_bytes=0|total_kv_model_payload_bytes=96|total_layout_metadata_lookup_count=15|total_layout_metadata_read_bytes=60|total_kv_accounted_bytes=156|total_scenario_accounted_bytes=156|minimum_worker_accounted_bytes=76|maximum_worker_accounted_bytes=80|worker_accounted_imbalance_bytes=4|worker_vector_count=2|worker_0_weight_shard_bytes=0|worker_0_kv_model_payload_bytes=48|worker_0_layout_metadata_lookup_count=7|worker_0_layout_metadata_read_bytes=28|worker_0_scenario_accounted_bytes=76|worker_1_weight_shard_bytes=0|worker_1_kv_model_payload_bytes=48|worker_1_layout_metadata_lookup_count=8|worker_1_layout_metadata_read_bytes=32|worker_1_scenario_accounted_bytes=80|assignment_count=2|assignment_0_range_rank=0|assignment_0_worker_index=0|assignment_0_first_unit=0|assignment_0_unit_count=1|assignment_0_kv_model_payload_bytes=48|assignment_0_layout_metadata_lookup_count=7|assignment_0_layout_metadata_read_bytes=28|assignment_0_kv_accounted_bytes=76|assignment_1_range_rank=1|assignment_1_worker_index=1|assignment_1_first_unit=1|assignment_1_unit_count=2|assignment_1_kv_model_payload_bytes=48|assignment_1_layout_metadata_lookup_count=8|assignment_1_layout_metadata_read_bytes=32|assignment_1_kv_accounted_bytes=80","llm-prefill-cpu-accounted-prefix-balanced-v1|planner_version=llm-prefill-planner-v1|schedule_version=llm-prefill-owner-local-write-then-tile-k-v-v1|unit_kind=paged_block|scenario=kv_only|active_weight_bytes=1048576|prompt_tokens=5|query_tile_tokens=2|tile_count=3|layer_count=2|batch_size=2|query_head_count=2|head_dimension=3|k_or_v_record_bytes_per_layer=3|paged=1|kv_block_tokens=2|blocks_per_sequence=3|worker_count=2|worker_rotation=1|logical_unit_count=3|active_worker_count=2|weight_shards_included=0|total_weight_shard_bytes=0|total_kv_model_payload_bytes=96|total_layout_metadata_lookup_count=15|total_layout_metadata_read_bytes=60|total_kv_accounted_bytes=156|total_scenario_accounted_bytes=156|minimum_worker_accounted_bytes=76|maximum_worker_accounted_bytes=80|worker_accounted_imbalance_bytes=4|worker_vector_count=2|worker_0_weight_shard_bytes=0|worker_0_kv_model_payload_bytes=48|worker_0_layout_metadata_lookup_count=8|worker_0_layout_metadata_read_bytes=32|worker_0_scenario_accounted_bytes=80|worker_1_weight_shard_bytes=0|worker_1_kv_model_payload_bytes=48|worker_1_layout_metadata_lookup_count=7|worker_1_layout_metadata_read_bytes=28|worker_1_scenario_accounted_bytes=76|assignment_count=2|assignment_0_range_rank=0|assignment_0_worker_index=1|assignment_0_first_unit=0|assignment_0_unit_count=1|assignment_0_kv_model_payload_bytes=48|assignment_0_layout_metadata_lookup_count=7|assignment_0_layout_metadata_read_bytes=28|assignment_0_kv_accounted_bytes=76|assignment_1_range_rank=1|assignment_1_worker_index=0|assignment_1_first_unit=1|assignment_1_unit_count=2|assignment_1_kv_model_payload_bytes=48|assignment_1_layout_metadata_lookup_count=8|assignment_1_layout_metadata_read_bytes=32|assignment_1_kv_accounted_bytes=80","llm-prefill-cpu-accounted-prefix-balanced-v1|planner_version=llm-prefill-planner-v1|schedule_version=llm-prefill-owner-local-write-then-tile-k-v-v1|unit_kind=paged_block|scenario=kv_only|active_weight_bytes=1048576|prompt_tokens=5|query_tile_tokens=2|tile_count=3|layer_count=2|batch_size=2|query_head_count=2|head_dimension=3|k_or_v_record_bytes_per_layer=3|paged=1|kv_block_tokens=2|blocks_per_sequence=3|worker_count=2|worker_rotation=0|logical_unit_count=3|active_worker_count=2|weight_shards_included=0|total_weight_shard_bytes=0|total_kv_model_payload_bytes=96|total_layout_metadata_lookup_count=15|total_layout_metadata_read_bytes=60|total_kv_accounted_bytes=156|total_scenario_accounted_bytes=156|minimum_worker_accounted_bytes=76|maximum_worker_accounted_bytes=80|worker_accounted_imbalance_bytes=4|worker_vector_count=2|worker_0_weight_shard_bytes=0|worker_0_kv_model_payload_bytes=48|worker_0_layout_metadata_lookup_count=7|worker_0_layout_metadata_read_bytes=28|worker_0_scenario_accounted_bytes=76|worker_1_weight_shard_bytes=0|worker_1_kv_model_payload_bytes=48|worker_1_layout_metadata_lookup_count=8|worker_1_layout_metadata_read_bytes=32|worker_1_scenario_accounted_bytes=80|assignment_count=2|assignment_0_range_rank=0|assignment_0_worker_index=0|assignment_0_first_unit=0|assignment_0_unit_count=1|assignment_0_kv_model_payload_bytes=48|assignment_0_layout_metadata_lookup_count=7|assignment_0_layout_metadata_read_bytes=28|assignment_0_kv_accounted_bytes=76|assignment_1_range_rank=1|assignment_1_worker_index=1|assignment_1_first_unit=1|assignment_1_unit_count=2|assignment_1_kv_model_payload_bytes=48|assignment_1_layout_metadata_lookup_count=8|assignment_1_layout_metadata_read_bytes=32|assignment_1_kv_accounted_bytes=80","llm-prefill-cpu-accounted-prefix-balanced-v1|planner_version=llm-prefill-planner-v1|schedule_version=llm-prefill-owner-local-write-then-tile-k-v-v1|unit_kind=paged_block|scenario=kv_only|active_weight_bytes=1048576|prompt_tokens=5|query_tile_tokens=2|tile_count=3|layer_count=2|batch_size=2|query_head_count=2|head_dimension=3|k_or_v_record_bytes_per_layer=3|paged=1|kv_block_tokens=2|blocks_per_sequence=3|worker_count=2|worker_rotation=1|logical_unit_count=3|active_worker_count=2|weight_shards_included=0|total_weight_shard_bytes=0|total_kv_model_payload_bytes=96|total_layout_metadata_lookup_count=15|total_layout_metadata_read_bytes=60|total_kv_accounted_bytes=156|total_scenario_accounted_bytes=156|minimum_worker_accounted_bytes=76|maximum_worker_accounted_bytes=80|worker_accounted_imbalance_bytes=4|worker_vector_count=2|worker_0_weight_shard_bytes=0|worker_0_kv_model_payload_bytes=48|worker_0_layout_metadata_lookup_count=8|worker_0_layout_metadata_read_bytes=32|worker_0_scenario_accounted_bytes=80|worker_1_weight_shard_bytes=0|worker_1_kv_model_payload_bytes=48|worker_1_layout_metadata_lookup_count=7|worker_1_layout_metadata_read_bytes=28|worker_1_scenario_accounted_bytes=76|assignment_count=2|assignment_0_range_rank=0|assignment_0_worker_index=1|assignment_0_first_unit=0|assignment_0_unit_count=1|assignment_0_kv_model_payload_bytes=48|assignment_0_layout_metadata_lookup_count=7|assignment_0_layout_metadata_read_bytes=28|assignment_0_kv_accounted_bytes=76|assignment_1_range_rank=1|assignment_1_worker_index=0|assignment_1_first_unit=1|assignment_1_unit_count=2|assignment_1_kv_model_payload_bytes=48|assignment_1_layout_metadata_lookup_count=8|assignment_1_layout_metadata_read_bytes=32|assignment_1_kv_accounted_bytes=80"],"worker_accounted_bytes_per_work_unit":["312","312"],"minimum_worker_accounted_bytes_per_work_unit":"312","maximum_worker_accounted_bytes_per_work_unit":"312","worker_accounted_imbalance_bytes_per_work_unit":"0","identity":"llm-prefill-cpu-scenario-execution-v1|scenario=kv_only|scope_count=4|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|worker_count=2|worker_accounted_bytes=312|worker_accounted_bytes=312|minimum_worker_accounted_bytes=312|maximum_worker_accounted_bytes=312|worker_accounted_imbalance_bytes=0"},{"scenario":"mixed","cost_unit":"worker-cost","scope_count":"4","scope_identities":["llm-prefill-cpu-accounted-prefix-balanced-v1|planner_version=llm-prefill-planner-v1|schedule_version=llm-prefill-owner-local-write-then-tile-k-v-v1|unit_kind=paged_block|scenario=mixed|active_weight_bytes=1048576|prompt_tokens=5|query_tile_tokens=2|tile_count=3|layer_count=2|batch_size=2|query_head_count=2|head_dimension=3|k_or_v_record_bytes_per_layer=3|paged=1|kv_block_tokens=2|blocks_per_sequence=3|worker_count=2|worker_rotation=0|logical_unit_count=3|active_worker_count=2|weight_shards_included=1|total_weight_shard_bytes=524288|total_kv_model_payload_bytes=96|total_layout_metadata_lookup_count=15|total_layout_metadata_read_bytes=60|total_kv_accounted_bytes=156|total_scenario_accounted_bytes=524444|minimum_worker_accounted_bytes=262220|maximum_worker_accounted_bytes=262224|worker_accounted_imbalance_bytes=4|worker_vector_count=2|worker_0_weight_shard_bytes=262144|worker_0_kv_model_payload_bytes=48|worker_0_layout_metadata_lookup_count=7|worker_0_layout_metadata_read_bytes=28|worker_0_scenario_accounted_bytes=262220|worker_1_weight_shard_bytes=262144|worker_1_kv_model_payload_bytes=48|worker_1_layout_metadata_lookup_count=8|worker_1_layout_metadata_read_bytes=32|worker_1_scenario_accounted_bytes=262224|assignment_count=2|assignment_0_range_rank=0|assignment_0_worker_index=0|assignment_0_first_unit=0|assignment_0_unit_count=1|assignment_0_kv_model_payload_bytes=48|assignment_0_layout_metadata_lookup_count=7|assignment_0_layout_metadata_read_bytes=28|assignment_0_kv_accounted_bytes=76|assignment_1_range_rank=1|assignment_1_worker_index=1|assignment_1_first_unit=1|assignment_1_unit_count=2|assignment_1_kv_model_payload_bytes=48|assignment_1_layout_metadata_lookup_count=8|assignment_1_layout_metadata_read_bytes=32|assignment_1_kv_accounted_bytes=80","llm-prefill-cpu-accounted-prefix-balanced-v1|planner_version=llm-prefill-planner-v1|schedule_version=llm-prefill-owner-local-write-then-tile-k-v-v1|unit_kind=paged_block|scenario=mixed|active_weight_bytes=1048576|prompt_tokens=5|query_tile_tokens=2|tile_count=3|layer_count=2|batch_size=2|query_head_count=2|head_dimension=3|k_or_v_record_bytes_per_layer=3|paged=1|kv_block_tokens=2|blocks_per_sequence=3|worker_count=2|worker_rotation=1|logical_unit_count=3|active_worker_count=2|weight_shards_included=0|total_weight_shard_bytes=0|total_kv_model_payload_bytes=96|total_layout_metadata_lookup_count=15|total_layout_metadata_read_bytes=60|total_kv_accounted_bytes=156|total_scenario_accounted_bytes=156|minimum_worker_accounted_bytes=76|maximum_worker_accounted_bytes=80|worker_accounted_imbalance_bytes=4|worker_vector_count=2|worker_0_weight_shard_bytes=0|worker_0_kv_model_payload_bytes=48|worker_0_layout_metadata_lookup_count=8|worker_0_layout_metadata_read_bytes=32|worker_0_scenario_accounted_bytes=80|worker_1_weight_shard_bytes=0|worker_1_kv_model_payload_bytes=48|worker_1_layout_metadata_lookup_count=7|worker_1_layout_metadata_read_bytes=28|worker_1_scenario_accounted_bytes=76|assignment_count=2|assignment_0_range_rank=0|assignment_0_worker_index=1|assignment_0_first_unit=0|assignment_0_unit_count=1|assignment_0_kv_model_payload_bytes=48|assignment_0_layout_metadata_lookup_count=7|assignment_0_layout_metadata_read_bytes=28|assignment_0_kv_accounted_bytes=76|assignment_1_range_rank=1|assignment_1_worker_index=0|assignment_1_first_unit=1|assignment_1_unit_count=2|assignment_1_kv_model_payload_bytes=48|assignment_1_layout_metadata_lookup_count=8|assignment_1_layout_metadata_read_bytes=32|assignment_1_kv_accounted_bytes=80","llm-prefill-cpu-accounted-prefix-balanced-v1|planner_version=llm-prefill-planner-v1|schedule_version=llm-prefill-owner-local-write-then-tile-k-v-v1|unit_kind=paged_block|scenario=mixed|active_weight_bytes=1048576|prompt_tokens=5|query_tile_tokens=2|tile_count=3|layer_count=2|batch_size=2|query_head_count=2|head_dimension=3|k_or_v_record_bytes_per_layer=3|paged=1|kv_block_tokens=2|blocks_per_sequence=3|worker_count=2|worker_rotation=0|logical_unit_count=3|active_worker_count=2|weight_shards_included=1|total_weight_shard_bytes=524288|total_kv_model_payload_bytes=96|total_layout_metadata_lookup_count=15|total_layout_metadata_read_bytes=60|total_kv_accounted_bytes=156|total_scenario_accounted_bytes=524444|minimum_worker_accounted_bytes=262220|maximum_worker_accounted_bytes=262224|worker_accounted_imbalance_bytes=4|worker_vector_count=2|worker_0_weight_shard_bytes=262144|worker_0_kv_model_payload_bytes=48|worker_0_layout_metadata_lookup_count=7|worker_0_layout_metadata_read_bytes=28|worker_0_scenario_accounted_bytes=262220|worker_1_weight_shard_bytes=262144|worker_1_kv_model_payload_bytes=48|worker_1_layout_metadata_lookup_count=8|worker_1_layout_metadata_read_bytes=32|worker_1_scenario_accounted_bytes=262224|assignment_count=2|assignment_0_range_rank=0|assignment_0_worker_index=0|assignment_0_first_unit=0|assignment_0_unit_count=1|assignment_0_kv_model_payload_bytes=48|assignment_0_layout_metadata_lookup_count=7|assignment_0_layout_metadata_read_bytes=28|assignment_0_kv_accounted_bytes=76|assignment_1_range_rank=1|assignment_1_worker_index=1|assignment_1_first_unit=1|assignment_1_unit_count=2|assignment_1_kv_model_payload_bytes=48|assignment_1_layout_metadata_lookup_count=8|assignment_1_layout_metadata_read_bytes=32|assignment_1_kv_accounted_bytes=80","llm-prefill-cpu-accounted-prefix-balanced-v1|planner_version=llm-prefill-planner-v1|schedule_version=llm-prefill-owner-local-write-then-tile-k-v-v1|unit_kind=paged_block|scenario=mixed|active_weight_bytes=1048576|prompt_tokens=5|query_tile_tokens=2|tile_count=3|layer_count=2|batch_size=2|query_head_count=2|head_dimension=3|k_or_v_record_bytes_per_layer=3|paged=1|kv_block_tokens=2|blocks_per_sequence=3|worker_count=2|worker_rotation=1|logical_unit_count=3|active_worker_count=2|weight_shards_included=0|total_weight_shard_bytes=0|total_kv_model_payload_bytes=96|total_layout_metadata_lookup_count=15|total_layout_metadata_read_bytes=60|total_kv_accounted_bytes=156|total_scenario_accounted_bytes=156|minimum_worker_accounted_bytes=76|maximum_worker_accounted_bytes=80|worker_accounted_imbalance_bytes=4|worker_vector_count=2|worker_0_weight_shard_bytes=0|worker_0_kv_model_payload_bytes=48|worker_0_layout_metadata_lookup_count=8|worker_0_layout_metadata_read_bytes=32|worker_0_scenario_accounted_bytes=80|worker_1_weight_shard_bytes=0|worker_1_kv_model_payload_bytes=48|worker_1_layout_metadata_lookup_count=7|worker_1_layout_metadata_read_bytes=28|worker_1_scenario_accounted_bytes=76|assignment_count=2|assignment_0_range_rank=0|assignment_0_worker_index=1|assignment_0_first_unit=0|assignment_0_unit_count=1|assignment_0_kv_model_payload_bytes=48|assignment_0_layout_metadata_lookup_count=7|assignment_0_layout_metadata_read_bytes=28|assignment_0_kv_accounted_bytes=76|assignment_1_range_rank=1|assignment_1_worker_index=0|assignment_1_first_unit=1|assignment_1_unit_count=2|assignment_1_kv_model_payload_bytes=48|assignment_1_layout_metadata_lookup_count=8|assignment_1_layout_metadata_read_bytes=32|assignment_1_kv_accounted_bytes=80"],"worker_accounted_bytes_per_work_unit":["524600","524600"],"minimum_worker_accounted_bytes_per_work_unit":"524600","maximum_worker_accounted_bytes_per_work_unit":"524600","worker_accounted_imbalance_bytes_per_work_unit":"0","identity":"llm-prefill-cpu-scenario-execution-v1|scenario=mixed|scope_count=4|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|worker_count=2|worker_accounted_bytes=524600|worker_accounted_bytes=524600|minimum_worker_accounted_bytes=524600|maximum_worker_accounted_bytes=524600|worker_accounted_imbalance_bytes=0"}],"identity":"llm-prefill-cpu-execution-v1|sequence_descriptors_per_scenario_per_worker=4|scenario_index=0|scenario=weights_only|scope_count=2|scope_identity_size=1240|scope_identity_sha256=ec65388b21acf11d327e80e42eec02024a5e4e713880a07d5ddc23ecce969e86|scope_identity_size=1240|scope_identity_sha256=576390a7fd86b32a77e0623f06256f3db8c05ebb58fc8e66c4f0edaa45931aa5|scenario_identity_size=483|scenario_identity_sha256=bdea7d1380a216d61b118c5100ae14508441714b38871a894449ff7382ac324e|worker_count=2|worker_accounted_bytes=524288|worker_accounted_bytes=524288|minimum_worker_accounted_bytes=524288|maximum_worker_accounted_bytes=524288|worker_accounted_imbalance_bytes=0|scenario_index=1|scenario=kv_only|scope_count=4|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|scenario_identity_size=690|scenario_identity_sha256=2d1bbb68fcf68dfd8ec91c87e6637be4cb17fd48ba9448dd7832fdbae001bea2|worker_count=2|worker_accounted_bytes=312|worker_accounted_bytes=312|minimum_worker_accounted_bytes=312|maximum_worker_accounted_bytes=312|worker_accounted_imbalance_bytes=0|scenario_index=2|scenario=mixed|scope_count=4|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|scenario_identity_size=700|scenario_identity_sha256=c6559156693601203bec3632ab45e8d8ce2358333fd24379902964ce03030ec1|worker_count=2|worker_accounted_bytes=524600|worker_accounted_bytes=524600|minimum_worker_accounted_bytes=524600|maximum_worker_accounted_bytes=524600|worker_accounted_imbalance_bytes=0"},"paged":{"layout_identity":"llm-kv-layout-plan-v1|geometry_identity=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24|permutation_algorithm_version=36:splitmix64-fisher-yates-rejection-v1|permutation_domain=5497854231078520625|permutation_domain_uint64_hex=18:0x4c4c4d4b56504731|permutation_seed=1732835419980629872|permutation_entry_count=6|permutation_sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c","execution_identity":"llm-prefill-cpu-execution-v1|sequence_descriptors_per_scenario_per_worker=4|scenario_index=0|scenario=weights_only|scope_count=2|scope_identity_size=1240|scope_identity_sha256=ec65388b21acf11d327e80e42eec02024a5e4e713880a07d5ddc23ecce969e86|scope_identity_size=1240|scope_identity_sha256=576390a7fd86b32a77e0623f06256f3db8c05ebb58fc8e66c4f0edaa45931aa5|scenario_identity_size=483|scenario_identity_sha256=bdea7d1380a216d61b118c5100ae14508441714b38871a894449ff7382ac324e|worker_count=2|worker_accounted_bytes=524288|worker_accounted_bytes=524288|minimum_worker_accounted_bytes=524288|maximum_worker_accounted_bytes=524288|worker_accounted_imbalance_bytes=0|scenario_index=1|scenario=kv_only|scope_count=4|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|scope_identity_size=1744|scope_identity_sha256=8863ba824c5bed966098849b9d84c5225e1a0343c75c0942177fd1f2fba2710d|scope_identity_size=1744|scope_identity_sha256=3631e975c3e7f61177842431f2ee100a2eb614f6e79ee77e9d8297ef666b4ae6|scenario_identity_size=690|scenario_identity_sha256=2d1bbb68fcf68dfd8ec91c87e6637be4cb17fd48ba9448dd7832fdbae001bea2|worker_count=2|worker_accounted_bytes=312|worker_accounted_bytes=312|minimum_worker_accounted_bytes=312|maximum_worker_accounted_bytes=312|worker_accounted_imbalance_bytes=0|scenario_index=2|scenario=mixed|scope_count=4|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|scope_identity_size=1776|scope_identity_sha256=08ba7daf16c451cbfe627810723ae6e7bec3282c0133039f87b2004a5398d640|scope_identity_size=1742|scope_identity_sha256=adf7e31d97728f83211c646877da24404eb62d2840681e1fda03653f714d0b36|scenario_identity_size=700|scenario_identity_sha256=c6559156693601203bec3632ab45e8d8ce2358333fd24379902964ce03030ec1|worker_count=2|worker_accounted_bytes=524600|worker_accounted_bytes=524600|minimum_worker_accounted_bytes=524600|maximum_worker_accounted_bytes=524600|worker_accounted_imbalance_bytes=0","block_table_logical_bytes":"24","block_table_page_rounded_bytes":"16384","block_table_read_only":true,"table_validation":{"valid":true,"interrupted":false,"reason_code":"valid","expected_entries":"6","examined_entries":"6","validation_bitset_bytes":"1"},"permutation":{"algorithm_version":"splitmix64-fisher-yates-rejection-v1","domain_uint64_hex":"0x4c4c4d4b56504731","resolved_seed_uint64_decimal":"1732835419980629872","entry_count":"6","sha256":"14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c","identity":"splitmix64-fisher-yates-rejection-v1|domain=5497854231078520625|domain_uint64_hex=18:0x4c4c4d4b56504731|resolved_seed=1732835419980629872|entry_count=6|sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c"},"ownership":null},"resources":{"valid":true,"reason_code":"valid","model_ref":"resolved_plan","mappings":{"policy":"private_anonymous_regular_cacheable","full_size_physical_mappings":true,"weight":{"requested_bytes":"1048576","committed_bytes":"1048576"},"k":{"requested_bytes":"72","committed_bytes":"16384"},"v":{"requested_bytes":"72","committed_bytes":"16384"},"block_table":{"requested_bytes":"24","committed_bytes":"16384"},"requested_data_bytes":"1048720","committed_data_bytes":"1081344"},"descriptors":{"abi_version":"llm-memory-prefill-paged-descriptor-abi-v1","layer_descriptors_per_worker":2,"sequence_descriptors_per_worker":12,"total_layer_descriptors":4,"total_sequence_descriptors":24,"descriptor_bytes":"2880"},"executor_auxiliary":{"valid":true,"reason_code":"valid","static_reference_bytes":"672","expected_checksum_bytes":"192","actual_checksum_bytes":"192","run_checksum_bytes":"32","worker_status_bytes":"2","thread_handle_bytes":"16","checksum_auxiliary_bytes":"1088","orchestration_auxiliary_bytes":"18","total_auxiliary_bytes":"1106"},"json_output_peak_estimate":{"enabled":true,"valid":true,"reason_code":"valid","policy":"conservative-live-dom-plus-serialized-transport","fixed_schema_bytes":"2097152","input_string_bytes":"3044656","measurement_record_bytes":"1179648","worker_checksum_bytes":"1966080","total_bytes":"8287536"},"allocation_memory_budget":{"resource_rounding_bytes":"48984","transient_peak_bytes":"8834522","layout_transient_bytes":"1","setup_peak_bytes":"57233","runtime_peak_bytes":"9932250","known_owned_peak_bytes":"9932250","admitted_budget_bytes":"9270250700","valid":true,"reason_code":"within-memory-budget","request":{"valid":true,"reason_code":"valid","mapping_granularity_bytes":"16384","requested_weight_mapping_bytes":"1048576","requested_k_mapping_bytes":"72","requested_v_mapping_bytes":"72","committed_weight_mapping_bytes":"1048576","committed_k_mapping_bytes":"16384","committed_v_mapping_bytes":"16384","requested_block_table_mapping_bytes":"24","committed_block_table_mapping_bytes":"16384","requested_data_bytes":"1048720","committed_data_bytes":"1081344","layout_transient_bytes":"1","setup_peak_bytes":"57233","runtime_peak_bytes":"9932250","descriptor_bytes":"2880","planner_storage_bytes":"40848","checksum_auxiliary_bytes":"4352","orchestration_auxiliary_bytes":"8786442","auxiliary_bytes":"8834522","required_total_bytes":"9932250"},"available_memory_bytes":"11587813376","allowed_memory_bytes":"9270250700","used_fallback":false},"initialization":{"complete":true,"pattern_version":"llm-paged-physical-buffer-pattern-v1","pre_touch_policy":"write-every-requested-mapping-byte-once","separate_reference_read_pass":false,"static_references_accumulated_during_initialization":true,"weight_bytes":"1048576","k_bytes":"72","v_bytes":"72","total_bytes":"1048720","non_empty_weight_spans":4,"non_empty_k_spans":12,"non_empty_v_spans":12,"block_table_logical_bytes":"24","block_table_page_rounded_bytes":"16384","block_table_read_only":true,"k_layout_padding_bytes":"12","v_layout_padding_bytes":"12"}}},"metal":null},"memory_budget":{"resource_rounding_bytes":"48984","transient_peak_bytes":"8834522","layout_transient_bytes":"1","setup_peak_bytes":"57233","runtime_peak_bytes":"9932250","known_owned_peak_bytes":"9932250","admitted_budget_bytes":"9270250700","valid":true,"reason_code":"within-memory-budget","request":{"valid":true,"reason_code":"valid","mapping_granularity_bytes":"16384","requested_weight_mapping_bytes":"1048576","requested_k_mapping_bytes":"72","requested_v_mapping_bytes":"72","committed_weight_mapping_bytes":"1048576","committed_k_mapping_bytes":"16384","committed_v_mapping_bytes":"16384","requested_block_table_mapping_bytes":"24","committed_block_table_mapping_bytes":"16384","requested_data_bytes":"1048720","committed_data_bytes":"1081344","layout_transient_bytes":"1","setup_peak_bytes":"57233","runtime_peak_bytes":"9932250","descriptor_bytes":"2880","planner_storage_bytes":"40848","checksum_auxiliary_bytes":"4352","orchestration_auxiliary_bytes":"8786442","auxiliary_bytes":"8834522","required_total_bytes":"9932250"},"available_memory_bytes":"11587813376","allowed_memory_bytes":"9270250700","used_fallback":false},"calibration":{"excluded_from_results":true,"attempts":{"weights_only":[{"attempt_index":0,"scenario":"weights_only","plan_ref":0,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":2.158333333333333e-05,"timing":{"cpu_raw":{"start_ticks":"36788195779112","stop_ticks":"36788195779630","delta_ticks":"518","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"state_a_uint64_decimal":"4693879361642038518","state_b_uint64_decimal":"15995465716993483296"}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}}}],"kv_only":[{"attempt_index":0,"scenario":"kv_only","plan_ref":1,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":5.166666666666667e-06,"timing":{"cpu_raw":{"start_ticks":"36788195781215","stop_ticks":"36788195781339","delta_ticks":"124","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"state_a_uint64_decimal":"3424005326438551271","state_b_uint64_decimal":"6237725668856807365"}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}}}],"mixed":[{"attempt_index":0,"scenario":"mixed","plan_ref":2,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":1.7208333333333333e-05,"timing":{"cpu_raw":{"start_ticks":"36788195782585","stop_ticks":"36788195782998","delta_ticks":"413","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"state_a_uint64_decimal":"14877702997802518770","state_b_uint64_decimal":"7409280873164736813"}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}}}]}},"measurements":[{"measurement_id":0,"plan_ref":0,"scenario":"weights_only","loop_index":0,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":"accepted-plan-derived","elapsed_seconds":1.5666666666666667e-05,"synthetic_work_unit_latency_seconds":7.833333333333333e-06,"synthetic_memory_work_units_per_second":127659.5744680851,"effective_model_payload_gb_s":133.8607659574468,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195784484","stop_ticks":"36788195784860","delta_ticks":"376","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"}}],"actual_run_checksum":{"state_a_uint64_decimal":"4693879361642038518","state_b_uint64_decimal":"15995465716993483296"}}},{"measurement_id":1,"plan_ref":1,"scenario":"kv_only","loop_index":0,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"768","completed_layout_metadata_lookup_count":"120","completed_layout_metadata_read_bytes":"480","completed_task_accounted_bytes":"1248","completion_derivation":"accepted-plan-derived","elapsed_seconds":3.0833333333333336e-06,"synthetic_work_unit_latency_seconds":1.5416666666666668e-06,"synthetic_memory_work_units_per_second":648648.6486486486,"effective_model_payload_gb_s":0.24908108108108107,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195786139","stop_ticks":"36788195786213","delta_ticks":"74","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"1140714041612029950","state_b_uint64_decimal":"8498857480141418907","exact_bytes_read":"132","span_count_uint64_decimal":"24"},"v":{"state_a_uint64_decimal":"16681237628322075608","state_b_uint64_decimal":"414276627016090771","exact_bytes_read":"132","span_count_uint64_decimal":"24"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"10639026457221265723","state_b_uint64_decimal":"11898353891714788459","exact_bytes_read":"132","span_count_uint64_decimal":"24"},"v":{"state_a_uint64_decimal":"18400180927322680104","state_b_uint64_decimal":"4392117935559101237","exact_bytes_read":"132","span_count_uint64_decimal":"24"}}],"actual_run_checksum":{"state_a_uint64_decimal":"3424005326438551271","state_b_uint64_decimal":"6237725668856807365"}}},{"measurement_id":2,"plan_ref":2,"scenario":"mixed","loop_index":0,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":0.999633923123856,"kv_read_payload_fraction":0.0002516778523489933,"kv_write_payload_fraction":0.00011439902379499695,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097920","completed_layout_metadata_lookup_count":"120","completed_layout_metadata_read_bytes":"480","completed_task_accounted_bytes":"2098400","completion_derivation":"accepted-plan-derived","elapsed_seconds":1.6083333333333332e-05,"synthetic_work_unit_latency_seconds":8.041666666666666e-06,"synthetic_memory_work_units_per_second":124352.33160621763,"effective_model_payload_gb_s":130.44062176165804,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195787465","stop_ticks":"36788195787851","delta_ticks":"386","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"11155577244390976892","state_b_uint64_decimal":"11131867310101203795","exact_bytes_read":"132","span_count_uint64_decimal":"24"},"v":{"state_a_uint64_decimal":"183736363622793439","state_b_uint64_decimal":"11344593588544046653","exact_bytes_read":"132","span_count_uint64_decimal":"24"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"4072218455961768215","state_b_uint64_decimal":"2950715978437760927","exact_bytes_read":"132","span_count_uint64_decimal":"24"},"v":{"state_a_uint64_decimal":"16361630500378501476","state_b_uint64_decimal":"10061267757372228855","exact_bytes_read":"132","span_count_uint64_decimal":"24"}}],"actual_run_checksum":{"state_a_uint64_decimal":"14877702997802518770","state_b_uint64_decimal":"7409280873164736813"}}},{"measurement_id":3,"plan_ref":1,"scenario":"kv_only","loop_index":1,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"768","completed_layout_metadata_lookup_count":"120","completed_layout_metadata_read_bytes":"480","completed_task_accounted_bytes":"1248","completion_derivation":"accepted-plan-derived","elapsed_seconds":4.791666666666667e-06,"synthetic_work_unit_latency_seconds":2.3958333333333334e-06,"synthetic_memory_work_units_per_second":417391.30434782605,"effective_model_payload_gb_s":0.16027826086956523,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195814308","stop_ticks":"36788195814423","delta_ticks":"115","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"1140714041612029950","state_b_uint64_decimal":"8498857480141418907","exact_bytes_read":"132","span_count_uint64_decimal":"24"},"v":{"state_a_uint64_decimal":"16681237628322075608","state_b_uint64_decimal":"414276627016090771","exact_bytes_read":"132","span_count_uint64_decimal":"24"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"10639026457221265723","state_b_uint64_decimal":"11898353891714788459","exact_bytes_read":"132","span_count_uint64_decimal":"24"},"v":{"state_a_uint64_decimal":"18400180927322680104","state_b_uint64_decimal":"4392117935559101237","exact_bytes_read":"132","span_count_uint64_decimal":"24"}}],"actual_run_checksum":{"state_a_uint64_decimal":"3424005326438551271","state_b_uint64_decimal":"6237725668856807365"}}},{"measurement_id":4,"plan_ref":2,"scenario":"mixed","loop_index":1,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":0.999633923123856,"kv_read_payload_fraction":0.0002516778523489933,"kv_write_payload_fraction":0.00011439902379499695,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097920","completed_layout_metadata_lookup_count":"120","completed_layout_metadata_read_bytes":"480","completed_task_accounted_bytes":"2098400","completion_derivation":"accepted-plan-derived","elapsed_seconds":1.5958333333333333e-05,"synthetic_work_unit_latency_seconds":7.979166666666666e-06,"synthetic_memory_work_units_per_second":125326.37075718016,"effective_model_payload_gb_s":131.4623498694517,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195815799","stop_ticks":"36788195816182","delta_ticks":"383","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"11155577244390976892","state_b_uint64_decimal":"11131867310101203795","exact_bytes_read":"132","span_count_uint64_decimal":"24"},"v":{"state_a_uint64_decimal":"183736363622793439","state_b_uint64_decimal":"11344593588544046653","exact_bytes_read":"132","span_count_uint64_decimal":"24"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"4072218455961768215","state_b_uint64_decimal":"2950715978437760927","exact_bytes_read":"132","span_count_uint64_decimal":"24"},"v":{"state_a_uint64_decimal":"16361630500378501476","state_b_uint64_decimal":"10061267757372228855","exact_bytes_read":"132","span_count_uint64_decimal":"24"}}],"actual_run_checksum":{"state_a_uint64_decimal":"14877702997802518770","state_b_uint64_decimal":"7409280873164736813"}}},{"measurement_id":5,"plan_ref":0,"scenario":"weights_only","loop_index":1,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":"accepted-plan-derived","elapsed_seconds":1.5458333333333334e-05,"synthetic_work_unit_latency_seconds":7.729166666666667e-06,"synthetic_memory_work_units_per_second":129380.05390835578,"effective_model_payload_gb_s":135.66481940700808,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195817428","stop_ticks":"36788195817799","delta_ticks":"371","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"}}],"actual_run_checksum":{"state_a_uint64_decimal":"4693879361642038518","state_b_uint64_decimal":"15995465716993483296"}}},{"measurement_id":6,"plan_ref":2,"scenario":"mixed","loop_index":2,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":0.999633923123856,"kv_read_payload_fraction":0.0002516778523489933,"kv_write_payload_fraction":0.00011439902379499695,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097920","completed_layout_metadata_lookup_count":"120","completed_layout_metadata_read_bytes":"480","completed_task_accounted_bytes":"2098400","completion_derivation":"accepted-plan-derived","elapsed_seconds":1.6208333333333335e-05,"synthetic_work_unit_latency_seconds":8.104166666666668e-06,"synthetic_memory_work_units_per_second":123393.31619537274,"effective_model_payload_gb_s":129.43465295629818,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195845555","stop_ticks":"36788195845944","delta_ticks":"389","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"11155577244390976892","state_b_uint64_decimal":"11131867310101203795","exact_bytes_read":"132","span_count_uint64_decimal":"24"},"v":{"state_a_uint64_decimal":"183736363622793439","state_b_uint64_decimal":"11344593588544046653","exact_bytes_read":"132","span_count_uint64_decimal":"24"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"4072218455961768215","state_b_uint64_decimal":"2950715978437760927","exact_bytes_read":"132","span_count_uint64_decimal":"24"},"v":{"state_a_uint64_decimal":"16361630500378501476","state_b_uint64_decimal":"10061267757372228855","exact_bytes_read":"132","span_count_uint64_decimal":"24"}}],"actual_run_checksum":{"state_a_uint64_decimal":"14877702997802518770","state_b_uint64_decimal":"7409280873164736813"}}},{"measurement_id":7,"plan_ref":0,"scenario":"weights_only","loop_index":2,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":"accepted-plan-derived","elapsed_seconds":1.575e-05,"synthetic_work_unit_latency_seconds":7.875e-06,"synthetic_memory_work_units_per_second":126984.12698412698,"effective_model_payload_gb_s":133.15250793650793,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195847160","stop_ticks":"36788195847538","delta_ticks":"378","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"11138128939664298039","state_b_uint64_decimal":"12734149885824874954","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"13269377719661686295","state_b_uint64_decimal":"14612894007163574370","exact_bytes_read":"1048576","span_count_uint64_decimal":"4"},"k":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"v":{"state_a_uint64_decimal":"0","state_b_uint64_decimal":"0","exact_bytes_read":"0","span_count_uint64_decimal":"0"}}],"actual_run_checksum":{"state_a_uint64_decimal":"4693879361642038518","state_b_uint64_decimal":"15995465716993483296"}}},{"measurement_id":8,"plan_ref":1,"scenario":"kv_only","loop_index":2,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":2,"effective_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"768","completed_layout_metadata_lookup_count":"120","completed_layout_metadata_read_bytes":"480","completed_task_accounted_bytes":"1248","completion_derivation":"accepted-plan-derived","elapsed_seconds":2.875e-06,"synthetic_work_unit_latency_seconds":1.4375e-06,"synthetic_memory_work_units_per_second":695652.1739130435,"effective_model_payload_gb_s":0.2671304347826087,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":{"start_ticks":"36788195848777","stop_ticks":"36788195848846","delta_ticks":"69","timebase_numer":125,"timebase_denom":3}},"requested_workers":2,"created_workers":2,"completed_workers":2,"qos_successful_workers":2,"qos_failed_workers":0,"worker_startup_failed":false,"kernel_succeeded":true,"timer_started":true,"timer_stopped":true,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":[{"worker_index":0,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"1140714041612029950","state_b_uint64_decimal":"8498857480141418907","exact_bytes_read":"132","span_count_uint64_decimal":"24"},"v":{"state_a_uint64_decimal":"16681237628322075608","state_b_uint64_decimal":"414276627016090771","exact_bytes_read":"132","span_count_uint64_decimal":"24"}},{"worker_index":1,"weight":{"state_a_uint64_decimal":"8321002636941154274","state_b_uint64_decimal":"7664796116566987381","exact_bytes_read":"0","span_count_uint64_decimal":"0"},"k":{"state_a_uint64_decimal":"10639026457221265723","state_b_uint64_decimal":"11898353891714788459","exact_bytes_read":"132","span_count_uint64_decimal":"24"},"v":{"state_a_uint64_decimal":"18400180927322680104","state_b_uint64_decimal":"4392117935559101237","exact_bytes_read":"132","span_count_uint64_decimal":"24"}}],"actual_run_checksum":{"state_a_uint64_decimal":"3424005326438551271","state_b_uint64_decimal":"6237725668856807365"}}}],"aggregates":{"scenarios":{"weights_only":{"scenario":"weights_only","status":"complete","observed_cv_classification":"below-threshold","accepted_measurement_ids":[0,5,7],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":7.833333333333333e-06,"statistics":{"sample_count":3,"average":7.8125e-06,"min":7.729166666666667e-06,"max":7.875e-06,"median":7.833333333333333e-06,"p90":7.866666666666667e-06,"p95":7.870833333333334e-06,"p99":7.874166666666667e-06,"stddev":7.511565157216624e-08,"coefficient_of_variation_pct":0.961480340123728,"median_absolute_deviation":4.166666666666655e-08}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":127659.5744680851,"statistics":{"sample_count":3,"average":128007.91845352262,"min":126984.12698412698,"max":129380.05390835578,"median":127659.5744680851,"p90":129035.95802030164,"p95":129208.00596432871,"p99":129345.64431955037,"stddev":1235.3639567771065,"coefficient_of_variation_pct":0.9650683892853433,"median_absolute_deviation":675.4474839581235}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":133.8607659574468,"statistics":{"sample_count":3,"average":134.22603110032094,"min":133.15250793650793,"max":135.66481940700808,"median":133.8607659574468,"p90":135.30400871709583,"p95":135.48441406205194,"p99":135.62873833801686,"stddev":1.2953729963415193,"coefficient_of_variation_pct":0.9650683892853493,"median_absolute_deviation":0.7082580209388709}}},"kv_only":{"scenario":"kv_only","status":"complete","observed_cv_classification":"above-threshold","accepted_measurement_ids":[1,3,8],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":1.5416666666666668e-06,"statistics":{"sample_count":3,"average":1.7916666666666667e-06,"min":1.4375e-06,"max":2.3958333333333334e-06,"median":1.5416666666666668e-06,"p90":2.2250000000000003e-06,"p95":2.3104166666666666e-06,"p99":2.37875e-06,"stddev":5.258095610051652e-07,"coefficient_of_variation_pct":29.347510381683634,"median_absolute_deviation":1.041666666666668e-07}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":648648.6486486486,"statistics":{"sample_count":3,"average":587230.7089698394,"min":417391.30434782605,"max":695652.1739130435,"median":648648.6486486486,"p90":686251.4688601645,"p95":690951.821386604,"p99":694712.1034077556,"stddev":148950.99989120624,"coefficient_of_variation_pct":25.364988175176734,"median_absolute_deviation":47003.52526439482}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":0.24908108108108107,"statistics":{"sample_count":3,"average":0.22549659224441831,"min":0.16027826086956523,"max":0.2671304347826087,"median":0.24908108108108107,"p90":0.2635205640423031,"p95":0.2653254994124559,"p99":0.26676944770857813,"stddev":0.05719718395822317,"coefficient_of_variation_pct":25.364988175176723,"median_absolute_deviation":0.018049353701527604}}},"mixed":{"scenario":"mixed","status":"complete","observed_cv_classification":"below-threshold","accepted_measurement_ids":[2,4,6],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":8.041666666666666e-06,"statistics":{"sample_count":3,"average":8.041666666666668e-06,"min":7.979166666666666e-06,"max":8.104166666666668e-06,"median":8.041666666666666e-06,"p90":8.091666666666667e-06,"p95":8.097916666666668e-06,"p99":8.102916666666668e-06,"stddev":6.250000000000067e-08,"coefficient_of_variation_pct":0.7772020725388683,"median_absolute_deviation":6.249999999999983e-08}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":124352.33160621763,"statistics":{"sample_count":3,"average":124357.33951959018,"min":123393.31619537274,"max":125326.37075718016,"median":124352.33160621763,"p90":125131.56292698767,"p95":125228.96684208392,"p99":125306.88997416092,"stddev":966.5370112563636,"coefficient_of_variation_pct":0.7772255461480854,"median_absolute_deviation":959.0154108448914}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":130.44062176165804,"statistics":{"sample_count":3,"average":130.4458748624693,"min":129.43465295629818,"max":131.4623498694517,"median":130.44062176165804,"p90":131.25800424789298,"p95":131.36017705867235,"p99":131.44191530729586,"stddev":1.013858663327484,"coefficient_of_variation_pct":0.7772255461480921,"median_absolute_deviation":1.0059688053598563}}}},"traffic_diagnostics":{"classification_version":"llm-exact-weight-vs-kv-read-payload-v1","traffic_crossover_numerator":null,"traffic_crossover_denominator":null,"traffic_crossover_context_tokens":null,"current_visible_context_tokens":null,"current_weight_read_payload_bytes_per_work_unit":"1048576","current_kv_read_payload_bytes_per_work_unit":"264","current_weight_to_kv_read_payload_ratio":null,"current_context_classification":null,"classification_is_payload_only":true,"scenario_headlines":{"weights_only":{"synthetic_work_unit_latency_seconds":7.833333333333333e-06,"synthetic_memory_work_units_per_second":127659.5744680851,"effective_model_payload_gb_s":133.8607659574468},"kv_only":{"synthetic_work_unit_latency_seconds":1.5416666666666668e-06,"synthetic_memory_work_units_per_second":648648.6486486486,"effective_model_payload_gb_s":0.24908108108108107},"mixed":{"synthetic_work_unit_latency_seconds":8.041666666666666e-06,"synthetic_memory_work_units_per_second":124352.33160621763,"effective_model_payload_gb_s":130.44062176165804}}}},"status":"complete","reason_code":"complete","results_complete":true,"run_accepted":true,"interpretation":{"result_scope":"synthetic-memory-only-work-unit-not-real-inference-token-or-prefill-latency","reported_rate":"synthetic_memory_work_units_per_second","backend":"cpu","phase":"prefill","work_unit_kind":"prefill_operation","transformer_math_included":false,"framework_scheduler_and_dispatch_included":false,"compute_memory_overlap_included":false,"physical_dram_traffic_measured":false,"dram_residency":"unverified","cache_residency":"unverified","fixed_context_includes_current_token_slot":null,"kv_layout":"paged","payload_semantics":"exact-logical-weight-plus-kv-read-plus-kv-write-bytes-divided-by-elapsed-time","layout_metadata_timed_but_excluded_from_effective_model_payload_gb_s":true,"prefill_transformer_compute_or_ttft_prediction_included":false,"private_metal_storage_implies_separate_vram":false,"cross_backend_performance_distributions_combined":false,"traffic_classification_semantics":"exact-weight-vs-kv-read-logical-payload-only-not-hardware-bottleneck","full_size_working_set_reduces_but_does_not_prove_dram_residency":true,"comparability_requires":["same-backend","same-methodology-version","same-frozen-work-plan-and-model-geometry","same-software-version","sufficiently-similar-hardware","sufficiently-similar-thermal-and-power-state"]},"diagnostic":null,"interruption_requested":false,"scenario_order_balance_complete":true,"seeds":{"base_seed_uint64_decimal":"424242","source":"user","buffer_domain_seeds":{"weight_uint64_decimal":"904846021374001452","k_uint64_decimal":"10718278915871169538","v_uint64_decimal":"2111147304547004201"},"scenario_domain_seeds":{"weights_only":"17410844953366850226","kv_only":"16590208891222127307","mixed":"1546516732201688391"}},"counters":{"planned_loops":3,"attempted_loops":3,"completed_loops":3,"planned_measurements":9,"attempted_measurements":9,"terminal_measurements":9,"measured_measurements":9,"planned_work_units":"18","completed_work_units":"18","planned_effective_model_payload_bytes":"12587520","completed_effective_model_payload_bytes":"12587520","planned_layout_metadata_lookup_count":"720","completed_layout_metadata_lookup_count":"720","planned_layout_metadata_read_bytes":"2880","completed_layout_metadata_read_bytes":"2880","planned_task_accounted_bytes":"12590400","completed_task_accounted_bytes":"12590400","runner_auxiliary":{"valid":true,"reason_code":"valid","measurement_record_bytes":"7056","loop_record_bytes":"144","calibration_record_bytes":"2784","calibration_identity_bytes":"214584","aggregate_value_bytes":"72","statistics_workspace_bytes":"72","warning_record_bytes":"64","fixed_metadata_bytes":"274112","retained_checksum_bytes":"3264","checksum_auxiliary_bytes":"3264","orchestration_auxiliary_bytes":"498888","total_auxiliary_bytes":"502152"}},"checkpoint_lifecycle":{"checkpoint_failed":false,"observation_point":"before-current-snapshot-preparation","prior_file_writer_attempts":3,"prior_successful_file_writes":3,"current_request":"terminal","current_persistence_success":null,"snapshot_interval_loops":1,"checkpoint_policy":"bounded-loop-snapshots","file_checkpoint_failure_is_terminal_and_not_retried":true,"stdout_intermediate_checkpoints_are_lazy":true},"loop_records":[{"loop_index":0,"planned_order":["weights_only","kv_only","mixed"],"realized_order":["weights_only","kv_only","mixed"],"realized_order_count":3,"measurement_indexes":[0,1,2]},{"loop_index":1,"planned_order":["kv_only","mixed","weights_only"],"realized_order":["kv_only","mixed","weights_only"],"realized_order_count":3,"measurement_indexes":[3,4,5]},{"loop_index":2,"planned_order":["mixed","weights_only","kv_only"],"realized_order":["mixed","weights_only","kv_only"],"realized_order_count":3,"measurement_indexes":[6,7,8]}],"environment":{"processor_name":"Apple M4","macos_version":"26.6.2","performance_core_count":4,"efficiency_core_count":6,"logical_core_count":10,"page_size_bytes":"16384","l1_data_cache_bytes":"131072","l2_data_cache_bytes":"16777216","available_memory_bytes":"11587813376","available_memory_source":"mach-free-plus-inactive-rounded-mib","main_thread_qos":{"requested":true,"applied":true,"code":0},"start":{"thermal_state":"nominal","low_power_mode_available":true,"low_power_mode_enabled":false,"physical_memory_bytes":"25769803776"},"end":{"thermal_state":"nominal","low_power_mode_available":true,"low_power_mode_enabled":false,"physical_memory_bytes":"25769803776"}},"quality_warnings":["kv_only-high-cv","weight-working-set-cache-dominant","kv-working-set-cache-dominant","weights_only-duration-below-target-window","kv_only-duration-below-target-window","mixed-duration-below-target-window"]} diff --git a/tests/fixtures/llm-schema-v2/metal-decode-contiguous.json b/tests/fixtures/llm-schema-v2/metal-decode-contiguous.json new file mode 100644 index 0000000..13593e1 --- /dev/null +++ b/tests/fixtures/llm-schema-v2/metal-decode-contiguous.json @@ -0,0 +1 @@ +{"schema_version":2,"mode":"llm_memory","backend":"metal","phase":"decode","kv_layout":"contiguous","methodology_version":"llm-memory-v2-metal-decode-contiguous","software":{"version":"0.63.1","timestamp":"2026-09-06T07:07:07Z"},"build_manifest":{"binary_sha256":"a9c0aa68c61fccf12bda222a0485a3eba1b642a54b9b1caf8292348f608a1100","compile_flags":{"asflags":"-arch arm64","cxxflags":"-Wall -O3 -std=c++17 -arch arm64 -pthread -Isrc","test_cxxflags":"-Wall -g -std=c++17 -arch arm64 -pthread -Isrc -I/opt/homebrew/opt/googletest/include"},"compiler":"Apple clang version 21.0.0 (clang-2100.1.1.101)\nTarget: arm64-apple-darwin25.6.0\nThread model: posix\nInstalledDir: /Library/Developer/CommandLineTools/usr/bin","git_commit":"870b175927d747fb1d90fadefebe8728da38407a","git_dirty":true,"link_flags":"-pthread -framework Metal -framework Foundation","manifest_version":1,"min_os":"26.0","reason_code":"valid","sdk":"26.5","status":"available","target_arch":"arm64-apple-darwin25.6.0"},"configuration":{"backend":"metal","phase":"decode","kv_layout":"contiguous","kv_block_tokens":null,"weight_size_mb":1,"layer_count":2,"query_head_count":2,"kv_head_count":1,"head_dimension":3,"kv_element_bytes":"1","visible_context_tokens":5,"prompt_tokens":null,"attention_query_tile_tokens":null,"batch_size":2,"requested_workers":null,"available_workers":null,"worker_source":null,"iterations":2,"work_policy":"explicit_fixed_work","loop_count":3,"base_seed_uint64_decimal":"424242","seed_source":"user","output_file":"plans/llm-benchmark-remediation-evidence/phase7/profiles/metal-decode-contiguous.json","argv":["./memory_benchmark","--llm-memory","--llm-memory-backend","metal","--phase","decode","--kv-layout","contiguous","--weight-size-mb","1","--layers","2","--query-heads","2","--kv-heads","1","--head-dim","3","--kv-element-bytes","1","--batch-size","2","--iterations","2","--count","3","--seed","424242","-o","plans/llm-benchmark-remediation-evidence/phase7/profiles/metal-decode-contiguous.json","--context-tokens","5"],"resolved_sources":{"backend":"explicit","phase":"explicit","kv_layout":"explicit","kv_block_tokens":null,"visible_context_tokens":"explicit","prompt_tokens":null,"attention_query_tile_tokens":null,"workers":null,"iterations":"explicit","seed":"explicit"}},"resolved_plan":{"valid":true,"reason_code":"valid","plan_identity":"llm-memory-work-plan-v1|backend=metal|phase=decode|kv_layout=contiguous|work_unit_kind=decode_step|methodology=llm-memory-v2-metal-decode-contiguous|component_identity_size=746|component_identity=llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=39:llm-metal-executor-v1-decode-contiguous|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=42:llm-metal-decode-contiguous-grid-stride-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=32:llm-metal-contiguous-affine32-v1|write_pattern_version=35:llm-metal-decode-append-affine32-v1|checksum_pattern_version=23:llm-metal-dual-mod32-v1|msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5|msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|decode_context=5|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=0|kv_blocks_per_sequence=0|physical_blocks_per_layer=0|total_physical_blocks=0|kv_block_bytes=0|last_block_tokens=0|last_block_valid_bytes=0|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=0|v_layout_padding_bytes=0|block_table_entries=0|block_table_bytes=0|layout_metadata_lookups_per_layer_sequence_per_work_unit=0|k_mapping_bytes=60|v_mapping_bytes=60|kv_capacity_bytes=120|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|kv_only_payload_bytes_per_work_unit=144|mixed_payload_bytes_per_work_unit=1048720|total_data_mapping_bytes=1048696|traffic_crossover_numerator=1048576|traffic_crossover_denominator=24|metal_execution_resolved=1|metal_execution_identity_size=3147|metal_execution_identity_sha256=a04abce0d73149cb29bdf4c5eeab7bee83bf9a73951dc6abc3811d15bb7cc3b5|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391","backend":"metal","phase":"decode","kv_layout":"contiguous","work_unit_kind":"decode_step","geometry":{"valid":true,"reason_code":"valid","phase":"decode","work_unit_kind":"decode_step","decode":{"visible_context_tokens":5},"prefill":null,"attention_kind":"mqa","active_weight_bytes_per_work_unit":"1048576","layer_count":2,"query_head_count":2,"kv_head_count":1,"query_heads_per_kv_head":2,"head_dimension":3,"kv_element_bytes":"1","batch_size":2,"kv_vector_bytes":"3","k_or_v_record_bytes_per_layer":"3","kv_record_bytes_per_layer":"6","kv_bytes_per_visible_token":"12","k_or_v_sequence_visible_bytes":"15","k_mapping_bytes":"60","v_mapping_bytes":"60","kv_capacity_bytes":"120","weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"120","kv_write_bytes_per_work_unit":"24","kv_only_effective_model_payload_bytes_per_work_unit":"144","mixed_effective_model_payload_bytes_per_work_unit":"1048720","total_data_mapping_bytes":"1048696","traffic_crossover_numerator":"1048576","traffic_crossover_denominator":"24","traffic_crossover_context_tokens":43690.666666666664},"model_context":{"prefill":null},"layout":{"kv_layout":"contiguous","kv_block_tokens":null,"blocks_per_sequence":null,"physical_blocks_per_layer":null,"total_physical_blocks":null,"block_bytes":null,"last_block_tokens":null,"last_block_valid_bytes":null,"decode_append_offset_in_last_block":null,"block_table_entries":null,"block_table_bytes":null,"layout_geometry_identity":null,"layout_identity":null,"permutation_domain_uint64_hex":null,"permutation_seed_uint64_decimal":null,"permutation_algorithm_version":null,"permutation_entry_count":null,"permutation_sha256":null,"permutation_identity":null},"resources":{"weight_logical_bytes":"1048576","k_logical_bytes":"60","v_logical_bytes":"60","k_physical_length_bytes":"60","v_physical_length_bytes":"60","k_layout_padding_bytes":"0","v_layout_padding_bytes":"0","block_table_bytes":null,"metal":{"valid":true,"reason_code":"valid","execution_identity":"llm-metal-resource-foundation-v1phase=6:decode;kv_layout=10:contiguous;segment_capacity_bytes=9:268435456;segment_slots_per_pool=3:256;weight_segments=310:whole-element-segments-v1|element_count=1048576|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=1048576;k_segments=300:whole-element-segments-v1|element_count=60|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=60;v_segments=300:whole-element-segments-v1|element_count=60|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=60;table_segments=14:not-applicable;paged_layout_geometry=14:not-applicable;argument_buffer=305:llm-metal-argument-buffer-tier2-v1segment_slot_cap=3:256;weight_slot_base=1:0;k_slot_base=3:256;v_slot_base=3:512;table_slot_base=3:768;status_slot=4:1024;encoded_resource_slot_count=4:1025;weight_segment_count=1:1;k_segment_count=1:1;v_segment_count=1:1;table_segment_count=1:0;active_resource_count=1:4;;argument_encoded_length=4:8200;argument_alignment=1:8;max_buffer_length=11:14302248960;host_mapping_granularity_bytes=5:16384;status_length=4:4096;staging_length=1:0;persistent_resource_length=7:1060992;host_permutation_mapping_bytes=1:0;permutation_validation_bitset_bytes=1:0;additional_owned_bytes=9:233978596;transient_peak_bytes=1:0;known_owned_peak_bytes=9:235039588;planned_resource_count=1:5;resource_0_pool=6:weight;resource_0_pool_index=1:0;resource_0_length_bytes=7:1048576;resource_0_persistent=1:1;resource_1_pool=1:k;resource_1_pool_index=1:0;resource_1_length_bytes=2:60;resource_1_persistent=1:1;resource_2_pool=1:v;resource_2_pool_index=1:0;resource_2_length_bytes=2:60;resource_2_persistent=1:1;resource_3_pool=15:argument_buffer;resource_3_pool_index=1:0;resource_3_length_bytes=4:8200;resource_3_persistent=1:1;resource_4_pool=15:status_checksum;resource_4_pool_index=1:0;resource_4_length_bytes=4:4096;resource_4_persistent=1:1;msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5;msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2;foundation_parameter_abi=34:llm-metal-foundation-parameters-v1;workload_parameter_abi=41:llm-metal-decode-contiguous-parameters-v1;resource_table_abi=27:llm-metal-resource-table-v1;checksum_algorithm=23:llm-metal-dual-mod32-v1;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;","resource_identity":"llm-metal-resource-foundation-v1phase=6:decode;kv_layout=10:contiguous;segment_capacity_bytes=9:268435456;segment_slots_per_pool=3:256;weight_segments=310:whole-element-segments-v1|element_count=1048576|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=1048576;k_segments=300:whole-element-segments-v1|element_count=60|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=60;v_segments=300:whole-element-segments-v1|element_count=60|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=60;table_segments=14:not-applicable;paged_layout_geometry=14:not-applicable;argument_buffer=305:llm-metal-argument-buffer-tier2-v1segment_slot_cap=3:256;weight_slot_base=1:0;k_slot_base=3:256;v_slot_base=3:512;table_slot_base=3:768;status_slot=4:1024;encoded_resource_slot_count=4:1025;weight_segment_count=1:1;k_segment_count=1:1;v_segment_count=1:1;table_segment_count=1:0;active_resource_count=1:4;;argument_encoded_length=4:8200;argument_alignment=1:8;max_buffer_length=11:14302248960;host_mapping_granularity_bytes=5:16384;status_length=4:4096;staging_length=1:0;persistent_resource_length=7:1060992;host_permutation_mapping_bytes=1:0;permutation_validation_bitset_bytes=1:0;additional_owned_bytes=9:233978596;transient_peak_bytes=1:0;known_owned_peak_bytes=9:235039588;planned_resource_count=1:5;resource_0_pool=6:weight;resource_0_pool_index=1:0;resource_0_length_bytes=7:1048576;resource_0_persistent=1:1;resource_1_pool=1:k;resource_1_pool_index=1:0;resource_1_length_bytes=2:60;resource_1_persistent=1:1;resource_2_pool=1:v;resource_2_pool_index=1:0;resource_2_length_bytes=2:60;resource_2_persistent=1:1;resource_3_pool=15:argument_buffer;resource_3_pool_index=1:0;resource_3_length_bytes=4:8200;resource_3_persistent=1:1;resource_4_pool=15:status_checksum;resource_4_pool_index=1:0;resource_4_length_bytes=4:4096;resource_4_persistent=1:1;","msl_revision":"llm-metal-decode-prefill-contiguous-paged-msl23-v5","msl_source_sha256":"2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2","segment_capacity_bytes":"268435456","max_buffer_length_bytes":"14302248960","weight_segments":{"valid":true,"reason_code":"valid","element_count":"1048576","element_bytes":"1","segment_capacity_bytes":"268435456","segment_slot_cap":256,"elements_per_segment":"268435456","segment_count":1,"maximum_addressable_elements":"68719476736","maximum_addressable_bytes":"68719476736","unused_nominal_segment_capacity_bytes":"0","total_length_bytes":"1048576","segment_lengths_bytes":["1048576"],"identity":"whole-element-segments-v1|element_count=1048576|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=1048576"},"k_segments":{"valid":true,"reason_code":"valid","element_count":"60","element_bytes":"1","segment_capacity_bytes":"268435456","segment_slot_cap":256,"elements_per_segment":"268435456","segment_count":1,"maximum_addressable_elements":"68719476736","maximum_addressable_bytes":"68719476736","unused_nominal_segment_capacity_bytes":"0","total_length_bytes":"60","segment_lengths_bytes":["60"],"identity":"whole-element-segments-v1|element_count=60|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=60"},"v_segments":{"valid":true,"reason_code":"valid","element_count":"60","element_bytes":"1","segment_capacity_bytes":"268435456","segment_slot_cap":256,"elements_per_segment":"268435456","segment_count":1,"maximum_addressable_elements":"68719476736","maximum_addressable_bytes":"68719476736","unused_nominal_segment_capacity_bytes":"0","total_length_bytes":"60","segment_lengths_bytes":["60"],"identity":"whole-element-segments-v1|element_count=60|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=60"},"table_segments":null,"argument_buffer":{"valid":true,"reason_code":"valid","encoded_length_bytes":"8200","alignment_bytes":"8","weight_slot_base":0,"k_slot_base":256,"v_slot_base":512,"table_slot_base":768,"status_slot":1024,"encoded_resource_slot_count":1025,"active_resource_count":4,"identity":"llm-metal-argument-buffer-tier2-v1segment_slot_cap=3:256;weight_slot_base=1:0;k_slot_base=3:256;v_slot_base=3:512;table_slot_base=3:768;status_slot=4:1024;encoded_resource_slot_count=4:1025;weight_segment_count=1:1;k_segment_count=1:1;v_segment_count=1:1;table_segment_count=1:0;active_resource_count=1:4;"},"status_buffer_length_bytes":"4096","staging_buffer_length_bytes":"0","persistent_resource_length_bytes":"1060992","transient_peak_bytes":"0","known_owned_peak_bytes":"235039588","admitted_budget_bytes":"9266895257"}},"component_identities":{"logical_profile_version":"decode_steady_fixed_context","kv_layout_version":"contiguous_layer_batch_token_head_dimension","permutation_version":null,"backend_executor_version":"llm-metal-executor-v1-decode-contiguous","resource_abi_version":"llm-metal-argument-buffer-tier2-v1","schedule_version":"llm-metal-decode-contiguous-grid-stride-v1","timer_policy_version":"metal-command-buffer-gpu-start-end-v1","buffer_pattern_version":"llm-metal-contiguous-affine32-v1","write_pattern_version":"llm-metal-decode-append-affine32-v1","checksum_pattern_version":"llm-metal-dual-mod32-v1","msl_revision":"llm-metal-decode-prefill-contiguous-paged-msl23-v5","msl_source_sha256":"2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2","identity":"llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=39:llm-metal-executor-v1-decode-contiguous|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=42:llm-metal-decode-contiguous-grid-stride-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=32:llm-metal-contiguous-affine32-v1|write_pattern_version=35:llm-metal-decode-append-affine32-v1|checksum_pattern_version=23:llm-metal-dual-mod32-v1|msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5|msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2","run_policy_version":"llm-run-policy-bounded-loop-snapshots-v1"},"methodology":{"backend":"metal","phase":"decode","kv_layout":"contiguous","work_unit_kind":"decode_step","weight_passes_per_work_unit":1,"kv_replay_factor":1,"warmup_policy":"same-shape-excluded-steady-state-warm-memory","context_policy":"fixed-visible-context-including-current-token-slot","scenario_order_policy":"cyclic-rotation-across-count-loops","timing_policy":"gpu-start-to-gpu-end-per-command-buffer-task","cache_policy":"metal-private-and-shared-cacheable-no-explicit-flush-between-scenarios","calibration_policy":"per-scenario-automatic-or-explicit-frozen-before-loop-zero","calibration_target_seconds":0.15,"calibration_min_seconds":0.1,"calibration_max_seconds":0.25,"calibration_max_corrections":2,"calibration_min_pilot_accounted_bytes":"8388608","maximum_work_units_per_measurement":65536,"maximum_serial_range_visits_per_lane_per_task":null,"maximum_accounted_bytes_per_task":"68719476736","repeatability_cv_warning_threshold_pct":5.0,"calibration_excluded_from_results":true,"snapshot_policy":"bounded-loop-snapshots","timed_region_exclusions":["mapping-allocation","buffer-initialization-and-pre-touch","argument-buffer-encoding-and-resource-validation","host-command-encoding-and-pre-gpu-queue-wait","status-reset-command-buffer","excluded-kv-write-and-padding-canary-post-validation","same-shape-warmup","calibration-attempts","checksum-reference-generation-and-validation","host-wait-aggregation-and-json-serialization"]},"model_work_plan":{"valid":true,"reason_code":"valid","backend":"metal","phase":"decode","kv_layout":"contiguous","work_unit_kind":"decode_step","weight_passes_per_work_unit":1,"kv_replay_factor":1,"requested_workers":null,"available_workers":null,"effective_workers":null,"worker_plan_count":null,"weight_layer_count":2,"layer_descriptors_per_worker":null,"sequence_descriptors_per_worker":null,"total_layer_descriptors":null,"total_sequence_descriptors":null,"descriptor_bytes":null,"planner_storage_bytes":null},"scenario_plans":[{"valid":true,"reason_code":"valid","scenario":"weights_only","work_unit_kind":"decode_step","kv_write_kind":"none","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"17410844953366850226","work_units":2,"weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"0","kv_write_bytes_per_work_unit":"0","effective_model_payload_bytes_per_work_unit":"1048576","layout_metadata_lookup_count_per_work_unit":"0","layout_metadata_read_bytes_per_work_unit":"0","accounted_bytes_per_work_unit":"1048576","weight_read_bytes":"2097152","kv_read_bytes":"0","kv_write_bytes":"0","effective_model_payload_bytes":"2097152","layout_metadata_lookup_count":"0","layout_metadata_read_bytes":"0","task_accounted_bytes":"2097152","maximum_work_units_by_work_unit_cap":65536,"maximum_work_units_by_guardrail":65536,"effective_maximum_work_units":65536,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=2445|model_plan_identity=llm-memory-work-plan-v1|backend=metal|phase=decode|kv_layout=contiguous|work_unit_kind=decode_step|methodology=llm-memory-v2-metal-decode-contiguous|component_identity_size=746|component_identity=llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=39:llm-metal-executor-v1-decode-contiguous|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=42:llm-metal-decode-contiguous-grid-stride-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=32:llm-metal-contiguous-affine32-v1|write_pattern_version=35:llm-metal-decode-append-affine32-v1|checksum_pattern_version=23:llm-metal-dual-mod32-v1|msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5|msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|decode_context=5|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=0|kv_blocks_per_sequence=0|physical_blocks_per_layer=0|total_physical_blocks=0|kv_block_bytes=0|last_block_tokens=0|last_block_valid_bytes=0|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=0|v_layout_padding_bytes=0|block_table_entries=0|block_table_bytes=0|layout_metadata_lookups_per_layer_sequence_per_work_unit=0|k_mapping_bytes=60|v_mapping_bytes=60|kv_capacity_bytes=120|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|kv_only_payload_bytes_per_work_unit=144|mixed_payload_bytes_per_work_unit=1048720|total_data_mapping_bytes=1048696|traffic_crossover_numerator=1048576|traffic_crossover_denominator=24|metal_execution_resolved=1|metal_execution_identity_size=3147|metal_execution_identity_sha256=a04abce0d73149cb29bdf4c5eeab7bee83bf9a73951dc6abc3811d15bb7cc3b5|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=weights_only|work_unit_kind=decode_step|kv_write_kind=none|scenario_seed=17410844953366850226|explicit=1|work_units=2|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=0|kv_write_bytes_per_work_unit=0|effective_model_payload_bytes_per_work_unit=1048576|layout_metadata_lookup_count_per_work_unit=0|layout_metadata_read_bytes_per_work_unit=0|accounted_bytes_per_work_unit=1048576|weight_read_bytes=2097152|kv_read_bytes=0|kv_write_bytes=0|effective_model_payload_bytes=2097152|layout_metadata_lookup_count=0|layout_metadata_read_bytes=0|task_accounted_bytes=2097152|maximum_work_units_by_work_unit_cap=65536|maximum_work_units_by_guardrail=65536|effective_maximum_work_units=65536","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":null,"expected_run_checksum":{"weight":{"a_uint32_decimal":"3970170880","b_uint32_decimal":"1443364864"},"k":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"v":{"a_uint32_decimal":"0","b_uint32_decimal":"0"}}}},{"valid":true,"reason_code":"valid","scenario":"kv_only","work_unit_kind":"decode_step","kv_write_kind":"current_token_append","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"16590208891222127307","work_units":2,"weight_read_bytes_per_work_unit":"0","kv_read_bytes_per_work_unit":"120","kv_write_bytes_per_work_unit":"24","effective_model_payload_bytes_per_work_unit":"144","layout_metadata_lookup_count_per_work_unit":"0","layout_metadata_read_bytes_per_work_unit":"0","accounted_bytes_per_work_unit":"144","weight_read_bytes":"0","kv_read_bytes":"240","kv_write_bytes":"48","effective_model_payload_bytes":"288","layout_metadata_lookup_count":"0","layout_metadata_read_bytes":"0","task_accounted_bytes":"288","maximum_work_units_by_work_unit_cap":65536,"maximum_work_units_by_guardrail":477218588,"effective_maximum_work_units":65536,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=2445|model_plan_identity=llm-memory-work-plan-v1|backend=metal|phase=decode|kv_layout=contiguous|work_unit_kind=decode_step|methodology=llm-memory-v2-metal-decode-contiguous|component_identity_size=746|component_identity=llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=39:llm-metal-executor-v1-decode-contiguous|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=42:llm-metal-decode-contiguous-grid-stride-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=32:llm-metal-contiguous-affine32-v1|write_pattern_version=35:llm-metal-decode-append-affine32-v1|checksum_pattern_version=23:llm-metal-dual-mod32-v1|msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5|msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|decode_context=5|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=0|kv_blocks_per_sequence=0|physical_blocks_per_layer=0|total_physical_blocks=0|kv_block_bytes=0|last_block_tokens=0|last_block_valid_bytes=0|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=0|v_layout_padding_bytes=0|block_table_entries=0|block_table_bytes=0|layout_metadata_lookups_per_layer_sequence_per_work_unit=0|k_mapping_bytes=60|v_mapping_bytes=60|kv_capacity_bytes=120|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|kv_only_payload_bytes_per_work_unit=144|mixed_payload_bytes_per_work_unit=1048720|total_data_mapping_bytes=1048696|traffic_crossover_numerator=1048576|traffic_crossover_denominator=24|metal_execution_resolved=1|metal_execution_identity_size=3147|metal_execution_identity_sha256=a04abce0d73149cb29bdf4c5eeab7bee83bf9a73951dc6abc3811d15bb7cc3b5|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=kv_only|work_unit_kind=decode_step|kv_write_kind=current_token_append|scenario_seed=16590208891222127307|explicit=1|work_units=2|weight_read_bytes_per_work_unit=0|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|effective_model_payload_bytes_per_work_unit=144|layout_metadata_lookup_count_per_work_unit=0|layout_metadata_read_bytes_per_work_unit=0|accounted_bytes_per_work_unit=144|weight_read_bytes=0|kv_read_bytes=240|kv_write_bytes=48|effective_model_payload_bytes=288|layout_metadata_lookup_count=0|layout_metadata_read_bytes=0|task_accounted_bytes=288|maximum_work_units_by_work_unit_cap=65536|maximum_work_units_by_guardrail=477218588|effective_maximum_work_units=65536","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":null,"expected_run_checksum":{"weight":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"k":{"a_uint32_decimal":"4040505722","b_uint32_decimal":"2801292184"},"v":{"a_uint32_decimal":"2744383266","b_uint32_decimal":"3717700800"}}}},{"valid":true,"reason_code":"valid","scenario":"mixed","work_unit_kind":"decode_step","kv_write_kind":"current_token_append","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"1546516732201688391","work_units":2,"weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"120","kv_write_bytes_per_work_unit":"24","effective_model_payload_bytes_per_work_unit":"1048720","layout_metadata_lookup_count_per_work_unit":"0","layout_metadata_read_bytes_per_work_unit":"0","accounted_bytes_per_work_unit":"1048720","weight_read_bytes":"2097152","kv_read_bytes":"240","kv_write_bytes":"48","effective_model_payload_bytes":"2097440","layout_metadata_lookup_count":"0","layout_metadata_read_bytes":"0","task_accounted_bytes":"2097440","maximum_work_units_by_work_unit_cap":65536,"maximum_work_units_by_guardrail":65527,"effective_maximum_work_units":65527,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=2445|model_plan_identity=llm-memory-work-plan-v1|backend=metal|phase=decode|kv_layout=contiguous|work_unit_kind=decode_step|methodology=llm-memory-v2-metal-decode-contiguous|component_identity_size=746|component_identity=llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=39:llm-metal-executor-v1-decode-contiguous|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=42:llm-metal-decode-contiguous-grid-stride-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=32:llm-metal-contiguous-affine32-v1|write_pattern_version=35:llm-metal-decode-append-affine32-v1|checksum_pattern_version=23:llm-metal-dual-mod32-v1|msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5|msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|decode_context=5|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=0|kv_blocks_per_sequence=0|physical_blocks_per_layer=0|total_physical_blocks=0|kv_block_bytes=0|last_block_tokens=0|last_block_valid_bytes=0|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=0|v_layout_padding_bytes=0|block_table_entries=0|block_table_bytes=0|layout_metadata_lookups_per_layer_sequence_per_work_unit=0|k_mapping_bytes=60|v_mapping_bytes=60|kv_capacity_bytes=120|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|kv_only_payload_bytes_per_work_unit=144|mixed_payload_bytes_per_work_unit=1048720|total_data_mapping_bytes=1048696|traffic_crossover_numerator=1048576|traffic_crossover_denominator=24|metal_execution_resolved=1|metal_execution_identity_size=3147|metal_execution_identity_sha256=a04abce0d73149cb29bdf4c5eeab7bee83bf9a73951dc6abc3811d15bb7cc3b5|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=mixed|work_unit_kind=decode_step|kv_write_kind=current_token_append|scenario_seed=1546516732201688391|explicit=1|work_units=2|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|effective_model_payload_bytes_per_work_unit=1048720|layout_metadata_lookup_count_per_work_unit=0|layout_metadata_read_bytes_per_work_unit=0|accounted_bytes_per_work_unit=1048720|weight_read_bytes=2097152|kv_read_bytes=240|kv_write_bytes=48|effective_model_payload_bytes=2097440|layout_metadata_lookup_count=0|layout_metadata_read_bytes=0|task_accounted_bytes=2097440|maximum_work_units_by_work_unit_cap=65536|maximum_work_units_by_guardrail=65527|effective_maximum_work_units=65527","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":null,"expected_run_checksum":{"weight":{"a_uint32_decimal":"3375628288","b_uint32_decimal":"3948412928"},"k":{"a_uint32_decimal":"1534629258","b_uint32_decimal":"1878915496"},"v":{"a_uint32_decimal":"238375730","b_uint32_decimal":"3007005392"}}}}],"frozen_plan_refs":{"weights_only":0,"kv_only":1,"mixed":2}},"backend_evidence":{"cpu":null,"metal":{"workers_applicable":false,"worker_qos_applicable":false,"lifecycle":{"initialization":{"status":"ready","reason_code":"valid"},"plan_resolution":{"status":"ready","reason_code":"valid"},"preparation":{"status":"ready","reason_code":"valid"},"release":{"status":"ready","reason_code":"valid"}},"capability":{"device_name":"Apple M4","registry_id_uint64_decimal":"4294968227","has_unified_memory":true,"required_apple7_family_supported":true,"argument_buffers_tier2_supported":true,"supported_families":["apple1","apple2","apple3","apple4","apple5","apple6","apple7","apple8","apple9","mac1","mac2","common1","common2","common3","metal3","metal4"],"max_buffer_length_bytes":"14302248960","recommended_max_working_set_size_bytes":"19069665280","compilation_mode":"runtime-source","msl_language_version":"2.3","kernel_revision":"llm-metal-decode-prefill-contiguous-paged-msl23-v5","kernel_source_sha256":"2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2","compiler_diagnostics":null,"compiler_identifier":"21.0.0 (clang-2100.1.1.101)","build_sdk":"26.5","deployment_target":"26.0","argument_buffer":{"encoded_length_bytes":"8200","alignment_bytes":"8"},"layout_probe":{"evaluated":true,"valid":true,"resource_count":4},"foundation_pipelines":[{"label":"membenchmark.llm-metal.pipeline.initialize","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.copy","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.layout-probe","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.decode-contiguous-layout-probe","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.validate-bytes","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.validate-table","thread_execution_width":32,"max_total_threads_per_threadgroup":1024}],"error":{"domain":null,"code":0,"description":null}},"resources":{"allocation_attempted":true,"allocation_completed":true,"initialization_completed":true,"table_upload_completed":true,"table_validation_completed":true,"table_permutation":null,"post_validation_completed":true,"cpu_sample_readback_validation_completed":true,"candidate_cleanup_completed":true,"resources_published":false,"persistent_resource_length_bytes":"1060992","committed_resource_bytes":"1060992","resource_rounding_bytes":"0","layout_padding_bytes":"0","transient_peak_bytes":"0","additional_owned_bytes":"233978596","known_owned_peak_bytes":"235039588","admitted_budget_bytes":"9266895257","current_allocated_size_before_bytes":"1785856","current_allocated_size_peak_bytes":"2834432","current_allocated_size_after_release_bytes":"1785856","recommended_working_set_available":true,"recommended_working_set_headroom_bytes":"18832839836","recommended_working_set_headroom_fraction":0.9875810382341436,"exceeds_recommended_working_set":false,"resources":[{"label":"membenchmark.llm-metal.weight-0","pool":"weight","pool_index":0,"storage_mode":"private","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"544","length_bytes":"1048576","allocated_size_available":true,"allocated_size_bytes":"1048576","committed_bytes":"1048576","allocation_rounding_bytes":"0"},{"label":"membenchmark.llm-metal.k-0","pool":"k","pool_index":0,"storage_mode":"private","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"544","length_bytes":"60","allocated_size_available":true,"allocated_size_bytes":"60","committed_bytes":"60","allocation_rounding_bytes":"0"},{"label":"membenchmark.llm-metal.v-0","pool":"v","pool_index":0,"storage_mode":"private","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"544","length_bytes":"60","allocated_size_available":true,"allocated_size_bytes":"60","committed_bytes":"60","allocation_rounding_bytes":"0"},{"label":"membenchmark.llm-metal.argument_buffer-0","pool":"argument_buffer","pool_index":0,"storage_mode":"shared","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"512","length_bytes":"8200","allocated_size_available":true,"allocated_size_bytes":"8200","committed_bytes":"8200","allocation_rounding_bytes":"0"},{"label":"membenchmark.llm-metal.status_checksum-0","pool":"status_checksum","pool_index":0,"storage_mode":"shared","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"512","length_bytes":"4096","allocated_size_available":true,"allocated_size_bytes":"4096","committed_bytes":"4096","allocation_rounding_bytes":"0"}],"error":{"domain":null,"code":0,"description":null}},"workload_pipelines":[{"label":"membenchmark.llm-metal.pipeline.decode-contiguous.weights-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.decode-contiguous.kv-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.decode-contiguous.mixed","thread_execution_width":32,"max_total_threads_per_threadgroup":1024}],"timed_results_available":true}},"memory_budget":{"resource_rounding_bytes":"0","transient_peak_bytes":"0","layout_transient_bytes":null,"setup_peak_bytes":"235039588","runtime_peak_bytes":"235039588","known_owned_peak_bytes":"235039588","admitted_budget_bytes":"9266895257","valid":true,"reason_code":"within-memory-budget","request":{"valid":true,"reason_code":"valid","mapping_granularity_bytes":"16384","requested_weight_mapping_bytes":"1048576","requested_k_mapping_bytes":"60","requested_v_mapping_bytes":"60","committed_weight_mapping_bytes":"1048576","committed_k_mapping_bytes":"60","committed_v_mapping_bytes":"60","requested_block_table_mapping_bytes":null,"committed_block_table_mapping_bytes":null,"requested_data_bytes":"1048696","committed_data_bytes":"1048696","layout_transient_bytes":null,"setup_peak_bytes":"235039588","runtime_peak_bytes":"235039588","descriptor_bytes":"0","planner_storage_bytes":"152192","checksum_auxiliary_bytes":"0","orchestration_auxiliary_bytes":"233826404","auxiliary_bytes":"233978596","required_total_bytes":"235039588"},"available_memory_bytes":"11583619072","allowed_memory_bytes":"9266895257","used_fallback":false},"calibration":{"excluded_from_results":true,"attempts":{"weights_only":[{"attempt_index":0,"scenario":"weights_only","plan_ref":0,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":0.0003211249131709337,"timing":{"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"weight":{"a_uint32_decimal":"3970170880","b_uint32_decimal":"1443364864"},"k":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"v":{"a_uint32_decimal":"0","b_uint32_decimal":"0"}}},"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-contiguous.weights-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"2097152","maximum_threadgroup_accounted_bytes":"2097152","threadgroup_accounted_imbalance_bytes":"0","threadgroup_accounted_bytes":["2097152"],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:0;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=7:2097152;maximum_threadgroup_accounted_bytes=7:2097152;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:1;threadgroup_accounted_bytes=7:2097152;weight_shard_cost_version=43:llm-metal-weight-vector-grid-stride-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.52666625,"gpu_end_seconds":1532841.526987375,"host_submit_to_completion_seconds":0.000521,"host_wait_seconds":0.000519,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-append-final","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}}}],"kv_only":[{"attempt_index":0,"scenario":"kv_only","plan_ref":1,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":0.00028200005181133747,"timing":{"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"weight":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"k":{"a_uint32_decimal":"4040505722","b_uint32_decimal":"2801292184"},"v":{"a_uint32_decimal":"2744383266","b_uint32_decimal":"3717700800"}}},"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-contiguous.kv-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"1","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"0","cost_unit":null,"minimum_threadgroup_accounted_bytes":null,"maximum_threadgroup_accounted_bytes":null,"threadgroup_accounted_imbalance_bytes":null,"threadgroup_accounted_bytes":[],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=1:1;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:0;visit_bytes=2:15;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=1:0;maximum_threadgroup_accounted_bytes=1:0;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:0;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.52758325,"gpu_end_seconds":1532841.5278652501,"host_submit_to_completion_seconds":0.00054525,"host_wait_seconds":0.000543459,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}}}],"mixed":[{"attempt_index":0,"scenario":"mixed","plan_ref":2,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":0.0005810416769236326,"timing":{"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"weight":{"a_uint32_decimal":"3375628288","b_uint32_decimal":"3948412928"},"k":{"a_uint32_decimal":"1534629258","b_uint32_decimal":"1878915496"},"v":{"a_uint32_decimal":"238375730","b_uint32_decimal":"3007005392"}}},"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-contiguous.mixed","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"0","cost_unit":null,"minimum_threadgroup_accounted_bytes":null,"maximum_threadgroup_accounted_bytes":null,"threadgroup_accounted_imbalance_bytes":null,"threadgroup_accounted_bytes":[],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:0;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=1:0;maximum_threadgroup_accounted_bytes=1:0;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:0;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.528540625,"gpu_end_seconds":1532841.5291216667,"host_submit_to_completion_seconds":0.000883625,"host_wait_seconds":0.000882,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}}}]}},"measurements":[{"measurement_id":0,"plan_ref":0,"scenario":"weights_only","loop_index":0,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":null,"elapsed_seconds":0.0003200001083314419,"synthetic_work_unit_latency_seconds":0.00016000005416572094,"synthetic_memory_work_units_per_second":6249.9978841522425,"effective_model_payload_gb_s":6.5535977813728215,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-contiguous.weights-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"2097152","maximum_threadgroup_accounted_bytes":"2097152","threadgroup_accounted_imbalance_bytes":"0","threadgroup_accounted_bytes":["2097152"],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:0;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=7:2097152;maximum_threadgroup_accounted_bytes=7:2097152;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:1;threadgroup_accounted_bytes=7:2097152;weight_shard_cost_version=43:llm-metal-weight-vector-grid-stride-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.5298849167,"gpu_end_seconds":1532841.5302049168,"host_submit_to_completion_seconds":0.000652416,"host_wait_seconds":0.000651083,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-append-final","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"3970170880","b_uint32_decimal":"1443364864"},"k":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"v":{"a_uint32_decimal":"0","b_uint32_decimal":"0"}}}},{"measurement_id":1,"plan_ref":1,"scenario":"kv_only","loop_index":0,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"288","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"288","completion_derivation":null,"elapsed_seconds":0.00027083326131105423,"synthetic_work_unit_latency_seconds":0.00013541663065552711,"synthetic_memory_work_units_per_second":7384.617348394972,"effective_model_payload_gb_s":0.001063384898168876,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-contiguous.kv-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"1","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"0","cost_unit":null,"minimum_threadgroup_accounted_bytes":null,"maximum_threadgroup_accounted_bytes":null,"threadgroup_accounted_imbalance_bytes":null,"threadgroup_accounted_bytes":[],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=1:1;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:0;visit_bytes=2:15;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=1:0;maximum_threadgroup_accounted_bytes=1:0;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:0;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.5309264169,"gpu_end_seconds":1532841.5311972501,"host_submit_to_completion_seconds":0.000530083,"host_wait_seconds":0.000528916,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"k":{"a_uint32_decimal":"4040505722","b_uint32_decimal":"2801292184"},"v":{"a_uint32_decimal":"2744383266","b_uint32_decimal":"3717700800"}}}},{"measurement_id":2,"plan_ref":2,"scenario":"mixed","loop_index":0,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":0.99986268975513,"kv_read_payload_fraction":0.00011442520405828057,"kv_write_payload_fraction":2.2885040811656113e-05,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097440","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097440","completion_derivation":null,"elapsed_seconds":0.0005654166452586651,"synthetic_work_unit_latency_seconds":0.00028270832262933254,"synthetic_memory_work_units_per_second":3537.214577552888,"effective_model_payload_gb_s":3.7095476717712645,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-contiguous.mixed","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"0","cost_unit":null,"minimum_threadgroup_accounted_bytes":null,"maximum_threadgroup_accounted_bytes":null,"threadgroup_accounted_imbalance_bytes":null,"threadgroup_accounted_bytes":[],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:0;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=1:0;maximum_threadgroup_accounted_bytes=1:0;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:0;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.5318645001,"gpu_end_seconds":1532841.5324299168,"host_submit_to_completion_seconds":0.000864583,"host_wait_seconds":0.000863458,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"3375628288","b_uint32_decimal":"3948412928"},"k":{"a_uint32_decimal":"1534629258","b_uint32_decimal":"1878915496"},"v":{"a_uint32_decimal":"238375730","b_uint32_decimal":"3007005392"}}}},{"measurement_id":3,"plan_ref":1,"scenario":"kv_only","loop_index":1,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"288","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"288","completion_derivation":null,"elapsed_seconds":0.0002680832985788584,"synthetic_work_unit_latency_seconds":0.0001340416492894292,"synthetic_memory_work_units_per_second":7460.367768534031,"effective_model_payload_gb_s":0.0010742929586689005,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-contiguous.kv-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"1","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"0","cost_unit":null,"minimum_threadgroup_accounted_bytes":null,"maximum_threadgroup_accounted_bytes":null,"threadgroup_accounted_imbalance_bytes":null,"threadgroup_accounted_bytes":[],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=1:1;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:0;visit_bytes=2:15;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=1:0;maximum_threadgroup_accounted_bytes=1:0;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:0;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.5341514167,"gpu_end_seconds":1532841.5344195,"host_submit_to_completion_seconds":0.000514583,"host_wait_seconds":0.000513292,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"k":{"a_uint32_decimal":"4040505722","b_uint32_decimal":"2801292184"},"v":{"a_uint32_decimal":"2744383266","b_uint32_decimal":"3717700800"}}}},{"measurement_id":4,"plan_ref":2,"scenario":"mixed","loop_index":1,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":0.99986268975513,"kv_read_payload_fraction":0.00011442520405828057,"kv_write_payload_fraction":2.2885040811656113e-05,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097440","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097440","completion_derivation":null,"elapsed_seconds":0.0005626250058412552,"synthetic_work_unit_latency_seconds":0.0002813125029206276,"synthetic_memory_work_units_per_second":3554.765570736649,"effective_model_payload_gb_s":3.727953749342938,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-contiguous.mixed","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"0","cost_unit":null,"minimum_threadgroup_accounted_bytes":null,"maximum_threadgroup_accounted_bytes":null,"threadgroup_accounted_imbalance_bytes":null,"threadgroup_accounted_bytes":[],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:0;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=1:0;maximum_threadgroup_accounted_bytes=1:0;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:0;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.5350282502,"gpu_end_seconds":1532841.5355908752,"host_submit_to_completion_seconds":0.000840916,"host_wait_seconds":0.000839708,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"3375628288","b_uint32_decimal":"3948412928"},"k":{"a_uint32_decimal":"1534629258","b_uint32_decimal":"1878915496"},"v":{"a_uint32_decimal":"238375730","b_uint32_decimal":"3007005392"}}}},{"measurement_id":5,"plan_ref":0,"scenario":"weights_only","loop_index":1,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":null,"elapsed_seconds":0.00032049999572336674,"synthetic_work_unit_latency_seconds":0.00016024999786168337,"synthetic_memory_work_units_per_second":6240.24969325198,"effective_model_payload_gb_s":6.543376062351388,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-contiguous.weights-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"2097152","maximum_threadgroup_accounted_bytes":"2097152","threadgroup_accounted_imbalance_bytes":"0","threadgroup_accounted_bytes":["2097152"],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:0;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=7:2097152;maximum_threadgroup_accounted_bytes=7:2097152;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:1;threadgroup_accounted_bytes=7:2097152;weight_shard_cost_version=43:llm-metal-weight-vector-grid-stride-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.5362835,"gpu_end_seconds":1532841.536604,"host_submit_to_completion_seconds":0.00063575,"host_wait_seconds":0.000633708,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-append-final","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"3970170880","b_uint32_decimal":"1443364864"},"k":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"v":{"a_uint32_decimal":"0","b_uint32_decimal":"0"}}}},{"measurement_id":6,"plan_ref":2,"scenario":"mixed","loop_index":2,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":0.99986268975513,"kv_read_payload_fraction":0.00011442520405828057,"kv_write_payload_fraction":2.2885040811656113e-05,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097440","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097440","completion_derivation":null,"elapsed_seconds":0.0005681251641362906,"synthetic_work_unit_latency_seconds":0.0002840625820681453,"synthetic_memory_work_units_per_second":3520.351018143265,"effective_model_payload_gb_s":3.691862519747205,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-contiguous.mixed","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"0","cost_unit":null,"minimum_threadgroup_accounted_bytes":null,"maximum_threadgroup_accounted_bytes":null,"threadgroup_accounted_imbalance_bytes":null,"threadgroup_accounted_bytes":[],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:0;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=1:0;maximum_threadgroup_accounted_bytes=1:0;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:0;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.5385,"gpu_end_seconds":1532841.5390681252,"host_submit_to_completion_seconds":0.000920417,"host_wait_seconds":0.000917875,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"3375628288","b_uint32_decimal":"3948412928"},"k":{"a_uint32_decimal":"1534629258","b_uint32_decimal":"1878915496"},"v":{"a_uint32_decimal":"238375730","b_uint32_decimal":"3007005392"}}}},{"measurement_id":7,"plan_ref":0,"scenario":"weights_only","loop_index":2,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":null,"elapsed_seconds":0.0003239165525883436,"synthetic_work_unit_latency_seconds":0.0001619582762941718,"synthetic_memory_work_units_per_second":6174.429753646284,"effective_model_payload_gb_s":6.474358853359406,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-contiguous.weights-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"2097152","maximum_threadgroup_accounted_bytes":"2097152","threadgroup_accounted_imbalance_bytes":"0","threadgroup_accounted_bytes":["2097152"],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:0;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=7:2097152;maximum_threadgroup_accounted_bytes=7:2097152;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:1;threadgroup_accounted_bytes=7:2097152;weight_shard_cost_version=43:llm-metal-weight-vector-grid-stride-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.5398463751,"gpu_end_seconds":1532841.5401702917,"host_submit_to_completion_seconds":0.0006625,"host_wait_seconds":0.0006605,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-append-final","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"3970170880","b_uint32_decimal":"1443364864"},"k":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"v":{"a_uint32_decimal":"0","b_uint32_decimal":"0"}}}},{"measurement_id":8,"plan_ref":1,"scenario":"kv_only","loop_index":2,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"288","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"288","completion_derivation":null,"elapsed_seconds":0.00027116667479276657,"synthetic_work_unit_latency_seconds":0.00013558333739638329,"synthetic_memory_work_units_per_second":7375.537578607172,"effective_model_payload_gb_s":0.0010620774113194328,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-contiguous.kv-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"1","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"0","cost_unit":null,"minimum_threadgroup_accounted_bytes":null,"maximum_threadgroup_accounted_bytes":null,"threadgroup_accounted_imbalance_bytes":null,"threadgroup_accounted_bytes":[],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=1:1;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:0;visit_bytes=2:15;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=1:0;maximum_threadgroup_accounted_bytes=1:0;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:0;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.540918125,"gpu_end_seconds":1532841.5411892917,"host_submit_to_completion_seconds":0.000545041,"host_wait_seconds":0.000543375,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"k":{"a_uint32_decimal":"4040505722","b_uint32_decimal":"2801292184"},"v":{"a_uint32_decimal":"2744383266","b_uint32_decimal":"3717700800"}}}}],"aggregates":{"scenarios":{"weights_only":{"scenario":"weights_only","status":"complete","observed_cv_classification":"below-threshold","accepted_measurement_ids":[0,5,7],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":0.00016024999786168337,"statistics":{"sample_count":3,"average":0.00016073610944052538,"min":0.00016000005416572094,"max":0.0001619582762941718,"median":0.00016024999786168337,"p90":0.00016161662060767412,"p95":0.00016178744845092298,"p99":0.00016192411072552204,"stddev":1.065779914617396e-06,"coefficient_of_variation_pct":0.663061908321073,"median_absolute_deviation":2.4994369596242905e-07}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":6240.24969325198,"statistics":{"sample_count":3,"average":6221.559110350168,"min":6174.429753646284,"max":6249.9978841522425,"median":6240.24969325198,"p90":6248.04824597219,"p95":6249.023065062217,"p99":6249.802920334238,"stddev":41.105218694814326,"coefficient_of_variation_pct":0.6606899969242725,"median_absolute_deviation":9.74819090026267}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":6.543376062351388,"statistics":{"sample_count":3,"average":6.523777565694538,"min":6.474358853359406,"max":6.5535977813728215,"median":6.543376062351388,"p90":6.5515534375685345,"p95":6.552575609470678,"p99":6.553393346992393,"stddev":0.043101945798133395,"coefficient_of_variation_pct":0.660689996924269,"median_absolute_deviation":0.010221719021433096}}},"kv_only":{"scenario":"kv_only","status":"complete","observed_cv_classification":"below-threshold","accepted_measurement_ids":[1,3,8],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":0.00013541663065552711,"statistics":{"sample_count":3,"average":0.0001350138724471132,"min":0.0001340416492894292,"max":0.00013558333739638329,"median":0.00013541663065552711,"p90":0.00013554999604821206,"p95":0.00013556666672229767,"p99":0.00013558000326156615,"stddev":8.4608580274965e-07,"coefficient_of_variation_pct":0.6266658287881296,"median_absolute_deviation":1.6670674085617065e-07}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":7384.617348394972,"statistics":{"sample_count":3,"average":7406.840898512059,"min":7375.537578607172,"max":7460.367768534031,"median":7384.617348394972,"p90":7445.2176845062195,"p95":7452.792726520125,"p99":7458.85276013125,"stddev":46.57740778115197,"coefficient_of_variation_pct":0.6288430981487504,"median_absolute_deviation":9.079769787800615}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":0.001063384898168876,"statistics":{"sample_count":3,"average":0.0010665850893857363,"min":0.0010620774113194328,"max":0.0010742929586689005,"median":0.001063384898168876,"p90":0.0010721113465688955,"p95":0.001073202152618898,"p99":0.0010740747974589,"stddev":6.707146720485908e-06,"coefficient_of_variation_pct":0.6288430981487527,"median_absolute_deviation":1.3074868494432259e-06}}},"mixed":{"scenario":"mixed","status":"complete","observed_cv_classification":"below-threshold","accepted_measurement_ids":[2,4,6],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":0.00028270832262933254,"statistics":{"sample_count":3,"average":0.00028269446920603514,"min":0.0002813125029206276,"max":0.0002840625820681453,"median":0.00028270832262933254,"p90":0.0002837917301803827,"p95":0.00028392715612426397,"p99":0.000284035496879369,"stddev":1.3750919123482935e-06,"coefficient_of_variation_pct":0.4864233517586403,"median_absolute_deviation":1.3542594388127327e-06}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":3537.214577552888,"statistics":{"sample_count":3,"average":3537.443722144267,"min":3520.351018143265,"max":3554.765570736649,"median":3537.214577552888,"p90":3551.2553720998967,"p95":3553.0104714182726,"p99":3554.4145508729735,"stddev":17.208420554586603,"coefficient_of_variation_pct":0.4864648572884008,"median_absolute_deviation":16.86355940962312}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":3.7095476717712645,"statistics":{"sample_count":3,"average":3.7097879802871354,"min":3.691862519747205,"max":3.727953749342938,"median":3.7095476717712645,"p90":3.724272533828603,"p95":3.726113141585771,"p99":3.727585627791505,"stddev":0.018046814804005887,"coefficient_of_variation_pct":0.48646485728839617,"median_absolute_deviation":0.017685152024059736}}}},"traffic_diagnostics":{"classification_version":"llm-exact-weight-vs-kv-read-payload-v1","traffic_crossover_numerator":"1048576","traffic_crossover_denominator":"24","traffic_crossover_context_tokens":43690.666666666664,"current_visible_context_tokens":5,"current_weight_read_payload_bytes_per_work_unit":"1048576","current_kv_read_payload_bytes_per_work_unit":"120","current_weight_to_kv_read_payload_ratio":8738.133333333333,"current_context_classification":"weight_payload_dominant","classification_is_payload_only":true,"scenario_headlines":{"weights_only":{"synthetic_work_unit_latency_seconds":0.00016024999786168337,"synthetic_memory_work_units_per_second":6240.24969325198,"effective_model_payload_gb_s":6.543376062351388},"kv_only":{"synthetic_work_unit_latency_seconds":0.00013541663065552711,"synthetic_memory_work_units_per_second":7384.617348394972,"effective_model_payload_gb_s":0.001063384898168876},"mixed":{"synthetic_work_unit_latency_seconds":0.00028270832262933254,"synthetic_memory_work_units_per_second":3537.214577552888,"effective_model_payload_gb_s":3.7095476717712645}}}},"status":"complete","reason_code":"complete","results_complete":true,"run_accepted":true,"interpretation":{"result_scope":"synthetic-memory-only-work-unit-not-real-inference-token-or-prefill-latency","reported_rate":"synthetic_memory_work_units_per_second","backend":"metal","phase":"decode","work_unit_kind":"decode_step","transformer_math_included":false,"framework_scheduler_and_dispatch_included":false,"compute_memory_overlap_included":false,"physical_dram_traffic_measured":false,"dram_residency":"unverified","cache_residency":"unverified","fixed_context_includes_current_token_slot":true,"kv_layout":"contiguous","payload_semantics":"exact-logical-weight-plus-kv-read-plus-kv-write-bytes-divided-by-elapsed-time","layout_metadata_timed_but_excluded_from_effective_model_payload_gb_s":true,"prefill_transformer_compute_or_ttft_prediction_included":false,"private_metal_storage_implies_separate_vram":false,"cross_backend_performance_distributions_combined":false,"traffic_classification_semantics":"exact-weight-vs-kv-read-logical-payload-only-not-hardware-bottleneck","full_size_working_set_reduces_but_does_not_prove_dram_residency":true,"comparability_requires":["same-backend","same-methodology-version","same-frozen-work-plan-and-model-geometry","same-software-version","sufficiently-similar-hardware","sufficiently-similar-thermal-and-power-state"]},"diagnostic":null,"interruption_requested":false,"scenario_order_balance_complete":true,"seeds":{"base_seed_uint64_decimal":"424242","source":"user","buffer_domain_seeds":{"weight_uint64_decimal":"904846021374001452","k_uint64_decimal":"10718278915871169538","v_uint64_decimal":"2111147304547004201"},"scenario_domain_seeds":{"weights_only":"17410844953366850226","kv_only":"16590208891222127307","mixed":"1546516732201688391"}},"counters":{"planned_loops":3,"attempted_loops":3,"completed_loops":3,"planned_measurements":9,"attempted_measurements":9,"terminal_measurements":9,"measured_measurements":9,"planned_work_units":"18","completed_work_units":"18","planned_effective_model_payload_bytes":"12584640","completed_effective_model_payload_bytes":"12584640","planned_layout_metadata_lookup_count":"0","completed_layout_metadata_lookup_count":"0","planned_layout_metadata_read_bytes":"0","completed_layout_metadata_read_bytes":"0","planned_task_accounted_bytes":"12584640","completed_task_accounted_bytes":"12584640","runner_auxiliary":{"valid":true,"reason_code":"valid","measurement_record_bytes":"7056","loop_record_bytes":"144","calibration_record_bytes":"2784","calibration_identity_bytes":"68352","aggregate_value_bytes":"72","statistics_workspace_bytes":"72","warning_record_bytes":"64","fixed_metadata_bytes":"13764204","retained_checksum_bytes":"0","checksum_auxiliary_bytes":"0","orchestration_auxiliary_bytes":"13842748","total_auxiliary_bytes":"13842748"}},"checkpoint_lifecycle":{"checkpoint_failed":false,"observation_point":"before-current-snapshot-preparation","prior_file_writer_attempts":3,"prior_successful_file_writes":3,"current_request":"terminal","current_persistence_success":null,"snapshot_interval_loops":1,"checkpoint_policy":"bounded-loop-snapshots","file_checkpoint_failure_is_terminal_and_not_retried":true,"stdout_intermediate_checkpoints_are_lazy":true},"loop_records":[{"loop_index":0,"planned_order":["weights_only","kv_only","mixed"],"realized_order":["weights_only","kv_only","mixed"],"realized_order_count":3,"measurement_indexes":[0,1,2]},{"loop_index":1,"planned_order":["kv_only","mixed","weights_only"],"realized_order":["kv_only","mixed","weights_only"],"realized_order_count":3,"measurement_indexes":[3,4,5]},{"loop_index":2,"planned_order":["mixed","weights_only","kv_only"],"realized_order":["mixed","weights_only","kv_only"],"realized_order_count":3,"measurement_indexes":[6,7,8]}],"environment":{"processor_name":"Apple M4","macos_version":"26.6.2","performance_core_count":4,"efficiency_core_count":6,"logical_core_count":10,"page_size_bytes":"16384","l1_data_cache_bytes":"131072","l2_data_cache_bytes":"16777216","available_memory_bytes":"11583619072","available_memory_source":"mach-free-plus-inactive-rounded-mib","main_thread_qos":{"requested":true,"applied":true,"code":0},"start":{"thermal_state":"nominal","low_power_mode_available":true,"low_power_mode_enabled":false,"physical_memory_bytes":"25769803776"},"end":{"thermal_state":"nominal","low_power_mode_available":true,"low_power_mode_enabled":false,"physical_memory_bytes":"25769803776"}},"quality_warnings":["weight-working-set-cache-dominant","kv-working-set-cache-dominant","weights_only-duration-below-target-window","kv_only-duration-below-target-window","mixed-duration-below-target-window"]} diff --git a/tests/fixtures/llm-schema-v2/metal-decode-paged.json b/tests/fixtures/llm-schema-v2/metal-decode-paged.json new file mode 100644 index 0000000..87a24a9 --- /dev/null +++ b/tests/fixtures/llm-schema-v2/metal-decode-paged.json @@ -0,0 +1 @@ +{"schema_version":2,"mode":"llm_memory","backend":"metal","phase":"decode","kv_layout":"paged","methodology_version":"llm-memory-v2-metal-decode-paged","software":{"version":"0.63.1","timestamp":"2026-09-06T07:07:07Z"},"build_manifest":{"binary_sha256":"a9c0aa68c61fccf12bda222a0485a3eba1b642a54b9b1caf8292348f608a1100","compile_flags":{"asflags":"-arch arm64","cxxflags":"-Wall -O3 -std=c++17 -arch arm64 -pthread -Isrc","test_cxxflags":"-Wall -g -std=c++17 -arch arm64 -pthread -Isrc -I/opt/homebrew/opt/googletest/include"},"compiler":"Apple clang version 21.0.0 (clang-2100.1.1.101)\nTarget: arm64-apple-darwin25.6.0\nThread model: posix\nInstalledDir: /Library/Developer/CommandLineTools/usr/bin","git_commit":"870b175927d747fb1d90fadefebe8728da38407a","git_dirty":true,"link_flags":"-pthread -framework Metal -framework Foundation","manifest_version":1,"min_os":"26.0","reason_code":"valid","sdk":"26.5","status":"available","target_arch":"arm64-apple-darwin25.6.0"},"configuration":{"backend":"metal","phase":"decode","kv_layout":"paged","kv_block_tokens":2,"weight_size_mb":1,"layer_count":2,"query_head_count":2,"kv_head_count":1,"head_dimension":3,"kv_element_bytes":"1","visible_context_tokens":5,"prompt_tokens":null,"attention_query_tile_tokens":null,"batch_size":2,"requested_workers":null,"available_workers":null,"worker_source":null,"iterations":2,"work_policy":"explicit_fixed_work","loop_count":3,"base_seed_uint64_decimal":"424242","seed_source":"user","output_file":"plans/llm-benchmark-remediation-evidence/phase7/profiles/metal-decode-paged.json","argv":["./memory_benchmark","--llm-memory","--llm-memory-backend","metal","--phase","decode","--kv-layout","paged","--weight-size-mb","1","--layers","2","--query-heads","2","--kv-heads","1","--head-dim","3","--kv-element-bytes","1","--batch-size","2","--iterations","2","--count","3","--seed","424242","-o","plans/llm-benchmark-remediation-evidence/phase7/profiles/metal-decode-paged.json","--context-tokens","5","--kv-block-tokens","2"],"resolved_sources":{"backend":"explicit","phase":"explicit","kv_layout":"explicit","kv_block_tokens":"explicit","visible_context_tokens":"explicit","prompt_tokens":null,"attention_query_tile_tokens":null,"workers":null,"iterations":"explicit","seed":"explicit"}},"resolved_plan":{"valid":true,"reason_code":"valid","plan_identity":"llm-memory-work-plan-v1|backend=metal|phase=decode|kv_layout=paged|work_unit_kind=decode_step|methodology=llm-memory-v2-metal-decode-paged|component_identity_size=806|component_identity=llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|permutation_version=36:splitmix64-fisher-yates-rejection-v1|backend_executor_version=34:llm-metal-executor-v1-decode-paged|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=49:llm-metal-decode-paged-block-owner-grid-stride-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=36:llm-paged-physical-buffer-pattern-v1|write_pattern_version=41:llm-metal-decode-paged-append-affine32-v1|checksum_pattern_version=40:llm-metal-paged-dual-mod32-lookup-mix-v1|msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5|msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|decode_context=5|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=2|kv_blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|kv_block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|layout_metadata_lookups_per_layer_sequence_per_work_unit=7|k_mapping_bytes=72|v_mapping_bytes=72|kv_capacity_bytes=144|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|kv_only_payload_bytes_per_work_unit=144|mixed_payload_bytes_per_work_unit=1048720|total_data_mapping_bytes=1048720|traffic_crossover_numerator=1048576|traffic_crossover_denominator=24|metal_execution_resolved=1|metal_execution_identity_size=4345|metal_execution_identity_sha256=8016a3f1ef96939bd4f8d9ec9896940d831322f5a67b8f04a466f182cf2e6f53|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391","backend":"metal","phase":"decode","kv_layout":"paged","work_unit_kind":"decode_step","geometry":{"valid":true,"reason_code":"valid","phase":"decode","work_unit_kind":"decode_step","decode":{"visible_context_tokens":5},"prefill":null,"attention_kind":"mqa","active_weight_bytes_per_work_unit":"1048576","layer_count":2,"query_head_count":2,"kv_head_count":1,"query_heads_per_kv_head":2,"head_dimension":3,"kv_element_bytes":"1","batch_size":2,"kv_vector_bytes":"3","k_or_v_record_bytes_per_layer":"3","kv_record_bytes_per_layer":"6","kv_bytes_per_visible_token":"12","k_or_v_sequence_visible_bytes":"15","k_mapping_bytes":"72","v_mapping_bytes":"72","kv_capacity_bytes":"144","weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"120","kv_write_bytes_per_work_unit":"24","kv_only_effective_model_payload_bytes_per_work_unit":"144","mixed_effective_model_payload_bytes_per_work_unit":"1048720","total_data_mapping_bytes":"1048720","traffic_crossover_numerator":"1048576","traffic_crossover_denominator":"24","traffic_crossover_context_tokens":43690.666666666664},"model_context":{"prefill":null},"layout":{"kv_layout":"paged","kv_block_tokens":2,"blocks_per_sequence":"3","physical_blocks_per_layer":"6","total_physical_blocks":"12","block_bytes":"6","last_block_tokens":"1","last_block_valid_bytes":"3","decode_append_offset_in_last_block":"0","block_table_entries":"6","block_table_bytes":"24","layout_geometry_identity":"llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24","layout_identity":"llm-kv-layout-plan-v1|geometry_identity=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24|permutation_algorithm_version=36:splitmix64-fisher-yates-rejection-v1|permutation_domain=5497854231078520625|permutation_domain_uint64_hex=18:0x4c4c4d4b56504731|permutation_seed=1732835419980629872|permutation_entry_count=6|permutation_sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c","permutation_domain_uint64_hex":"0x4c4c4d4b56504731","permutation_seed_uint64_decimal":"1732835419980629872","permutation_algorithm_version":"splitmix64-fisher-yates-rejection-v1","permutation_entry_count":"6","permutation_sha256":"14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c","permutation_identity":"splitmix64-fisher-yates-rejection-v1|domain=5497854231078520625|domain_uint64_hex=18:0x4c4c4d4b56504731|resolved_seed=1732835419980629872|entry_count=6|sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c"},"resources":{"weight_logical_bytes":"1048576","k_logical_bytes":"60","v_logical_bytes":"60","k_physical_length_bytes":"72","v_physical_length_bytes":"72","k_layout_padding_bytes":"12","v_layout_padding_bytes":"12","block_table_bytes":"24","metal":{"valid":true,"reason_code":"valid","execution_identity":"llm-metal-resource-foundation-v1phase=6:decode;kv_layout=5:paged;segment_capacity_bytes=9:268435456;segment_slots_per_pool=3:256;weight_segments=310:whole-element-segments-v1|element_count=1048576|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=1048576;k_segments=302:whole-element-segments-v1|element_count=12|element_bytes=6|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=44739242|segment_count=1|maximum_addressable_elements=11453245952|maximum_addressable_bytes=68719475712|unused_nominal_segment_capacity_bytes=1024|total_length_bytes=72;v_segments=302:whole-element-segments-v1|element_count=12|element_bytes=6|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=44739242|segment_count=1|maximum_addressable_elements=11453245952|maximum_addressable_bytes=68719475712|unused_nominal_segment_capacity_bytes=1024|total_length_bytes=72;table_segments=298:whole-element-segments-v1|element_count=6|element_bytes=4|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=67108864|segment_count=1|maximum_addressable_elements=17179869184|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=24;paged_layout_geometry=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24;argument_buffer=305:llm-metal-argument-buffer-tier2-v1segment_slot_cap=3:256;weight_slot_base=1:0;k_slot_base=3:256;v_slot_base=3:512;table_slot_base=3:768;status_slot=4:1024;encoded_resource_slot_count=4:1025;weight_segment_count=1:1;k_segment_count=1:1;v_segment_count=1:1;table_segment_count=1:1;active_resource_count=1:5;;argument_encoded_length=4:8200;argument_alignment=1:8;max_buffer_length=11:14302248960;host_mapping_granularity_bytes=5:16384;status_length=4:4096;staging_length=2:24;persistent_resource_length=7:1061040;host_permutation_mapping_bytes=5:16384;permutation_validation_bitset_bytes=1:1;additional_owned_bytes=9:234085333;transient_peak_bytes=5:16409;known_owned_peak_bytes=9:235162782;planned_resource_count=1:7;resource_0_pool=6:weight;resource_0_pool_index=1:0;resource_0_length_bytes=7:1048576;resource_0_persistent=1:1;resource_1_pool=1:k;resource_1_pool_index=1:0;resource_1_length_bytes=2:72;resource_1_persistent=1:1;resource_2_pool=1:v;resource_2_pool_index=1:0;resource_2_length_bytes=2:72;resource_2_persistent=1:1;resource_3_pool=11:block_table;resource_3_pool_index=1:0;resource_3_length_bytes=2:24;resource_3_persistent=1:1;resource_4_pool=15:argument_buffer;resource_4_pool_index=1:0;resource_4_length_bytes=4:8200;resource_4_persistent=1:1;resource_5_pool=15:status_checksum;resource_5_pool_index=1:0;resource_5_length_bytes=4:4096;resource_5_persistent=1:1;resource_6_pool=7:staging;resource_6_pool_index=1:0;resource_6_length_bytes=2:24;resource_6_persistent=1:0;msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5;msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2;foundation_parameter_abi=34:llm-metal-foundation-parameters-v1;workload_parameter_abi=36:llm-metal-decode-paged-parameters-v1;resource_table_abi=27:llm-metal-resource-table-v1;checksum_algorithm=23:llm-metal-dual-mod32-v1;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;","resource_identity":"llm-metal-resource-foundation-v1phase=6:decode;kv_layout=5:paged;segment_capacity_bytes=9:268435456;segment_slots_per_pool=3:256;weight_segments=310:whole-element-segments-v1|element_count=1048576|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=1048576;k_segments=302:whole-element-segments-v1|element_count=12|element_bytes=6|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=44739242|segment_count=1|maximum_addressable_elements=11453245952|maximum_addressable_bytes=68719475712|unused_nominal_segment_capacity_bytes=1024|total_length_bytes=72;v_segments=302:whole-element-segments-v1|element_count=12|element_bytes=6|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=44739242|segment_count=1|maximum_addressable_elements=11453245952|maximum_addressable_bytes=68719475712|unused_nominal_segment_capacity_bytes=1024|total_length_bytes=72;table_segments=298:whole-element-segments-v1|element_count=6|element_bytes=4|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=67108864|segment_count=1|maximum_addressable_elements=17179869184|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=24;paged_layout_geometry=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24;argument_buffer=305:llm-metal-argument-buffer-tier2-v1segment_slot_cap=3:256;weight_slot_base=1:0;k_slot_base=3:256;v_slot_base=3:512;table_slot_base=3:768;status_slot=4:1024;encoded_resource_slot_count=4:1025;weight_segment_count=1:1;k_segment_count=1:1;v_segment_count=1:1;table_segment_count=1:1;active_resource_count=1:5;;argument_encoded_length=4:8200;argument_alignment=1:8;max_buffer_length=11:14302248960;host_mapping_granularity_bytes=5:16384;status_length=4:4096;staging_length=2:24;persistent_resource_length=7:1061040;host_permutation_mapping_bytes=5:16384;permutation_validation_bitset_bytes=1:1;additional_owned_bytes=9:234085333;transient_peak_bytes=5:16409;known_owned_peak_bytes=9:235162782;planned_resource_count=1:7;resource_0_pool=6:weight;resource_0_pool_index=1:0;resource_0_length_bytes=7:1048576;resource_0_persistent=1:1;resource_1_pool=1:k;resource_1_pool_index=1:0;resource_1_length_bytes=2:72;resource_1_persistent=1:1;resource_2_pool=1:v;resource_2_pool_index=1:0;resource_2_length_bytes=2:72;resource_2_persistent=1:1;resource_3_pool=11:block_table;resource_3_pool_index=1:0;resource_3_length_bytes=2:24;resource_3_persistent=1:1;resource_4_pool=15:argument_buffer;resource_4_pool_index=1:0;resource_4_length_bytes=4:8200;resource_4_persistent=1:1;resource_5_pool=15:status_checksum;resource_5_pool_index=1:0;resource_5_length_bytes=4:4096;resource_5_persistent=1:1;resource_6_pool=7:staging;resource_6_pool_index=1:0;resource_6_length_bytes=2:24;resource_6_persistent=1:0;","msl_revision":"llm-metal-decode-prefill-contiguous-paged-msl23-v5","msl_source_sha256":"2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2","segment_capacity_bytes":"268435456","max_buffer_length_bytes":"14302248960","weight_segments":{"valid":true,"reason_code":"valid","element_count":"1048576","element_bytes":"1","segment_capacity_bytes":"268435456","segment_slot_cap":256,"elements_per_segment":"268435456","segment_count":1,"maximum_addressable_elements":"68719476736","maximum_addressable_bytes":"68719476736","unused_nominal_segment_capacity_bytes":"0","total_length_bytes":"1048576","segment_lengths_bytes":["1048576"],"identity":"whole-element-segments-v1|element_count=1048576|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=1048576"},"k_segments":{"valid":true,"reason_code":"valid","element_count":"12","element_bytes":"6","segment_capacity_bytes":"268435456","segment_slot_cap":256,"elements_per_segment":"44739242","segment_count":1,"maximum_addressable_elements":"11453245952","maximum_addressable_bytes":"68719475712","unused_nominal_segment_capacity_bytes":"1024","total_length_bytes":"72","segment_lengths_bytes":["72"],"identity":"whole-element-segments-v1|element_count=12|element_bytes=6|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=44739242|segment_count=1|maximum_addressable_elements=11453245952|maximum_addressable_bytes=68719475712|unused_nominal_segment_capacity_bytes=1024|total_length_bytes=72"},"v_segments":{"valid":true,"reason_code":"valid","element_count":"12","element_bytes":"6","segment_capacity_bytes":"268435456","segment_slot_cap":256,"elements_per_segment":"44739242","segment_count":1,"maximum_addressable_elements":"11453245952","maximum_addressable_bytes":"68719475712","unused_nominal_segment_capacity_bytes":"1024","total_length_bytes":"72","segment_lengths_bytes":["72"],"identity":"whole-element-segments-v1|element_count=12|element_bytes=6|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=44739242|segment_count=1|maximum_addressable_elements=11453245952|maximum_addressable_bytes=68719475712|unused_nominal_segment_capacity_bytes=1024|total_length_bytes=72"},"table_segments":{"valid":true,"reason_code":"valid","element_count":"6","element_bytes":"4","segment_capacity_bytes":"268435456","segment_slot_cap":256,"elements_per_segment":"67108864","segment_count":1,"maximum_addressable_elements":"17179869184","maximum_addressable_bytes":"68719476736","unused_nominal_segment_capacity_bytes":"0","total_length_bytes":"24","segment_lengths_bytes":["24"],"identity":"whole-element-segments-v1|element_count=6|element_bytes=4|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=67108864|segment_count=1|maximum_addressable_elements=17179869184|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=24"},"argument_buffer":{"valid":true,"reason_code":"valid","encoded_length_bytes":"8200","alignment_bytes":"8","weight_slot_base":0,"k_slot_base":256,"v_slot_base":512,"table_slot_base":768,"status_slot":1024,"encoded_resource_slot_count":1025,"active_resource_count":5,"identity":"llm-metal-argument-buffer-tier2-v1segment_slot_cap=3:256;weight_slot_base=1:0;k_slot_base=3:256;v_slot_base=3:512;table_slot_base=3:768;status_slot=4:1024;encoded_resource_slot_count=4:1025;weight_segment_count=1:1;k_segment_count=1:1;v_segment_count=1:1;table_segment_count=1:1;active_resource_count=1:5;"},"status_buffer_length_bytes":"4096","staging_buffer_length_bytes":"24","persistent_resource_length_bytes":"1061040","transient_peak_bytes":"16409","known_owned_peak_bytes":"235162782","admitted_budget_bytes":"9266895257"}},"component_identities":{"logical_profile_version":"decode_steady_fixed_context","kv_layout_version":"paged-uint32-block-table-full-blocks-v1","permutation_version":"splitmix64-fisher-yates-rejection-v1","backend_executor_version":"llm-metal-executor-v1-decode-paged","resource_abi_version":"llm-metal-argument-buffer-tier2-v1","schedule_version":"llm-metal-decode-paged-block-owner-grid-stride-v1","timer_policy_version":"metal-command-buffer-gpu-start-end-v1","buffer_pattern_version":"llm-paged-physical-buffer-pattern-v1","write_pattern_version":"llm-metal-decode-paged-append-affine32-v1","checksum_pattern_version":"llm-metal-paged-dual-mod32-lookup-mix-v1","msl_revision":"llm-metal-decode-prefill-contiguous-paged-msl23-v5","msl_source_sha256":"2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2","identity":"llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|permutation_version=36:splitmix64-fisher-yates-rejection-v1|backend_executor_version=34:llm-metal-executor-v1-decode-paged|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=49:llm-metal-decode-paged-block-owner-grid-stride-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=36:llm-paged-physical-buffer-pattern-v1|write_pattern_version=41:llm-metal-decode-paged-append-affine32-v1|checksum_pattern_version=40:llm-metal-paged-dual-mod32-lookup-mix-v1|msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5|msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2","run_policy_version":"llm-run-policy-bounded-loop-snapshots-v1"},"methodology":{"backend":"metal","phase":"decode","kv_layout":"paged","work_unit_kind":"decode_step","weight_passes_per_work_unit":1,"kv_replay_factor":1,"warmup_policy":"same-shape-excluded-steady-state-warm-memory","context_policy":"fixed-visible-context-including-current-token-slot","scenario_order_policy":"cyclic-rotation-across-count-loops","timing_policy":"gpu-start-to-gpu-end-per-command-buffer-task","cache_policy":"metal-private-and-shared-cacheable-no-explicit-flush-between-scenarios","calibration_policy":"per-scenario-automatic-or-explicit-frozen-before-loop-zero","calibration_target_seconds":0.15,"calibration_min_seconds":0.1,"calibration_max_seconds":0.25,"calibration_max_corrections":2,"calibration_min_pilot_accounted_bytes":"8388608","maximum_work_units_per_measurement":65536,"maximum_serial_range_visits_per_lane_per_task":null,"maximum_accounted_bytes_per_task":"68719476736","repeatability_cv_warning_threshold_pct":5.0,"calibration_excluded_from_results":true,"snapshot_policy":"bounded-loop-snapshots","timed_region_exclusions":["mapping-allocation","buffer-initialization-and-pre-touch","argument-buffer-encoding-and-resource-validation","host-command-encoding-and-pre-gpu-queue-wait","status-reset-command-buffer","excluded-kv-write-and-padding-canary-post-validation","same-shape-warmup","calibration-attempts","checksum-reference-generation-and-validation","host-wait-aggregation-and-json-serialization"]},"model_work_plan":{"valid":true,"reason_code":"valid","backend":"metal","phase":"decode","kv_layout":"paged","work_unit_kind":"decode_step","weight_passes_per_work_unit":1,"kv_replay_factor":1,"requested_workers":null,"available_workers":null,"effective_workers":null,"worker_plan_count":null,"weight_layer_count":2,"layer_descriptors_per_worker":null,"sequence_descriptors_per_worker":null,"total_layer_descriptors":null,"total_sequence_descriptors":null,"descriptor_bytes":null,"planner_storage_bytes":null},"scenario_plans":[{"valid":true,"reason_code":"valid","scenario":"weights_only","work_unit_kind":"decode_step","kv_write_kind":"none","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"17410844953366850226","work_units":2,"weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"0","kv_write_bytes_per_work_unit":"0","effective_model_payload_bytes_per_work_unit":"1048576","layout_metadata_lookup_count_per_work_unit":"0","layout_metadata_read_bytes_per_work_unit":"0","accounted_bytes_per_work_unit":"1048576","weight_read_bytes":"2097152","kv_read_bytes":"0","kv_write_bytes":"0","effective_model_payload_bytes":"2097152","layout_metadata_lookup_count":"0","layout_metadata_read_bytes":"0","task_accounted_bytes":"2097152","maximum_work_units_by_work_unit_cap":65536,"maximum_work_units_by_guardrail":65536,"effective_maximum_work_units":65536,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=2499|model_plan_identity=llm-memory-work-plan-v1|backend=metal|phase=decode|kv_layout=paged|work_unit_kind=decode_step|methodology=llm-memory-v2-metal-decode-paged|component_identity_size=806|component_identity=llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|permutation_version=36:splitmix64-fisher-yates-rejection-v1|backend_executor_version=34:llm-metal-executor-v1-decode-paged|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=49:llm-metal-decode-paged-block-owner-grid-stride-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=36:llm-paged-physical-buffer-pattern-v1|write_pattern_version=41:llm-metal-decode-paged-append-affine32-v1|checksum_pattern_version=40:llm-metal-paged-dual-mod32-lookup-mix-v1|msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5|msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|decode_context=5|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=2|kv_blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|kv_block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|layout_metadata_lookups_per_layer_sequence_per_work_unit=7|k_mapping_bytes=72|v_mapping_bytes=72|kv_capacity_bytes=144|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|kv_only_payload_bytes_per_work_unit=144|mixed_payload_bytes_per_work_unit=1048720|total_data_mapping_bytes=1048720|traffic_crossover_numerator=1048576|traffic_crossover_denominator=24|metal_execution_resolved=1|metal_execution_identity_size=4345|metal_execution_identity_sha256=8016a3f1ef96939bd4f8d9ec9896940d831322f5a67b8f04a466f182cf2e6f53|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=weights_only|work_unit_kind=decode_step|kv_write_kind=none|scenario_seed=17410844953366850226|explicit=1|work_units=2|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=0|kv_write_bytes_per_work_unit=0|effective_model_payload_bytes_per_work_unit=1048576|layout_metadata_lookup_count_per_work_unit=0|layout_metadata_read_bytes_per_work_unit=0|accounted_bytes_per_work_unit=1048576|weight_read_bytes=2097152|kv_read_bytes=0|kv_write_bytes=0|effective_model_payload_bytes=2097152|layout_metadata_lookup_count=0|layout_metadata_read_bytes=0|task_accounted_bytes=2097152|maximum_work_units_by_work_unit_cap=65536|maximum_work_units_by_guardrail=65536|effective_maximum_work_units=65536","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":null,"expected_run_checksum":{"weight":{"a_uint32_decimal":"1420034048","b_uint32_decimal":"2651324416"},"k":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"v":{"a_uint32_decimal":"0","b_uint32_decimal":"0"}}}},{"valid":true,"reason_code":"valid","scenario":"kv_only","work_unit_kind":"decode_step","kv_write_kind":"current_token_append","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"16590208891222127307","work_units":2,"weight_read_bytes_per_work_unit":"0","kv_read_bytes_per_work_unit":"120","kv_write_bytes_per_work_unit":"24","effective_model_payload_bytes_per_work_unit":"144","layout_metadata_lookup_count_per_work_unit":"28","layout_metadata_read_bytes_per_work_unit":"112","accounted_bytes_per_work_unit":"256","weight_read_bytes":"0","kv_read_bytes":"240","kv_write_bytes":"48","effective_model_payload_bytes":"288","layout_metadata_lookup_count":"56","layout_metadata_read_bytes":"224","task_accounted_bytes":"512","maximum_work_units_by_work_unit_cap":65536,"maximum_work_units_by_guardrail":268435456,"effective_maximum_work_units":65536,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=2499|model_plan_identity=llm-memory-work-plan-v1|backend=metal|phase=decode|kv_layout=paged|work_unit_kind=decode_step|methodology=llm-memory-v2-metal-decode-paged|component_identity_size=806|component_identity=llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|permutation_version=36:splitmix64-fisher-yates-rejection-v1|backend_executor_version=34:llm-metal-executor-v1-decode-paged|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=49:llm-metal-decode-paged-block-owner-grid-stride-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=36:llm-paged-physical-buffer-pattern-v1|write_pattern_version=41:llm-metal-decode-paged-append-affine32-v1|checksum_pattern_version=40:llm-metal-paged-dual-mod32-lookup-mix-v1|msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5|msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|decode_context=5|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=2|kv_blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|kv_block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|layout_metadata_lookups_per_layer_sequence_per_work_unit=7|k_mapping_bytes=72|v_mapping_bytes=72|kv_capacity_bytes=144|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|kv_only_payload_bytes_per_work_unit=144|mixed_payload_bytes_per_work_unit=1048720|total_data_mapping_bytes=1048720|traffic_crossover_numerator=1048576|traffic_crossover_denominator=24|metal_execution_resolved=1|metal_execution_identity_size=4345|metal_execution_identity_sha256=8016a3f1ef96939bd4f8d9ec9896940d831322f5a67b8f04a466f182cf2e6f53|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=kv_only|work_unit_kind=decode_step|kv_write_kind=current_token_append|scenario_seed=16590208891222127307|explicit=1|work_units=2|weight_read_bytes_per_work_unit=0|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|effective_model_payload_bytes_per_work_unit=144|layout_metadata_lookup_count_per_work_unit=28|layout_metadata_read_bytes_per_work_unit=112|accounted_bytes_per_work_unit=256|weight_read_bytes=0|kv_read_bytes=240|kv_write_bytes=48|effective_model_payload_bytes=288|layout_metadata_lookup_count=56|layout_metadata_read_bytes=224|task_accounted_bytes=512|maximum_work_units_by_work_unit_cap=65536|maximum_work_units_by_guardrail=268435456|effective_maximum_work_units=65536","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":null,"expected_run_checksum":{"weight":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"k":{"a_uint32_decimal":"2103918284","b_uint32_decimal":"2599709732"},"v":{"a_uint32_decimal":"2897527724","b_uint32_decimal":"3534973700"}}}},{"valid":true,"reason_code":"valid","scenario":"mixed","work_unit_kind":"decode_step","kv_write_kind":"current_token_append","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"1546516732201688391","work_units":2,"weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"120","kv_write_bytes_per_work_unit":"24","effective_model_payload_bytes_per_work_unit":"1048720","layout_metadata_lookup_count_per_work_unit":"28","layout_metadata_read_bytes_per_work_unit":"112","accounted_bytes_per_work_unit":"1048832","weight_read_bytes":"2097152","kv_read_bytes":"240","kv_write_bytes":"48","effective_model_payload_bytes":"2097440","layout_metadata_lookup_count":"56","layout_metadata_read_bytes":"224","task_accounted_bytes":"2097664","maximum_work_units_by_work_unit_cap":65536,"maximum_work_units_by_guardrail":65520,"effective_maximum_work_units":65520,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=2499|model_plan_identity=llm-memory-work-plan-v1|backend=metal|phase=decode|kv_layout=paged|work_unit_kind=decode_step|methodology=llm-memory-v2-metal-decode-paged|component_identity_size=806|component_identity=llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|permutation_version=36:splitmix64-fisher-yates-rejection-v1|backend_executor_version=34:llm-metal-executor-v1-decode-paged|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=49:llm-metal-decode-paged-block-owner-grid-stride-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=36:llm-paged-physical-buffer-pattern-v1|write_pattern_version=41:llm-metal-decode-paged-append-affine32-v1|checksum_pattern_version=40:llm-metal-paged-dual-mod32-lookup-mix-v1|msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5|msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|decode_context=5|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=2|kv_blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|kv_block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|layout_metadata_lookups_per_layer_sequence_per_work_unit=7|k_mapping_bytes=72|v_mapping_bytes=72|kv_capacity_bytes=144|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|kv_only_payload_bytes_per_work_unit=144|mixed_payload_bytes_per_work_unit=1048720|total_data_mapping_bytes=1048720|traffic_crossover_numerator=1048576|traffic_crossover_denominator=24|metal_execution_resolved=1|metal_execution_identity_size=4345|metal_execution_identity_sha256=8016a3f1ef96939bd4f8d9ec9896940d831322f5a67b8f04a466f182cf2e6f53|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=mixed|work_unit_kind=decode_step|kv_write_kind=current_token_append|scenario_seed=1546516732201688391|explicit=1|work_units=2|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=120|kv_write_bytes_per_work_unit=24|effective_model_payload_bytes_per_work_unit=1048720|layout_metadata_lookup_count_per_work_unit=28|layout_metadata_read_bytes_per_work_unit=112|accounted_bytes_per_work_unit=1048832|weight_read_bytes=2097152|kv_read_bytes=240|kv_write_bytes=48|effective_model_payload_bytes=2097440|layout_metadata_lookup_count=56|layout_metadata_read_bytes=224|task_accounted_bytes=2097664|maximum_work_units_by_work_unit_cap=65536|maximum_work_units_by_guardrail=65520|effective_maximum_work_units=65520","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":null,"expected_run_checksum":{"weight":{"a_uint32_decimal":"825491456","b_uint32_decimal":"861405184"},"k":{"a_uint32_decimal":"3659169868","b_uint32_decimal":"618917540"},"v":{"a_uint32_decimal":"157812012","b_uint32_decimal":"1554181508"}}}}],"frozen_plan_refs":{"weights_only":0,"kv_only":1,"mixed":2}},"backend_evidence":{"cpu":null,"metal":{"workers_applicable":false,"worker_qos_applicable":false,"lifecycle":{"initialization":{"status":"ready","reason_code":"valid"},"plan_resolution":{"status":"ready","reason_code":"valid"},"preparation":{"status":"ready","reason_code":"valid"},"release":{"status":"ready","reason_code":"valid"}},"capability":{"device_name":"Apple M4","registry_id_uint64_decimal":"4294968227","has_unified_memory":true,"required_apple7_family_supported":true,"argument_buffers_tier2_supported":true,"supported_families":["apple1","apple2","apple3","apple4","apple5","apple6","apple7","apple8","apple9","mac1","mac2","common1","common2","common3","metal3","metal4"],"max_buffer_length_bytes":"14302248960","recommended_max_working_set_size_bytes":"19069665280","compilation_mode":"runtime-source","msl_language_version":"2.3","kernel_revision":"llm-metal-decode-prefill-contiguous-paged-msl23-v5","kernel_source_sha256":"2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2","compiler_diagnostics":null,"compiler_identifier":"21.0.0 (clang-2100.1.1.101)","build_sdk":"26.5","deployment_target":"26.0","argument_buffer":{"encoded_length_bytes":"8200","alignment_bytes":"8"},"layout_probe":{"evaluated":true,"valid":true,"resource_count":5},"foundation_pipelines":[{"label":"membenchmark.llm-metal.pipeline.initialize","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.copy","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.layout-probe","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.decode-paged-layout-probe","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.validate-bytes","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.validate-table","thread_execution_width":32,"max_total_threads_per_threadgroup":1024}],"error":{"domain":null,"code":0,"description":null}},"resources":{"allocation_attempted":true,"allocation_completed":true,"initialization_completed":true,"table_upload_completed":true,"table_validation_completed":true,"table_permutation":{"algorithm_version":"splitmix64-fisher-yates-rejection-v1","domain_uint64_hex":"0x4c4c4d4b56504731","resolved_seed_uint64_decimal":"1732835419980629872","entry_count":"6","sha256":"14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c","identity":"splitmix64-fisher-yates-rejection-v1|domain=5497854231078520625|domain_uint64_hex=18:0x4c4c4d4b56504731|resolved_seed=1732835419980629872|entry_count=6|sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c"},"post_validation_completed":true,"cpu_sample_readback_validation_completed":true,"candidate_cleanup_completed":true,"resources_published":false,"persistent_resource_length_bytes":"1061040","committed_resource_bytes":"1061064","resource_rounding_bytes":"0","layout_padding_bytes":"24","transient_peak_bytes":"16409","additional_owned_bytes":"234085333","known_owned_peak_bytes":"235162782","admitted_budget_bytes":"9266895257","current_allocated_size_before_bytes":"737280","current_allocated_size_peak_bytes":"1785856","current_allocated_size_after_release_bytes":"737280","recommended_working_set_available":true,"recommended_working_set_headroom_bytes":"18833765218","recommended_working_set_headroom_fraction":0.9876295646233807,"exceeds_recommended_working_set":false,"resources":[{"label":"membenchmark.llm-metal.weight-0","pool":"weight","pool_index":0,"storage_mode":"private","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"544","length_bytes":"1048576","allocated_size_available":true,"allocated_size_bytes":"1048576","committed_bytes":"1048576","allocation_rounding_bytes":"0"},{"label":"membenchmark.llm-metal.k-0","pool":"k","pool_index":0,"storage_mode":"private","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"544","length_bytes":"72","allocated_size_available":true,"allocated_size_bytes":"72","committed_bytes":"72","allocation_rounding_bytes":"0"},{"label":"membenchmark.llm-metal.v-0","pool":"v","pool_index":0,"storage_mode":"private","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"544","length_bytes":"72","allocated_size_available":true,"allocated_size_bytes":"72","committed_bytes":"72","allocation_rounding_bytes":"0"},{"label":"membenchmark.llm-metal.block_table-0","pool":"block_table","pool_index":0,"storage_mode":"private","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"544","length_bytes":"24","allocated_size_available":true,"allocated_size_bytes":"24","committed_bytes":"24","allocation_rounding_bytes":"0"},{"label":"membenchmark.llm-metal.argument_buffer-0","pool":"argument_buffer","pool_index":0,"storage_mode":"shared","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"512","length_bytes":"8200","allocated_size_available":true,"allocated_size_bytes":"8200","committed_bytes":"8200","allocation_rounding_bytes":"0"},{"label":"membenchmark.llm-metal.status_checksum-0","pool":"status_checksum","pool_index":0,"storage_mode":"shared","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"512","length_bytes":"4096","allocated_size_available":true,"allocated_size_bytes":"4096","committed_bytes":"4096","allocation_rounding_bytes":"0"},{"label":"membenchmark.llm-metal.staging-0","pool":"staging","pool_index":0,"storage_mode":"shared","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"512","length_bytes":"24","allocated_size_available":true,"allocated_size_bytes":"24","committed_bytes":"24","allocation_rounding_bytes":"0"}],"error":{"domain":null,"code":0,"description":null}},"workload_pipelines":[{"label":"membenchmark.llm-metal.pipeline.decode-paged.weights-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.decode-paged.kv-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.decode-paged.mixed","thread_execution_width":32,"max_total_threads_per_threadgroup":1024}],"timed_results_available":true}},"memory_budget":{"resource_rounding_bytes":"0","transient_peak_bytes":"16409","layout_transient_bytes":"16409","setup_peak_bytes":"235162782","runtime_peak_bytes":"235162782","known_owned_peak_bytes":"235162782","admitted_budget_bytes":"9266895257","valid":true,"reason_code":"within-memory-budget","request":{"valid":true,"reason_code":"valid","mapping_granularity_bytes":"16384","requested_weight_mapping_bytes":"1048576","requested_k_mapping_bytes":"72","requested_v_mapping_bytes":"72","committed_weight_mapping_bytes":"1048576","committed_k_mapping_bytes":"72","committed_v_mapping_bytes":"72","requested_block_table_mapping_bytes":"24","committed_block_table_mapping_bytes":"24","requested_data_bytes":"1048720","committed_data_bytes":"1048720","layout_transient_bytes":"16409","setup_peak_bytes":"235162782","runtime_peak_bytes":"235162782","descriptor_bytes":"0","planner_storage_bytes":"158752","checksum_auxiliary_bytes":"0","orchestration_auxiliary_bytes":"233926581","auxiliary_bytes":"234085333","required_total_bytes":"235162782"},"available_memory_bytes":"11583619072","allowed_memory_bytes":"9266895257","used_fallback":false},"calibration":{"excluded_from_results":true,"attempts":{"weights_only":[{"attempt_index":0,"scenario":"weights_only","plan_ref":0,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":0.00034004170447587967,"timing":{"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"weight":{"a_uint32_decimal":"1420034048","b_uint32_decimal":"2651324416"},"k":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"v":{"a_uint32_decimal":"0","b_uint32_decimal":"0"}}},"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-paged.weights-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"2097152","maximum_threadgroup_accounted_bytes":"2097152","threadgroup_accounted_imbalance_bytes":"0","threadgroup_accounted_bytes":["2097152"],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:0;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=7:2097152;maximum_threadgroup_accounted_bytes=7:2097152;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:1;threadgroup_accounted_bytes=7:2097152;weight_shard_cost_version=43:llm-metal-weight-vector-grid-stride-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.57722575,"gpu_end_seconds":1532841.5775657918,"host_submit_to_completion_seconds":0.000641875,"host_wait_seconds":0.000639709,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-append-final","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}}}],"kv_only":[{"attempt_index":0,"scenario":"kv_only","plan_ref":1,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":5.7750148698687553e-05,"timing":{"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"weight":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"k":{"a_uint32_decimal":"2103918284","b_uint32_decimal":"2599709732"},"v":{"a_uint32_decimal":"2897527724","b_uint32_decimal":"3534973700"}}},"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-paged.kv-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"12","threads_per_threadgroup":256,"actual_threadgroups":12,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"1","work_units":2,"paged_semantic_lookups":"56","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"40","maximum_threadgroup_accounted_bytes":"48","threadgroup_accounted_imbalance_bytes":"8","threadgroup_accounted_bytes":["40","40","48","40","40","48","40","40","48","40","40","48"],"identity":"llm-metal-capped-grid-stride-v1owner_count=2:12;threads_per_threadgroup=3:256;actual_threadgroups=2:12;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=1:1;work_units=1:2;paged_semantic_lookups=2:56;serial_range_visits_per_lane=1:0;visit_bytes=1:6;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=2:40;maximum_threadgroup_accounted_bytes=2:48;threadgroup_accounted_imbalance_bytes=1:8;threadgroup_accounted_bytes_count=2:12;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;paged_owner_cost_version=29:llm-metal-paged-owner-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.57854925,"gpu_end_seconds":1532841.5786070002,"host_submit_to_completion_seconds":0.000245041,"host_wait_seconds":0.00024375,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}}}],"mixed":[{"attempt_index":0,"scenario":"mixed","plan_ref":2,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":0.00019900011830031872,"timing":{"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"weight":{"a_uint32_decimal":"825491456","b_uint32_decimal":"861405184"},"k":{"a_uint32_decimal":"3659169868","b_uint32_decimal":"618917540"},"v":{"a_uint32_decimal":"157812012","b_uint32_decimal":"1554181508"}}},"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-paged.mixed","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"12","threads_per_threadgroup":256,"actual_threadgroups":12,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"56","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"40","maximum_threadgroup_accounted_bytes":"1048616","threadgroup_accounted_imbalance_bytes":"1048576","threadgroup_accounted_bytes":["1048616","40","48","40","40","48","1048616","40","48","40","40","48"],"identity":"llm-metal-capped-grid-stride-v1owner_count=2:12;threads_per_threadgroup=3:256;actual_threadgroups=2:12;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=2:56;serial_range_visits_per_lane=1:0;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=2:40;maximum_threadgroup_accounted_bytes=7:1048616;threadgroup_accounted_imbalance_bytes=7:1048576;threadgroup_accounted_bytes_count=2:12;threadgroup_accounted_bytes=7:1048616;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=7:1048616;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;paged_owner_cost_version=29:llm-metal-paged-owner-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.57919125,"gpu_end_seconds":1532841.5793902501,"host_submit_to_completion_seconds":0.000515875,"host_wait_seconds":0.00051375,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}}}]}},"measurements":[{"measurement_id":0,"plan_ref":0,"scenario":"weights_only","loop_index":0,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":null,"elapsed_seconds":0.0003328749444335699,"synthetic_work_unit_latency_seconds":0.00016643747221678495,"synthetic_memory_work_units_per_second":6008.262362321263,"effective_model_payload_gb_s":6.3001197148333805,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-paged.weights-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"2097152","maximum_threadgroup_accounted_bytes":"2097152","threadgroup_accounted_imbalance_bytes":"0","threadgroup_accounted_bytes":["2097152"],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:0;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=7:2097152;maximum_threadgroup_accounted_bytes=7:2097152;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:1;threadgroup_accounted_bytes=7:2097152;weight_shard_cost_version=43:llm-metal-weight-vector-grid-stride-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.5800760002,"gpu_end_seconds":1532841.5804088751,"host_submit_to_completion_seconds":0.000667792,"host_wait_seconds":0.000666417,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-append-final","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"1420034048","b_uint32_decimal":"2651324416"},"k":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"v":{"a_uint32_decimal":"0","b_uint32_decimal":"0"}}}},{"measurement_id":1,"plan_ref":1,"scenario":"kv_only","loop_index":0,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"288","completed_layout_metadata_lookup_count":"56","completed_layout_metadata_read_bytes":"224","completed_task_accounted_bytes":"512","completion_derivation":null,"elapsed_seconds":5.2291667088866234e-05,"synthetic_work_unit_latency_seconds":2.6145833544433117e-05,"synthetic_memory_work_units_per_second":38247.01164338731,"effective_model_payload_gb_s":0.005507569676647773,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-paged.kv-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"12","threads_per_threadgroup":256,"actual_threadgroups":12,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"1","work_units":2,"paged_semantic_lookups":"56","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"40","maximum_threadgroup_accounted_bytes":"48","threadgroup_accounted_imbalance_bytes":"8","threadgroup_accounted_bytes":["40","40","48","40","40","48","40","40","48","40","40","48"],"identity":"llm-metal-capped-grid-stride-v1owner_count=2:12;threads_per_threadgroup=3:256;actual_threadgroups=2:12;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=1:1;work_units=1:2;paged_semantic_lookups=2:56;serial_range_visits_per_lane=1:0;visit_bytes=1:6;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=2:40;maximum_threadgroup_accounted_bytes=2:48;threadgroup_accounted_imbalance_bytes=1:8;threadgroup_accounted_bytes_count=2:12;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;paged_owner_cost_version=29:llm-metal-paged-owner-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.58112525,"gpu_end_seconds":1532841.5811775418,"host_submit_to_completion_seconds":0.000237625,"host_wait_seconds":0.00023625,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"k":{"a_uint32_decimal":"2103918284","b_uint32_decimal":"2599709732"},"v":{"a_uint32_decimal":"2897527724","b_uint32_decimal":"3534973700"}}}},{"measurement_id":2,"plan_ref":2,"scenario":"mixed","loop_index":0,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":0.99986268975513,"kv_read_payload_fraction":0.00011442520405828057,"kv_write_payload_fraction":2.2885040811656113e-05,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097440","completed_layout_metadata_lookup_count":"56","completed_layout_metadata_read_bytes":"224","completed_task_accounted_bytes":"2097664","completion_derivation":null,"elapsed_seconds":0.00019341660663485527,"synthetic_work_unit_latency_seconds":9.670830331742764e-05,"synthetic_memory_work_units_per_second":10340.373739343555,"effective_model_payload_gb_s":10.844156747924373,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-paged.mixed","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"12","threads_per_threadgroup":256,"actual_threadgroups":12,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"56","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"40","maximum_threadgroup_accounted_bytes":"1048616","threadgroup_accounted_imbalance_bytes":"1048576","threadgroup_accounted_bytes":["1048616","40","48","40","40","48","1048616","40","48","40","40","48"],"identity":"llm-metal-capped-grid-stride-v1owner_count=2:12;threads_per_threadgroup=3:256;actual_threadgroups=2:12;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=2:56;serial_range_visits_per_lane=1:0;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=2:40;maximum_threadgroup_accounted_bytes=7:1048616;threadgroup_accounted_imbalance_bytes=7:1048576;threadgroup_accounted_bytes_count=2:12;threadgroup_accounted_bytes=7:1048616;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=7:1048616;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;paged_owner_cost_version=29:llm-metal-paged-owner-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.58172375,"gpu_end_seconds":1532841.5819171667,"host_submit_to_completion_seconds":0.000455125,"host_wait_seconds":0.000453791,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"825491456","b_uint32_decimal":"861405184"},"k":{"a_uint32_decimal":"3659169868","b_uint32_decimal":"618917540"},"v":{"a_uint32_decimal":"157812012","b_uint32_decimal":"1554181508"}}}},{"measurement_id":3,"plan_ref":1,"scenario":"kv_only","loop_index":1,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"288","completed_layout_metadata_lookup_count":"56","completed_layout_metadata_read_bytes":"224","completed_task_accounted_bytes":"512","completion_derivation":null,"elapsed_seconds":5.204160697758198e-05,"synthetic_work_unit_latency_seconds":2.602080348879099e-05,"synthetic_memory_work_units_per_second":38430.78867379215,"effective_model_payload_gb_s":0.00553403356902607,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-paged.kv-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"12","threads_per_threadgroup":256,"actual_threadgroups":12,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"1","work_units":2,"paged_semantic_lookups":"56","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"40","maximum_threadgroup_accounted_bytes":"48","threadgroup_accounted_imbalance_bytes":"8","threadgroup_accounted_bytes":["40","40","48","40","40","48","40","40","48","40","40","48"],"identity":"llm-metal-capped-grid-stride-v1owner_count=2:12;threads_per_threadgroup=3:256;actual_threadgroups=2:12;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=1:1;work_units=1:2;paged_semantic_lookups=2:56;serial_range_visits_per_lane=1:0;visit_bytes=1:6;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=2:40;maximum_threadgroup_accounted_bytes=2:48;threadgroup_accounted_imbalance_bytes=1:8;threadgroup_accounted_bytes_count=2:12;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;paged_owner_cost_version=29:llm-metal-paged-owner-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.5836565,"gpu_end_seconds":1532841.5837085417,"host_submit_to_completion_seconds":0.000240167,"host_wait_seconds":0.000238542,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"k":{"a_uint32_decimal":"2103918284","b_uint32_decimal":"2599709732"},"v":{"a_uint32_decimal":"2897527724","b_uint32_decimal":"3534973700"}}}},{"measurement_id":4,"plan_ref":2,"scenario":"mixed","loop_index":1,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":0.99986268975513,"kv_read_payload_fraction":0.00011442520405828057,"kv_write_payload_fraction":2.2885040811656113e-05,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097440","completed_layout_metadata_lookup_count":"56","completed_layout_metadata_read_bytes":"224","completed_task_accounted_bytes":"2097664","completion_derivation":null,"elapsed_seconds":0.00019424990750849247,"synthetic_work_unit_latency_seconds":9.712495375424623e-05,"synthetic_memory_work_units_per_second":10296.015198424542,"effective_model_payload_gb_s":10.797637058891784,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-paged.mixed","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"12","threads_per_threadgroup":256,"actual_threadgroups":12,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"56","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"40","maximum_threadgroup_accounted_bytes":"1048616","threadgroup_accounted_imbalance_bytes":"1048576","threadgroup_accounted_bytes":["1048616","40","48","40","40","48","1048616","40","48","40","40","48"],"identity":"llm-metal-capped-grid-stride-v1owner_count=2:12;threads_per_threadgroup=3:256;actual_threadgroups=2:12;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=2:56;serial_range_visits_per_lane=1:0;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=2:40;maximum_threadgroup_accounted_bytes=7:1048616;threadgroup_accounted_imbalance_bytes=7:1048576;threadgroup_accounted_bytes_count=2:12;threadgroup_accounted_bytes=7:1048616;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=7:1048616;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;paged_owner_cost_version=29:llm-metal-paged-owner-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.58424525,"gpu_end_seconds":1532841.5844395,"host_submit_to_completion_seconds":0.000443167,"host_wait_seconds":0.000441,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"825491456","b_uint32_decimal":"861405184"},"k":{"a_uint32_decimal":"3659169868","b_uint32_decimal":"618917540"},"v":{"a_uint32_decimal":"157812012","b_uint32_decimal":"1554181508"}}}},{"measurement_id":5,"plan_ref":0,"scenario":"weights_only","loop_index":1,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":null,"elapsed_seconds":0.00032962486147880554,"synthetic_work_unit_latency_seconds":0.00016481243073940277,"synthetic_memory_work_units_per_second":6067.503497847044,"effective_model_payload_gb_s":6.362238547758462,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-paged.weights-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"2097152","maximum_threadgroup_accounted_bytes":"2097152","threadgroup_accounted_imbalance_bytes":"0","threadgroup_accounted_bytes":["2097152"],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:0;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=7:2097152;maximum_threadgroup_accounted_bytes=7:2097152;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:1;threadgroup_accounted_bytes=7:2097152;weight_shard_cost_version=43:llm-metal-weight-vector-grid-stride-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.5850512502,"gpu_end_seconds":1532841.585380875,"host_submit_to_completion_seconds":0.000544583,"host_wait_seconds":0.00054275,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-append-final","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"1420034048","b_uint32_decimal":"2651324416"},"k":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"v":{"a_uint32_decimal":"0","b_uint32_decimal":"0"}}}},{"measurement_id":6,"plan_ref":2,"scenario":"mixed","loop_index":2,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":0.99986268975513,"kv_read_payload_fraction":0.00011442520405828057,"kv_write_payload_fraction":2.2885040811656113e-05,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097440","completed_layout_metadata_lookup_count":"56","completed_layout_metadata_read_bytes":"224","completed_task_accounted_bytes":"2097664","completion_derivation":null,"elapsed_seconds":0.0001900000497698784,"synthetic_work_unit_latency_seconds":9.50000248849392e-05,"synthetic_memory_work_units_per_second":10526.313032140424,"effective_model_payload_gb_s":11.039155003066305,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-paged.mixed","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"12","threads_per_threadgroup":256,"actual_threadgroups":12,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"56","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"40","maximum_threadgroup_accounted_bytes":"1048616","threadgroup_accounted_imbalance_bytes":"1048576","threadgroup_accounted_bytes":["1048616","40","48","40","40","48","1048616","40","48","40","40","48"],"identity":"llm-metal-capped-grid-stride-v1owner_count=2:12;threads_per_threadgroup=3:256;actual_threadgroups=2:12;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=2:56;serial_range_visits_per_lane=1:0;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=2:40;maximum_threadgroup_accounted_bytes=7:1048616;threadgroup_accounted_imbalance_bytes=7:1048576;threadgroup_accounted_bytes_count=2:12;threadgroup_accounted_bytes=7:1048616;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=7:1048616;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;paged_owner_cost_version=29:llm-metal-paged-owner-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.58690725,"gpu_end_seconds":1532841.5870972502,"host_submit_to_completion_seconds":0.000441875,"host_wait_seconds":0.000437791,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"825491456","b_uint32_decimal":"861405184"},"k":{"a_uint32_decimal":"3659169868","b_uint32_decimal":"618917540"},"v":{"a_uint32_decimal":"157812012","b_uint32_decimal":"1554181508"}}}},{"measurement_id":7,"plan_ref":0,"scenario":"weights_only","loop_index":2,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":null,"elapsed_seconds":0.00033624982461333275,"synthetic_work_unit_latency_seconds":0.00016812491230666637,"synthetic_memory_work_units_per_second":5947.958492766147,"effective_model_payload_gb_s":6.236886524510756,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-paged.weights-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"2097152","maximum_threadgroup_accounted_bytes":"2097152","threadgroup_accounted_imbalance_bytes":"0","threadgroup_accounted_bytes":["2097152"],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:0;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=7:2097152;maximum_threadgroup_accounted_bytes=7:2097152;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:1;threadgroup_accounted_bytes=7:2097152;weight_shard_cost_version=43:llm-metal-weight-vector-grid-stride-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.5878062502,"gpu_end_seconds":1532841.5881425,"host_submit_to_completion_seconds":0.000717375,"host_wait_seconds":0.000714958,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-append-final","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"1420034048","b_uint32_decimal":"2651324416"},"k":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"v":{"a_uint32_decimal":"0","b_uint32_decimal":"0"}}}},{"measurement_id":8,"plan_ref":1,"scenario":"kv_only","loop_index":2,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":5,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"288","completed_layout_metadata_lookup_count":"56","completed_layout_metadata_read_bytes":"224","completed_task_accounted_bytes":"512","completion_derivation":null,"elapsed_seconds":5.1625072956085205e-05,"synthetic_work_unit_latency_seconds":2.5812536478042603e-05,"synthetic_memory_work_units_per_second":38740.86534853514,"effective_model_payload_gb_s":0.00557868461018906,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.decode-paged.kv-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"12","threads_per_threadgroup":256,"actual_threadgroups":12,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"1","work_units":2,"paged_semantic_lookups":"56","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"40","maximum_threadgroup_accounted_bytes":"48","threadgroup_accounted_imbalance_bytes":"8","threadgroup_accounted_bytes":["40","40","48","40","40","48","40","40","48","40","40","48"],"identity":"llm-metal-capped-grid-stride-v1owner_count=2:12;threads_per_threadgroup=3:256;actual_threadgroups=2:12;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=1:1;work_units=1:2;paged_semantic_lookups=2:56;serial_range_visits_per_lane=1:0;visit_bytes=1:6;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=2:40;maximum_threadgroup_accounted_bytes=2:48;threadgroup_accounted_imbalance_bytes=1:8;threadgroup_accounted_bytes_count=2:12;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:40;threadgroup_accounted_bytes=2:48;paged_owner_cost_version=29:llm-metal-paged-owner-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.58876525,"gpu_end_seconds":1532841.5888168751,"host_submit_to_completion_seconds":0.000224291,"host_wait_seconds":0.000222333,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-append-final","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"k":{"a_uint32_decimal":"2103918284","b_uint32_decimal":"2599709732"},"v":{"a_uint32_decimal":"2897527724","b_uint32_decimal":"3534973700"}}}}],"aggregates":{"scenarios":{"weights_only":{"scenario":"weights_only","status":"complete","observed_cv_classification":"below-threshold","accepted_measurement_ids":[0,5,7],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":0.00016643747221678495,"statistics":{"sample_count":3,"average":0.0001664582717542847,"min":0.00016481243073940277,"max":0.00016812491230666637,"median":0.00016643747221678495,"p90":0.0001677874242886901,"p95":0.00016795616829767823,"p99":0.00016809116350486873,"stddev":1.6563387331507517e-06,"coefficient_of_variation_pct":0.9950474168058979,"median_absolute_deviation":1.625041477382183e-06}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":6008.262362321263,"statistics":{"sample_count":3,"average":6007.908117644819,"min":5947.958492766147,"max":6067.503497847044,"median":6008.262362321263,"p90":6055.655270741888,"p95":6061.579384294466,"p99":6066.318675136528,"stddev":59.77328982845098,"coefficient_of_variation_pct":0.9949101860080196,"median_absolute_deviation":59.24113552578092}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":6.3001197148333805,"statistics":{"sample_count":3,"average":6.299748262367533,"min":6.236886524510756,"max":6.362238547758462,"median":6.3001197148333805,"p90":6.349814781173446,"p95":6.356026664465954,"p99":6.360996171099961,"stddev":0.06267683715515776,"coefficient_of_variation_pct":0.994910186008019,"median_absolute_deviation":0.06211883292508169}}},"kv_only":{"scenario":"kv_only","status":"complete","observed_cv_classification":"below-threshold","accepted_measurement_ids":[1,3,8],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":2.602080348879099e-05,"statistics":{"sample_count":3,"average":2.59930578370889e-05,"min":2.5812536478042603e-05,"max":2.6145833544433117e-05,"median":2.602080348879099e-05,"p90":2.6120827533304693e-05,"p95":2.6133330538868905e-05,"p99":2.6143332943320273e-05,"stddev":1.683719083083958e-07,"coefficient_of_variation_pct":0.6477572179605193,"median_absolute_deviation":1.25030055642128e-07}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":38430.78867379215,"statistics":{"sample_count":3,"average":38472.888555238205,"min":38247.01164338731,"max":38740.86534853514,"median":38430.78867379215,"p90":38678.85001358654,"p95":38709.85768106084,"p99":38734.66381504028,"stddev":249.60402748230604,"coefficient_of_variation_pct":0.6487790151860632,"median_absolute_deviation":183.77703040483902}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":0.00553403356902607,"statistics":{"sample_count":3,"average":0.005540095951954301,"min":0.005507569676647773,"max":0.00557868461018906,"median":0.00553403356902607,"p90":0.005569754401956463,"p95":0.005574219506072761,"p99":0.0055777915893658,"stddev":3.594297995745221e-05,"coefficient_of_variation_pct":0.6487790151860658,"median_absolute_deviation":2.6463892378297132e-05}}},"mixed":{"scenario":"mixed","status":"complete","observed_cv_classification":"below-threshold","accepted_measurement_ids":[2,4,6],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":9.670830331742764e-05,"statistics":{"sample_count":3,"average":9.627776065220435e-05,"min":9.50000248849392e-05,"max":9.712495375424623e-05,"median":9.670830331742764e-05,"p90":9.704162366688252e-05,"p95":9.708328871056437e-05,"p99":9.711662074550985e-05,"stddev":1.1259910811533098e-06,"coefficient_of_variation_pct":1.169523546793804,"median_absolute_deviation":4.166504368185997e-07}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":10340.373739343555,"statistics":{"sample_count":3,"average":10387.567323302841,"min":10296.015198424542,"max":10526.313032140424,"median":10340.373739343555,"p90":10489.12517358105,"p95":10507.719102860736,"p99":10522.594246284485,"stddev":122.18714674117163,"coefficient_of_variation_pct":1.1762825976305769,"median_absolute_deviation":44.35854091901274}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":10.844156747924373,"statistics":{"sample_count":3,"average":10.893649603294156,"min":10.797637058891784,"max":11.039155003066305,"median":10.844156747924373,"p90":11.000155352037918,"p95":11.019655177552112,"p99":11.035255037963466,"stddev":0.12814010453040217,"coefficient_of_variation_pct":1.1762825976305826,"median_absolute_deviation":0.046519689032589184}}}},"traffic_diagnostics":{"classification_version":"llm-exact-weight-vs-kv-read-payload-v1","traffic_crossover_numerator":"1048576","traffic_crossover_denominator":"24","traffic_crossover_context_tokens":43690.666666666664,"current_visible_context_tokens":5,"current_weight_read_payload_bytes_per_work_unit":"1048576","current_kv_read_payload_bytes_per_work_unit":"120","current_weight_to_kv_read_payload_ratio":8738.133333333333,"current_context_classification":"weight_payload_dominant","classification_is_payload_only":true,"scenario_headlines":{"weights_only":{"synthetic_work_unit_latency_seconds":0.00016643747221678495,"synthetic_memory_work_units_per_second":6008.262362321263,"effective_model_payload_gb_s":6.3001197148333805},"kv_only":{"synthetic_work_unit_latency_seconds":2.602080348879099e-05,"synthetic_memory_work_units_per_second":38430.78867379215,"effective_model_payload_gb_s":0.00553403356902607},"mixed":{"synthetic_work_unit_latency_seconds":9.670830331742764e-05,"synthetic_memory_work_units_per_second":10340.373739343555,"effective_model_payload_gb_s":10.844156747924373}}}},"status":"complete","reason_code":"complete","results_complete":true,"run_accepted":true,"interpretation":{"result_scope":"synthetic-memory-only-work-unit-not-real-inference-token-or-prefill-latency","reported_rate":"synthetic_memory_work_units_per_second","backend":"metal","phase":"decode","work_unit_kind":"decode_step","transformer_math_included":false,"framework_scheduler_and_dispatch_included":false,"compute_memory_overlap_included":false,"physical_dram_traffic_measured":false,"dram_residency":"unverified","cache_residency":"unverified","fixed_context_includes_current_token_slot":true,"kv_layout":"paged","payload_semantics":"exact-logical-weight-plus-kv-read-plus-kv-write-bytes-divided-by-elapsed-time","layout_metadata_timed_but_excluded_from_effective_model_payload_gb_s":true,"prefill_transformer_compute_or_ttft_prediction_included":false,"private_metal_storage_implies_separate_vram":false,"cross_backend_performance_distributions_combined":false,"traffic_classification_semantics":"exact-weight-vs-kv-read-logical-payload-only-not-hardware-bottleneck","full_size_working_set_reduces_but_does_not_prove_dram_residency":true,"comparability_requires":["same-backend","same-methodology-version","same-frozen-work-plan-and-model-geometry","same-software-version","sufficiently-similar-hardware","sufficiently-similar-thermal-and-power-state"]},"diagnostic":null,"interruption_requested":false,"scenario_order_balance_complete":true,"seeds":{"base_seed_uint64_decimal":"424242","source":"user","buffer_domain_seeds":{"weight_uint64_decimal":"904846021374001452","k_uint64_decimal":"10718278915871169538","v_uint64_decimal":"2111147304547004201"},"scenario_domain_seeds":{"weights_only":"17410844953366850226","kv_only":"16590208891222127307","mixed":"1546516732201688391"}},"counters":{"planned_loops":3,"attempted_loops":3,"completed_loops":3,"planned_measurements":9,"attempted_measurements":9,"terminal_measurements":9,"measured_measurements":9,"planned_work_units":"18","completed_work_units":"18","planned_effective_model_payload_bytes":"12584640","completed_effective_model_payload_bytes":"12584640","planned_layout_metadata_lookup_count":"336","completed_layout_metadata_lookup_count":"336","planned_layout_metadata_read_bytes":"1344","completed_layout_metadata_read_bytes":"1344","planned_task_accounted_bytes":"12585984","completed_task_accounted_bytes":"12585984","runner_auxiliary":{"valid":true,"reason_code":"valid","measurement_record_bytes":"7056","loop_record_bytes":"144","calibration_record_bytes":"2784","calibration_identity_bytes":"69772","aggregate_value_bytes":"72","statistics_workspace_bytes":"72","warning_record_bytes":"64","fixed_metadata_bytes":"13782417","retained_checksum_bytes":"0","checksum_auxiliary_bytes":"0","orchestration_auxiliary_bytes":"13862381","total_auxiliary_bytes":"13862381"}},"checkpoint_lifecycle":{"checkpoint_failed":false,"observation_point":"before-current-snapshot-preparation","prior_file_writer_attempts":3,"prior_successful_file_writes":3,"current_request":"terminal","current_persistence_success":null,"snapshot_interval_loops":1,"checkpoint_policy":"bounded-loop-snapshots","file_checkpoint_failure_is_terminal_and_not_retried":true,"stdout_intermediate_checkpoints_are_lazy":true},"loop_records":[{"loop_index":0,"planned_order":["weights_only","kv_only","mixed"],"realized_order":["weights_only","kv_only","mixed"],"realized_order_count":3,"measurement_indexes":[0,1,2]},{"loop_index":1,"planned_order":["kv_only","mixed","weights_only"],"realized_order":["kv_only","mixed","weights_only"],"realized_order_count":3,"measurement_indexes":[3,4,5]},{"loop_index":2,"planned_order":["mixed","weights_only","kv_only"],"realized_order":["mixed","weights_only","kv_only"],"realized_order_count":3,"measurement_indexes":[6,7,8]}],"environment":{"processor_name":"Apple M4","macos_version":"26.6.2","performance_core_count":4,"efficiency_core_count":6,"logical_core_count":10,"page_size_bytes":"16384","l1_data_cache_bytes":"131072","l2_data_cache_bytes":"16777216","available_memory_bytes":"11583619072","available_memory_source":"mach-free-plus-inactive-rounded-mib","main_thread_qos":{"requested":true,"applied":true,"code":0},"start":{"thermal_state":"nominal","low_power_mode_available":true,"low_power_mode_enabled":false,"physical_memory_bytes":"25769803776"},"end":{"thermal_state":"nominal","low_power_mode_available":true,"low_power_mode_enabled":false,"physical_memory_bytes":"25769803776"}},"quality_warnings":["weight-working-set-cache-dominant","kv-working-set-cache-dominant","weights_only-duration-below-target-window","kv_only-duration-below-target-window","mixed-duration-below-target-window"]} diff --git a/tests/fixtures/llm-schema-v2/metal-prefill-contiguous.json b/tests/fixtures/llm-schema-v2/metal-prefill-contiguous.json new file mode 100644 index 0000000..1270578 --- /dev/null +++ b/tests/fixtures/llm-schema-v2/metal-prefill-contiguous.json @@ -0,0 +1 @@ +{"schema_version":2,"mode":"llm_memory","backend":"metal","phase":"prefill","kv_layout":"contiguous","methodology_version":"llm-memory-v2-metal-prefill-contiguous","software":{"version":"0.63.1","timestamp":"2026-09-06T07:07:07Z"},"build_manifest":{"binary_sha256":"a9c0aa68c61fccf12bda222a0485a3eba1b642a54b9b1caf8292348f608a1100","compile_flags":{"asflags":"-arch arm64","cxxflags":"-Wall -O3 -std=c++17 -arch arm64 -pthread -Isrc","test_cxxflags":"-Wall -g -std=c++17 -arch arm64 -pthread -Isrc -I/opt/homebrew/opt/googletest/include"},"compiler":"Apple clang version 21.0.0 (clang-2100.1.1.101)\nTarget: arm64-apple-darwin25.6.0\nThread model: posix\nInstalledDir: /Library/Developer/CommandLineTools/usr/bin","git_commit":"870b175927d747fb1d90fadefebe8728da38407a","git_dirty":true,"link_flags":"-pthread -framework Metal -framework Foundation","manifest_version":1,"min_os":"26.0","reason_code":"valid","sdk":"26.5","status":"available","target_arch":"arm64-apple-darwin25.6.0"},"configuration":{"backend":"metal","phase":"prefill","kv_layout":"contiguous","kv_block_tokens":null,"weight_size_mb":1,"layer_count":2,"query_head_count":2,"kv_head_count":1,"head_dimension":3,"kv_element_bytes":"1","visible_context_tokens":null,"prompt_tokens":5,"attention_query_tile_tokens":2,"batch_size":2,"requested_workers":null,"available_workers":null,"worker_source":null,"iterations":2,"work_policy":"explicit_fixed_work","loop_count":3,"base_seed_uint64_decimal":"424242","seed_source":"user","output_file":"plans/llm-benchmark-remediation-evidence/phase7/profiles/metal-prefill-contiguous.json","argv":["./memory_benchmark","--llm-memory","--llm-memory-backend","metal","--phase","prefill","--kv-layout","contiguous","--weight-size-mb","1","--layers","2","--query-heads","2","--kv-heads","1","--head-dim","3","--kv-element-bytes","1","--batch-size","2","--iterations","2","--count","3","--seed","424242","-o","plans/llm-benchmark-remediation-evidence/phase7/profiles/metal-prefill-contiguous.json","--prompt-tokens","5","--attention-query-tile-tokens","2"],"resolved_sources":{"backend":"explicit","phase":"explicit","kv_layout":"explicit","kv_block_tokens":null,"visible_context_tokens":null,"prompt_tokens":"explicit","attention_query_tile_tokens":"explicit","workers":null,"iterations":"explicit","seed":"explicit"}},"resolved_plan":{"valid":true,"reason_code":"valid","plan_identity":"llm-memory-work-plan-v1|backend=metal|phase=prefill|kv_layout=contiguous|work_unit_kind=prefill_operation|methodology=llm-memory-v2-metal-prefill-contiguous|component_identity_size=782|component_identity=llm-memory-components-v1|logical_profile_version=31:full_prompt_causal_prefix_tiled|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=40:llm-metal-executor-v1-prefill-contiguous|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=56:llm-metal-prefill-contiguous-lane-local-vector-stripe-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=32:llm-metal-contiguous-affine32-v1|write_pattern_version=52:llm-metal-prefill-contiguous-full-prompt-affine32-v1|checksum_pattern_version=23:llm-metal-dual-mod32-v1|msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5|msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|prefill_planner_version=llm-prefill-planner-v1|prefill_prompt_tokens=5|prefill_query_tile_tokens=2|prefill_tile_count=3|prefill_prefix_token_visits_per_sequence=11|prefill_causal_token_pairs=15|prefill_logical_attention_pairs=120|prefill_logical_attention_fma_terms=360|prefill_prefix_block_visits_per_sequence=0|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=0|kv_blocks_per_sequence=0|physical_blocks_per_layer=0|total_physical_blocks=0|kv_block_bytes=0|last_block_tokens=0|last_block_valid_bytes=0|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=0|v_layout_padding_bytes=0|block_table_entries=0|block_table_bytes=0|layout_metadata_lookups_per_layer_sequence_per_work_unit=0|k_mapping_bytes=60|v_mapping_bytes=60|kv_capacity_bytes=120|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|kv_only_payload_bytes_per_work_unit=384|mixed_payload_bytes_per_work_unit=1048960|total_data_mapping_bytes=1048696|traffic_crossover_numerator=0|traffic_crossover_denominator=0|metal_execution_resolved=1|metal_execution_identity_size=3149|metal_execution_identity_sha256=fbf07727309e770f627285f9f96c4b6d472e573626d673f9c48668073dbd2906|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391","backend":"metal","phase":"prefill","kv_layout":"contiguous","work_unit_kind":"prefill_operation","geometry":{"valid":true,"reason_code":"valid","phase":"prefill","work_unit_kind":"prefill_operation","decode":null,"prefill":{"prompt_tokens":5,"attention_query_tile_tokens":2,"tile_count":"3","attention_prefix_token_visits_per_sequence":"11"},"attention_kind":"mqa","active_weight_bytes_per_work_unit":"1048576","layer_count":2,"query_head_count":2,"kv_head_count":1,"query_heads_per_kv_head":2,"head_dimension":3,"kv_element_bytes":"1","batch_size":2,"kv_vector_bytes":"3","k_or_v_record_bytes_per_layer":"3","kv_record_bytes_per_layer":"6","kv_bytes_per_visible_token":"12","k_or_v_sequence_visible_bytes":"15","k_mapping_bytes":"60","v_mapping_bytes":"60","kv_capacity_bytes":"120","weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"264","kv_write_bytes_per_work_unit":"120","kv_only_effective_model_payload_bytes_per_work_unit":"384","mixed_effective_model_payload_bytes_per_work_unit":"1048960","total_data_mapping_bytes":"1048696","traffic_crossover_numerator":null,"traffic_crossover_denominator":null,"traffic_crossover_context_tokens":null},"model_context":{"prefill":{"causal_token_pairs_per_sequence":"15","causal_token_pairs_per_sequence_reason_code":"valid","logical_attention_pairs":"120","logical_attention_pairs_reason_code":"valid","logical_attention_fma_terms":"360","logical_attention_fma_terms_reason_code":"valid"}},"layout":{"kv_layout":"contiguous","kv_block_tokens":null,"blocks_per_sequence":null,"physical_blocks_per_layer":null,"total_physical_blocks":null,"block_bytes":null,"last_block_tokens":null,"last_block_valid_bytes":null,"decode_append_offset_in_last_block":null,"block_table_entries":null,"block_table_bytes":null,"layout_geometry_identity":null,"layout_identity":null,"permutation_domain_uint64_hex":null,"permutation_seed_uint64_decimal":null,"permutation_algorithm_version":null,"permutation_entry_count":null,"permutation_sha256":null,"permutation_identity":null},"resources":{"weight_logical_bytes":"1048576","k_logical_bytes":"60","v_logical_bytes":"60","k_physical_length_bytes":"60","v_physical_length_bytes":"60","k_layout_padding_bytes":"0","v_layout_padding_bytes":"0","block_table_bytes":null,"metal":{"valid":true,"reason_code":"valid","execution_identity":"llm-metal-resource-foundation-v1phase=7:prefill;kv_layout=10:contiguous;segment_capacity_bytes=9:268435456;segment_slots_per_pool=3:256;weight_segments=310:whole-element-segments-v1|element_count=1048576|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=1048576;k_segments=300:whole-element-segments-v1|element_count=60|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=60;v_segments=300:whole-element-segments-v1|element_count=60|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=60;table_segments=14:not-applicable;paged_layout_geometry=14:not-applicable;argument_buffer=305:llm-metal-argument-buffer-tier2-v1segment_slot_cap=3:256;weight_slot_base=1:0;k_slot_base=3:256;v_slot_base=3:512;table_slot_base=3:768;status_slot=4:1024;encoded_resource_slot_count=4:1025;weight_segment_count=1:1;k_segment_count=1:1;v_segment_count=1:1;table_segment_count=1:0;active_resource_count=1:4;;argument_encoded_length=4:8200;argument_alignment=1:8;max_buffer_length=11:14302248960;host_mapping_granularity_bytes=5:16384;status_length=4:4096;staging_length=1:0;persistent_resource_length=7:1060992;host_permutation_mapping_bytes=1:0;permutation_validation_bitset_bytes=1:0;additional_owned_bytes=9:234096992;transient_peak_bytes=1:0;known_owned_peak_bytes=9:235157984;planned_resource_count=1:5;resource_0_pool=6:weight;resource_0_pool_index=1:0;resource_0_length_bytes=7:1048576;resource_0_persistent=1:1;resource_1_pool=1:k;resource_1_pool_index=1:0;resource_1_length_bytes=2:60;resource_1_persistent=1:1;resource_2_pool=1:v;resource_2_pool_index=1:0;resource_2_length_bytes=2:60;resource_2_persistent=1:1;resource_3_pool=15:argument_buffer;resource_3_pool_index=1:0;resource_3_length_bytes=4:8200;resource_3_persistent=1:1;resource_4_pool=15:status_checksum;resource_4_pool_index=1:0;resource_4_length_bytes=4:4096;resource_4_persistent=1:1;msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5;msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2;foundation_parameter_abi=34:llm-metal-foundation-parameters-v1;workload_parameter_abi=42:llm-metal-prefill-contiguous-parameters-v1;resource_table_abi=27:llm-metal-resource-table-v1;checksum_algorithm=23:llm-metal-dual-mod32-v1;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;","resource_identity":"llm-metal-resource-foundation-v1phase=7:prefill;kv_layout=10:contiguous;segment_capacity_bytes=9:268435456;segment_slots_per_pool=3:256;weight_segments=310:whole-element-segments-v1|element_count=1048576|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=1048576;k_segments=300:whole-element-segments-v1|element_count=60|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=60;v_segments=300:whole-element-segments-v1|element_count=60|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=60;table_segments=14:not-applicable;paged_layout_geometry=14:not-applicable;argument_buffer=305:llm-metal-argument-buffer-tier2-v1segment_slot_cap=3:256;weight_slot_base=1:0;k_slot_base=3:256;v_slot_base=3:512;table_slot_base=3:768;status_slot=4:1024;encoded_resource_slot_count=4:1025;weight_segment_count=1:1;k_segment_count=1:1;v_segment_count=1:1;table_segment_count=1:0;active_resource_count=1:4;;argument_encoded_length=4:8200;argument_alignment=1:8;max_buffer_length=11:14302248960;host_mapping_granularity_bytes=5:16384;status_length=4:4096;staging_length=1:0;persistent_resource_length=7:1060992;host_permutation_mapping_bytes=1:0;permutation_validation_bitset_bytes=1:0;additional_owned_bytes=9:234096992;transient_peak_bytes=1:0;known_owned_peak_bytes=9:235157984;planned_resource_count=1:5;resource_0_pool=6:weight;resource_0_pool_index=1:0;resource_0_length_bytes=7:1048576;resource_0_persistent=1:1;resource_1_pool=1:k;resource_1_pool_index=1:0;resource_1_length_bytes=2:60;resource_1_persistent=1:1;resource_2_pool=1:v;resource_2_pool_index=1:0;resource_2_length_bytes=2:60;resource_2_persistent=1:1;resource_3_pool=15:argument_buffer;resource_3_pool_index=1:0;resource_3_length_bytes=4:8200;resource_3_persistent=1:1;resource_4_pool=15:status_checksum;resource_4_pool_index=1:0;resource_4_length_bytes=4:4096;resource_4_persistent=1:1;","msl_revision":"llm-metal-decode-prefill-contiguous-paged-msl23-v5","msl_source_sha256":"2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2","segment_capacity_bytes":"268435456","max_buffer_length_bytes":"14302248960","weight_segments":{"valid":true,"reason_code":"valid","element_count":"1048576","element_bytes":"1","segment_capacity_bytes":"268435456","segment_slot_cap":256,"elements_per_segment":"268435456","segment_count":1,"maximum_addressable_elements":"68719476736","maximum_addressable_bytes":"68719476736","unused_nominal_segment_capacity_bytes":"0","total_length_bytes":"1048576","segment_lengths_bytes":["1048576"],"identity":"whole-element-segments-v1|element_count=1048576|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=1048576"},"k_segments":{"valid":true,"reason_code":"valid","element_count":"60","element_bytes":"1","segment_capacity_bytes":"268435456","segment_slot_cap":256,"elements_per_segment":"268435456","segment_count":1,"maximum_addressable_elements":"68719476736","maximum_addressable_bytes":"68719476736","unused_nominal_segment_capacity_bytes":"0","total_length_bytes":"60","segment_lengths_bytes":["60"],"identity":"whole-element-segments-v1|element_count=60|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=60"},"v_segments":{"valid":true,"reason_code":"valid","element_count":"60","element_bytes":"1","segment_capacity_bytes":"268435456","segment_slot_cap":256,"elements_per_segment":"268435456","segment_count":1,"maximum_addressable_elements":"68719476736","maximum_addressable_bytes":"68719476736","unused_nominal_segment_capacity_bytes":"0","total_length_bytes":"60","segment_lengths_bytes":["60"],"identity":"whole-element-segments-v1|element_count=60|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=60"},"table_segments":null,"argument_buffer":{"valid":true,"reason_code":"valid","encoded_length_bytes":"8200","alignment_bytes":"8","weight_slot_base":0,"k_slot_base":256,"v_slot_base":512,"table_slot_base":768,"status_slot":1024,"encoded_resource_slot_count":1025,"active_resource_count":4,"identity":"llm-metal-argument-buffer-tier2-v1segment_slot_cap=3:256;weight_slot_base=1:0;k_slot_base=3:256;v_slot_base=3:512;table_slot_base=3:768;status_slot=4:1024;encoded_resource_slot_count=4:1025;weight_segment_count=1:1;k_segment_count=1:1;v_segment_count=1:1;table_segment_count=1:0;active_resource_count=1:4;"},"status_buffer_length_bytes":"4096","staging_buffer_length_bytes":"0","persistent_resource_length_bytes":"1060992","transient_peak_bytes":"0","known_owned_peak_bytes":"235157984","admitted_budget_bytes":"9266895257"}},"component_identities":{"logical_profile_version":"full_prompt_causal_prefix_tiled","kv_layout_version":"contiguous_layer_batch_token_head_dimension","permutation_version":null,"backend_executor_version":"llm-metal-executor-v1-prefill-contiguous","resource_abi_version":"llm-metal-argument-buffer-tier2-v1","schedule_version":"llm-metal-prefill-contiguous-lane-local-vector-stripe-v1","timer_policy_version":"metal-command-buffer-gpu-start-end-v1","buffer_pattern_version":"llm-metal-contiguous-affine32-v1","write_pattern_version":"llm-metal-prefill-contiguous-full-prompt-affine32-v1","checksum_pattern_version":"llm-metal-dual-mod32-v1","msl_revision":"llm-metal-decode-prefill-contiguous-paged-msl23-v5","msl_source_sha256":"2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2","identity":"llm-memory-components-v1|logical_profile_version=31:full_prompt_causal_prefix_tiled|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=40:llm-metal-executor-v1-prefill-contiguous|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=56:llm-metal-prefill-contiguous-lane-local-vector-stripe-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=32:llm-metal-contiguous-affine32-v1|write_pattern_version=52:llm-metal-prefill-contiguous-full-prompt-affine32-v1|checksum_pattern_version=23:llm-metal-dual-mod32-v1|msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5|msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2","run_policy_version":"llm-run-policy-bounded-loop-snapshots-v1"},"methodology":{"backend":"metal","phase":"prefill","kv_layout":"contiguous","work_unit_kind":"prefill_operation","weight_passes_per_work_unit":1,"kv_replay_factor":1,"warmup_policy":"same-shape-excluded-steady-state-warm-memory","context_policy":"full-prompt-population-with-tiled-causal-prefix-scans","scenario_order_policy":"cyclic-rotation-across-count-loops","timing_policy":"gpu-start-to-gpu-end-per-command-buffer-task","cache_policy":"metal-private-and-shared-cacheable-no-explicit-flush-between-scenarios","calibration_policy":"per-scenario-automatic-or-explicit-frozen-before-loop-zero","calibration_target_seconds":0.15,"calibration_min_seconds":0.1,"calibration_max_seconds":0.25,"calibration_max_corrections":2,"calibration_min_pilot_accounted_bytes":"8388608","maximum_work_units_per_measurement":65536,"maximum_serial_range_visits_per_lane_per_task":1048576,"maximum_accounted_bytes_per_task":"68719476736","repeatability_cv_warning_threshold_pct":5.0,"calibration_excluded_from_results":true,"snapshot_policy":"bounded-loop-snapshots","timed_region_exclusions":["mapping-allocation","buffer-initialization-and-pre-touch","argument-buffer-encoding-and-resource-validation","host-command-encoding-and-pre-gpu-queue-wait","status-reset-command-buffer","excluded-kv-write-and-padding-canary-post-validation","same-shape-warmup","calibration-attempts","checksum-reference-generation-and-validation","host-wait-aggregation-and-json-serialization"]},"model_work_plan":{"valid":true,"reason_code":"valid","backend":"metal","phase":"prefill","kv_layout":"contiguous","work_unit_kind":"prefill_operation","weight_passes_per_work_unit":1,"kv_replay_factor":1,"requested_workers":null,"available_workers":null,"effective_workers":null,"worker_plan_count":null,"weight_layer_count":2,"layer_descriptors_per_worker":null,"sequence_descriptors_per_worker":null,"total_layer_descriptors":null,"total_sequence_descriptors":null,"descriptor_bytes":null,"planner_storage_bytes":null},"scenario_plans":[{"valid":true,"reason_code":"valid","scenario":"weights_only","work_unit_kind":"prefill_operation","kv_write_kind":"none","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"17410844953366850226","work_units":2,"weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"0","kv_write_bytes_per_work_unit":"0","effective_model_payload_bytes_per_work_unit":"1048576","layout_metadata_lookup_count_per_work_unit":"0","layout_metadata_read_bytes_per_work_unit":"0","accounted_bytes_per_work_unit":"1048576","weight_read_bytes":"2097152","kv_read_bytes":"0","kv_write_bytes":"0","effective_model_payload_bytes":"2097152","layout_metadata_lookup_count":"0","layout_metadata_read_bytes":"0","task_accounted_bytes":"2097152","maximum_work_units_by_work_unit_cap":65536,"maximum_work_units_by_guardrail":65536,"effective_maximum_work_units":65536,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=2779|model_plan_identity=llm-memory-work-plan-v1|backend=metal|phase=prefill|kv_layout=contiguous|work_unit_kind=prefill_operation|methodology=llm-memory-v2-metal-prefill-contiguous|component_identity_size=782|component_identity=llm-memory-components-v1|logical_profile_version=31:full_prompt_causal_prefix_tiled|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=40:llm-metal-executor-v1-prefill-contiguous|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=56:llm-metal-prefill-contiguous-lane-local-vector-stripe-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=32:llm-metal-contiguous-affine32-v1|write_pattern_version=52:llm-metal-prefill-contiguous-full-prompt-affine32-v1|checksum_pattern_version=23:llm-metal-dual-mod32-v1|msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5|msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|prefill_planner_version=llm-prefill-planner-v1|prefill_prompt_tokens=5|prefill_query_tile_tokens=2|prefill_tile_count=3|prefill_prefix_token_visits_per_sequence=11|prefill_causal_token_pairs=15|prefill_logical_attention_pairs=120|prefill_logical_attention_fma_terms=360|prefill_prefix_block_visits_per_sequence=0|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=0|kv_blocks_per_sequence=0|physical_blocks_per_layer=0|total_physical_blocks=0|kv_block_bytes=0|last_block_tokens=0|last_block_valid_bytes=0|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=0|v_layout_padding_bytes=0|block_table_entries=0|block_table_bytes=0|layout_metadata_lookups_per_layer_sequence_per_work_unit=0|k_mapping_bytes=60|v_mapping_bytes=60|kv_capacity_bytes=120|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|kv_only_payload_bytes_per_work_unit=384|mixed_payload_bytes_per_work_unit=1048960|total_data_mapping_bytes=1048696|traffic_crossover_numerator=0|traffic_crossover_denominator=0|metal_execution_resolved=1|metal_execution_identity_size=3149|metal_execution_identity_sha256=fbf07727309e770f627285f9f96c4b6d472e573626d673f9c48668073dbd2906|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=weights_only|work_unit_kind=prefill_operation|kv_write_kind=none|scenario_seed=17410844953366850226|explicit=1|work_units=2|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=0|kv_write_bytes_per_work_unit=0|effective_model_payload_bytes_per_work_unit=1048576|layout_metadata_lookup_count_per_work_unit=0|layout_metadata_read_bytes_per_work_unit=0|accounted_bytes_per_work_unit=1048576|weight_read_bytes=2097152|kv_read_bytes=0|kv_write_bytes=0|effective_model_payload_bytes=2097152|layout_metadata_lookup_count=0|layout_metadata_read_bytes=0|task_accounted_bytes=2097152|maximum_work_units_by_work_unit_cap=65536|maximum_work_units_by_guardrail=65536|effective_maximum_work_units=65536","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":null,"expected_run_checksum":{"weight":{"a_uint32_decimal":"3970170880","b_uint32_decimal":"1443364864"},"k":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"v":{"a_uint32_decimal":"0","b_uint32_decimal":"0"}}}},{"valid":true,"reason_code":"valid","scenario":"kv_only","work_unit_kind":"prefill_operation","kv_write_kind":"full_prompt_population","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"16590208891222127307","work_units":2,"weight_read_bytes_per_work_unit":"0","kv_read_bytes_per_work_unit":"264","kv_write_bytes_per_work_unit":"120","effective_model_payload_bytes_per_work_unit":"384","layout_metadata_lookup_count_per_work_unit":"0","layout_metadata_read_bytes_per_work_unit":"0","accounted_bytes_per_work_unit":"384","weight_read_bytes":"0","kv_read_bytes":"528","kv_write_bytes":"240","effective_model_payload_bytes":"768","layout_metadata_lookup_count":"0","layout_metadata_read_bytes":"0","task_accounted_bytes":"768","maximum_work_units_by_work_unit_cap":16384,"maximum_work_units_by_guardrail":178956970,"effective_maximum_work_units":16384,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=2779|model_plan_identity=llm-memory-work-plan-v1|backend=metal|phase=prefill|kv_layout=contiguous|work_unit_kind=prefill_operation|methodology=llm-memory-v2-metal-prefill-contiguous|component_identity_size=782|component_identity=llm-memory-components-v1|logical_profile_version=31:full_prompt_causal_prefix_tiled|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=40:llm-metal-executor-v1-prefill-contiguous|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=56:llm-metal-prefill-contiguous-lane-local-vector-stripe-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=32:llm-metal-contiguous-affine32-v1|write_pattern_version=52:llm-metal-prefill-contiguous-full-prompt-affine32-v1|checksum_pattern_version=23:llm-metal-dual-mod32-v1|msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5|msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|prefill_planner_version=llm-prefill-planner-v1|prefill_prompt_tokens=5|prefill_query_tile_tokens=2|prefill_tile_count=3|prefill_prefix_token_visits_per_sequence=11|prefill_causal_token_pairs=15|prefill_logical_attention_pairs=120|prefill_logical_attention_fma_terms=360|prefill_prefix_block_visits_per_sequence=0|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=0|kv_blocks_per_sequence=0|physical_blocks_per_layer=0|total_physical_blocks=0|kv_block_bytes=0|last_block_tokens=0|last_block_valid_bytes=0|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=0|v_layout_padding_bytes=0|block_table_entries=0|block_table_bytes=0|layout_metadata_lookups_per_layer_sequence_per_work_unit=0|k_mapping_bytes=60|v_mapping_bytes=60|kv_capacity_bytes=120|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|kv_only_payload_bytes_per_work_unit=384|mixed_payload_bytes_per_work_unit=1048960|total_data_mapping_bytes=1048696|traffic_crossover_numerator=0|traffic_crossover_denominator=0|metal_execution_resolved=1|metal_execution_identity_size=3149|metal_execution_identity_sha256=fbf07727309e770f627285f9f96c4b6d472e573626d673f9c48668073dbd2906|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=kv_only|work_unit_kind=prefill_operation|kv_write_kind=full_prompt_population|scenario_seed=16590208891222127307|explicit=1|work_units=2|weight_read_bytes_per_work_unit=0|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|effective_model_payload_bytes_per_work_unit=384|layout_metadata_lookup_count_per_work_unit=0|layout_metadata_read_bytes_per_work_unit=0|accounted_bytes_per_work_unit=384|weight_read_bytes=0|kv_read_bytes=528|kv_write_bytes=240|effective_model_payload_bytes=768|layout_metadata_lookup_count=0|layout_metadata_read_bytes=0|task_accounted_bytes=768|maximum_work_units_by_work_unit_cap=16384|maximum_work_units_by_guardrail=178956970|effective_maximum_work_units=16384","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":null,"expected_run_checksum":{"weight":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"k":{"a_uint32_decimal":"959095562","b_uint32_decimal":"606908354"},"v":{"a_uint32_decimal":"1795379978","b_uint32_decimal":"4097023938"}}}},{"valid":true,"reason_code":"valid","scenario":"mixed","work_unit_kind":"prefill_operation","kv_write_kind":"full_prompt_population","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"1546516732201688391","work_units":2,"weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"264","kv_write_bytes_per_work_unit":"120","effective_model_payload_bytes_per_work_unit":"1048960","layout_metadata_lookup_count_per_work_unit":"0","layout_metadata_read_bytes_per_work_unit":"0","accounted_bytes_per_work_unit":"1048960","weight_read_bytes":"2097152","kv_read_bytes":"528","kv_write_bytes":"240","effective_model_payload_bytes":"2097920","layout_metadata_lookup_count":"0","layout_metadata_read_bytes":"0","task_accounted_bytes":"2097920","maximum_work_units_by_work_unit_cap":15887,"maximum_work_units_by_guardrail":65512,"effective_maximum_work_units":15887,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=2779|model_plan_identity=llm-memory-work-plan-v1|backend=metal|phase=prefill|kv_layout=contiguous|work_unit_kind=prefill_operation|methodology=llm-memory-v2-metal-prefill-contiguous|component_identity_size=782|component_identity=llm-memory-components-v1|logical_profile_version=31:full_prompt_causal_prefix_tiled|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=40:llm-metal-executor-v1-prefill-contiguous|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=56:llm-metal-prefill-contiguous-lane-local-vector-stripe-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=32:llm-metal-contiguous-affine32-v1|write_pattern_version=52:llm-metal-prefill-contiguous-full-prompt-affine32-v1|checksum_pattern_version=23:llm-metal-dual-mod32-v1|msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5|msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|prefill_planner_version=llm-prefill-planner-v1|prefill_prompt_tokens=5|prefill_query_tile_tokens=2|prefill_tile_count=3|prefill_prefix_token_visits_per_sequence=11|prefill_causal_token_pairs=15|prefill_logical_attention_pairs=120|prefill_logical_attention_fma_terms=360|prefill_prefix_block_visits_per_sequence=0|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=0|kv_blocks_per_sequence=0|physical_blocks_per_layer=0|total_physical_blocks=0|kv_block_bytes=0|last_block_tokens=0|last_block_valid_bytes=0|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=0|v_layout_padding_bytes=0|block_table_entries=0|block_table_bytes=0|layout_metadata_lookups_per_layer_sequence_per_work_unit=0|k_mapping_bytes=60|v_mapping_bytes=60|kv_capacity_bytes=120|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|kv_only_payload_bytes_per_work_unit=384|mixed_payload_bytes_per_work_unit=1048960|total_data_mapping_bytes=1048696|traffic_crossover_numerator=0|traffic_crossover_denominator=0|metal_execution_resolved=1|metal_execution_identity_size=3149|metal_execution_identity_sha256=fbf07727309e770f627285f9f96c4b6d472e573626d673f9c48668073dbd2906|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=mixed|work_unit_kind=prefill_operation|kv_write_kind=full_prompt_population|scenario_seed=1546516732201688391|explicit=1|work_units=2|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|effective_model_payload_bytes_per_work_unit=1048960|layout_metadata_lookup_count_per_work_unit=0|layout_metadata_read_bytes_per_work_unit=0|accounted_bytes_per_work_unit=1048960|weight_read_bytes=2097152|kv_read_bytes=528|kv_write_bytes=240|effective_model_payload_bytes=2097920|layout_metadata_lookup_count=0|layout_metadata_read_bytes=0|task_accounted_bytes=2097920|maximum_work_units_by_work_unit_cap=15887|maximum_work_units_by_guardrail=65512|effective_maximum_work_units=15887","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":null,"expected_run_checksum":{"weight":{"a_uint32_decimal":"3375628288","b_uint32_decimal":"3948412928"},"k":{"a_uint32_decimal":"2542249290","b_uint32_decimal":"3836332802"},"v":{"a_uint32_decimal":"3378533706","b_uint32_decimal":"3031481090"}}}}],"frozen_plan_refs":{"weights_only":0,"kv_only":1,"mixed":2}},"backend_evidence":{"cpu":null,"metal":{"workers_applicable":false,"worker_qos_applicable":false,"lifecycle":{"initialization":{"status":"ready","reason_code":"valid"},"plan_resolution":{"status":"ready","reason_code":"valid"},"preparation":{"status":"ready","reason_code":"valid"},"release":{"status":"ready","reason_code":"valid"}},"capability":{"device_name":"Apple M4","registry_id_uint64_decimal":"4294968227","has_unified_memory":true,"required_apple7_family_supported":true,"argument_buffers_tier2_supported":true,"supported_families":["apple1","apple2","apple3","apple4","apple5","apple6","apple7","apple8","apple9","mac1","mac2","common1","common2","common3","metal3","metal4"],"max_buffer_length_bytes":"14302248960","recommended_max_working_set_size_bytes":"19069665280","compilation_mode":"runtime-source","msl_language_version":"2.3","kernel_revision":"llm-metal-decode-prefill-contiguous-paged-msl23-v5","kernel_source_sha256":"2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2","compiler_diagnostics":null,"compiler_identifier":"21.0.0 (clang-2100.1.1.101)","build_sdk":"26.5","deployment_target":"26.0","argument_buffer":{"encoded_length_bytes":"8200","alignment_bytes":"8"},"layout_probe":{"evaluated":true,"valid":true,"resource_count":4},"foundation_pipelines":[{"label":"membenchmark.llm-metal.pipeline.initialize","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.copy","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.layout-probe","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.prefill-contiguous-layout-probe","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.validate-bytes","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.validate-table","thread_execution_width":32,"max_total_threads_per_threadgroup":1024}],"error":{"domain":null,"code":0,"description":null}},"resources":{"allocation_attempted":true,"allocation_completed":true,"initialization_completed":true,"table_upload_completed":true,"table_validation_completed":true,"table_permutation":null,"post_validation_completed":true,"cpu_sample_readback_validation_completed":true,"candidate_cleanup_completed":true,"resources_published":false,"persistent_resource_length_bytes":"1060992","committed_resource_bytes":"1060992","resource_rounding_bytes":"0","layout_padding_bytes":"0","transient_peak_bytes":"0","additional_owned_bytes":"234096992","known_owned_peak_bytes":"235157984","admitted_budget_bytes":"9266895257","current_allocated_size_before_bytes":"1785856","current_allocated_size_peak_bytes":"2834432","current_allocated_size_after_release_bytes":"1785856","recommended_working_set_available":true,"recommended_working_set_headroom_bytes":"18832721440","recommended_working_set_headroom_fraction":0.9875748296301508,"exceeds_recommended_working_set":false,"resources":[{"label":"membenchmark.llm-metal.weight-0","pool":"weight","pool_index":0,"storage_mode":"private","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"544","length_bytes":"1048576","allocated_size_available":true,"allocated_size_bytes":"1048576","committed_bytes":"1048576","allocation_rounding_bytes":"0"},{"label":"membenchmark.llm-metal.k-0","pool":"k","pool_index":0,"storage_mode":"private","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"544","length_bytes":"60","allocated_size_available":true,"allocated_size_bytes":"60","committed_bytes":"60","allocation_rounding_bytes":"0"},{"label":"membenchmark.llm-metal.v-0","pool":"v","pool_index":0,"storage_mode":"private","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"544","length_bytes":"60","allocated_size_available":true,"allocated_size_bytes":"60","committed_bytes":"60","allocation_rounding_bytes":"0"},{"label":"membenchmark.llm-metal.argument_buffer-0","pool":"argument_buffer","pool_index":0,"storage_mode":"shared","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"512","length_bytes":"8200","allocated_size_available":true,"allocated_size_bytes":"8200","committed_bytes":"8200","allocation_rounding_bytes":"0"},{"label":"membenchmark.llm-metal.status_checksum-0","pool":"status_checksum","pool_index":0,"storage_mode":"shared","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"512","length_bytes":"4096","allocated_size_available":true,"allocated_size_bytes":"4096","committed_bytes":"4096","allocation_rounding_bytes":"0"}],"error":{"domain":null,"code":0,"description":null}},"workload_pipelines":[{"label":"membenchmark.llm-metal.pipeline.prefill-contiguous.weights-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.prefill-contiguous.kv-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.prefill-contiguous.mixed","thread_execution_width":32,"max_total_threads_per_threadgroup":1024}],"timed_results_available":true}},"memory_budget":{"resource_rounding_bytes":"0","transient_peak_bytes":"0","layout_transient_bytes":null,"setup_peak_bytes":"235157984","runtime_peak_bytes":"235157984","known_owned_peak_bytes":"235157984","admitted_budget_bytes":"9266895257","valid":true,"reason_code":"within-memory-budget","request":{"valid":true,"reason_code":"valid","mapping_granularity_bytes":"16384","requested_weight_mapping_bytes":"1048576","requested_k_mapping_bytes":"60","requested_v_mapping_bytes":"60","committed_weight_mapping_bytes":"1048576","committed_k_mapping_bytes":"60","committed_v_mapping_bytes":"60","requested_block_table_mapping_bytes":null,"committed_block_table_mapping_bytes":null,"requested_data_bytes":"1048696","committed_data_bytes":"1048696","layout_transient_bytes":null,"setup_peak_bytes":"235157984","runtime_peak_bytes":"235157984","descriptor_bytes":"0","planner_storage_bytes":"152192","checksum_auxiliary_bytes":"0","orchestration_auxiliary_bytes":"233944800","auxiliary_bytes":"234096992","required_total_bytes":"235157984"},"available_memory_bytes":"11583619072","allowed_memory_bytes":"9266895257","used_fallback":false},"calibration":{"excluded_from_results":true,"attempts":{"weights_only":[{"attempt_index":0,"scenario":"weights_only","plan_ref":0,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":0.00032808328978717327,"timing":{"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"weight":{"a_uint32_decimal":"3970170880","b_uint32_decimal":"1443364864"},"k":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"v":{"a_uint32_decimal":"0","b_uint32_decimal":"0"}}},"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-contiguous.weights-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"4","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"2097152","maximum_threadgroup_accounted_bytes":"2097152","threadgroup_accounted_imbalance_bytes":"0","threadgroup_accounted_bytes":["2097152"],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:4;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=7:2097152;maximum_threadgroup_accounted_bytes=7:2097152;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:1;threadgroup_accounted_bytes=7:2097152;weight_shard_cost_version=43:llm-metal-weight-vector-grid-stride-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.6223616668,"gpu_end_seconds":1532841.62268975,"host_submit_to_completion_seconds":0.00059375,"host_wait_seconds":0.000591792,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-prefill-final-samples","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}}}],"kv_only":[{"attempt_index":0,"scenario":"kv_only","plan_ref":1,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":0.0007768750656396151,"timing":{"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"weight":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"k":{"a_uint32_decimal":"959095562","b_uint32_decimal":"606908354"},"v":{"a_uint32_decimal":"1795379978","b_uint32_decimal":"4097023938"}}},"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-contiguous.kv-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"1","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"128","cost_unit":null,"minimum_threadgroup_accounted_bytes":null,"maximum_threadgroup_accounted_bytes":null,"threadgroup_accounted_imbalance_bytes":null,"threadgroup_accounted_bytes":[],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=1:1;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=3:128;visit_bytes=2:32;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=1:0;maximum_threadgroup_accounted_bytes=1:0;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:0;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.62335875,"gpu_end_seconds":1532841.624135625,"host_submit_to_completion_seconds":0.001130917,"host_wait_seconds":0.00112925,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}}}],"mixed":[{"attempt_index":0,"scenario":"mixed","plan_ref":2,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":0.0011404165998101234,"timing":{"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"weight":{"a_uint32_decimal":"3375628288","b_uint32_decimal":"3948412928"},"k":{"a_uint32_decimal":"2542249290","b_uint32_decimal":"3836332802"},"v":{"a_uint32_decimal":"3378533706","b_uint32_decimal":"3031481090"}}},"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-contiguous.mixed","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"132","cost_unit":null,"minimum_threadgroup_accounted_bytes":null,"maximum_threadgroup_accounted_bytes":null,"threadgroup_accounted_imbalance_bytes":null,"threadgroup_accounted_bytes":[],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=3:132;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=1:0;maximum_threadgroup_accounted_bytes=1:0;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:0;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.6255237502,"gpu_end_seconds":1532841.6266641668,"host_submit_to_completion_seconds":0.001709542,"host_wait_seconds":0.001708,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}}}]}},"measurements":[{"measurement_id":0,"plan_ref":0,"scenario":"weights_only","loop_index":0,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":null,"elapsed_seconds":0.0003470415249466896,"synthetic_work_unit_latency_seconds":0.0001735207624733448,"synthetic_memory_work_units_per_second":5762.999111726552,"effective_model_payload_gb_s":6.04294255657778,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-contiguous.weights-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"4","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"2097152","maximum_threadgroup_accounted_bytes":"2097152","threadgroup_accounted_imbalance_bytes":"0","threadgroup_accounted_bytes":["2097152"],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:4;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=7:2097152;maximum_threadgroup_accounted_bytes=7:2097152;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:1;threadgroup_accounted_bytes=7:2097152;weight_shard_cost_version=43:llm-metal-weight-vector-grid-stride-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.6278011252,"gpu_end_seconds":1532841.6281481667,"host_submit_to_completion_seconds":0.000698167,"host_wait_seconds":0.000696917,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-prefill-final-samples","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"3970170880","b_uint32_decimal":"1443364864"},"k":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"v":{"a_uint32_decimal":"0","b_uint32_decimal":"0"}}}},{"measurement_id":1,"plan_ref":1,"scenario":"kv_only","loop_index":0,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"768","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"768","completion_derivation":null,"elapsed_seconds":0.0007618749514222145,"synthetic_work_unit_latency_seconds":0.00038093747571110725,"synthetic_memory_work_units_per_second":2625.102710446827,"effective_model_payload_gb_s":0.0010080394408115815,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-contiguous.kv-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"1","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"128","cost_unit":null,"minimum_threadgroup_accounted_bytes":null,"maximum_threadgroup_accounted_bytes":null,"threadgroup_accounted_imbalance_bytes":null,"threadgroup_accounted_bytes":[],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=1:1;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=3:128;visit_bytes=2:32;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=1:0;maximum_threadgroup_accounted_bytes=1:0;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:0;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.62892125,"gpu_end_seconds":1532841.629683125,"host_submit_to_completion_seconds":0.001042709,"host_wait_seconds":0.0010415,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"k":{"a_uint32_decimal":"959095562","b_uint32_decimal":"606908354"},"v":{"a_uint32_decimal":"1795379978","b_uint32_decimal":"4097023938"}}}},{"measurement_id":2,"plan_ref":2,"scenario":"mixed","loop_index":0,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":0.999633923123856,"kv_read_payload_fraction":0.0002516778523489933,"kv_write_payload_fraction":0.00011439902379499695,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097920","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097920","completion_derivation":null,"elapsed_seconds":0.0010604166891425848,"synthetic_work_unit_latency_seconds":0.0005302083445712924,"synthetic_memory_work_units_per_second":1886.0510405745583,"effective_model_payload_gb_s":1.9783920995210886,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-contiguous.mixed","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"132","cost_unit":null,"minimum_threadgroup_accounted_bytes":null,"maximum_threadgroup_accounted_bytes":null,"threadgroup_accounted_imbalance_bytes":null,"threadgroup_accounted_bytes":[],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=3:132;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=1:0;maximum_threadgroup_accounted_bytes=1:0;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:0;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.630407375,"gpu_end_seconds":1532841.6314677917,"host_submit_to_completion_seconds":0.001352417,"host_wait_seconds":0.001351208,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"3375628288","b_uint32_decimal":"3948412928"},"k":{"a_uint32_decimal":"2542249290","b_uint32_decimal":"3836332802"},"v":{"a_uint32_decimal":"3378533706","b_uint32_decimal":"3031481090"}}}},{"measurement_id":3,"plan_ref":1,"scenario":"kv_only","loop_index":1,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"768","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"768","completion_derivation":null,"elapsed_seconds":0.0007625001017004251,"synthetic_work_unit_latency_seconds":0.0003812500508502126,"synthetic_memory_work_units_per_second":2622.950469829275,"effective_model_payload_gb_s":0.0010072129804144416,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-contiguous.kv-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"1","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"128","cost_unit":null,"minimum_threadgroup_accounted_bytes":null,"maximum_threadgroup_accounted_bytes":null,"threadgroup_accounted_imbalance_bytes":null,"threadgroup_accounted_bytes":[],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=1:1;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=3:128;visit_bytes=2:32;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=1:0;maximum_threadgroup_accounted_bytes=1:0;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:0;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.633225125,"gpu_end_seconds":1532841.6339876251,"host_submit_to_completion_seconds":0.001022417,"host_wait_seconds":0.001021125,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"k":{"a_uint32_decimal":"959095562","b_uint32_decimal":"606908354"},"v":{"a_uint32_decimal":"1795379978","b_uint32_decimal":"4097023938"}}}},{"measurement_id":4,"plan_ref":2,"scenario":"mixed","loop_index":1,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":0.999633923123856,"kv_read_payload_fraction":0.0002516778523489933,"kv_write_payload_fraction":0.00011439902379499695,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097920","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097920","completion_derivation":null,"elapsed_seconds":0.0010601666290313005,"synthetic_work_unit_latency_seconds":0.0005300833145156503,"synthetic_memory_work_units_per_second":1886.4959009579914,"effective_model_payload_gb_s":1.9788587402688946,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-contiguous.mixed","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"132","cost_unit":null,"minimum_threadgroup_accounted_bytes":null,"maximum_threadgroup_accounted_bytes":null,"threadgroup_accounted_imbalance_bytes":null,"threadgroup_accounted_bytes":[],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=3:132;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=1:0;maximum_threadgroup_accounted_bytes=1:0;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:0;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.6346393751,"gpu_end_seconds":1532841.6356995418,"host_submit_to_completion_seconds":0.001408708,"host_wait_seconds":0.0014075,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"3375628288","b_uint32_decimal":"3948412928"},"k":{"a_uint32_decimal":"2542249290","b_uint32_decimal":"3836332802"},"v":{"a_uint32_decimal":"3378533706","b_uint32_decimal":"3031481090"}}}},{"measurement_id":5,"plan_ref":0,"scenario":"weights_only","loop_index":1,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":null,"elapsed_seconds":0.0003255833871662617,"synthetic_work_unit_latency_seconds":0.00016279169358313084,"synthetic_memory_work_units_per_second":6142.819562776662,"effective_model_payload_gb_s":6.441213165858101,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-contiguous.weights-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"4","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"2097152","maximum_threadgroup_accounted_bytes":"2097152","threadgroup_accounted_imbalance_bytes":"0","threadgroup_accounted_bytes":["2097152"],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:4;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=7:2097152;maximum_threadgroup_accounted_bytes=7:2097152;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:1;threadgroup_accounted_bytes=7:2097152;weight_shard_cost_version=43:llm-metal-weight-vector-grid-stride-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.6364819168,"gpu_end_seconds":1532841.6368075002,"host_submit_to_completion_seconds":0.000675625,"host_wait_seconds":0.000674459,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-prefill-final-samples","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"3970170880","b_uint32_decimal":"1443364864"},"k":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"v":{"a_uint32_decimal":"0","b_uint32_decimal":"0"}}}},{"measurement_id":6,"plan_ref":2,"scenario":"mixed","loop_index":2,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":0.999633923123856,"kv_read_payload_fraction":0.0002516778523489933,"kv_write_payload_fraction":0.00011439902379499695,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097920","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097920","completion_derivation":null,"elapsed_seconds":0.00057787518016994,"synthetic_work_unit_latency_seconds":0.00028893759008497,"synthetic_memory_work_units_per_second":3460.955010062632,"effective_model_payload_gb_s":3.6304033673552984,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-contiguous.mixed","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"132","cost_unit":null,"minimum_threadgroup_accounted_bytes":null,"maximum_threadgroup_accounted_bytes":null,"threadgroup_accounted_imbalance_bytes":null,"threadgroup_accounted_bytes":[],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=3:132;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=1:0;maximum_threadgroup_accounted_bytes=1:0;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:0;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.638520875,"gpu_end_seconds":1532841.6390987502,"host_submit_to_completion_seconds":0.000855541,"host_wait_seconds":0.0008535,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"3375628288","b_uint32_decimal":"3948412928"},"k":{"a_uint32_decimal":"2542249290","b_uint32_decimal":"3836332802"},"v":{"a_uint32_decimal":"3378533706","b_uint32_decimal":"3031481090"}}}},{"measurement_id":7,"plan_ref":0,"scenario":"weights_only","loop_index":2,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":null,"elapsed_seconds":0.00017899996601045132,"synthetic_work_unit_latency_seconds":8.949998300522566e-05,"synthetic_memory_work_units_per_second":11173.18647917076,"effective_model_payload_gb_s":11.715935185582959,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-contiguous.weights-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"4","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"2097152","maximum_threadgroup_accounted_bytes":"2097152","threadgroup_accounted_imbalance_bytes":"0","threadgroup_accounted_bytes":["2097152"],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:4;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=7:2097152;maximum_threadgroup_accounted_bytes=7:2097152;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:1;threadgroup_accounted_bytes=7:2097152;weight_shard_cost_version=43:llm-metal-weight-vector-grid-stride-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.6397637501,"gpu_end_seconds":1532841.63994275,"host_submit_to_completion_seconds":0.000357292,"host_wait_seconds":0.00035475,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-prefill-final-samples","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"3970170880","b_uint32_decimal":"1443364864"},"k":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"v":{"a_uint32_decimal":"0","b_uint32_decimal":"0"}}}},{"measurement_id":8,"plan_ref":1,"scenario":"kv_only","loop_index":2,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048696","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"768","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"768","completion_derivation":null,"elapsed_seconds":0.00041562505066394806,"synthetic_work_unit_latency_seconds":0.00020781252533197403,"synthetic_memory_work_units_per_second":4812.0294886101365,"effective_model_payload_gb_s":0.0018478193236262923,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-contiguous.kv-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"1","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"128","cost_unit":null,"minimum_threadgroup_accounted_bytes":null,"maximum_threadgroup_accounted_bytes":null,"threadgroup_accounted_imbalance_bytes":null,"threadgroup_accounted_bytes":[],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=1:1;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=3:128;visit_bytes=2:32;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=1:0;maximum_threadgroup_accounted_bytes=1:0;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:0;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.640441125,"gpu_end_seconds":1532841.64085675,"host_submit_to_completion_seconds":0.000641541,"host_wait_seconds":0.000639958,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"k":{"a_uint32_decimal":"959095562","b_uint32_decimal":"606908354"},"v":{"a_uint32_decimal":"1795379978","b_uint32_decimal":"4097023938"}}}}],"aggregates":{"scenarios":{"weights_only":{"scenario":"weights_only","status":"complete","observed_cv_classification":"above-threshold","accepted_measurement_ids":[0,5,7],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":0.00016279169358313084,"statistics":{"sample_count":3,"average":0.00014193747968723377,"min":8.949998300522566e-05,"max":0.0001735207624733448,"median":0.00016279169358313084,"p90":0.00017137494869530202,"p95":0.0001724478555843234,"p99":0.00017330618109554052,"stddev":4.5727962162329605e-05,"coefficient_of_variation_pct":32.21697487026923,"median_absolute_deviation":1.0729068890213966e-05}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":6142.819562776662,"statistics":{"sample_count":3,"average":7693.001717891325,"min":5762.999111726552,"max":11173.18647917076,"median":6142.819562776662,"p90":10167.11309589194,"p95":10670.149787531349,"p99":11072.579140842878,"stddev":3019.9056894612086,"coefficient_of_variation_pct":39.25523222538645,"median_absolute_deviation":379.82045105011}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":6.441213165858101,"statistics":{"sample_count":3,"average":8.066696969339613,"min":6.04294255657778,"max":11.715935185582959,"median":6.441213165858101,"p90":10.660990781637988,"p95":11.188462983610473,"p99":11.61044074518846,"stddev":3.166600628232476,"coefficient_of_variation_pct":39.25523222538645,"median_absolute_deviation":0.39827060928032054}}},"kv_only":{"scenario":"kv_only","status":"complete","observed_cv_classification":"above-threshold","accepted_measurement_ids":[1,3,8],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":0.00038093747571110725,"statistics":{"sample_count":3,"average":0.00032333335063109797,"min":0.00020781252533197403,"max":0.0003812500508502126,"median":0.00038093747571110725,"p90":0.0003811875358223915,"p95":0.00038121879333630204,"p99":0.00038124379934743045,"stddev":0.00010004409145045713,"coefficient_of_variation_pct":30.94146992730139,"median_absolute_deviation":3.1257513910532e-07}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":2625.102710446827,"statistics":{"sample_count":3,"average":3353.360889628746,"min":2622.950469829275,"max":4812.0294886101365,"median":2625.102710446827,"p90":4374.644132977474,"p95":4593.336810793805,"p99":4768.2909530468705,"stddev":1263.2445207780115,"coefficient_of_variation_pct":37.670998212121056,"median_absolute_deviation":2.152240617552252}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":0.0010080394408115815,"statistics":{"sample_count":3,"average":0.0012876905816174384,"min":0.0010072129804144416,"max":0.0018478193236262923,"median":0.0010080394408115815,"p90":0.00167986334706335,"p95":0.0017638413353448213,"p99":0.001831023725969998,"stddev":0.00048508589597875637,"coefficient_of_variation_pct":37.67099821212105,"median_absolute_deviation":8.26460397139895e-07}}},"mixed":{"scenario":"mixed","status":"complete","observed_cv_classification":"above-threshold","accepted_measurement_ids":[2,4,6],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":0.0005300833145156503,"statistics":{"sample_count":3,"average":0.0004497430830573042,"min":0.00028893759008497,"max":0.0005302083445712924,"median":0.0005300833145156503,"p90":0.000530183338560164,"p95":0.0005301958415657281,"p99":0.0005302058439701796,"stddev":0.00013926165601372574,"coefficient_of_variation_pct":30.964713246291698,"median_absolute_deviation":1.25030055642128e-07}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":1886.4959009579914,"statistics":{"sample_count":3,"average":2411.167317198394,"min":1886.0510405745583,"max":3460.955010062632,"median":1886.4959009579914,"p90":3146.063188241704,"p95":3303.509099152168,"p99":3429.465827880539,"stddev":909.1428378104868,"coefficient_of_variation_pct":37.70550601469028,"median_absolute_deviation":0.44486038343302425}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":1.9788587402688946,"statistics":{"sample_count":3,"average":2.529218069048427,"min":1.9783920995210886,"max":3.6304033673552984,"median":1.9788587402688946,"p90":3.300094441938018,"p95":3.465248904646658,"p99":3.5973724748135707,"stddev":0.9536544711496883,"coefficient_of_variation_pct":37.705506014690286,"median_absolute_deviation":0.00046664074780600195}}}},"traffic_diagnostics":{"classification_version":"llm-exact-weight-vs-kv-read-payload-v1","traffic_crossover_numerator":null,"traffic_crossover_denominator":null,"traffic_crossover_context_tokens":null,"current_visible_context_tokens":null,"current_weight_read_payload_bytes_per_work_unit":"1048576","current_kv_read_payload_bytes_per_work_unit":"264","current_weight_to_kv_read_payload_ratio":null,"current_context_classification":null,"classification_is_payload_only":true,"scenario_headlines":{"weights_only":{"synthetic_work_unit_latency_seconds":0.00016279169358313084,"synthetic_memory_work_units_per_second":6142.819562776662,"effective_model_payload_gb_s":6.441213165858101},"kv_only":{"synthetic_work_unit_latency_seconds":0.00038093747571110725,"synthetic_memory_work_units_per_second":2625.102710446827,"effective_model_payload_gb_s":0.0010080394408115815},"mixed":{"synthetic_work_unit_latency_seconds":0.0005300833145156503,"synthetic_memory_work_units_per_second":1886.4959009579914,"effective_model_payload_gb_s":1.9788587402688946}}}},"status":"complete","reason_code":"complete","results_complete":true,"run_accepted":true,"interpretation":{"result_scope":"synthetic-memory-only-work-unit-not-real-inference-token-or-prefill-latency","reported_rate":"synthetic_memory_work_units_per_second","backend":"metal","phase":"prefill","work_unit_kind":"prefill_operation","transformer_math_included":false,"framework_scheduler_and_dispatch_included":false,"compute_memory_overlap_included":false,"physical_dram_traffic_measured":false,"dram_residency":"unverified","cache_residency":"unverified","fixed_context_includes_current_token_slot":null,"kv_layout":"contiguous","payload_semantics":"exact-logical-weight-plus-kv-read-plus-kv-write-bytes-divided-by-elapsed-time","layout_metadata_timed_but_excluded_from_effective_model_payload_gb_s":true,"prefill_transformer_compute_or_ttft_prediction_included":false,"private_metal_storage_implies_separate_vram":false,"cross_backend_performance_distributions_combined":false,"traffic_classification_semantics":"exact-weight-vs-kv-read-logical-payload-only-not-hardware-bottleneck","full_size_working_set_reduces_but_does_not_prove_dram_residency":true,"comparability_requires":["same-backend","same-methodology-version","same-frozen-work-plan-and-model-geometry","same-software-version","sufficiently-similar-hardware","sufficiently-similar-thermal-and-power-state"]},"diagnostic":null,"interruption_requested":false,"scenario_order_balance_complete":true,"seeds":{"base_seed_uint64_decimal":"424242","source":"user","buffer_domain_seeds":{"weight_uint64_decimal":"904846021374001452","k_uint64_decimal":"10718278915871169538","v_uint64_decimal":"2111147304547004201"},"scenario_domain_seeds":{"weights_only":"17410844953366850226","kv_only":"16590208891222127307","mixed":"1546516732201688391"}},"counters":{"planned_loops":3,"attempted_loops":3,"completed_loops":3,"planned_measurements":9,"attempted_measurements":9,"terminal_measurements":9,"measured_measurements":9,"planned_work_units":"18","completed_work_units":"18","planned_effective_model_payload_bytes":"12587520","completed_effective_model_payload_bytes":"12587520","planned_layout_metadata_lookup_count":"0","completed_layout_metadata_lookup_count":"0","planned_layout_metadata_read_bytes":"0","completed_layout_metadata_read_bytes":"0","planned_task_accounted_bytes":"12587520","completed_task_accounted_bytes":"12587520","runner_auxiliary":{"valid":true,"reason_code":"valid","measurement_record_bytes":"7056","loop_record_bytes":"144","calibration_record_bytes":"2784","calibration_identity_bytes":"76464","aggregate_value_bytes":"72","statistics_workspace_bytes":"72","warning_record_bytes":"64","fixed_metadata_bytes":"13774356","retained_checksum_bytes":"0","checksum_auxiliary_bytes":"0","orchestration_auxiliary_bytes":"13861012","total_auxiliary_bytes":"13861012"}},"checkpoint_lifecycle":{"checkpoint_failed":false,"observation_point":"before-current-snapshot-preparation","prior_file_writer_attempts":3,"prior_successful_file_writes":3,"current_request":"terminal","current_persistence_success":null,"snapshot_interval_loops":1,"checkpoint_policy":"bounded-loop-snapshots","file_checkpoint_failure_is_terminal_and_not_retried":true,"stdout_intermediate_checkpoints_are_lazy":true},"loop_records":[{"loop_index":0,"planned_order":["weights_only","kv_only","mixed"],"realized_order":["weights_only","kv_only","mixed"],"realized_order_count":3,"measurement_indexes":[0,1,2]},{"loop_index":1,"planned_order":["kv_only","mixed","weights_only"],"realized_order":["kv_only","mixed","weights_only"],"realized_order_count":3,"measurement_indexes":[3,4,5]},{"loop_index":2,"planned_order":["mixed","weights_only","kv_only"],"realized_order":["mixed","weights_only","kv_only"],"realized_order_count":3,"measurement_indexes":[6,7,8]}],"environment":{"processor_name":"Apple M4","macos_version":"26.6.2","performance_core_count":4,"efficiency_core_count":6,"logical_core_count":10,"page_size_bytes":"16384","l1_data_cache_bytes":"131072","l2_data_cache_bytes":"16777216","available_memory_bytes":"11583619072","available_memory_source":"mach-free-plus-inactive-rounded-mib","main_thread_qos":{"requested":true,"applied":true,"code":0},"start":{"thermal_state":"nominal","low_power_mode_available":true,"low_power_mode_enabled":false,"physical_memory_bytes":"25769803776"},"end":{"thermal_state":"nominal","low_power_mode_available":true,"low_power_mode_enabled":false,"physical_memory_bytes":"25769803776"}},"quality_warnings":["weights_only-high-cv","kv_only-high-cv","mixed-high-cv","weight-working-set-cache-dominant","kv-working-set-cache-dominant","weights_only-duration-below-target-window","kv_only-duration-below-target-window","mixed-duration-below-target-window"]} diff --git a/tests/fixtures/llm-schema-v2/metal-prefill-paged.json b/tests/fixtures/llm-schema-v2/metal-prefill-paged.json new file mode 100644 index 0000000..4eb0449 --- /dev/null +++ b/tests/fixtures/llm-schema-v2/metal-prefill-paged.json @@ -0,0 +1 @@ +{"schema_version":2,"mode":"llm_memory","backend":"metal","phase":"prefill","kv_layout":"paged","methodology_version":"llm-memory-v2-metal-prefill-paged","software":{"version":"0.63.1","timestamp":"2026-09-06T07:07:07Z"},"build_manifest":{"binary_sha256":"a9c0aa68c61fccf12bda222a0485a3eba1b642a54b9b1caf8292348f608a1100","compile_flags":{"asflags":"-arch arm64","cxxflags":"-Wall -O3 -std=c++17 -arch arm64 -pthread -Isrc","test_cxxflags":"-Wall -g -std=c++17 -arch arm64 -pthread -Isrc -I/opt/homebrew/opt/googletest/include"},"compiler":"Apple clang version 21.0.0 (clang-2100.1.1.101)\nTarget: arm64-apple-darwin25.6.0\nThread model: posix\nInstalledDir: /Library/Developer/CommandLineTools/usr/bin","git_commit":"870b175927d747fb1d90fadefebe8728da38407a","git_dirty":true,"link_flags":"-pthread -framework Metal -framework Foundation","manifest_version":1,"min_os":"26.0","reason_code":"valid","sdk":"26.5","status":"available","target_arch":"arm64-apple-darwin25.6.0"},"configuration":{"backend":"metal","phase":"prefill","kv_layout":"paged","kv_block_tokens":2,"weight_size_mb":1,"layer_count":2,"query_head_count":2,"kv_head_count":1,"head_dimension":3,"kv_element_bytes":"1","visible_context_tokens":null,"prompt_tokens":5,"attention_query_tile_tokens":2,"batch_size":2,"requested_workers":null,"available_workers":null,"worker_source":null,"iterations":2,"work_policy":"explicit_fixed_work","loop_count":3,"base_seed_uint64_decimal":"424242","seed_source":"user","output_file":"plans/llm-benchmark-remediation-evidence/phase7/profiles/metal-prefill-paged.json","argv":["./memory_benchmark","--llm-memory","--llm-memory-backend","metal","--phase","prefill","--kv-layout","paged","--weight-size-mb","1","--layers","2","--query-heads","2","--kv-heads","1","--head-dim","3","--kv-element-bytes","1","--batch-size","2","--iterations","2","--count","3","--seed","424242","-o","plans/llm-benchmark-remediation-evidence/phase7/profiles/metal-prefill-paged.json","--prompt-tokens","5","--attention-query-tile-tokens","2","--kv-block-tokens","2"],"resolved_sources":{"backend":"explicit","phase":"explicit","kv_layout":"explicit","kv_block_tokens":"explicit","visible_context_tokens":null,"prompt_tokens":"explicit","attention_query_tile_tokens":"explicit","workers":null,"iterations":"explicit","seed":"explicit"}},"resolved_plan":{"valid":true,"reason_code":"valid","plan_identity":"llm-memory-work-plan-v1|backend=metal|phase=prefill|kv_layout=paged|work_unit_kind=prefill_operation|methodology=llm-memory-v2-metal-prefill-paged|component_identity_size=841|component_identity=llm-memory-components-v1|logical_profile_version=31:full_prompt_causal_prefix_tiled|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|permutation_version=36:splitmix64-fisher-yates-rejection-v1|backend_executor_version=35:llm-metal-executor-v1-prefill-paged|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=57:llm-metal-prefill-paged-cyclic-block-owner-grid-stride-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=36:llm-paged-physical-buffer-pattern-v1|write_pattern_version=47:llm-metal-prefill-paged-full-prompt-affine32-v1|checksum_pattern_version=56:llm-metal-paged-prefill-dual-mod32-lookup-address-mix-v1|msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5|msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|prefill_planner_version=llm-prefill-planner-v1|prefill_prompt_tokens=5|prefill_query_tile_tokens=2|prefill_tile_count=3|prefill_prefix_token_visits_per_sequence=11|prefill_causal_token_pairs=15|prefill_logical_attention_pairs=120|prefill_logical_attention_fma_terms=360|prefill_prefix_block_visits_per_sequence=6|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=2|kv_blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|kv_block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|layout_metadata_lookups_per_layer_sequence_per_work_unit=15|k_mapping_bytes=72|v_mapping_bytes=72|kv_capacity_bytes=144|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|kv_only_payload_bytes_per_work_unit=384|mixed_payload_bytes_per_work_unit=1048960|total_data_mapping_bytes=1048720|traffic_crossover_numerator=0|traffic_crossover_denominator=0|metal_execution_resolved=1|metal_execution_identity_size=4347|metal_execution_identity_sha256=af1834ec5df2c17ed0497c39063a71159fac2822836b30184386f22a139b8ae9|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391","backend":"metal","phase":"prefill","kv_layout":"paged","work_unit_kind":"prefill_operation","geometry":{"valid":true,"reason_code":"valid","phase":"prefill","work_unit_kind":"prefill_operation","decode":null,"prefill":{"prompt_tokens":5,"attention_query_tile_tokens":2,"tile_count":"3","attention_prefix_token_visits_per_sequence":"11"},"attention_kind":"mqa","active_weight_bytes_per_work_unit":"1048576","layer_count":2,"query_head_count":2,"kv_head_count":1,"query_heads_per_kv_head":2,"head_dimension":3,"kv_element_bytes":"1","batch_size":2,"kv_vector_bytes":"3","k_or_v_record_bytes_per_layer":"3","kv_record_bytes_per_layer":"6","kv_bytes_per_visible_token":"12","k_or_v_sequence_visible_bytes":"15","k_mapping_bytes":"72","v_mapping_bytes":"72","kv_capacity_bytes":"144","weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"264","kv_write_bytes_per_work_unit":"120","kv_only_effective_model_payload_bytes_per_work_unit":"384","mixed_effective_model_payload_bytes_per_work_unit":"1048960","total_data_mapping_bytes":"1048720","traffic_crossover_numerator":null,"traffic_crossover_denominator":null,"traffic_crossover_context_tokens":null},"model_context":{"prefill":{"causal_token_pairs_per_sequence":"15","causal_token_pairs_per_sequence_reason_code":"valid","logical_attention_pairs":"120","logical_attention_pairs_reason_code":"valid","logical_attention_fma_terms":"360","logical_attention_fma_terms_reason_code":"valid"}},"layout":{"kv_layout":"paged","kv_block_tokens":2,"blocks_per_sequence":"3","physical_blocks_per_layer":"6","total_physical_blocks":"12","block_bytes":"6","last_block_tokens":"1","last_block_valid_bytes":"3","decode_append_offset_in_last_block":null,"block_table_entries":"6","block_table_bytes":"24","layout_geometry_identity":"llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24","layout_identity":"llm-kv-layout-plan-v1|geometry_identity=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24|permutation_algorithm_version=36:splitmix64-fisher-yates-rejection-v1|permutation_domain=5497854231078520625|permutation_domain_uint64_hex=18:0x4c4c4d4b56504731|permutation_seed=1732835419980629872|permutation_entry_count=6|permutation_sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c","permutation_domain_uint64_hex":"0x4c4c4d4b56504731","permutation_seed_uint64_decimal":"1732835419980629872","permutation_algorithm_version":"splitmix64-fisher-yates-rejection-v1","permutation_entry_count":"6","permutation_sha256":"14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c","permutation_identity":"splitmix64-fisher-yates-rejection-v1|domain=5497854231078520625|domain_uint64_hex=18:0x4c4c4d4b56504731|resolved_seed=1732835419980629872|entry_count=6|sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c"},"resources":{"weight_logical_bytes":"1048576","k_logical_bytes":"60","v_logical_bytes":"60","k_physical_length_bytes":"72","v_physical_length_bytes":"72","k_layout_padding_bytes":"12","v_layout_padding_bytes":"12","block_table_bytes":"24","metal":{"valid":true,"reason_code":"valid","execution_identity":"llm-metal-resource-foundation-v1phase=7:prefill;kv_layout=5:paged;segment_capacity_bytes=9:268435456;segment_slots_per_pool=3:256;weight_segments=310:whole-element-segments-v1|element_count=1048576|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=1048576;k_segments=302:whole-element-segments-v1|element_count=12|element_bytes=6|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=44739242|segment_count=1|maximum_addressable_elements=11453245952|maximum_addressable_bytes=68719475712|unused_nominal_segment_capacity_bytes=1024|total_length_bytes=72;v_segments=302:whole-element-segments-v1|element_count=12|element_bytes=6|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=44739242|segment_count=1|maximum_addressable_elements=11453245952|maximum_addressable_bytes=68719475712|unused_nominal_segment_capacity_bytes=1024|total_length_bytes=72;table_segments=298:whole-element-segments-v1|element_count=6|element_bytes=4|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=67108864|segment_count=1|maximum_addressable_elements=17179869184|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=24;paged_layout_geometry=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24;argument_buffer=305:llm-metal-argument-buffer-tier2-v1segment_slot_cap=3:256;weight_slot_base=1:0;k_slot_base=3:256;v_slot_base=3:512;table_slot_base=3:768;status_slot=4:1024;encoded_resource_slot_count=4:1025;weight_segment_count=1:1;k_segment_count=1:1;v_segment_count=1:1;table_segment_count=1:1;active_resource_count=1:5;;argument_encoded_length=4:8200;argument_alignment=1:8;max_buffer_length=11:14302248960;host_mapping_granularity_bytes=5:16384;status_length=4:4096;staging_length=2:24;persistent_resource_length=7:1061040;host_permutation_mapping_bytes=5:16384;permutation_validation_bitset_bytes=1:1;additional_owned_bytes=9:234204057;transient_peak_bytes=5:16409;known_owned_peak_bytes=9:235281506;planned_resource_count=1:7;resource_0_pool=6:weight;resource_0_pool_index=1:0;resource_0_length_bytes=7:1048576;resource_0_persistent=1:1;resource_1_pool=1:k;resource_1_pool_index=1:0;resource_1_length_bytes=2:72;resource_1_persistent=1:1;resource_2_pool=1:v;resource_2_pool_index=1:0;resource_2_length_bytes=2:72;resource_2_persistent=1:1;resource_3_pool=11:block_table;resource_3_pool_index=1:0;resource_3_length_bytes=2:24;resource_3_persistent=1:1;resource_4_pool=15:argument_buffer;resource_4_pool_index=1:0;resource_4_length_bytes=4:8200;resource_4_persistent=1:1;resource_5_pool=15:status_checksum;resource_5_pool_index=1:0;resource_5_length_bytes=4:4096;resource_5_persistent=1:1;resource_6_pool=7:staging;resource_6_pool_index=1:0;resource_6_length_bytes=2:24;resource_6_persistent=1:0;msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5;msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2;foundation_parameter_abi=34:llm-metal-foundation-parameters-v1;workload_parameter_abi=37:llm-metal-prefill-paged-parameters-v1;resource_table_abi=27:llm-metal-resource-table-v1;checksum_algorithm=23:llm-metal-dual-mod32-v1;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;","resource_identity":"llm-metal-resource-foundation-v1phase=7:prefill;kv_layout=5:paged;segment_capacity_bytes=9:268435456;segment_slots_per_pool=3:256;weight_segments=310:whole-element-segments-v1|element_count=1048576|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=1048576;k_segments=302:whole-element-segments-v1|element_count=12|element_bytes=6|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=44739242|segment_count=1|maximum_addressable_elements=11453245952|maximum_addressable_bytes=68719475712|unused_nominal_segment_capacity_bytes=1024|total_length_bytes=72;v_segments=302:whole-element-segments-v1|element_count=12|element_bytes=6|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=44739242|segment_count=1|maximum_addressable_elements=11453245952|maximum_addressable_bytes=68719475712|unused_nominal_segment_capacity_bytes=1024|total_length_bytes=72;table_segments=298:whole-element-segments-v1|element_count=6|element_bytes=4|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=67108864|segment_count=1|maximum_addressable_elements=17179869184|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=24;paged_layout_geometry=705:llm-kv-layout-geometry-v1|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|sequence_tokens=5|kv_block_tokens=2|layer_count=2|batch_size=2|k_or_v_record_bytes_per_layer=3|blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_physical_bytes=72|v_physical_bytes=72|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|validation_bitset_bytes=1|transient_peak_bytes=25|resident_layout_bytes=168|known_owned_peak_bytes=169|permutation_iterations=5|validation_entries=6|hash_entries=6|upload_bytes=24;argument_buffer=305:llm-metal-argument-buffer-tier2-v1segment_slot_cap=3:256;weight_slot_base=1:0;k_slot_base=3:256;v_slot_base=3:512;table_slot_base=3:768;status_slot=4:1024;encoded_resource_slot_count=4:1025;weight_segment_count=1:1;k_segment_count=1:1;v_segment_count=1:1;table_segment_count=1:1;active_resource_count=1:5;;argument_encoded_length=4:8200;argument_alignment=1:8;max_buffer_length=11:14302248960;host_mapping_granularity_bytes=5:16384;status_length=4:4096;staging_length=2:24;persistent_resource_length=7:1061040;host_permutation_mapping_bytes=5:16384;permutation_validation_bitset_bytes=1:1;additional_owned_bytes=9:234204057;transient_peak_bytes=5:16409;known_owned_peak_bytes=9:235281506;planned_resource_count=1:7;resource_0_pool=6:weight;resource_0_pool_index=1:0;resource_0_length_bytes=7:1048576;resource_0_persistent=1:1;resource_1_pool=1:k;resource_1_pool_index=1:0;resource_1_length_bytes=2:72;resource_1_persistent=1:1;resource_2_pool=1:v;resource_2_pool_index=1:0;resource_2_length_bytes=2:72;resource_2_persistent=1:1;resource_3_pool=11:block_table;resource_3_pool_index=1:0;resource_3_length_bytes=2:24;resource_3_persistent=1:1;resource_4_pool=15:argument_buffer;resource_4_pool_index=1:0;resource_4_length_bytes=4:8200;resource_4_persistent=1:1;resource_5_pool=15:status_checksum;resource_5_pool_index=1:0;resource_5_length_bytes=4:4096;resource_5_persistent=1:1;resource_6_pool=7:staging;resource_6_pool_index=1:0;resource_6_length_bytes=2:24;resource_6_persistent=1:0;","msl_revision":"llm-metal-decode-prefill-contiguous-paged-msl23-v5","msl_source_sha256":"2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2","segment_capacity_bytes":"268435456","max_buffer_length_bytes":"14302248960","weight_segments":{"valid":true,"reason_code":"valid","element_count":"1048576","element_bytes":"1","segment_capacity_bytes":"268435456","segment_slot_cap":256,"elements_per_segment":"268435456","segment_count":1,"maximum_addressable_elements":"68719476736","maximum_addressable_bytes":"68719476736","unused_nominal_segment_capacity_bytes":"0","total_length_bytes":"1048576","segment_lengths_bytes":["1048576"],"identity":"whole-element-segments-v1|element_count=1048576|element_bytes=1|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=268435456|segment_count=1|maximum_addressable_elements=68719476736|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=1048576"},"k_segments":{"valid":true,"reason_code":"valid","element_count":"12","element_bytes":"6","segment_capacity_bytes":"268435456","segment_slot_cap":256,"elements_per_segment":"44739242","segment_count":1,"maximum_addressable_elements":"11453245952","maximum_addressable_bytes":"68719475712","unused_nominal_segment_capacity_bytes":"1024","total_length_bytes":"72","segment_lengths_bytes":["72"],"identity":"whole-element-segments-v1|element_count=12|element_bytes=6|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=44739242|segment_count=1|maximum_addressable_elements=11453245952|maximum_addressable_bytes=68719475712|unused_nominal_segment_capacity_bytes=1024|total_length_bytes=72"},"v_segments":{"valid":true,"reason_code":"valid","element_count":"12","element_bytes":"6","segment_capacity_bytes":"268435456","segment_slot_cap":256,"elements_per_segment":"44739242","segment_count":1,"maximum_addressable_elements":"11453245952","maximum_addressable_bytes":"68719475712","unused_nominal_segment_capacity_bytes":"1024","total_length_bytes":"72","segment_lengths_bytes":["72"],"identity":"whole-element-segments-v1|element_count=12|element_bytes=6|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=44739242|segment_count=1|maximum_addressable_elements=11453245952|maximum_addressable_bytes=68719475712|unused_nominal_segment_capacity_bytes=1024|total_length_bytes=72"},"table_segments":{"valid":true,"reason_code":"valid","element_count":"6","element_bytes":"4","segment_capacity_bytes":"268435456","segment_slot_cap":256,"elements_per_segment":"67108864","segment_count":1,"maximum_addressable_elements":"17179869184","maximum_addressable_bytes":"68719476736","unused_nominal_segment_capacity_bytes":"0","total_length_bytes":"24","segment_lengths_bytes":["24"],"identity":"whole-element-segments-v1|element_count=6|element_bytes=4|segment_capacity_bytes=268435456|segment_slot_cap=256|elements_per_segment=67108864|segment_count=1|maximum_addressable_elements=17179869184|maximum_addressable_bytes=68719476736|unused_nominal_segment_capacity_bytes=0|total_length_bytes=24"},"argument_buffer":{"valid":true,"reason_code":"valid","encoded_length_bytes":"8200","alignment_bytes":"8","weight_slot_base":0,"k_slot_base":256,"v_slot_base":512,"table_slot_base":768,"status_slot":1024,"encoded_resource_slot_count":1025,"active_resource_count":5,"identity":"llm-metal-argument-buffer-tier2-v1segment_slot_cap=3:256;weight_slot_base=1:0;k_slot_base=3:256;v_slot_base=3:512;table_slot_base=3:768;status_slot=4:1024;encoded_resource_slot_count=4:1025;weight_segment_count=1:1;k_segment_count=1:1;v_segment_count=1:1;table_segment_count=1:1;active_resource_count=1:5;"},"status_buffer_length_bytes":"4096","staging_buffer_length_bytes":"24","persistent_resource_length_bytes":"1061040","transient_peak_bytes":"16409","known_owned_peak_bytes":"235281506","admitted_budget_bytes":"9268572979"}},"component_identities":{"logical_profile_version":"full_prompt_causal_prefix_tiled","kv_layout_version":"paged-uint32-block-table-full-blocks-v1","permutation_version":"splitmix64-fisher-yates-rejection-v1","backend_executor_version":"llm-metal-executor-v1-prefill-paged","resource_abi_version":"llm-metal-argument-buffer-tier2-v1","schedule_version":"llm-metal-prefill-paged-cyclic-block-owner-grid-stride-v1","timer_policy_version":"metal-command-buffer-gpu-start-end-v1","buffer_pattern_version":"llm-paged-physical-buffer-pattern-v1","write_pattern_version":"llm-metal-prefill-paged-full-prompt-affine32-v1","checksum_pattern_version":"llm-metal-paged-prefill-dual-mod32-lookup-address-mix-v1","msl_revision":"llm-metal-decode-prefill-contiguous-paged-msl23-v5","msl_source_sha256":"2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2","identity":"llm-memory-components-v1|logical_profile_version=31:full_prompt_causal_prefix_tiled|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|permutation_version=36:splitmix64-fisher-yates-rejection-v1|backend_executor_version=35:llm-metal-executor-v1-prefill-paged|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=57:llm-metal-prefill-paged-cyclic-block-owner-grid-stride-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=36:llm-paged-physical-buffer-pattern-v1|write_pattern_version=47:llm-metal-prefill-paged-full-prompt-affine32-v1|checksum_pattern_version=56:llm-metal-paged-prefill-dual-mod32-lookup-address-mix-v1|msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5|msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2","run_policy_version":"llm-run-policy-bounded-loop-snapshots-v1"},"methodology":{"backend":"metal","phase":"prefill","kv_layout":"paged","work_unit_kind":"prefill_operation","weight_passes_per_work_unit":1,"kv_replay_factor":1,"warmup_policy":"same-shape-excluded-steady-state-warm-memory","context_policy":"full-prompt-population-with-tiled-causal-prefix-scans","scenario_order_policy":"cyclic-rotation-across-count-loops","timing_policy":"gpu-start-to-gpu-end-per-command-buffer-task","cache_policy":"metal-private-and-shared-cacheable-no-explicit-flush-between-scenarios","calibration_policy":"per-scenario-automatic-or-explicit-frozen-before-loop-zero","calibration_target_seconds":0.15,"calibration_min_seconds":0.1,"calibration_max_seconds":0.25,"calibration_max_corrections":2,"calibration_min_pilot_accounted_bytes":"8388608","maximum_work_units_per_measurement":65536,"maximum_serial_range_visits_per_lane_per_task":1048576,"maximum_accounted_bytes_per_task":"68719476736","repeatability_cv_warning_threshold_pct":5.0,"calibration_excluded_from_results":true,"snapshot_policy":"bounded-loop-snapshots","timed_region_exclusions":["mapping-allocation","buffer-initialization-and-pre-touch","argument-buffer-encoding-and-resource-validation","host-command-encoding-and-pre-gpu-queue-wait","status-reset-command-buffer","excluded-kv-write-and-padding-canary-post-validation","same-shape-warmup","calibration-attempts","checksum-reference-generation-and-validation","host-wait-aggregation-and-json-serialization"]},"model_work_plan":{"valid":true,"reason_code":"valid","backend":"metal","phase":"prefill","kv_layout":"paged","work_unit_kind":"prefill_operation","weight_passes_per_work_unit":1,"kv_replay_factor":1,"requested_workers":null,"available_workers":null,"effective_workers":null,"worker_plan_count":null,"weight_layer_count":2,"layer_descriptors_per_worker":null,"sequence_descriptors_per_worker":null,"total_layer_descriptors":null,"total_sequence_descriptors":null,"descriptor_bytes":null,"planner_storage_bytes":null},"scenario_plans":[{"valid":true,"reason_code":"valid","scenario":"weights_only","work_unit_kind":"prefill_operation","kv_write_kind":"none","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"17410844953366850226","work_units":2,"weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"0","kv_write_bytes_per_work_unit":"0","effective_model_payload_bytes_per_work_unit":"1048576","layout_metadata_lookup_count_per_work_unit":"0","layout_metadata_read_bytes_per_work_unit":"0","accounted_bytes_per_work_unit":"1048576","weight_read_bytes":"2097152","kv_read_bytes":"0","kv_write_bytes":"0","effective_model_payload_bytes":"2097152","layout_metadata_lookup_count":"0","layout_metadata_read_bytes":"0","task_accounted_bytes":"2097152","maximum_work_units_by_work_unit_cap":65536,"maximum_work_units_by_guardrail":65536,"effective_maximum_work_units":65536,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=2833|model_plan_identity=llm-memory-work-plan-v1|backend=metal|phase=prefill|kv_layout=paged|work_unit_kind=prefill_operation|methodology=llm-memory-v2-metal-prefill-paged|component_identity_size=841|component_identity=llm-memory-components-v1|logical_profile_version=31:full_prompt_causal_prefix_tiled|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|permutation_version=36:splitmix64-fisher-yates-rejection-v1|backend_executor_version=35:llm-metal-executor-v1-prefill-paged|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=57:llm-metal-prefill-paged-cyclic-block-owner-grid-stride-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=36:llm-paged-physical-buffer-pattern-v1|write_pattern_version=47:llm-metal-prefill-paged-full-prompt-affine32-v1|checksum_pattern_version=56:llm-metal-paged-prefill-dual-mod32-lookup-address-mix-v1|msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5|msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|prefill_planner_version=llm-prefill-planner-v1|prefill_prompt_tokens=5|prefill_query_tile_tokens=2|prefill_tile_count=3|prefill_prefix_token_visits_per_sequence=11|prefill_causal_token_pairs=15|prefill_logical_attention_pairs=120|prefill_logical_attention_fma_terms=360|prefill_prefix_block_visits_per_sequence=6|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=2|kv_blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|kv_block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|layout_metadata_lookups_per_layer_sequence_per_work_unit=15|k_mapping_bytes=72|v_mapping_bytes=72|kv_capacity_bytes=144|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|kv_only_payload_bytes_per_work_unit=384|mixed_payload_bytes_per_work_unit=1048960|total_data_mapping_bytes=1048720|traffic_crossover_numerator=0|traffic_crossover_denominator=0|metal_execution_resolved=1|metal_execution_identity_size=4347|metal_execution_identity_sha256=af1834ec5df2c17ed0497c39063a71159fac2822836b30184386f22a139b8ae9|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=weights_only|work_unit_kind=prefill_operation|kv_write_kind=none|scenario_seed=17410844953366850226|explicit=1|work_units=2|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=0|kv_write_bytes_per_work_unit=0|effective_model_payload_bytes_per_work_unit=1048576|layout_metadata_lookup_count_per_work_unit=0|layout_metadata_read_bytes_per_work_unit=0|accounted_bytes_per_work_unit=1048576|weight_read_bytes=2097152|kv_read_bytes=0|kv_write_bytes=0|effective_model_payload_bytes=2097152|layout_metadata_lookup_count=0|layout_metadata_read_bytes=0|task_accounted_bytes=2097152|maximum_work_units_by_work_unit_cap=65536|maximum_work_units_by_guardrail=65536|effective_maximum_work_units=65536","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":null,"expected_run_checksum":{"weight":{"a_uint32_decimal":"1420034048","b_uint32_decimal":"2651324416"},"k":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"v":{"a_uint32_decimal":"0","b_uint32_decimal":"0"}}}},{"valid":true,"reason_code":"valid","scenario":"kv_only","work_unit_kind":"prefill_operation","kv_write_kind":"full_prompt_population","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"16590208891222127307","work_units":2,"weight_read_bytes_per_work_unit":"0","kv_read_bytes_per_work_unit":"264","kv_write_bytes_per_work_unit":"120","effective_model_payload_bytes_per_work_unit":"384","layout_metadata_lookup_count_per_work_unit":"60","layout_metadata_read_bytes_per_work_unit":"240","accounted_bytes_per_work_unit":"624","weight_read_bytes":"0","kv_read_bytes":"528","kv_write_bytes":"240","effective_model_payload_bytes":"768","layout_metadata_lookup_count":"120","layout_metadata_read_bytes":"480","task_accounted_bytes":"1248","maximum_work_units_by_work_unit_cap":65536,"maximum_work_units_by_guardrail":110127366,"effective_maximum_work_units":65536,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=2833|model_plan_identity=llm-memory-work-plan-v1|backend=metal|phase=prefill|kv_layout=paged|work_unit_kind=prefill_operation|methodology=llm-memory-v2-metal-prefill-paged|component_identity_size=841|component_identity=llm-memory-components-v1|logical_profile_version=31:full_prompt_causal_prefix_tiled|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|permutation_version=36:splitmix64-fisher-yates-rejection-v1|backend_executor_version=35:llm-metal-executor-v1-prefill-paged|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=57:llm-metal-prefill-paged-cyclic-block-owner-grid-stride-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=36:llm-paged-physical-buffer-pattern-v1|write_pattern_version=47:llm-metal-prefill-paged-full-prompt-affine32-v1|checksum_pattern_version=56:llm-metal-paged-prefill-dual-mod32-lookup-address-mix-v1|msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5|msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|prefill_planner_version=llm-prefill-planner-v1|prefill_prompt_tokens=5|prefill_query_tile_tokens=2|prefill_tile_count=3|prefill_prefix_token_visits_per_sequence=11|prefill_causal_token_pairs=15|prefill_logical_attention_pairs=120|prefill_logical_attention_fma_terms=360|prefill_prefix_block_visits_per_sequence=6|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=2|kv_blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|kv_block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|layout_metadata_lookups_per_layer_sequence_per_work_unit=15|k_mapping_bytes=72|v_mapping_bytes=72|kv_capacity_bytes=144|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|kv_only_payload_bytes_per_work_unit=384|mixed_payload_bytes_per_work_unit=1048960|total_data_mapping_bytes=1048720|traffic_crossover_numerator=0|traffic_crossover_denominator=0|metal_execution_resolved=1|metal_execution_identity_size=4347|metal_execution_identity_sha256=af1834ec5df2c17ed0497c39063a71159fac2822836b30184386f22a139b8ae9|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=kv_only|work_unit_kind=prefill_operation|kv_write_kind=full_prompt_population|scenario_seed=16590208891222127307|explicit=1|work_units=2|weight_read_bytes_per_work_unit=0|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|effective_model_payload_bytes_per_work_unit=384|layout_metadata_lookup_count_per_work_unit=60|layout_metadata_read_bytes_per_work_unit=240|accounted_bytes_per_work_unit=624|weight_read_bytes=0|kv_read_bytes=528|kv_write_bytes=240|effective_model_payload_bytes=768|layout_metadata_lookup_count=120|layout_metadata_read_bytes=480|task_accounted_bytes=1248|maximum_work_units_by_work_unit_cap=65536|maximum_work_units_by_guardrail=110127366|effective_maximum_work_units=65536","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":null,"expected_run_checksum":{"weight":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"k":{"a_uint32_decimal":"2986220311","b_uint32_decimal":"1047917595"},"v":{"a_uint32_decimal":"1016915223","b_uint32_decimal":"2326885915"}}}},{"valid":true,"reason_code":"valid","scenario":"mixed","work_unit_kind":"prefill_operation","kv_write_kind":"full_prompt_population","explicit_iterations":true,"work_policy":"explicit_fixed_work","model_ref":"resolved_plan","scenario_seed_uint64_decimal":"1546516732201688391","work_units":2,"weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"264","kv_write_bytes_per_work_unit":"120","effective_model_payload_bytes_per_work_unit":"1048960","layout_metadata_lookup_count_per_work_unit":"60","layout_metadata_read_bytes_per_work_unit":"240","accounted_bytes_per_work_unit":"1049200","weight_read_bytes":"2097152","kv_read_bytes":"528","kv_write_bytes":"240","effective_model_payload_bytes":"2097920","layout_metadata_lookup_count":"120","layout_metadata_read_bytes":"480","task_accounted_bytes":"2098400","maximum_work_units_by_work_unit_cap":65536,"maximum_work_units_by_guardrail":65497,"effective_maximum_work_units":65497,"plan_identity":"llm-memory-work-plan-v1|model_plan_identity_size=2833|model_plan_identity=llm-memory-work-plan-v1|backend=metal|phase=prefill|kv_layout=paged|work_unit_kind=prefill_operation|methodology=llm-memory-v2-metal-prefill-paged|component_identity_size=841|component_identity=llm-memory-components-v1|logical_profile_version=31:full_prompt_causal_prefix_tiled|kv_layout_version=39:paged-uint32-block-table-full-blocks-v1|permutation_version=36:splitmix64-fisher-yates-rejection-v1|backend_executor_version=35:llm-metal-executor-v1-prefill-paged|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=57:llm-metal-prefill-paged-cyclic-block-owner-grid-stride-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=36:llm-paged-physical-buffer-pattern-v1|write_pattern_version=47:llm-metal-prefill-paged-full-prompt-affine32-v1|checksum_pattern_version=56:llm-metal-paged-prefill-dual-mod32-lookup-address-mix-v1|msl_revision=50:llm-metal-decode-prefill-contiguous-paged-msl23-v5|msl_source_sha256=64:2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=2|query_heads=2|kv_heads=1|query_heads_per_kv_head=2|attention_kind=mqa|head_dim=3|kv_element_bytes=1|prefill_planner_version=llm-prefill-planner-v1|prefill_prompt_tokens=5|prefill_query_tile_tokens=2|prefill_tile_count=3|prefill_prefix_token_visits_per_sequence=11|prefill_causal_token_pairs=15|prefill_logical_attention_pairs=120|prefill_logical_attention_fma_terms=360|prefill_prefix_block_visits_per_sequence=6|batch=2|kv_vector_bytes=3|k_or_v_record_bytes_per_layer=3|kv_record_bytes_per_layer=6|kv_bytes_per_visible_token=12|k_or_v_sequence_visible_bytes=15|kv_block_tokens=2|kv_blocks_per_sequence=3|physical_blocks_per_layer=6|total_physical_blocks=12|kv_block_bytes=6|last_block_tokens=1|last_block_valid_bytes=3|decode_append_offset_in_last_block=0|k_logical_bytes=60|v_logical_bytes=60|k_layout_padding_bytes=12|v_layout_padding_bytes=12|block_table_entries=6|block_table_bytes=24|layout_metadata_lookups_per_layer_sequence_per_work_unit=15|k_mapping_bytes=72|v_mapping_bytes=72|kv_capacity_bytes=144|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|kv_only_payload_bytes_per_work_unit=384|mixed_payload_bytes_per_work_unit=1048960|total_data_mapping_bytes=1048720|traffic_crossover_numerator=0|traffic_crossover_denominator=0|metal_execution_resolved=1|metal_execution_identity_size=4347|metal_execution_identity_sha256=af1834ec5df2c17ed0497c39063a71159fac2822836b30184386f22a139b8ae9|base_seed=424242|weight_buffer_seed=904846021374001452|k_buffer_seed=10718278915871169538|v_buffer_seed=2111147304547004201|weights_only_scenario_seed=17410844953366850226|kv_only_scenario_seed=16590208891222127307|mixed_scenario_seed=1546516732201688391|scenario=mixed|work_unit_kind=prefill_operation|kv_write_kind=full_prompt_population|scenario_seed=1546516732201688391|explicit=1|work_units=2|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=264|kv_write_bytes_per_work_unit=120|effective_model_payload_bytes_per_work_unit=1048960|layout_metadata_lookup_count_per_work_unit=60|layout_metadata_read_bytes_per_work_unit=240|accounted_bytes_per_work_unit=1049200|weight_read_bytes=2097152|kv_read_bytes=528|kv_write_bytes=240|effective_model_payload_bytes=2097920|layout_metadata_lookup_count=120|layout_metadata_read_bytes=480|task_accounted_bytes=2098400|maximum_work_units_by_work_unit_cap=65536|maximum_work_units_by_guardrail=65497|effective_maximum_work_units=65497","expected_checksum":{"status":"available","reason_code":"valid","expected_worker_checksums":null,"expected_run_checksum":{"weight":{"a_uint32_decimal":"825491456","b_uint32_decimal":"861405184"},"k":{"a_uint32_decimal":"1788818239","b_uint32_decimal":"694939939"},"v":{"a_uint32_decimal":"4114480447","b_uint32_decimal":"1973908259"}}}}],"frozen_plan_refs":{"weights_only":0,"kv_only":1,"mixed":2}},"backend_evidence":{"cpu":null,"metal":{"workers_applicable":false,"worker_qos_applicable":false,"lifecycle":{"initialization":{"status":"ready","reason_code":"valid"},"plan_resolution":{"status":"ready","reason_code":"valid"},"preparation":{"status":"ready","reason_code":"valid"},"release":{"status":"ready","reason_code":"valid"}},"capability":{"device_name":"Apple M4","registry_id_uint64_decimal":"4294968227","has_unified_memory":true,"required_apple7_family_supported":true,"argument_buffers_tier2_supported":true,"supported_families":["apple1","apple2","apple3","apple4","apple5","apple6","apple7","apple8","apple9","mac1","mac2","common1","common2","common3","metal3","metal4"],"max_buffer_length_bytes":"14302248960","recommended_max_working_set_size_bytes":"19069665280","compilation_mode":"runtime-source","msl_language_version":"2.3","kernel_revision":"llm-metal-decode-prefill-contiguous-paged-msl23-v5","kernel_source_sha256":"2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2","compiler_diagnostics":null,"compiler_identifier":"21.0.0 (clang-2100.1.1.101)","build_sdk":"26.5","deployment_target":"26.0","argument_buffer":{"encoded_length_bytes":"8200","alignment_bytes":"8"},"layout_probe":{"evaluated":true,"valid":true,"resource_count":5},"foundation_pipelines":[{"label":"membenchmark.llm-metal.pipeline.initialize","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.copy","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.layout-probe","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.prefill-paged-layout-probe","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.validate-bytes","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.validate-table","thread_execution_width":32,"max_total_threads_per_threadgroup":1024}],"error":{"domain":null,"code":0,"description":null}},"resources":{"allocation_attempted":true,"allocation_completed":true,"initialization_completed":true,"table_upload_completed":true,"table_validation_completed":true,"table_permutation":{"algorithm_version":"splitmix64-fisher-yates-rejection-v1","domain_uint64_hex":"0x4c4c4d4b56504731","resolved_seed_uint64_decimal":"1732835419980629872","entry_count":"6","sha256":"14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c","identity":"splitmix64-fisher-yates-rejection-v1|domain=5497854231078520625|domain_uint64_hex=18:0x4c4c4d4b56504731|resolved_seed=1732835419980629872|entry_count=6|sha256=64:14f8f84e0b3b0be0ed2fd32a3f810b02b1ac7e32a802327ceabf3ce53661855c"},"post_validation_completed":true,"cpu_sample_readback_validation_completed":true,"candidate_cleanup_completed":true,"resources_published":false,"persistent_resource_length_bytes":"1061040","committed_resource_bytes":"1061064","resource_rounding_bytes":"0","layout_padding_bytes":"24","transient_peak_bytes":"16409","additional_owned_bytes":"234204057","known_owned_peak_bytes":"235281506","admitted_budget_bytes":"9268572979","current_allocated_size_before_bytes":"1785856","current_allocated_size_peak_bytes":"2834432","current_allocated_size_after_release_bytes":"1785856","recommended_working_set_available":true,"recommended_working_set_headroom_bytes":"18832597918","recommended_working_set_headroom_fraction":0.9875683522222788,"exceeds_recommended_working_set":false,"resources":[{"label":"membenchmark.llm-metal.weight-0","pool":"weight","pool_index":0,"storage_mode":"private","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"544","length_bytes":"1048576","allocated_size_available":true,"allocated_size_bytes":"1048576","committed_bytes":"1048576","allocation_rounding_bytes":"0"},{"label":"membenchmark.llm-metal.k-0","pool":"k","pool_index":0,"storage_mode":"private","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"544","length_bytes":"72","allocated_size_available":true,"allocated_size_bytes":"72","committed_bytes":"72","allocation_rounding_bytes":"0"},{"label":"membenchmark.llm-metal.v-0","pool":"v","pool_index":0,"storage_mode":"private","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"544","length_bytes":"72","allocated_size_available":true,"allocated_size_bytes":"72","committed_bytes":"72","allocation_rounding_bytes":"0"},{"label":"membenchmark.llm-metal.block_table-0","pool":"block_table","pool_index":0,"storage_mode":"private","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"544","length_bytes":"24","allocated_size_available":true,"allocated_size_bytes":"24","committed_bytes":"24","allocation_rounding_bytes":"0"},{"label":"membenchmark.llm-metal.argument_buffer-0","pool":"argument_buffer","pool_index":0,"storage_mode":"shared","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"512","length_bytes":"8200","allocated_size_available":true,"allocated_size_bytes":"8200","committed_bytes":"8200","allocation_rounding_bytes":"0"},{"label":"membenchmark.llm-metal.status_checksum-0","pool":"status_checksum","pool_index":0,"storage_mode":"shared","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"512","length_bytes":"4096","allocated_size_available":true,"allocated_size_bytes":"4096","committed_bytes":"4096","allocation_rounding_bytes":"0"},{"label":"membenchmark.llm-metal.staging-0","pool":"staging","pool_index":0,"storage_mode":"shared","cpu_cache_mode":"default-cache","hazard_tracking_mode":"tracked","resource_options_uint64_decimal":"512","length_bytes":"24","allocated_size_available":true,"allocated_size_bytes":"24","committed_bytes":"24","allocation_rounding_bytes":"0"}],"error":{"domain":null,"code":0,"description":null}},"workload_pipelines":[{"label":"membenchmark.llm-metal.pipeline.prefill-paged.weights-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.prefill-paged.kv-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},{"label":"membenchmark.llm-metal.pipeline.prefill-paged.mixed","thread_execution_width":32,"max_total_threads_per_threadgroup":1024}],"timed_results_available":true}},"memory_budget":{"resource_rounding_bytes":"0","transient_peak_bytes":"16409","layout_transient_bytes":"16409","setup_peak_bytes":"235281506","runtime_peak_bytes":"235281506","known_owned_peak_bytes":"235281506","admitted_budget_bytes":"9268572979","valid":true,"reason_code":"within-memory-budget","request":{"valid":true,"reason_code":"valid","mapping_granularity_bytes":"16384","requested_weight_mapping_bytes":"1048576","requested_k_mapping_bytes":"72","requested_v_mapping_bytes":"72","committed_weight_mapping_bytes":"1048576","committed_k_mapping_bytes":"72","committed_v_mapping_bytes":"72","requested_block_table_mapping_bytes":"24","committed_block_table_mapping_bytes":"24","requested_data_bytes":"1048720","committed_data_bytes":"1048720","layout_transient_bytes":"16409","setup_peak_bytes":"235281506","runtime_peak_bytes":"235281506","descriptor_bytes":"0","planner_storage_bytes":"158752","checksum_auxiliary_bytes":"0","orchestration_auxiliary_bytes":"234045305","auxiliary_bytes":"234204057","required_total_bytes":"235281506"},"available_memory_bytes":"11585716224","allowed_memory_bytes":"9268572979","used_fallback":false},"calibration":{"excluded_from_results":true,"attempts":{"weights_only":[{"attempt_index":0,"scenario":"weights_only","plan_ref":0,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":0.0003327499143779278,"timing":{"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"weight":{"a_uint32_decimal":"1420034048","b_uint32_decimal":"2651324416"},"k":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"v":{"a_uint32_decimal":"0","b_uint32_decimal":"0"}}},"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-paged.weights-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"4","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"2097152","maximum_threadgroup_accounted_bytes":"2097152","threadgroup_accounted_imbalance_bytes":"0","threadgroup_accounted_bytes":["2097152"],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:4;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=7:2097152;maximum_threadgroup_accounted_bytes=7:2097152;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:1;threadgroup_accounted_bytes=7:2097152;weight_shard_cost_version=43:llm-metal-weight-vector-grid-stride-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.6751216252,"gpu_end_seconds":1532841.675454375,"host_submit_to_completion_seconds":0.000687959,"host_wait_seconds":0.000686084,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-prefill-final-samples","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}}}],"kv_only":[{"attempt_index":0,"scenario":"kv_only","plan_ref":1,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":0.00017291679978370667,"timing":{"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"weight":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"k":{"a_uint32_decimal":"2986220311","b_uint32_decimal":"1047917595"},"v":{"a_uint32_decimal":"1016915223","b_uint32_decimal":"2326885915"}}},"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-paged.kv-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"12","threads_per_threadgroup":256,"actual_threadgroups":12,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"1","work_units":2,"paged_semantic_lookups":"120","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"48","maximum_threadgroup_accounted_bytes":"152","threadgroup_accounted_imbalance_bytes":"104","threadgroup_accounted_bytes":["152","112","48","152","112","48","152","112","48","152","112","48"],"identity":"llm-metal-capped-grid-stride-v1owner_count=2:12;threads_per_threadgroup=3:256;actual_threadgroups=2:12;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=1:1;work_units=1:2;paged_semantic_lookups=3:120;serial_range_visits_per_lane=1:0;visit_bytes=2:32;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=2:48;maximum_threadgroup_accounted_bytes=3:152;threadgroup_accounted_imbalance_bytes=3:104;threadgroup_accounted_bytes_count=2:12;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;paged_owner_cost_version=37:llm-metal-prefill-paged-owner-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.67686575,"gpu_end_seconds":1532841.6770386668,"host_submit_to_completion_seconds":0.00069875,"host_wait_seconds":0.000697041,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}}}],"mixed":[{"attempt_index":0,"scenario":"mixed","plan_ref":2,"purpose":"frozen_measurement_warmup","duration_quality":"below-target-window","terminal":true,"valid":true,"reason_code":"valid","execution":{"status":"valid","reason_code":"valid","valid":true,"elapsed_seconds":0.0003082500770688057,"timing":{"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_run_checksum":{"weight":{"a_uint32_decimal":"825491456","b_uint32_decimal":"861405184"},"k":{"a_uint32_decimal":"1788818239","b_uint32_decimal":"694939939"},"v":{"a_uint32_decimal":"4114480447","b_uint32_decimal":"1973908259"}}},"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-paged.mixed","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"12","threads_per_threadgroup":256,"actual_threadgroups":12,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"120","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"48","maximum_threadgroup_accounted_bytes":"1048728","threadgroup_accounted_imbalance_bytes":"1048680","threadgroup_accounted_bytes":["1048728","112","48","152","112","48","1048728","112","48","152","112","48"],"identity":"llm-metal-capped-grid-stride-v1owner_count=2:12;threads_per_threadgroup=3:256;actual_threadgroups=2:12;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=3:120;serial_range_visits_per_lane=1:0;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=2:48;maximum_threadgroup_accounted_bytes=7:1048728;threadgroup_accounted_imbalance_bytes=7:1048680;threadgroup_accounted_bytes_count=2:12;threadgroup_accounted_bytes=7:1048728;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=7:1048728;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;paged_owner_cost_version=37:llm-metal-prefill-paged-owner-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.678122,"gpu_end_seconds":1532841.6784302501,"host_submit_to_completion_seconds":0.000764042,"host_wait_seconds":0.000762334,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}}}]}},"measurements":[{"measurement_id":0,"plan_ref":0,"scenario":"weights_only","loop_index":0,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":null,"elapsed_seconds":0.00032983324490487576,"synthetic_work_unit_latency_seconds":0.00016491662245243788,"synthetic_memory_work_units_per_second":6063.670145126826,"effective_model_payload_gb_s":6.3582189860965075,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-paged.weights-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"4","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"2097152","maximum_threadgroup_accounted_bytes":"2097152","threadgroup_accounted_imbalance_bytes":"0","threadgroup_accounted_bytes":["2097152"],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:4;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=7:2097152;maximum_threadgroup_accounted_bytes=7:2097152;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:1;threadgroup_accounted_bytes=7:2097152;weight_shard_cost_version=43:llm-metal-weight-vector-grid-stride-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.6792397918,"gpu_end_seconds":1532841.679569625,"host_submit_to_completion_seconds":0.000622875,"host_wait_seconds":0.000621416,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-prefill-final-samples","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"1420034048","b_uint32_decimal":"2651324416"},"k":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"v":{"a_uint32_decimal":"0","b_uint32_decimal":"0"}}}},{"measurement_id":1,"plan_ref":1,"scenario":"kv_only","loop_index":0,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"768","completed_layout_metadata_lookup_count":"120","completed_layout_metadata_read_bytes":"480","completed_task_accounted_bytes":"1248","completion_derivation":null,"elapsed_seconds":0.0001502500381320715,"synthetic_work_unit_latency_seconds":7.512501906603575e-05,"synthetic_memory_work_units_per_second":13311.144708276062,"effective_model_payload_gb_s":0.005111479567978008,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-paged.kv-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"12","threads_per_threadgroup":256,"actual_threadgroups":12,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"1","work_units":2,"paged_semantic_lookups":"120","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"48","maximum_threadgroup_accounted_bytes":"152","threadgroup_accounted_imbalance_bytes":"104","threadgroup_accounted_bytes":["152","112","48","152","112","48","152","112","48","152","112","48"],"identity":"llm-metal-capped-grid-stride-v1owner_count=2:12;threads_per_threadgroup=3:256;actual_threadgroups=2:12;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=1:1;work_units=1:2;paged_semantic_lookups=3:120;serial_range_visits_per_lane=1:0;visit_bytes=2:32;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=2:48;maximum_threadgroup_accounted_bytes=3:152;threadgroup_accounted_imbalance_bytes=3:104;threadgroup_accounted_bytes_count=2:12;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;paged_owner_cost_version=37:llm-metal-prefill-paged-owner-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.68031225,"gpu_end_seconds":1532841.6804625,"host_submit_to_completion_seconds":0.000404417,"host_wait_seconds":0.000402917,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"k":{"a_uint32_decimal":"2986220311","b_uint32_decimal":"1047917595"},"v":{"a_uint32_decimal":"1016915223","b_uint32_decimal":"2326885915"}}}},{"measurement_id":2,"plan_ref":2,"scenario":"mixed","loop_index":0,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":0.999633923123856,"kv_read_payload_fraction":0.0002516778523489933,"kv_write_payload_fraction":0.00011439902379499695,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097920","completed_layout_metadata_lookup_count":"120","completed_layout_metadata_read_bytes":"480","completed_task_accounted_bytes":"2098400","completion_derivation":null,"elapsed_seconds":0.00029762485064566135,"synthetic_work_unit_latency_seconds":0.00014881242532283068,"synthetic_memory_work_units_per_second":6719.868974856234,"effective_model_payload_gb_s":7.048873759865195,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-paged.mixed","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"12","threads_per_threadgroup":256,"actual_threadgroups":12,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"120","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"48","maximum_threadgroup_accounted_bytes":"1048728","threadgroup_accounted_imbalance_bytes":"1048680","threadgroup_accounted_bytes":["1048728","112","48","152","112","48","1048728","112","48","152","112","48"],"identity":"llm-metal-capped-grid-stride-v1owner_count=2:12;threads_per_threadgroup=3:256;actual_threadgroups=2:12;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=3:120;serial_range_visits_per_lane=1:0;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=2:48;maximum_threadgroup_accounted_bytes=7:1048728;threadgroup_accounted_imbalance_bytes=7:1048680;threadgroup_accounted_bytes_count=2:12;threadgroup_accounted_bytes=7:1048728;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=7:1048728;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;paged_owner_cost_version=37:llm-metal-prefill-paged-owner-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.6811727502,"gpu_end_seconds":1532841.681470375,"host_submit_to_completion_seconds":0.000563208,"host_wait_seconds":0.000561708,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"825491456","b_uint32_decimal":"861405184"},"k":{"a_uint32_decimal":"1788818239","b_uint32_decimal":"694939939"},"v":{"a_uint32_decimal":"4114480447","b_uint32_decimal":"1973908259"}}}},{"measurement_id":3,"plan_ref":1,"scenario":"kv_only","loop_index":1,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"768","completed_layout_metadata_lookup_count":"120","completed_layout_metadata_read_bytes":"480","completed_task_accounted_bytes":"1248","completion_derivation":null,"elapsed_seconds":0.00015983334742486477,"synthetic_work_unit_latency_seconds":7.991667371243238e-05,"synthetic_memory_work_units_per_second":12513.033307646701,"effective_model_payload_gb_s":0.004805004790136334,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-paged.kv-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"12","threads_per_threadgroup":256,"actual_threadgroups":12,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"1","work_units":2,"paged_semantic_lookups":"120","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"48","maximum_threadgroup_accounted_bytes":"152","threadgroup_accounted_imbalance_bytes":"104","threadgroup_accounted_bytes":["152","112","48","152","112","48","152","112","48","152","112","48"],"identity":"llm-metal-capped-grid-stride-v1owner_count=2:12;threads_per_threadgroup=3:256;actual_threadgroups=2:12;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=1:1;work_units=1:2;paged_semantic_lookups=3:120;serial_range_visits_per_lane=1:0;visit_bytes=2:32;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=2:48;maximum_threadgroup_accounted_bytes=3:152;threadgroup_accounted_imbalance_bytes=3:104;threadgroup_accounted_bytes_count=2:12;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;paged_owner_cost_version=37:llm-metal-prefill-paged-owner-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.6832590417,"gpu_end_seconds":1532841.683418875,"host_submit_to_completion_seconds":0.00042225,"host_wait_seconds":0.000420625,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"k":{"a_uint32_decimal":"2986220311","b_uint32_decimal":"1047917595"},"v":{"a_uint32_decimal":"1016915223","b_uint32_decimal":"2326885915"}}}},{"measurement_id":4,"plan_ref":2,"scenario":"mixed","loop_index":1,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":0.999633923123856,"kv_read_payload_fraction":0.0002516778523489933,"kv_write_payload_fraction":0.00011439902379499695,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097920","completed_layout_metadata_lookup_count":"120","completed_layout_metadata_read_bytes":"480","completed_task_accounted_bytes":"2098400","completion_derivation":null,"elapsed_seconds":0.0003006665501743555,"synthetic_work_unit_latency_seconds":0.00015033327508717775,"synthetic_memory_work_units_per_second":6651.887277917037,"effective_model_payload_gb_s":6.977563679043856,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-paged.mixed","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"12","threads_per_threadgroup":256,"actual_threadgroups":12,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"120","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"48","maximum_threadgroup_accounted_bytes":"1048728","threadgroup_accounted_imbalance_bytes":"1048680","threadgroup_accounted_bytes":["1048728","112","48","152","112","48","1048728","112","48","152","112","48"],"identity":"llm-metal-capped-grid-stride-v1owner_count=2:12;threads_per_threadgroup=3:256;actual_threadgroups=2:12;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=3:120;serial_range_visits_per_lane=1:0;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=2:48;maximum_threadgroup_accounted_bytes=7:1048728;threadgroup_accounted_imbalance_bytes=7:1048680;threadgroup_accounted_bytes_count=2:12;threadgroup_accounted_bytes=7:1048728;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=7:1048728;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;paged_owner_cost_version=37:llm-metal-prefill-paged-owner-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.6840757502,"gpu_end_seconds":1532841.6843764167,"host_submit_to_completion_seconds":0.000650375,"host_wait_seconds":0.000648834,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"825491456","b_uint32_decimal":"861405184"},"k":{"a_uint32_decimal":"1788818239","b_uint32_decimal":"694939939"},"v":{"a_uint32_decimal":"4114480447","b_uint32_decimal":"1973908259"}}}},{"measurement_id":5,"plan_ref":0,"scenario":"weights_only","loop_index":1,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":null,"elapsed_seconds":0.0003278751391917467,"synthetic_work_unit_latency_seconds":0.00016393756959587336,"synthetic_memory_work_units_per_second":6099.883037580253,"effective_model_payload_gb_s":6.396190956013751,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-paged.weights-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"4","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"2097152","maximum_threadgroup_accounted_bytes":"2097152","threadgroup_accounted_imbalance_bytes":"0","threadgroup_accounted_bytes":["2097152"],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:4;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=7:2097152;maximum_threadgroup_accounted_bytes=7:2097152;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:1;threadgroup_accounted_bytes=7:2097152;weight_shard_cost_version=43:llm-metal-weight-vector-grid-stride-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.685152,"gpu_end_seconds":1532841.6854798752,"host_submit_to_completion_seconds":0.0006795,"host_wait_seconds":0.000676792,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-prefill-final-samples","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"1420034048","b_uint32_decimal":"2651324416"},"k":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"v":{"a_uint32_decimal":"0","b_uint32_decimal":"0"}}}},{"measurement_id":6,"plan_ref":2,"scenario":"mixed","loop_index":2,"order_position":0,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":0.999633923123856,"kv_read_payload_fraction":0.0002516778523489933,"kv_write_payload_fraction":0.00011439902379499695,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097920","completed_layout_metadata_lookup_count":"120","completed_layout_metadata_read_bytes":"480","completed_task_accounted_bytes":"2098400","completion_derivation":null,"elapsed_seconds":0.0002965833991765976,"synthetic_work_unit_latency_seconds":0.0001482916995882988,"synthetic_memory_work_units_per_second":6743.465768996464,"effective_model_payload_gb_s":7.073625853046531,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-paged.mixed","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"12","threads_per_threadgroup":256,"actual_threadgroups":12,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"120","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"48","maximum_threadgroup_accounted_bytes":"1048728","threadgroup_accounted_imbalance_bytes":"1048680","threadgroup_accounted_bytes":["1048728","112","48","152","112","48","1048728","112","48","152","112","48"],"identity":"llm-metal-capped-grid-stride-v1owner_count=2:12;threads_per_threadgroup=3:256;actual_threadgroups=2:12;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=3:120;serial_range_visits_per_lane=1:0;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=2:48;maximum_threadgroup_accounted_bytes=7:1048728;threadgroup_accounted_imbalance_bytes=7:1048680;threadgroup_accounted_bytes_count=2:12;threadgroup_accounted_bytes=7:1048728;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=7:1048728;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;paged_owner_cost_version=37:llm-metal-prefill-paged-owner-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.6873151667,"gpu_end_seconds":1532841.68761175,"host_submit_to_completion_seconds":0.000824083,"host_wait_seconds":0.00082175,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"825491456","b_uint32_decimal":"861405184"},"k":{"a_uint32_decimal":"1788818239","b_uint32_decimal":"694939939"},"v":{"a_uint32_decimal":"4114480447","b_uint32_decimal":"1973908259"}}}},{"measurement_id":7,"plan_ref":0,"scenario":"weights_only","loop_index":2,"order_position":1,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"2097152","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"2097152","completion_derivation":null,"elapsed_seconds":0.00032483343966305256,"synthetic_work_unit_latency_seconds":0.00016241671983152628,"synthetic_memory_work_units_per_second":6157.001576175735,"effective_model_payload_gb_s":6.456084084740048,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-paged.weights-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"1","threads_per_threadgroup":256,"actual_threadgroups":1,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"128","work_units":2,"paged_semantic_lookups":"0","serial_range_visits_per_lane":"4","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"2097152","maximum_threadgroup_accounted_bytes":"2097152","threadgroup_accounted_imbalance_bytes":"0","threadgroup_accounted_bytes":["2097152"],"identity":"llm-metal-capped-grid-stride-v1owner_count=1:1;threads_per_threadgroup=3:256;actual_threadgroups=1:1;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=3:128;work_units=1:2;paged_semantic_lookups=1:0;serial_range_visits_per_lane=1:4;visit_bytes=6:524288;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=7:2097152;maximum_threadgroup_accounted_bytes=7:2097152;threadgroup_accounted_imbalance_bytes=1:0;threadgroup_accounted_bytes_count=1:1;threadgroup_accounted_bytes=7:2097152;weight_shard_cost_version=43:llm-metal-weight-vector-grid-stride-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.6885321666,"gpu_end_seconds":1532841.688857,"host_submit_to_completion_seconds":0.00067375,"host_wait_seconds":0.000671583,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-prefill-final-samples","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"1420034048","b_uint32_decimal":"2651324416"},"k":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"v":{"a_uint32_decimal":"0","b_uint32_decimal":"0"}}}},{"measurement_id":8,"plan_ref":1,"scenario":"kv_only","loop_index":2,"order_position":2,"status":"measured","reason_code":"measured","attempted":true,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":1,"calibration_attempt_indexes":[0],"working_set":{"bytes":"1048720","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"paged","fixed_visible_context_tokens":null,"current_token_slot_included":null},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"below-target-window","completed_work_units":2,"completed_effective_model_payload_bytes":"768","completed_layout_metadata_lookup_count":"120","completed_layout_metadata_read_bytes":"480","completed_task_accounted_bytes":"1248","completion_derivation":null,"elapsed_seconds":0.00015199999324977398,"synthetic_work_unit_latency_seconds":7.599999662488699e-05,"synthetic_memory_work_units_per_second":13157.895321176102,"effective_model_payload_gb_s":0.005052631803331623,"execution":{"status":"valid","reason_code":"valid","valid":true,"timing":{"evaluated":true,"valid":true,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"metal":{"pipeline":{"available":true,"label":"membenchmark.llm-metal.pipeline.prefill-paged.kv-only","thread_execution_width":32,"max_total_threads_per_threadgroup":1024},"grid":{"valid":true,"reason_code":"valid","owner_count":"12","threads_per_threadgroup":256,"actual_threadgroups":12,"owner_ordinals_per_threadgroup":"1","vector_iterations_per_lane_per_visit":"1","work_units":2,"paged_semantic_lookups":"120","serial_range_visits_per_lane":"0","cost_unit":"actual-threadgroup-cost","minimum_threadgroup_accounted_bytes":"48","maximum_threadgroup_accounted_bytes":"152","threadgroup_accounted_imbalance_bytes":"104","threadgroup_accounted_bytes":["152","112","48","152","112","48","152","112","48","152","112","48"],"identity":"llm-metal-capped-grid-stride-v1owner_count=2:12;threads_per_threadgroup=3:256;actual_threadgroups=2:12;owner_ordinals_per_threadgroup=1:1;vector_iterations_per_lane_per_visit=1:1;work_units=1:2;paged_semantic_lookups=3:120;serial_range_visits_per_lane=1:0;visit_bytes=2:32;thread_execution_width=2:32;max_total_threads_per_threadgroup=4:1024;threads_per_threadgroup_cap=3:256;maximum_threadgroups_per_grid=4:8192;maximum_owner_ordinals_per_threadgroup=5:65536;maximum_vector_iterations_per_lane_per_visit=7:1048576;maximum_serial_range_visits_per_lane_per_task=7:1048576;maximum_paged_semantic_lookups_per_task=9:268435456;maximum_work_units_per_dispatch=5:65536;minimum_threadgroup_accounted_bytes=2:48;maximum_threadgroup_accounted_bytes=3:152;threadgroup_accounted_imbalance_bytes=3:104;threadgroup_accounted_bytes_count=2:12;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;threadgroup_accounted_bytes=3:152;threadgroup_accounted_bytes=3:112;threadgroup_accounted_bytes=2:48;paged_owner_cost_version=37:llm-metal-prefill-paged-owner-cost-v1;"},"timing":{"evaluated":true,"valid":true,"gpu_start_seconds":1532841.6895996667,"gpu_end_seconds":1532841.6897516667,"host_submit_to_completion_seconds":0.000403958,"host_wait_seconds":0.000401416,"queue_delay_seconds":null},"commands":{"reset_command_buffers":1,"reset_status":"completed","timed_command_buffers":1,"timed_status":"completed","post_validation_command_buffers":1,"post_validation_status":"completed","timed_compute_encoders":1,"timed_workload_dispatches":1},"error":{"domain":null,"code":0,"description":null}},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-prefill-final-samples","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"},{"kind":"kv-padding-canary","applicable":true,"evaluated":true,"valid":true,"reason_code":"valid"}]}},"checksum":{"status":"valid","reason_code":"valid","checksum_valid":true,"actual_worker_checksums":null,"actual_run_checksum":{"weight":{"a_uint32_decimal":"0","b_uint32_decimal":"0"},"k":{"a_uint32_decimal":"2986220311","b_uint32_decimal":"1047917595"},"v":{"a_uint32_decimal":"1016915223","b_uint32_decimal":"2326885915"}}}}],"aggregates":{"scenarios":{"weights_only":{"scenario":"weights_only","status":"complete","observed_cv_classification":"below-threshold","accepted_measurement_ids":[0,5,7],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":0.00016393756959587336,"statistics":{"sample_count":3,"average":0.0001637569706266125,"min":0.00016241671983152628,"max":0.00016491662245243788,"median":0.00016393756959587336,"p90":0.00016472081188112497,"p95":0.00016481871716678143,"p99":0.0001648970413953066,"stddev":1.2596984834807618e-06,"coefficient_of_variation_pct":0.7692487706999908,"median_absolute_deviation":9.790528565645218e-07}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":6099.883037580253,"statistics":{"sample_count":3,"average":6106.85158629427,"min":6063.670145126826,"max":6157.001576175735,"median":6099.883037580253,"p90":6145.5778684566385,"p95":6151.289722316186,"p99":6155.859205403825,"stddev":47.05432508063455,"coefficient_of_variation_pct":0.7705169253865529,"median_absolute_deviation":36.21289245342632}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":6.396190956013751,"statistics":{"sample_count":3,"average":6.403498008950102,"min":6.3582189860965075,"max":6.456084084740048,"median":6.396190956013751,"p90":6.444105458994788,"p95":6.450094771867418,"p99":6.454886222165522,"stddev":0.04934003597575135,"coefficient_of_variation_pct":0.7705169253865513,"median_absolute_deviation":0.037971969917243165}}},"kv_only":{"scenario":"kv_only","status":"complete","observed_cv_classification":"below-threshold","accepted_measurement_ids":[1,3,8],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":7.599999662488699e-05,"statistics":{"sample_count":3,"average":7.701389646778505e-05,"min":7.512501906603575e-05,"max":7.991667371243238e-05,"median":7.599999662488699e-05,"p90":7.913333829492331e-05,"p95":7.952500600367784e-05,"p99":7.983834017068148e-05,"stddev":2.551662836496064e-06,"coefficient_of_variation_pct":3.313249885445578,"median_absolute_deviation":8.749775588512421e-07}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":13157.895321176102,"statistics":{"sample_count":3,"average":12994.024445699622,"min":12513.033307646701,"max":13311.144708276062,"median":13157.895321176102,"p90":13280.49483085607,"p95":13295.819769566066,"p99":13308.079720534064,"stddev":423.53949027890076,"coefficient_of_variation_pct":3.259494331789343,"median_absolute_deviation":153.2493870999606}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":0.005052631803331623,"statistics":{"sample_count":3,"average":0.004989705387148655,"min":0.004805004790136334,"max":0.005111479567978008,"median":0.005052631803331623,"p90":0.005099710015048731,"p95":0.005105594791513369,"p99":0.00511030261268508,"stddev":0.0001626391642670975,"coefficient_of_variation_pct":3.2594943317893352,"median_absolute_deviation":5.8847764646385164e-05}}},"mixed":{"scenario":"mixed","status":"complete","observed_cv_classification":"below-threshold","accepted_measurement_ids":[2,4,6],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":3,"headline_semantics":"median_p50","headline":0.00014881242532283068,"statistics":{"sample_count":3,"average":0.00014914579999943575,"min":0.0001482916995882988,"max":0.00015033327508717775,"median":0.00014881242532283068,"p90":0.00015002910513430833,"p95":0.00015018119011074306,"p99":0.0001503028580918908,"stddev":1.060830634765448e-06,"coefficient_of_variation_pct":0.7112708737151575,"median_absolute_deviation":5.207257345318794e-07}},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":3,"headline_semantics":"median_p50","headline":6719.868974856234,"statistics":{"sample_count":3,"average":6705.074007256578,"min":6651.887277917037,"max":6743.465768996464,"median":6719.868974856234,"p90":6738.746410168418,"p95":6741.106089582441,"p99":6742.993833113659,"stddev":47.54811570191112,"coefficient_of_variation_pct":0.7091363294491917,"median_absolute_deviation":23.596794140230486}},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":3,"headline_semantics":"median_p50","headline":7.048873759865195,"statistics":{"sample_count":3,"average":7.033354430651861,"min":6.977563679043856,"max":7.073625853046531,"median":7.048873759865195,"p90":7.068675434410264,"p95":7.071150643728398,"p99":7.073130811182905,"stddev":0.04987607144667628,"coefficient_of_variation_pct":0.7091363294491857,"median_absolute_deviation":0.024752093181335866}}}},"traffic_diagnostics":{"classification_version":"llm-exact-weight-vs-kv-read-payload-v1","traffic_crossover_numerator":null,"traffic_crossover_denominator":null,"traffic_crossover_context_tokens":null,"current_visible_context_tokens":null,"current_weight_read_payload_bytes_per_work_unit":"1048576","current_kv_read_payload_bytes_per_work_unit":"264","current_weight_to_kv_read_payload_ratio":null,"current_context_classification":null,"classification_is_payload_only":true,"scenario_headlines":{"weights_only":{"synthetic_work_unit_latency_seconds":0.00016393756959587336,"synthetic_memory_work_units_per_second":6099.883037580253,"effective_model_payload_gb_s":6.396190956013751},"kv_only":{"synthetic_work_unit_latency_seconds":7.599999662488699e-05,"synthetic_memory_work_units_per_second":13157.895321176102,"effective_model_payload_gb_s":0.005052631803331623},"mixed":{"synthetic_work_unit_latency_seconds":0.00014881242532283068,"synthetic_memory_work_units_per_second":6719.868974856234,"effective_model_payload_gb_s":7.048873759865195}}}},"status":"complete","reason_code":"complete","results_complete":true,"run_accepted":true,"interpretation":{"result_scope":"synthetic-memory-only-work-unit-not-real-inference-token-or-prefill-latency","reported_rate":"synthetic_memory_work_units_per_second","backend":"metal","phase":"prefill","work_unit_kind":"prefill_operation","transformer_math_included":false,"framework_scheduler_and_dispatch_included":false,"compute_memory_overlap_included":false,"physical_dram_traffic_measured":false,"dram_residency":"unverified","cache_residency":"unverified","fixed_context_includes_current_token_slot":null,"kv_layout":"paged","payload_semantics":"exact-logical-weight-plus-kv-read-plus-kv-write-bytes-divided-by-elapsed-time","layout_metadata_timed_but_excluded_from_effective_model_payload_gb_s":true,"prefill_transformer_compute_or_ttft_prediction_included":false,"private_metal_storage_implies_separate_vram":false,"cross_backend_performance_distributions_combined":false,"traffic_classification_semantics":"exact-weight-vs-kv-read-logical-payload-only-not-hardware-bottleneck","full_size_working_set_reduces_but_does_not_prove_dram_residency":true,"comparability_requires":["same-backend","same-methodology-version","same-frozen-work-plan-and-model-geometry","same-software-version","sufficiently-similar-hardware","sufficiently-similar-thermal-and-power-state"]},"diagnostic":null,"interruption_requested":false,"scenario_order_balance_complete":true,"seeds":{"base_seed_uint64_decimal":"424242","source":"user","buffer_domain_seeds":{"weight_uint64_decimal":"904846021374001452","k_uint64_decimal":"10718278915871169538","v_uint64_decimal":"2111147304547004201"},"scenario_domain_seeds":{"weights_only":"17410844953366850226","kv_only":"16590208891222127307","mixed":"1546516732201688391"}},"counters":{"planned_loops":3,"attempted_loops":3,"completed_loops":3,"planned_measurements":9,"attempted_measurements":9,"terminal_measurements":9,"measured_measurements":9,"planned_work_units":"18","completed_work_units":"18","planned_effective_model_payload_bytes":"12587520","completed_effective_model_payload_bytes":"12587520","planned_layout_metadata_lookup_count":"720","completed_layout_metadata_lookup_count":"720","planned_layout_metadata_read_bytes":"2880","completed_layout_metadata_read_bytes":"2880","planned_task_accounted_bytes":"12590400","completed_task_accounted_bytes":"12590400","runner_auxiliary":{"valid":true,"reason_code":"valid","measurement_record_bytes":"7056","loop_record_bytes":"144","calibration_record_bytes":"2784","calibration_identity_bytes":"77904","aggregate_value_bytes":"72","statistics_workspace_bytes":"72","warning_record_bytes":"64","fixed_metadata_bytes":"13792597","retained_checksum_bytes":"0","checksum_auxiliary_bytes":"0","orchestration_auxiliary_bytes":"13880693","total_auxiliary_bytes":"13880693"}},"checkpoint_lifecycle":{"checkpoint_failed":false,"observation_point":"before-current-snapshot-preparation","prior_file_writer_attempts":3,"prior_successful_file_writes":3,"current_request":"terminal","current_persistence_success":null,"snapshot_interval_loops":1,"checkpoint_policy":"bounded-loop-snapshots","file_checkpoint_failure_is_terminal_and_not_retried":true,"stdout_intermediate_checkpoints_are_lazy":true},"loop_records":[{"loop_index":0,"planned_order":["weights_only","kv_only","mixed"],"realized_order":["weights_only","kv_only","mixed"],"realized_order_count":3,"measurement_indexes":[0,1,2]},{"loop_index":1,"planned_order":["kv_only","mixed","weights_only"],"realized_order":["kv_only","mixed","weights_only"],"realized_order_count":3,"measurement_indexes":[3,4,5]},{"loop_index":2,"planned_order":["mixed","weights_only","kv_only"],"realized_order":["mixed","weights_only","kv_only"],"realized_order_count":3,"measurement_indexes":[6,7,8]}],"environment":{"processor_name":"Apple M4","macos_version":"26.6.2","performance_core_count":4,"efficiency_core_count":6,"logical_core_count":10,"page_size_bytes":"16384","l1_data_cache_bytes":"131072","l2_data_cache_bytes":"16777216","available_memory_bytes":"11585716224","available_memory_source":"mach-free-plus-inactive-rounded-mib","main_thread_qos":{"requested":true,"applied":true,"code":0},"start":{"thermal_state":"nominal","low_power_mode_available":true,"low_power_mode_enabled":false,"physical_memory_bytes":"25769803776"},"end":{"thermal_state":"nominal","low_power_mode_available":true,"low_power_mode_enabled":false,"physical_memory_bytes":"25769803776"}},"quality_warnings":["weight-working-set-cache-dominant","kv-working-set-cache-dominant","weights_only-duration-below-target-window","kv_only-duration-below-target-window","mixed-duration-below-target-window"]} diff --git a/tests/fixtures/llm-schema-v2/unsupported.json b/tests/fixtures/llm-schema-v2/unsupported.json new file mode 100644 index 0000000..524941d --- /dev/null +++ b/tests/fixtures/llm-schema-v2/unsupported.json @@ -0,0 +1 @@ +{"schema_version":2,"mode":"llm_memory","backend":"metal","phase":"decode","kv_layout":"contiguous","methodology_version":"llm-memory-v2-metal-decode-contiguous","software":{"version":"0.63.1","timestamp":"2026-09-06T07:19:28Z"},"build_manifest":{"binary_sha256":"a9c0aa68c61fccf12bda222a0485a3eba1b642a54b9b1caf8292348f608a1100","compile_flags":{"asflags":"-arch arm64","cxxflags":"-Wall -O3 -std=c++17 -arch arm64 -pthread -Isrc","test_cxxflags":"-Wall -g -std=c++17 -arch arm64 -pthread -Isrc -I/opt/homebrew/opt/googletest/include"},"compiler":"Apple clang version 21.0.0 (clang-2100.1.1.101)\nTarget: arm64-apple-darwin25.6.0\nThread model: posix\nInstalledDir: /Library/Developer/CommandLineTools/usr/bin","git_commit":"870b175927d747fb1d90fadefebe8728da38407a","git_dirty":true,"link_flags":"-pthread -framework Metal -framework Foundation","manifest_version":1,"min_os":"26.0","reason_code":"valid","sdk":"26.5","status":"available","target_arch":"arm64-apple-darwin25.6.0"},"configuration":{"backend":"metal","phase":"decode","kv_layout":"contiguous","kv_block_tokens":null,"weight_size_mb":1,"layer_count":1,"query_head_count":1,"kv_head_count":1,"head_dimension":1,"kv_element_bytes":"2","visible_context_tokens":1,"prompt_tokens":null,"attention_query_tile_tokens":null,"batch_size":1,"requested_workers":null,"available_workers":null,"worker_source":null,"iterations":1,"work_policy":"explicit_fixed_work","loop_count":1,"base_seed_uint64_decimal":"0","seed_source":"user","output_file":"/tmp/llm-phase7-unsupported.json","argv":["./memory_benchmark","--llm-memory","--llm-memory-backend","metal","--weight-size-mb","1","--layers","1","--query-heads","1","--kv-heads","1","--head-dim","1","--context-tokens","1","--iterations","1","--count","1","--seed","0","-o","/tmp/llm-phase7-unsupported.json"],"resolved_sources":{"backend":"explicit","phase":"default","kv_layout":"default","kv_block_tokens":null,"visible_context_tokens":"explicit","prompt_tokens":null,"attention_query_tile_tokens":null,"workers":null,"iterations":"explicit","seed":"explicit"}},"resolved_plan":{"valid":true,"reason_code":"valid","plan_identity":"llm-memory-work-plan-v1|backend=metal|phase=decode|kv_layout=contiguous|work_unit_kind=decode_step|methodology=llm-memory-v2-metal-decode-contiguous|component_identity_size=634|component_identity=llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=39:llm-metal-executor-v1-decode-contiguous|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=42:llm-metal-decode-contiguous-grid-stride-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=32:llm-metal-contiguous-affine32-v1|write_pattern_version=35:llm-metal-decode-append-affine32-v1|checksum_pattern_version=23:llm-metal-dual-mod32-v1|msl_revision=null|msl_source_sha256=null|range_alignment=32|weight_passes_per_work_unit=1|kv_replay_factor=1|weight=1048576|layers=1|query_heads=1|kv_heads=1|query_heads_per_kv_head=1|attention_kind=mha|head_dim=1|kv_element_bytes=2|decode_context=1|batch=1|kv_vector_bytes=2|k_or_v_record_bytes_per_layer=2|kv_record_bytes_per_layer=4|kv_bytes_per_visible_token=4|k_or_v_sequence_visible_bytes=2|kv_block_tokens=0|kv_blocks_per_sequence=0|physical_blocks_per_layer=0|total_physical_blocks=0|kv_block_bytes=0|last_block_tokens=0|last_block_valid_bytes=0|decode_append_offset_in_last_block=0|k_logical_bytes=2|v_logical_bytes=2|k_layout_padding_bytes=0|v_layout_padding_bytes=0|block_table_entries=0|block_table_bytes=0|layout_metadata_lookups_per_layer_sequence_per_work_unit=0|k_mapping_bytes=2|v_mapping_bytes=2|kv_capacity_bytes=4|weight_read_bytes_per_work_unit=1048576|kv_read_bytes_per_work_unit=4|kv_write_bytes_per_work_unit=4|kv_only_payload_bytes_per_work_unit=8|mixed_payload_bytes_per_work_unit=1048584|total_data_mapping_bytes=1048580|traffic_crossover_numerator=1048576|traffic_crossover_denominator=4|metal_execution_resolved=0|metal_execution_identity_size=0|metal_execution_identity_sha256=0000000000000000000000000000000000000000000000000000000000000000|base_seed=0|weight_buffer_seed=14674248913597854790|k_buffer_seed=9483971696087744821|v_buffer_seed=13783932941173899578|weights_only_scenario_seed=4306708088547548925|kv_only_scenario_seed=14535642120272398313|mixed_scenario_seed=539580404444253766","backend":"metal","phase":"decode","kv_layout":"contiguous","work_unit_kind":"decode_step","geometry":{"valid":true,"reason_code":"valid","phase":"decode","work_unit_kind":"decode_step","decode":{"visible_context_tokens":1},"prefill":null,"attention_kind":"mha","active_weight_bytes_per_work_unit":"1048576","layer_count":1,"query_head_count":1,"kv_head_count":1,"query_heads_per_kv_head":1,"head_dimension":1,"kv_element_bytes":"2","batch_size":1,"kv_vector_bytes":"2","k_or_v_record_bytes_per_layer":"2","kv_record_bytes_per_layer":"4","kv_bytes_per_visible_token":"4","k_or_v_sequence_visible_bytes":"2","k_mapping_bytes":"2","v_mapping_bytes":"2","kv_capacity_bytes":"4","weight_read_bytes_per_work_unit":"1048576","kv_read_bytes_per_work_unit":"4","kv_write_bytes_per_work_unit":"4","kv_only_effective_model_payload_bytes_per_work_unit":"8","mixed_effective_model_payload_bytes_per_work_unit":"1048584","total_data_mapping_bytes":"1048580","traffic_crossover_numerator":"1048576","traffic_crossover_denominator":"4","traffic_crossover_context_tokens":262144.0},"model_context":{"prefill":null},"layout":{"kv_layout":"contiguous","kv_block_tokens":null,"blocks_per_sequence":null,"physical_blocks_per_layer":null,"total_physical_blocks":null,"block_bytes":null,"last_block_tokens":null,"last_block_valid_bytes":null,"decode_append_offset_in_last_block":null,"block_table_entries":null,"block_table_bytes":null,"layout_geometry_identity":null,"layout_identity":null,"permutation_domain_uint64_hex":null,"permutation_seed_uint64_decimal":null,"permutation_algorithm_version":null,"permutation_entry_count":null,"permutation_sha256":null,"permutation_identity":null},"resources":{"weight_logical_bytes":"1048576","k_logical_bytes":"2","v_logical_bytes":"2","k_physical_length_bytes":"2","v_physical_length_bytes":"2","k_layout_padding_bytes":"0","v_layout_padding_bytes":"0","block_table_bytes":null,"metal":{"valid":false,"reason_code":"invalid-metal-geometry","execution_identity":null,"resource_identity":null,"msl_revision":null,"msl_source_sha256":null,"segment_capacity_bytes":"268435456","max_buffer_length_bytes":"0","weight_segments":{"valid":false,"reason_code":"segment-capacity-zero","element_count":"0","element_bytes":"0","segment_capacity_bytes":"0","segment_slot_cap":0,"elements_per_segment":"0","segment_count":0,"maximum_addressable_elements":"0","maximum_addressable_bytes":"0","unused_nominal_segment_capacity_bytes":"0","total_length_bytes":"0","segment_lengths_bytes":[],"identity":null},"k_segments":{"valid":false,"reason_code":"segment-capacity-zero","element_count":"0","element_bytes":"0","segment_capacity_bytes":"0","segment_slot_cap":0,"elements_per_segment":"0","segment_count":0,"maximum_addressable_elements":"0","maximum_addressable_bytes":"0","unused_nominal_segment_capacity_bytes":"0","total_length_bytes":"0","segment_lengths_bytes":[],"identity":null},"v_segments":{"valid":false,"reason_code":"segment-capacity-zero","element_count":"0","element_bytes":"0","segment_capacity_bytes":"0","segment_slot_cap":0,"elements_per_segment":"0","segment_count":0,"maximum_addressable_elements":"0","maximum_addressable_bytes":"0","unused_nominal_segment_capacity_bytes":"0","total_length_bytes":"0","segment_lengths_bytes":[],"identity":null},"table_segments":null,"argument_buffer":{"valid":false,"reason_code":"metal-argument-buffer-layout-invalid","encoded_length_bytes":"0","alignment_bytes":"0","weight_slot_base":0,"k_slot_base":256,"v_slot_base":512,"table_slot_base":768,"status_slot":1024,"encoded_resource_slot_count":1025,"active_resource_count":0,"identity":null},"status_buffer_length_bytes":"4096","staging_buffer_length_bytes":"0","persistent_resource_length_bytes":"0","transient_peak_bytes":"0","known_owned_peak_bytes":"0","admitted_budget_bytes":"0"}},"component_identities":{"logical_profile_version":"decode_steady_fixed_context","kv_layout_version":"contiguous_layer_batch_token_head_dimension","permutation_version":null,"backend_executor_version":"llm-metal-executor-v1-decode-contiguous","resource_abi_version":"llm-metal-argument-buffer-tier2-v1","schedule_version":"llm-metal-decode-contiguous-grid-stride-v1","timer_policy_version":"metal-command-buffer-gpu-start-end-v1","buffer_pattern_version":"llm-metal-contiguous-affine32-v1","write_pattern_version":"llm-metal-decode-append-affine32-v1","checksum_pattern_version":"llm-metal-dual-mod32-v1","msl_revision":null,"msl_source_sha256":null,"identity":"llm-memory-components-v1|logical_profile_version=27:decode_steady_fixed_context|kv_layout_version=43:contiguous_layer_batch_token_head_dimension|permutation_version=null|backend_executor_version=39:llm-metal-executor-v1-decode-contiguous|resource_abi_version=34:llm-metal-argument-buffer-tier2-v1|schedule_version=42:llm-metal-decode-contiguous-grid-stride-v1|timer_policy_version=37:metal-command-buffer-gpu-start-end-v1|buffer_pattern_version=32:llm-metal-contiguous-affine32-v1|write_pattern_version=35:llm-metal-decode-append-affine32-v1|checksum_pattern_version=23:llm-metal-dual-mod32-v1|msl_revision=null|msl_source_sha256=null","run_policy_version":"llm-run-policy-bounded-loop-snapshots-v1"},"methodology":{"backend":"metal","phase":"decode","kv_layout":"contiguous","work_unit_kind":"decode_step","weight_passes_per_work_unit":1,"kv_replay_factor":1,"warmup_policy":"same-shape-excluded-steady-state-warm-memory","context_policy":"fixed-visible-context-including-current-token-slot","scenario_order_policy":"cyclic-rotation-across-count-loops","timing_policy":"gpu-start-to-gpu-end-per-command-buffer-task","cache_policy":"metal-private-and-shared-cacheable-no-explicit-flush-between-scenarios","calibration_policy":"per-scenario-automatic-or-explicit-frozen-before-loop-zero","calibration_target_seconds":0.15,"calibration_min_seconds":0.1,"calibration_max_seconds":0.25,"calibration_max_corrections":2,"calibration_min_pilot_accounted_bytes":"8388608","maximum_work_units_per_measurement":65536,"maximum_serial_range_visits_per_lane_per_task":null,"maximum_accounted_bytes_per_task":"68719476736","repeatability_cv_warning_threshold_pct":5.0,"calibration_excluded_from_results":true,"snapshot_policy":"bounded-loop-snapshots","timed_region_exclusions":["mapping-allocation","buffer-initialization-and-pre-touch","argument-buffer-encoding-and-resource-validation","host-command-encoding-and-pre-gpu-queue-wait","status-reset-command-buffer","excluded-kv-write-and-padding-canary-post-validation","same-shape-warmup","calibration-attempts","checksum-reference-generation-and-validation","host-wait-aggregation-and-json-serialization"]},"model_work_plan":{"valid":true,"reason_code":"valid","backend":"metal","phase":"decode","kv_layout":"contiguous","work_unit_kind":"decode_step","weight_passes_per_work_unit":1,"kv_replay_factor":1,"requested_workers":null,"available_workers":null,"effective_workers":null,"worker_plan_count":null,"weight_layer_count":1,"layer_descriptors_per_worker":null,"sequence_descriptors_per_worker":null,"total_layer_descriptors":null,"total_sequence_descriptors":null,"descriptor_bytes":null,"planner_storage_bytes":null},"scenario_plans":[],"frozen_plan_refs":{"weights_only":null,"kv_only":null,"mixed":null}},"backend_evidence":{"cpu":null,"metal":{"workers_applicable":false,"worker_qos_applicable":false,"lifecycle":{"initialization":{"status":"unsupported","reason_code":"metal-device-unavailable"},"plan_resolution":{"status":"not_started","reason_code":"backend-not-initialized"},"preparation":{"status":"not_started","reason_code":"backend-not-initialized"},"release":{"status":"ready","reason_code":"valid"}},"capability":{"device_name":null,"registry_id_uint64_decimal":"0","has_unified_memory":false,"required_apple7_family_supported":false,"argument_buffers_tier2_supported":false,"supported_families":[],"max_buffer_length_bytes":"0","recommended_max_working_set_size_bytes":"0","compilation_mode":"runtime-source","msl_language_version":"2.3","kernel_revision":"llm-metal-decode-prefill-contiguous-paged-msl23-v5","kernel_source_sha256":"2fb49f1f8045aa01a74329d7b9d6b04134502d845e9f70c53b8f6fd0baf8fdd2","compiler_diagnostics":null,"compiler_identifier":"21.0.0 (clang-2100.1.1.101)","build_sdk":"26.5","deployment_target":"26.0","argument_buffer":{"encoded_length_bytes":"0","alignment_bytes":"0"},"layout_probe":{"evaluated":false,"valid":false,"resource_count":0},"foundation_pipelines":[],"error":{"domain":null,"code":0,"description":null}},"resources":{"allocation_attempted":false,"allocation_completed":false,"initialization_completed":false,"table_upload_completed":false,"table_validation_completed":false,"table_permutation":null,"post_validation_completed":false,"cpu_sample_readback_validation_completed":false,"candidate_cleanup_completed":true,"resources_published":false,"persistent_resource_length_bytes":"0","committed_resource_bytes":"0","resource_rounding_bytes":"0","layout_padding_bytes":"0","transient_peak_bytes":"0","additional_owned_bytes":"0","known_owned_peak_bytes":"0","admitted_budget_bytes":"0","current_allocated_size_before_bytes":"0","current_allocated_size_peak_bytes":"0","current_allocated_size_after_release_bytes":"0","recommended_working_set_available":false,"recommended_working_set_headroom_bytes":null,"recommended_working_set_headroom_fraction":null,"exceeds_recommended_working_set":null,"resources":[],"error":{"domain":null,"code":0,"description":null}},"workload_pipelines":[],"timed_results_available":false}},"memory_budget":{"resource_rounding_bytes":"32764","transient_peak_bytes":"171514946","layout_transient_bytes":null,"setup_peak_bytes":"56","runtime_peak_bytes":"172596290","known_owned_peak_bytes":"172596290","admitted_budget_bytes":"9292061081","valid":true,"reason_code":"within-memory-budget","request":{"valid":true,"reason_code":"valid","mapping_granularity_bytes":"16384","requested_weight_mapping_bytes":"1048576","requested_k_mapping_bytes":"2","requested_v_mapping_bytes":"2","committed_weight_mapping_bytes":"1048576","committed_k_mapping_bytes":"16384","committed_v_mapping_bytes":"16384","requested_block_table_mapping_bytes":null,"committed_block_table_mapping_bytes":null,"requested_data_bytes":"1048580","committed_data_bytes":"1081344","layout_transient_bytes":null,"setup_peak_bytes":"56","runtime_peak_bytes":"172596290","descriptor_bytes":"0","planner_storage_bytes":"56","checksum_auxiliary_bytes":"0","orchestration_auxiliary_bytes":"171514890","auxiliary_bytes":"171514946","required_total_bytes":"172596290"},"available_memory_bytes":"11615076352","allowed_memory_bytes":"9292061081","used_fallback":false},"calibration":{"excluded_from_results":true,"attempts":{"weights_only":[{"attempt_index":0,"scenario":"weights_only","plan_ref":null,"purpose":"not-run","duration_quality":"not-run","terminal":false,"valid":false,"reason_code":"not-started","execution":{"status":"unavailable","reason_code":"not-started","valid":null,"elapsed_seconds":null,"timing":{"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"checksum":{"status":"not_evaluated","reason_code":"not-started","checksum_valid":null,"actual_run_checksum":null},"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-append-final","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}}}],"kv_only":[{"attempt_index":0,"scenario":"kv_only","plan_ref":null,"purpose":"not-run","duration_quality":"not-run","terminal":false,"valid":false,"reason_code":"not-started","execution":{"status":"unavailable","reason_code":"not-started","valid":null,"elapsed_seconds":null,"timing":{"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"checksum":{"status":"not_evaluated","reason_code":"not-started","checksum_valid":null,"actual_run_checksum":null},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":false,"valid":null,"reason_code":"execution-evidence-unavailable"},{"kind":"kv-append-final","applicable":true,"evaluated":false,"valid":null,"reason_code":"execution-evidence-unavailable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}}}],"mixed":[{"attempt_index":0,"scenario":"mixed","plan_ref":null,"purpose":"not-run","duration_quality":"not-run","terminal":false,"valid":false,"reason_code":"not-started","execution":{"status":"unavailable","reason_code":"not-started","valid":null,"elapsed_seconds":null,"timing":{"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"checksum":{"status":"not_evaluated","reason_code":"not-started","checksum_valid":null,"actual_run_checksum":null},"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":false,"valid":null,"reason_code":"execution-evidence-unavailable"},{"kind":"kv-append-final","applicable":true,"evaluated":false,"valid":null,"reason_code":"execution-evidence-unavailable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}}}]}},"measurements":[{"measurement_id":0,"plan_ref":null,"scenario":"weights_only","loop_index":0,"order_position":0,"status":"not_run","reason_code":"not-run","attempted":false,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":0,"calibration_attempt_indexes":[],"working_set":{"bytes":"1048580","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":1,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"not-run","completed_work_units":0,"completed_effective_model_payload_bytes":"0","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"0","completion_derivation":null,"elapsed_seconds":null,"synthetic_work_unit_latency_seconds":null,"synthetic_memory_work_units_per_second":null,"effective_model_payload_gb_s":null,"execution":{"status":"not_run","reason_code":"not-run","valid":null,"timing":{"evaluated":false,"valid":null,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"validation":{"checks":[{"kind":"post-validation-structure","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-append-final","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"not_evaluated","reason_code":"not-run","checksum_valid":null,"actual_worker_checksums":null,"actual_run_checksum":null}},{"measurement_id":1,"plan_ref":null,"scenario":"kv_only","loop_index":0,"order_position":1,"status":"not_run","reason_code":"not-run","attempted":false,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":0,"calibration_attempt_indexes":[],"working_set":{"bytes":"1048580","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":1,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"not-run","completed_work_units":0,"completed_effective_model_payload_bytes":"0","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"0","completion_derivation":null,"elapsed_seconds":null,"synthetic_work_unit_latency_seconds":null,"synthetic_memory_work_units_per_second":null,"effective_model_payload_gb_s":null,"execution":{"status":"not_run","reason_code":"not-run","valid":null,"timing":{"evaluated":false,"valid":null,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":false,"valid":null,"reason_code":"execution-evidence-unavailable"},{"kind":"kv-append-final","applicable":true,"evaluated":false,"valid":null,"reason_code":"execution-evidence-unavailable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"not_evaluated","reason_code":"not-run","checksum_valid":null,"actual_worker_checksums":null,"actual_run_checksum":null}},{"measurement_id":2,"plan_ref":null,"scenario":"mixed","loop_index":0,"order_position":2,"status":"not_run","reason_code":"not-run","attempted":false,"requested_workers":null,"effective_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"calibration_attempt_count":0,"calibration_attempt_indexes":[],"working_set":{"bytes":"1048580","full_size_physical_mappings":true,"cacheable":true,"kv_layout":"contiguous","fixed_visible_context_tokens":1,"current_token_slot_included":true},"weight_payload_fraction":null,"kv_read_payload_fraction":null,"kv_write_payload_fraction":null,"duration_quality":"not-run","completed_work_units":0,"completed_effective_model_payload_bytes":"0","completed_layout_metadata_lookup_count":"0","completed_layout_metadata_read_bytes":"0","completed_task_accounted_bytes":"0","completion_derivation":null,"elapsed_seconds":null,"synthetic_work_unit_latency_seconds":null,"synthetic_memory_work_units_per_second":null,"effective_model_payload_gb_s":null,"execution":{"status":"not_run","reason_code":"not-run","valid":null,"timing":{"evaluated":false,"valid":null,"diagnostic_elapsed_seconds":null,"cpu_raw":null},"requested_workers":null,"created_workers":null,"completed_workers":null,"qos_successful_workers":null,"qos_failed_workers":null,"worker_startup_failed":null,"kernel_succeeded":null,"timer_started":null,"timer_stopped":null,"validation":{"checks":[{"kind":"post-validation-structure","applicable":true,"evaluated":false,"valid":null,"reason_code":"execution-evidence-unavailable"},{"kind":"kv-append-final","applicable":true,"evaluated":false,"valid":null,"reason_code":"execution-evidence-unavailable"},{"kind":"kv-padding-canary","applicable":false,"evaluated":null,"valid":null,"reason_code":"not-applicable"}]}},"checksum":{"status":"not_evaluated","reason_code":"not-run","checksum_valid":null,"actual_worker_checksums":null,"actual_run_checksum":null}}],"aggregates":{"scenarios":{"weights_only":{"scenario":"weights_only","status":"unavailable","observed_cv_classification":"insufficient-samples","accepted_measurement_ids":[],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":0,"headline_semantics":"unavailable","headline":null,"statistics":null},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":0,"headline_semantics":"unavailable","headline":null,"statistics":null},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":0,"headline_semantics":"unavailable","headline":null,"statistics":null}},"kv_only":{"scenario":"kv_only","status":"unavailable","observed_cv_classification":"insufficient-samples","accepted_measurement_ids":[],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":0,"headline_semantics":"unavailable","headline":null,"statistics":null},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":0,"headline_semantics":"unavailable","headline":null,"statistics":null},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":0,"headline_semantics":"unavailable","headline":null,"statistics":null}},"mixed":{"scenario":"mixed","status":"unavailable","observed_cv_classification":"insufficient-samples","accepted_measurement_ids":[],"cv_warning_threshold_pct":5.0,"synthetic_work_unit_latency_seconds":{"units":"seconds_per_synthetic_memory_work_unit","sample_count":0,"headline_semantics":"unavailable","headline":null,"statistics":null},"synthetic_memory_work_units_per_second":{"units":"synthetic_memory_work_units_per_second","sample_count":0,"headline_semantics":"unavailable","headline":null,"statistics":null},"effective_model_payload_gb_s":{"units":"GB/s","sample_count":0,"headline_semantics":"unavailable","headline":null,"statistics":null}}},"traffic_diagnostics":{"classification_version":"llm-exact-weight-vs-kv-read-payload-v1","traffic_crossover_numerator":"1048576","traffic_crossover_denominator":"4","traffic_crossover_context_tokens":262144.0,"current_visible_context_tokens":1,"current_weight_read_payload_bytes_per_work_unit":"1048576","current_kv_read_payload_bytes_per_work_unit":"4","current_weight_to_kv_read_payload_ratio":262144.0,"current_context_classification":"weight_payload_dominant","classification_is_payload_only":true,"scenario_headlines":{"weights_only":{"synthetic_work_unit_latency_seconds":null,"synthetic_memory_work_units_per_second":null,"effective_model_payload_gb_s":null},"kv_only":{"synthetic_work_unit_latency_seconds":null,"synthetic_memory_work_units_per_second":null,"effective_model_payload_gb_s":null},"mixed":{"synthetic_work_unit_latency_seconds":null,"synthetic_memory_work_units_per_second":null,"effective_model_payload_gb_s":null}}}},"status":"unsupported","reason_code":"metal-device-unavailable","results_complete":false,"run_accepted":false,"interpretation":{"result_scope":"synthetic-memory-only-work-unit-not-real-inference-token-or-prefill-latency","reported_rate":"synthetic_memory_work_units_per_second","backend":"metal","phase":"decode","work_unit_kind":"decode_step","transformer_math_included":false,"framework_scheduler_and_dispatch_included":false,"compute_memory_overlap_included":false,"physical_dram_traffic_measured":false,"dram_residency":"unverified","cache_residency":"unverified","fixed_context_includes_current_token_slot":true,"kv_layout":"contiguous","payload_semantics":"exact-logical-weight-plus-kv-read-plus-kv-write-bytes-divided-by-elapsed-time","layout_metadata_timed_but_excluded_from_effective_model_payload_gb_s":true,"prefill_transformer_compute_or_ttft_prediction_included":false,"private_metal_storage_implies_separate_vram":false,"cross_backend_performance_distributions_combined":false,"traffic_classification_semantics":"exact-weight-vs-kv-read-logical-payload-only-not-hardware-bottleneck","full_size_working_set_reduces_but_does_not_prove_dram_residency":true,"comparability_requires":["same-backend","same-methodology-version","same-frozen-work-plan-and-model-geometry","same-software-version","sufficiently-similar-hardware","sufficiently-similar-thermal-and-power-state"]},"diagnostic":null,"interruption_requested":false,"scenario_order_balance_complete":false,"seeds":{"base_seed_uint64_decimal":"0","source":"user","buffer_domain_seeds":{"weight_uint64_decimal":"14674248913597854790","k_uint64_decimal":"9483971696087744821","v_uint64_decimal":"13783932941173899578"},"scenario_domain_seeds":{"weights_only":"4306708088547548925","kv_only":"14535642120272398313","mixed":"539580404444253766"}},"counters":{"planned_loops":1,"attempted_loops":0,"completed_loops":0,"planned_measurements":3,"attempted_measurements":0,"terminal_measurements":0,"measured_measurements":0,"planned_work_units":"0","completed_work_units":"0","planned_effective_model_payload_bytes":"0","completed_effective_model_payload_bytes":"0","planned_layout_metadata_lookup_count":"0","completed_layout_metadata_lookup_count":"0","planned_layout_metadata_read_bytes":"0","completed_layout_metadata_read_bytes":"0","planned_task_accounted_bytes":"0","completed_task_accounted_bytes":"0","runner_auxiliary":{"valid":true,"reason_code":"valid","measurement_record_bytes":"2352","loop_record_bytes":"48","calibration_record_bytes":"2784","calibration_identity_bytes":"65068","aggregate_value_bytes":"24","statistics_workspace_bytes":"24","warning_record_bytes":"64","fixed_metadata_bytes":"6922852","retained_checksum_bytes":"0","checksum_auxiliary_bytes":"0","orchestration_auxiliary_bytes":"6993216","total_auxiliary_bytes":"6993216"}},"checkpoint_lifecycle":{"checkpoint_failed":false,"observation_point":"before-current-snapshot-preparation","prior_file_writer_attempts":0,"prior_successful_file_writes":0,"current_request":"terminal","current_persistence_success":null,"snapshot_interval_loops":1,"checkpoint_policy":"bounded-loop-snapshots","file_checkpoint_failure_is_terminal_and_not_retried":true,"stdout_intermediate_checkpoints_are_lazy":true},"loop_records":[{"loop_index":0,"planned_order":["weights_only","kv_only","mixed"],"realized_order":[],"realized_order_count":0,"measurement_indexes":[0,1,2]}],"environment":{"processor_name":"Apple M4","macos_version":"26.6.2","performance_core_count":4,"efficiency_core_count":6,"logical_core_count":10,"page_size_bytes":"16384","l1_data_cache_bytes":"131072","l2_data_cache_bytes":"16777216","available_memory_bytes":"11615076352","available_memory_source":"mach-free-plus-inactive-rounded-mib","main_thread_qos":{"requested":true,"applied":true,"code":0},"start":{"thermal_state":"nominal","low_power_mode_available":true,"low_power_mode_enabled":false,"physical_memory_bytes":"25769803776"},"end":{"thermal_state":"nominal","low_power_mode_available":true,"low_power_mode_enabled":false,"physical_memory_bytes":"25769803776"}},"quality_warnings":["weight-working-set-cache-dominant","kv-working-set-cache-dominant"]} diff --git a/tests/test_llm_result_verifier.py b/tests/test_llm_result_verifier.py new file mode 100644 index 0000000..5e26ad1 --- /dev/null +++ b/tests/test_llm_result_verifier.py @@ -0,0 +1,489 @@ +"""Independent numeric goldens, current producer artifacts, and semantic mutations.""" + +import copy +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "script-examples")) +import verify_llm_result as verifier +import llm_verify_oracles as oracle + +FIXTURES = ROOT / "tests/fixtures/llm-schema-v2" + + +def fixture(name="cpu-decode-contiguous"): + return json.loads((FIXTURES / (name + ".json")).read_text()) + + +def truncated(status): + """Author the status contract explicitly around a captured two-row prefix.""" + doc = fixture() + measured = 2 + for index, row in enumerate(doc["measurements"]): + if index < measured: + continue + row["attempted"] = False + row["status"] = ( + "not_run" if status == "partial" else "interrupted" if status == "interrupted" else "failed" + ) + row["reason_code"] = ( + "not-run" + if status == "partial" + else "interruption-before-task" if status == "interrupted" else "not-run-after-runtime-failure" + ) + for key in verifier.METRICS + ("elapsed_seconds",): + row[key] = None + row["completed_work_units"] = 0 + for key in ( + "effective_model_payload_bytes", + "layout_metadata_lookup_count", + "layout_metadata_read_bytes", + "task_accounted_bytes", + ): + row["completed_" + key] = "0" + row["completion_derivation"] = None + ex = row["execution"] + ex.update(status="not_run", valid=None) + ex["timing"] = dict(evaluated=False, valid=None, diagnostic_elapsed_seconds=None, cpu_raw=None) + for check in ex["validation"]["checks"]: + check["evaluated"] = False if check["applicable"] else None + check["valid"] = None + row["checksum"].update( + status="not_evaluated", + checksum_valid=None, + actual_worker_checksums=None, + actual_run_checksum=None, + ) + doc.update( + status=status, + reason_code=( + "partial-results" + if status == "partial" + else "interruption-requested" if status == "interrupted" else "runtime-failure" + ), + results_complete=False, + run_accepted=False, + scenario_order_balance_complete=False, + interruption_requested=status == "interrupted", + ) + for i, loop in enumerate(doc["loop_records"]): + loop["realized_order"] = loop["planned_order"][:2] if i == 0 else [] + loop["realized_order_count"] = len(loop["realized_order"]) + refresh(doc) + return doc + + +def refresh(doc): + """Fixture authoring: rebuild only population/counter metadata after a status edit.""" + rows = doc["measurements"] + c = doc["counters"] + c.update( + attempted_loops=len({r["loop_index"] for r in rows if r["attempted"]}), + completed_loops=sum( + all(r["status"] == "measured" for r in rows[i : i + 3]) for i in range(0, len(rows), 3) + ), + attempted_measurements=sum(r["attempted"] for r in rows), + terminal_measurements=sum(r["status"] != "not_run" for r in rows), + measured_measurements=sum(r["status"] == "measured" for r in rows), + ) + for key in ( + "work_units", + "effective_model_payload_bytes", + "layout_metadata_lookup_count", + "layout_metadata_read_bytes", + "task_accounted_bytes", + ): + c["completed_" + key] = str(sum(int(r["completed_" + key]) for r in rows)) + for scenario, a in doc["aggregates"]["scenarios"].items(): + ids = [i for i, r in enumerate(rows) if r["status"] == "measured" and r["scenario"] == scenario] + a.update( + accepted_measurement_ids=ids, + status="partial" if ids else "unavailable", + observed_cv_classification="insufficient-samples", + ) + for metric in verifier.METRICS: + block = a[metric] + values = [rows[i][metric] for i in ids] + # These fixtures deliberately have at most one member per scenario. + assert len(values) <= 1 + v = values[0] if values else None + block.update( + sample_count=len(values), + headline=v, + headline_semantics="single_measurement" if values else "unavailable", + ) + block["statistics"] = ( + None + if not values + else dict( + sample_count=1, + average=v, + min=v, + max=v, + median=v, + p90=v, + p95=v, + p99=v, + stddev=0.0, + coefficient_of_variation_pct=0.0, + median_absolute_deviation=0.0, + ) + ) + + +class VerifierTests(unittest.TestCase): + def assert_reason(self, doc, reason): + with self.assertRaises(verifier.Rejected) as caught: + verifier.verify(doc) + self.assertEqual(caught.exception.reason, reason) + + def test_all_eight_real_profiles(self): + for backend in ("cpu", "metal"): + for phase in ("decode", "prefill"): + for layout in ("contiguous", "paged"): + with self.subTest(backend=backend, phase=phase, layout=layout): + self.assertTrue( + verifier.verify(fixture("-".join((backend, phase, layout))))["run_accepted"] + ) + + def test_status_artifacts_are_consistent_but_not_accepted(self): + for status in ("partial", "interrupted", "failed"): + with self.subTest(status=status): + verdict = verifier.verify(truncated(status)) + self.assertTrue(verdict["artifact_consistent"]) + self.assertFalse(verdict["run_accepted"]) + verdict = verifier.verify(fixture("unsupported")) + self.assertTrue(verdict["artifact_consistent"]) + self.assertFalse(verdict["run_accepted"]) + + def test_invalid_checksum_keeps_actual_evidence_out_of_population(self): + doc = truncated("failed") + row = doc["measurements"][1] + row["status"] = "invalid" + row["execution"]["valid"] = False + row["execution"]["status"] = "invalid" + row["checksum"]["actual_worker_checksums"][0]["k"]["state_a_uint64_decimal"] = "0" + states = [ + [ + [ + int(worker[pool][key]) + for key in ( + "state_a_uint64_decimal", + "state_b_uint64_decimal", + "exact_bytes_read", + "span_count_uint64_decimal", + ) + ] + for pool in ("weight", "k", "v") + ] + for worker in row["checksum"]["actual_worker_checksums"] + ] + row["checksum"]["actual_run_checksum"] = oracle.fold(states) + row["checksum"].update(status="invalid", checksum_valid=False) + row["execution"]["timing"]["diagnostic_elapsed_seconds"] = row["elapsed_seconds"] + for key in verifier.METRICS + ("elapsed_seconds",): + row[key] = None + row["completed_work_units"] = 0 + for key in ( + "effective_model_payload_bytes", + "layout_metadata_lookup_count", + "layout_metadata_read_bytes", + "task_accounted_bytes", + ): + row["completed_" + key] = "0" + refresh(doc) + verdict = verifier.verify(doc) + self.assertTrue(verdict["artifact_consistent"]) + self.assertFalse(verdict["run_accepted"]) + doc["aggregates"]["scenarios"]["kv_only"]["accepted_measurement_ids"] = [1] + self.assert_reason(doc, "aggregate-population-mismatch") + + def test_mutation_reasons(self): + cases = [ + ("plan-reference", lambda d: d["measurements"][0].update(plan_ref=999)), + ( + "plan-identity-mismatch", + lambda d: d["resolved_plan"]["scenario_plans"][0].update( + plan_identity=d["resolved_plan"]["scenario_plans"][0]["plan_identity"] + "|extra=1" + ), + ), + ( + "model-identity-mismatch", + lambda d: d["resolved_plan"].update( + plan_identity=d["resolved_plan"]["plan_identity"].replace( + "|base_seed=424242", "|base_seed=424243" + ) + ), + ), + ("seed-mismatch", lambda d: d["seeds"]["buffer_domain_seeds"].update(k_uint64_decimal="1")), + ( + "configuration-geometry-mismatch", + lambda d: d["resolved_plan"]["geometry"].update(layer_count=3), + ), + ("plan-bytes-mismatch", lambda d: d["resolved_plan"]["scenario_plans"][0].update(work_units=3)), + ( + "plan-bytes-mismatch", + lambda d: d["resolved_plan"]["scenario_plans"][0].update(effective_model_payload_bytes="1"), + ), + ( + "tick-delta-mismatch", + lambda d: d["measurements"][0]["execution"]["timing"]["cpu_raw"].update(stop_ticks="1"), + ), + ( + "elapsed-mismatch", + lambda d: d["measurements"][0]["execution"]["timing"]["cpu_raw"].update(timebase_numer=126), + ), + ("rate-mismatch", lambda d: d["measurements"][0].update(effective_model_payload_gb_s=1.0)), + ( + "expected-run-checksum-mismatch", + lambda d: d["resolved_plan"]["scenario_plans"][0]["expected_checksum"][ + "expected_run_checksum" + ].update(state_a_uint64_decimal="0"), + ), + ( + "checksum-valid-mismatch", + lambda d: d["measurements"][0]["checksum"].update(checksum_valid=False), + ), + ("measurement-valid-mismatch", lambda d: d["measurements"][0]["execution"].update(valid=False)), + ("measurement-id-mismatch", lambda d: d["measurements"][1].update(measurement_id=0)), + ("measurement-count-mismatch", lambda d: d["measurements"].pop()), + ( + "aggregate-population-mismatch", + lambda d: d["aggregates"]["scenarios"]["weights_only"]["accepted_measurement_ids"].append(1), + ), + ( + "aggregate-statistic-mismatch", + lambda d: d["aggregates"]["scenarios"]["weights_only"]["effective_model_payload_gb_s"][ + "statistics" + ].update(average=1.0), + ), + ( + "required-validation-failed", + lambda d: d["measurements"][1]["execution"]["validation"]["checks"][1].update(valid=False), + ), + ("unsupported-schema", lambda d: d.update(schema_version=1)), + ("unsupported-methodology", lambda d: d.update(methodology_version="unknown")), + ("run-acceptance-mismatch", lambda d: d.update(run_accepted=False)), + ("counter-mismatch", lambda d: d["counters"].update(completed_work_units="0")), + ] + for reason, mutate in cases: + with self.subTest(reason=reason): + doc = fixture() + mutate(doc) + self.assert_reason(doc, reason) + doc = fixture("metal-prefill-paged") + doc["resolved_plan"]["layout"]["permutation_sha256"] = "0" * 64 + self.assert_reason(doc, "permutation-digest-mismatch") + doc = fixture("cpu-decode-paged") + doc["measurements"][1]["completed_layout_metadata_lookup_count"] = "0" + self.assert_reason(doc, "completion-mismatch") + + def test_interruption_prefix_rejects_resumption(self): + doc = truncated("interrupted") + doc["measurements"][3] = fixture()["measurements"][3] + self.assert_reason(doc, "invalid-execution-prefix") + + def test_missing_raw_timing_is_explicitly_limited(self): + doc = fixture() + for row in doc["measurements"]: + row["execution"]["timing"].pop("cpu_raw") + verdict = verifier.verify(doc) + self.assertTrue(verdict["artifact_consistent"]) + self.assertEqual(verdict["checks"]["timing"], "elapsed-only") + + def test_work_budget_never_means_success(self): + budget = verifier.Budget() + with self.assertRaises(verifier.Rejected) as caught: + budget(verifier.MAX_WORK + 1) + self.assertEqual(caught.exception.reason, "unsupported-work-limit") + self.assertTrue(caught.exception.unsupported) + + def test_strict_json_and_cli_exit_contract(self): + with tempfile.TemporaryDirectory(prefix="llm-verifier-", dir="/tmp") as directory: + path = Path(directory) / "input.json" + path.write_text('{"schema_version":2,"schema_version":2}') + with self.assertRaises(verifier.Rejected) as caught: + verifier.load(path) + self.assertEqual(caught.exception.reason, "duplicate-json-key") + for doc, expected in ((fixture(), 0), (truncated("failed"), 1), (dict(schema_version=1), 2)): + path.write_text(json.dumps(doc)) + result = subprocess.run( + [sys.executable, str(ROOT / "script-examples/verify_llm_result.py"), str(path)], + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, expected, result.stdout + result.stderr) + json.loads(result.stdout) + + def test_integer_encoding_and_binary_binding(self): + doc = fixture() + doc["measurements"][0]["plan_ref"] = True + self.assert_reason(doc, "integer-encoding") + with tempfile.NamedTemporaryFile(dir="/tmp") as binary: + binary.write(b"known binary bytes") + binary.flush() + import hashlib + + doc = fixture() + doc["build_manifest"]["binary_sha256"] = hashlib.sha256(b"known binary bytes").hexdigest() + self.assertEqual(verifier.verify(doc, binary.name)["checks"]["build"], "binary-bound") + doc["build_manifest"]["binary_sha256"] = "0" * 64 + with self.assertRaises(verifier.Rejected) as caught: + verifier.verify(doc, binary.name) + self.assertEqual(caught.exception.reason, "binary-hash-mismatch") + + +class ArithmeticGoldens(unittest.TestCase): + def test_published_cpu_golden_and_wrap(self): + # Independent published 19-byte vector 00..12; parity restarts in the span. + state = [ + 0x243F6A8885A308D3 ^ oracle.DOMAINS[0], + (0x13198A2E03707344 + oracle.DOMAINS[0]) & oracle.U64, + 0, + 0, + ] + oracle.absorb(state, 0x0706050403141210, 0x0F0E0D0C0B0A0908, 19) + self.assertEqual(state, [0x451AA0ABCC0E316F, 0x37A51B246A0ABBE7, 19, 1]) + self.assertEqual( + (oracle.A + oracle.B + oracle.C + oracle.D + 0x4B4B4B4B4B4B4B4B) & oracle.U64, 0x149454E56105BC97 + ) + self.assertEqual(oracle.splitmix(0), 0xE220A8397B1DCDAF) + + def test_hand_derived_one_byte_goldens_for_all_eight_profiles(self): + # W=K=V=one byte, seed=0, one layer/batch/operation, P=Q=block=1. + # W low byte is 0x15 (CPU) / 0xb9 (Metal). CPU decode append K/V + # are 0x97/0xa2; prefill adds phase-domain low byte 0x31 -> 0xc8/0xd3. + # Paged lookup has logical=physical=0; its pair term is exactly one. + cpu_runs = { + (False, False): (8604485311877256335, 11711443974649127621), + (False, True): (12314371692794693905, 18049950523347819890), + (True, False): (4538172094021051140, 2453338796290460398), + (True, True): (2839643602375233441, 6185486332617838998), + } + metal_runs = { + (False, False): [(3495835427, 20476811), (1723508762, 3471944986), (2092076570, 539171610)], + (False, True): [(3495838755, 4075535499), (1694068630, 572221926), (2431925142, 225265126)], + (True, False): [(3496621859, 2136241035), (1200551007, 2254724667), (1569118815, 3616918587)], + (True, True): [(3496625187, 1896332427), (3181621096, 3137674192), (3918759528, 2024859344)], + } + for prefill, paged in cpu_runs: + model = dict( + prefill=prefill, + paged=paged, + workers=1, + layers=1, + batch=1, + record=1, + tokens=1, + q=1, + tile_ends=[1], + blocks=1 if paged else 0, + block_tokens=1 if paged else 0, + block_bytes=1 if paged else 0, + weight=1, + buffer_seeds=[0, 0, 0], + scenario_seeds={"mixed": 0}, + ownership={"mixed": [[(0, 1)]]}, + table=[0], + ) + with self.subTest(backend="cpu", prefill=prefill, paged=paged): + workers, run = oracle.cpu_checksum(model, "mixed", 1, verifier.Budget()) + self.assertEqual(tuple(map(int, run.values())), cpu_runs[prefill, paged]) + for pool in ("weight", "k", "v"): + self.assertEqual(workers[0][pool]["exact_bytes_read"], "1") + self.assertEqual(workers[0][pool]["span_count_uint64_decimal"], "1") + with self.subTest(backend="metal", prefill=prefill, paged=paged): + _, run = oracle.metal_checksum(model, "mixed", 1, verifier.Budget()) + self.assertEqual( + [tuple(map(int, run[p].values())) for p in ("weight", "k", "v")], + metal_runs[prefill, paged], + ) + + def test_affine_closed_form_matches_independent_byte_enumeration(self): + for bits in (32, 64): + width = bits // 8 + mask = (1 << bits) - 1 + data = b"".join( + ((mask - 3 + 0x13579 * (i + 1)) & mask).to_bytes(width, "little") for i in range(10) + ) + for first in range(2 * width): + for length in range(1, 3 * width): + for relative in (True, False): + expected = [0, 0] + if relative: + for i in range(0, length, width): + expected[(i // width) % 2] += int.from_bytes( + data[first + i : first + min(length, i + width)], "little" + ) + else: + for i in range(first, first + length): + expected[(i // width) % 2] += data[i] << (8 * (i % width)) + self.assertEqual( + oracle.affine_slice( + mask - 3, 0x13579, first, length, bits, first if relative else None + ), + [v & mask for v in expected], + ) + + +class BuildProvenanceTests(unittest.TestCase): + def test_tarball_does_not_inherit_enclosing_checkout_and_header_is_stable(self): + spec = importlib.util.spec_from_file_location( + "build_provenance", ROOT / "build-support/generate_provenance.py" + ) + generator = importlib.util.module_from_spec(spec) + spec.loader.exec_module(generator) + environment = { + "PROVENANCE_" + key: "test" for key in ("CXX", "CXXFLAGS", "TEST_CXXFLAGS", "ASFLAGS", "LDFLAGS") + } + with tempfile.TemporaryDirectory() as directory: + header = Path(directory) / "manifest.h" + + def command(args): + if args[:2] == ["git", "rev-parse"]: + self.assertEqual(args, ["git", "rev-parse", "--show-toplevel"]) + return str(ROOT.parent) + return None + + with mock.patch.object(generator, "command", side_effect=command), mock.patch.dict( + os.environ, environment, clear=True + ), mock.patch.object(sys, "argv", ["generator", str(header)]): + generator.main() + manifest = json.loads( + json.loads(header.read_text().split("#define LLM_BUILD_MANIFEST_JSON ")[1]) + ) + self.assertIsNone(manifest["git_commit"]) + self.assertIsNone(manifest["git_dirty"]) + self.assertEqual(manifest["status"], "partial") + before = header.stat().st_mtime_ns + generator.main() + self.assertEqual(header.stat().st_mtime_ns, before) + + def test_reader_enforces_structure_and_utf8_limits(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "input.json" + path.write_text("[" * 66 + "0" + "]" * 66) + with self.assertRaises(verifier.Rejected) as caught: + verifier.load(path) + self.assertEqual(caught.exception.reason, "unsupported-structure-limit") + path.write_bytes("{}".encode("utf-16")) + with self.assertRaises(UnicodeError): + verifier.load(path) + path.write_bytes(b" " * 101) + with mock.patch.object(verifier, "MAX_INPUT", 100), self.assertRaises( + verifier.Rejected + ) as caught: + verifier.load(path) + self.assertEqual(caught.exception.reason, "unsupported-input-limit") + + +if __name__ == "__main__": + unittest.main() From 79ad48926851766f25f045839e43b061ee6a4923 Mon Sep 17 00:00:00 2001 From: timoheimonen Date: Sun, 6 Sep 2026 10:55:24 +0300 Subject: [PATCH 13/16] Complete LLM acceptance and simplify redundant contract tests --- README.md | 5 +- documents/LLM_MEMORY_PROFILE_WHITEPAPER.md | 4 +- documents/MANUAL.md | 6 +- .../console/messages/llm_memory_messages.cpp | 3 +- tests/test_llm_memory_contract.cpp | 31 ---------- tests/test_llm_metal_backend.cpp | 56 ++----------------- tests/test_messages.cpp | 3 +- 7 files changed, 18 insertions(+), 90 deletions(-) diff --git a/README.md b/README.md index 0872c78..846e9c5 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,9 @@ memory_benchmark --llm-memory --weight-size-mb 64 --layers 4 \ --kv-layout paged --kv-block-tokens 16 --iterations 1 --count 3 --seed 42 ``` +Query heads classify the model; they do not add executed attention math. The prefill query tile +defines synthetic prefix rereads, not a real inference kernel’s cache or SRAM tiling. + Run one full-prompt prefill operation per scenario with two-token attention query tiles: ```bash @@ -405,7 +408,7 @@ make coverage-unit make coverage-all ``` -See [CONTRIBUTING.md](documents/CONTRIBUTING.md) for contribution guidance and [Project Structure](documents/PROJECT_STRUCTURE.md) for +See [CONTRIBUTING.md](CONTRIBUTING.md) for contribution guidance and [Project Structure](documents/PROJECT_STRUCTURE.md) for repository navigation and the current test-suite map. C++ reference documentation can be generated with `make docs`. CPU LLM results retain the original Mach tick boundaries and timebase for duration reconstruction. diff --git a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md index 92112ff..87a4e82 100644 --- a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md +++ b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md @@ -1100,8 +1100,8 @@ component cohorts must not be pooled as one distribution. The console identifies backend, phase/work unit, KV layout, phase geometry, warm/cacheable semantics, exact weight/KV-read/KV-write bytes, and up to one measured headline per scenario. Decode prints context and crossover; -prefill prints P/Q/C, prefix visits, causal pairs, and logical attention/FMA audit counts. It uses phase-specific labels -such as `ms/decode step` or `ms/prefill operation`; JSON remains backend-neutral with +prefill prints P/Q/C, prefix visits, causal pairs, and theoretical logical attention/FMA model-context counts. +It uses phase-specific labels such as `ms/decode step` or `ms/prefill operation`; JSON remains backend-neutral with `synthetic_work_unit_latency_seconds`, `synthetic_memory_work_units_per_second`, and `effective_model_payload_gb_s`. Metal task output uses `kv_write=valid|invalid|not-evaluated|not-applicable`, summarizing the phase-specific named JSON checks. The report never uses bare `tokens/s` and diff --git a/documents/MANUAL.md b/documents/MANUAL.md index 6d65aa2..01f653e 100644 --- a/documents/MANUAL.md +++ b/documents/MANUAL.md @@ -304,9 +304,9 @@ KV read / prefill operation = B * S(P,Q) * K Prefill payload is `W`, `B*(P+S(P,Q))*K`, or `W+B*(P+S(P,Q))*K` for weights-only, KV-only, or mixed. Each owner writes its prompt tokens in ascending order, K then V for each token, before that owner's reads; no global worker barrier is implied. Each increasing tile then reads the complete owned K prefix followed by the complete owned V prefix. `Q=P` -scans one full prefix; `Q=1` gives `S=triangular(P)`. Reported causal-pair and logical attention/FMA counts are audit -metadata, not executed compute. The intended checksum accumulation includes each tile-read visit; equality alone -does not establish every executed visit. For a KV-bearing CPU task, excluded +scans one full prefix; `Q=1` gives `S=triangular(P)`. Reported causal-pair and logical attention/FMA counts are theoretical +model context, not executed compute or measured memory traffic. The intended checksum accumulation includes each +tile-read visit; equality alone does not establish every executed visit. For a KV-bearing CPU task, excluded post-validation checks each owner's deterministic first/middle/last canonical-word samples, including bytes clipped to owner boundaries, against the final operation ordinal `T-1`. For a KV-bearing Metal prefill task, it checks representative/boundary byte locations per layer/batch sequence in both K and V against that ordinal; paged Metal diff --git a/src/output/console/messages/llm_memory_messages.cpp b/src/output/console/messages/llm_memory_messages.cpp index 210db5a..16d57a4 100644 --- a/src/output/console/messages/llm_memory_messages.cpp +++ b/src/output/console/messages/llm_memory_messages.cpp @@ -123,7 +123,7 @@ std::string llm_memory_usage_options(const std::string& prog_name) { << " --layers Required transformer layer count.\n" << " --query-heads \n" << " Required query-head count; must be at least as large as KV heads\n" - << " and divisible by them.\n" + << " and divisible by them. Model classification, not executed attention.\n" << " --kv-heads Required physical KV-head count.\n" << " --head-dim Required elements per K or V head vector.\n" << " --kv-element-bytes <1|2|4>\n" @@ -137,6 +137,7 @@ std::string llm_memory_usage_options(const std::string& prog_name) { << " Rejected for decode.\n" << " --attention-query-tile-tokens \n" << " Required only for prefill; query tile Q, 1 <= Q <= P.\n" + << " Defines synthetic prefix rereads, not inference-kernel tiling.\n" << " Rejected for decode.\n" << " --kv-layout \n" << " KV storage layout (default: contiguous). CPU and Metal support both layouts.\n" diff --git a/tests/test_llm_memory_contract.cpp b/tests/test_llm_memory_contract.cpp index b1f66df..5e9e9c0 100644 --- a/tests/test_llm_memory_contract.cpp +++ b/tests/test_llm_memory_contract.cpp @@ -700,37 +700,6 @@ std::string sha256_little_endian_entries( return encoded; } -std::string sha256_text(std::string_view input) { - CC_SHA256_CTX context; - if (CC_SHA256_Init(&context) != 1) { - return {}; - } - constexpr size_t kBytesPerUpdateCap = 4096; - size_t offset = 0; - while (offset < input.size()) { - const size_t byte_count = - std::min(kBytesPerUpdateCap, input.size() - offset); - if (CC_SHA256_Update(&context, input.data() + offset, - static_cast(byte_count)) != 1) { - return {}; - } - offset += byte_count; - } - - std::array digest{}; - if (CC_SHA256_Final(digest.data(), &context) != 1) { - return {}; - } - constexpr char kLowercaseHex[] = "0123456789abcdef"; - std::string encoded(digest.size() * 2, '0'); - for (size_t index = 0; index < digest.size(); ++index) { - encoded[index * 2] = kLowercaseHex[digest[index] >> 4U]; - encoded[index * 2 + 1] = - kLowercaseHex[digest[index] & 0x0fU]; - } - return encoded; -} - std::vector contiguous_segment_lengths(uint64_t logical_bytes) { std::vector lengths; uint64_t remaining = logical_bytes; diff --git a/tests/test_llm_metal_backend.cpp b/tests/test_llm_metal_backend.cpp index 1c6ec3e..0e99d23 100644 --- a/tests/test_llm_metal_backend.cpp +++ b/tests/test_llm_metal_backend.cpp @@ -3021,57 +3021,11 @@ TEST(LlmMetalBackendTest, } TEST(LlmMetalBackendTest, - PrefillContiguousMslSourceLocksFullPromptThenPerTileKThenVContract) { - const std::string_view source = LlmMetalKernelContract::kSource; - EXPECT_EQ(canonical_llm_metal_kernel_source_sha256(), - kCanonicalKernelSourceSha256); - EXPECT_NE(source.find("#if LLM_METAL_PREFILL_CONTIGUOUS"), - std::string_view::npos); - EXPECT_NE(source.find("llm_metal_prefill_contiguous_weights_only"), - std::string_view::npos); - EXPECT_NE(source.find("llm_metal_prefill_contiguous_kv_only"), - std::string_view::npos); - EXPECT_NE(source.find("llm_metal_prefill_contiguous_mixed"), - std::string_view::npos); - EXPECT_NE(source.find("llm_metal_validate_prefill_contiguous_writes"), - std::string_view::npos); - - const size_t run_start = source.find("inline void run_prefill_kv"); - const size_t run_end = source.find( - "kernel void llm_metal_prefill_contiguous_weights_only", run_start); - ASSERT_NE(run_start, std::string_view::npos); - ASSERT_NE(run_end, std::string_view::npos); - const std::string_view run = source.substr(run_start, run_end - run_start); - const size_t token_loop = run.find( - "for (ulong prompt_token = 0ul; prompt_token < params.prompt_tokens;"); - const size_t write_key = run.find("write_prefill_key_range", token_loop); - const size_t write_value = run.find("write_prefill_value_range", write_key); - const size_t remaining = run.find( - "ulong remaining_tokens = params.prompt_tokens", write_value); - const size_t tile_loop = run.find("while (remaining_tokens != 0ul)", - remaining); - const size_t remaining_distance = run.find( - "min(params.attention_query_tile_tokens,", tile_loop); - const size_t scan_key = run.find("scan_key_range", tile_loop); - const size_t scan_value = run.find("scan_value_range", scan_key); - ASSERT_NE(token_loop, std::string_view::npos); - ASSERT_LT(token_loop, write_key); - ASSERT_LT(write_key, write_value); - ASSERT_LT(write_value, remaining); - ASSERT_LT(remaining, tile_loop); - ASSERT_LT(tile_loop, remaining_distance); - ASSERT_LT(remaining_distance, scan_key); - ASSERT_LT(scan_key, scan_value); - EXPECT_EQ(run.find("(tile_ordinal + 1ul) *"), std::string_view::npos); - EXPECT_EQ(run.find("threadgroup_barrier(mem_flags::mem_device)"), - std::string_view::npos); - EXPECT_NE(source.find( - "const ulong remainder = first_vector % ulong(grid_size)"), - std::string_view::npos); - EXPECT_NE(source.find( - "return first_vector + delta;"), - std::string_view::npos); - + PrefillContiguousSemanticTraceWritesFullPromptBeforePerTileKThenV) { + // This freezes the bounded schedule oracle. Actual MSL pipeline/layout, + // Q=1/Q=P/tail execution, and corruption rejection have real-device tests; + // the separate canonical source hash retains provenance. A trace alone + // does not attest the GPU's temporal access order. const LlmPrefillPlan plan = resolve_llm_prefill_plan( {64, 2, 1, 1, 1, 1, 1, 4, 0}); ASSERT_TRUE(plan.valid) << plan.reason_code; diff --git a/tests/test_messages.cpp b/tests/test_messages.cpp index 968f68d..36a0463 100644 --- a/tests/test_messages.cpp +++ b/tests/test_messages.cpp @@ -216,7 +216,7 @@ TEST(MessagesTest, LlmMemoryCliMessagesHaveExactOutput) { " --layers Required transformer layer count.\n" " --query-heads \n" " Required query-head count; must be at least as large as KV heads\n" - " and divisible by them.\n" + " and divisible by them. Model classification, not executed attention.\n" " --kv-heads Required physical KV-head count.\n" " --head-dim Required elements per K or V head vector.\n" " --kv-element-bytes <1|2|4>\n" @@ -231,6 +231,7 @@ TEST(MessagesTest, LlmMemoryCliMessagesHaveExactOutput) { " Rejected for decode.\n" " --attention-query-tile-tokens \n" " Required only for prefill; query tile Q, 1 <= Q <= P.\n" + " Defines synthetic prefix rereads, not inference-kernel tiling.\n" " Rejected for decode.\n" " --kv-layout \n" " KV storage layout (default: contiguous). CPU and Metal support both layouts.\n" From 45c38557b2a57b89986105e26281952ab6b1ef58 Mon Sep 17 00:00:00 2001 From: timoheimonen Date: Sun, 6 Sep 2026 11:37:58 +0300 Subject: [PATCH 14/16] docs: update version: 0.64.0 --- CHANGELOG.md | 21 ++- CITATION.cff | 4 +- README.md | 2 +- documents/API.md | 2 +- documents/CAPABILITIES.md | 75 +++++++--- documents/GPU_BANDWIDTH_WHITEPAPER.md | 8 +- documents/LLM_MEMORY_PROFILE_WHITEPAPER.md | 14 +- documents/MANUAL.md | 14 +- documents/PARAMETER_MATRIX.md | 28 ++-- documents/PROJECT_STRUCTURE.md | 32 +++-- documents/TECHNICAL_SPECIFICATION.md | 157 ++++++++++++++------- src/core/config/version.h | 2 +- 12 files changed, 246 insertions(+), 113 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5664078..0f72aaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,26 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.63.1] - Unreleased +## [0.64.0] - Unreleased + +### Added + - **Independent LLM artifact verifier**: Added `script-examples/verify_llm_result.py` for all eight current CPU/Metal decode/prefill and contiguous/paged profiles. Independent Python arithmetic checks geometry, work, checksums, timing, rates, accepted sample populations, and statistics. Optional `--binary` checks the executable file's SHA-256, and `--require-raw-timing` requires original CPU timing evidence. Verdicts distinguish consistent accepted results, inconsistent or unaccepted results, and unsupported evidence or resource limits. Added `make test-llm-verifier` to the aggregate `make test-all` gate. + - **LLM build provenance and original CPU timing**: Results now retain a build manifest with available Git revision and dirty state, compiler, build flags, architecture, SDK, deployment target, and executable SHA-256. CPU measurements and excluded attempts retain original Mach start/stop/delta ticks and timebase for elapsed-time reconstruction. Python 3 is now required for build provenance generation; missing evidence remains explicitly unavailable or partial. + - **CPU and Metal checksum fault coverage**: Added independent arithmetic goldens, corruption and work-mutation tests, real-GPU reduction-boundary tests, and wider prefill boundary coverage. Documented profile-specific checksum collisions and final-state sampling limits, with acceptance validation across all eight profiles, file/stdout output, graceful interruption, and persistent multi-GiB Metal resources. ### Changed - - **CLI help updated**. - - **Test coverage cleaned up**: Removed redundant tests. + - **LLM output advances to JSON schema 2 and v2 methodologies**: All eight profiles now use `llm-memory-v2---`. Canonical scenario plans and expected checksums are stored once, measurements and calibration attempts reference those plans, and aggregates identify their accepted measurements. Geometry, model context, layout, resources, and component identities have explicit owners, with strict reference and null/status semantics. The API documents the schema-1-to-schema-2 field map; no compatibility aliases or fallback reader are provided. + - **LLM validation retains separate observations**: Named checks now preserve applicability, completion, validity, and reasons for structure, final KV writes or unchanged append state, and applicable padding checks. A checksum match cannot hide a failed final-state check, and unevaluated checks remain unresolved instead of appearing successful. + - **LLM file checkpoints are bounded by loop count**: Files now receive progress snapshots every `K=max(1,ceil(count/8))` completed loops, normally at most eight progress snapshots plus one terminal snapshot. Abrupt termination can lose up to `3K` completed attempts since the last successful snapshot. Exact `--output -` prepares only the terminal document, and disabled output skips JSON construction. Writer counters report actual prior persistence attempts; checkpoint failures remain terminal and are not retried. + - **LLM result collection avoids repeated retained data and statistics work**: Calibration keeps compact actual checksum evidence, canonical plans own expected values, and exact statistics are prepared at snapshot boundaries from accepted measurement IDs. Memory admission accounts for retained evidence, canonical-plan storage, statistics scratch, and simultaneous JSON construction and serialization peaks. + - **CPU task preparation validates inputs once per call**: Removed duplicate plan/resource validation within expected-checksum calculation while retaining fresh validation of borrowed plans and materialized resources before each executor call starts workers or timing. + - **LLM acceptance and comparison quality are separate**: `results_complete` describes the measured population, while `run_accepted` also requires valid execution evidence and no known command or checkpoint failure. Position balance, sample count, CV, duration, and environment remain separate comparison criteria. Documentation clarifies that cyclic order balances scenario positions without guaranteeing predecessor-pair balance, and that comparisons must account for conditioning and output cadence. + - **CLI help and reference documentation updated**: Refreshed general and LLM help, the machine API, manual, whitepaper, and supporting references for schema 2, verification, checkpoint behavior, and comparison requirements. Clarified the synthetic meaning of query heads, prefill tiles, and theoretical attention quantities. + - **Test coverage cleaned up**: Removed redundant contract tests and replaced brittle source-text assertions with focused semantic coverage. Hardware-dependent LLM cases are classified as integration tests, and the Makefile now builds Objective-C++ test sources with test flags and ARC. + +### Fixed + - **CPU contiguous decode validates final KV append bytes**: After timing stops and workers join, KV-bearing scenarios now verify every final K/V append byte against the expected pattern. Corruption produces an invalid measurement with retained diagnostics and cannot enter accepted aggregates, even when the timed checksum matches. + - **Theoretical prefill overflow no longer rejects valid memory work**: Overflowing model-context attention-pair or FMA quantities become nullable values with an explicit arithmetic-overflow reason. Exact byte, work, allocation, and execution guardrails remain enforced. ## [0.63.0] - 2026-08-23 diff --git a/CITATION.cff b/CITATION.cff index 1690a3b..3bd21a7 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -9,8 +9,8 @@ authors: family-names: "Heimonen" orcid: "https://orcid.org/0009-0004-0023-2407" -version: "0.63.0" -date-released: 2026-08-23 +version: "0.64.0" +date-released: 2026-09-06 repository-code: "https://github.com/timoheimonen/macOS-memory-benchmark" url: "https://github.com/timoheimonen/macOS-memory-benchmark" diff --git a/README.md b/README.md index 846e9c5..9ce3a9b 100644 --- a/README.md +++ b/README.md @@ -374,7 +374,7 @@ recognizes the current console labels only and is neither JSON-schema nor histor - [TLB Analysis Whitepaper](documents/TLB_ANALYSIS_WHITEPAPER.md): paired analysis, boundary rules, confidence model, and JSON verification contract. - [Core-to-Core Whitepaper](documents/CORE_TO_CORE_WHITEPAPER.md): LDAR/STLR handoff protocol, scheduler-hint scenarios, and JSON schema. - [GPU Bandwidth Whitepaper](documents/GPU_BANDWIDTH_WHITEPAPER.md): Metal methodology, timing, validation, resource model, and interpretation limits. -- [LLM Memory Profile Whitepaper](documents/LLM_MEMORY_PROFILE_WHITEPAPER.md): generic schema-v1 vocabulary plus the +- [LLM Memory Profile Whitepaper](documents/LLM_MEMORY_PROFILE_WHITEPAPER.md): schema-2 vocabulary plus the active CPU and Metal decode/prefill traffic, timing, checksum, and interpretation contracts. - [Apple M5 LLM CPU-decode working-set samples](results/0.63.0/AppleM5_LLM_working_set_scaling.md): two complete 0.63.0 JSON runs and their observed working-set scaling. diff --git a/documents/API.md b/documents/API.md index 657cf13..9a06f92 100644 --- a/documents/API.md +++ b/documents/API.md @@ -170,7 +170,7 @@ a sequence of checkpoint documents. For LLM, let `N=planned_loops` and `K=max(1,ceil(N/8))`, calculated without addition overflow. File targets write progress after every Kth fully completed loop, at most eight times, and one command-terminal snapshot on -success, graceful interruption or representable failure. The current unmerged cadence gives at most 4/7/9 normal +success, graceful interruption or representable failure. The current cadence gives at most 4/7/9 normal snapshots for N=3/12/48. One late command exception after successful terminal persistence may write one corrective failure snapshot; a failed checkpoint is terminal and never retried or followed by a disguised final write. diff --git a/documents/CAPABILITIES.md b/documents/CAPABILITIES.md index 83ecff1..d25f46e 100644 --- a/documents/CAPABILITIES.md +++ b/documents/CAPABILITIES.md @@ -40,11 +40,11 @@ CPU and GPU GB/s should not be compared as if they were the same workload: their ## Synthetic LLM Memory Profile Standalone `--llm-memory` uses generic backend/phase/layout/work-unit vocabulary. Eight profiles are active: CPU and -Metal each support decode and prefill with contiguous or paged KV. Exact methodologies are -`llm-memory-v1-cpu-decode-contiguous`, `llm-memory-v1-cpu-decode-paged`, -`llm-memory-v1-cpu-prefill-contiguous`, `llm-memory-v1-cpu-prefill-paged`, -`llm-memory-v1-metal-decode-contiguous`, `llm-memory-v1-metal-decode-paged`, -`llm-memory-v1-metal-prefill-contiguous`, and `llm-memory-v1-metal-prefill-paged`. Metal is explicitly selected with +Metal each support decode and prefill with contiguous or paged KV. Results use JSON schema 2. Exact methodologies are +`llm-memory-v2-cpu-decode-contiguous`, `llm-memory-v2-cpu-decode-paged`, +`llm-memory-v2-cpu-prefill-contiguous`, `llm-memory-v2-cpu-prefill-paged`, +`llm-memory-v2-metal-decode-contiguous`, `llm-memory-v2-metal-decode-paged`, +`llm-memory-v2-metal-prefill-contiguous`, and `llm-memory-v2-metal-prefill-paged`. Metal is explicitly selected with `--llm-memory-backend metal`, rejects `--threads`, and never receives an implicit CPU fallback. All four active Metal profiles are governed by runtime capability checks. @@ -131,8 +131,11 @@ semantic visit kind, and work-unit ordinal. Decode current-token writes or prefi padding canaries, are validated after each task. Metal paged prefill therefore requires both full-prompt write validity and applicable terminal-padding validity before publishing a measured result. Permutation generation, initialization, expected-checksum construction, and post-validation remain outside the -synchronized CPU timing interval or Metal GPU interval. Metal publishes the generic `kv_write_evaluated` and -`kv_write_valid` validation fields for both phases. +synchronized CPU timing interval or Metal GPU interval. Both backends publish independent observations in +`execution.validation.checks[]`, with named structure, phase/scenario-specific write, and padding checks. Each check +records applicability, evaluation, validity, and a reason; every applicable check must be evaluated and valid for a +measurement to be accepted. These bounded checks and the timed checksum do not certify the complete memory-access +trace or prove that every possible corruption would be detected. The reported decimal GB/s is exact logical effective model payload divided by backend-authoritative elapsed time: synchronized worker time for CPU or `GPUStartTime`/`GPUEndTime` for Metal. A synthetic work unit is one `decode_step` @@ -148,18 +151,49 @@ resources reduce the risk of accidentally benchmarking a recycled proxy buffer, service. The three scenarios are independently calibrated when `--iterations` is omitted and then frozen before loop zero. -Their order rotates across count loops; the default count of three gives each scenario one first, middle, and last -position. Only measured, validly timed, checksum-accepted records enter aggregates. A file target receives an atomic -checkpoint after each terminal scenario measurement and at command terminal; exact `--output -` performs the same -logical state transitions but emits only one final schema 1 document. See the -[LLM Memory Profile Whitepaper](LLM_MEMORY_PROFILE_WHITEPAPER.md) for formulas, timing, validation, and interpretation. +Canonical frozen-plan warmups run once before loop zero. Scenario order rotates across count loops; the default count +of three gives each scenario one first, middle, and last position. This balances positions, not directed predecessor +pairs or carryover effects across scenario and loop boundaries. Only measured records with accepted timing, checksum, +applicable validation checks, and backend completion evidence enter aggregates. Each scenario's +`accepted_measurement_ids` defines the shared population for all its metrics; rates are calculated per measurement +before aggregation. + +For file output, let `K=max(1,ceil(count/8))`. An atomic progress snapshot is written after every Kth fully completed +loop, at most eight times, followed by a command-terminal snapshot on success, graceful interruption, or representable +failure. An abrupt exit can lose up to `3K` completed measurement attempts since the last successful snapshot. +A failed checkpoint is terminal and is not retried. Exact `--output -` skips intermediate snapshot preparation, +preserves task-boundary stop observations, and emits only one final schema 2 document. + +Schema 2 stores canonical plans and expected checksums once, with measurement and calibration references to those +plans. `build_manifest` records available build-time Git, compiler, flags, target, SDK, and minimum-OS provenance, +plus the executable file's SHA-256 when available. CPU `execution.timing.cpu_raw` retains original Mach start, stop, +and delta ticks and the timebase for independent elapsed-time reconstruction; unavailable evidence is null. +Metal retains GPU timestamps. Build hashes bind reported content to a file; they do not authenticate execution. + +The independent [LLM artifact verifier](../script-examples/verify_llm_result.py) checks the eight current schema-2 +profiles using independent arithmetic for geometry, work, checksums, timing, rates, accepted populations, and +statistics. Run `python3 script-examples/verify_llm_result.py result.json`; optional `--require-raw-timing` requires +CPU raw timing evidence and `--binary PATH` checks the supplied file's hash without executing it. The verdict +separates artifact consistency from run acceptance. Unsupported profiles, unavailable required evidence, and resource +limits produce an unsupported verdict, not successful verification. An artifact cannot establish the original process +exit status or prove physical DRAM service. See the [Machine-Readable CLI API](API.md#independent-llm-artifact-verifier) +for limits and exit codes. + +`results_complete` requires every planned measurement to be measured. `run_accepted` additionally requires complete +run status and no known checkpoint or command failure, with required timing, checksum, validation, and backend +completion/lifecycle evidence accepted. Acceptance is independent of position balance, sample count, CV, duration, +and environment: a correct count-one run can be accepted while `scenario_order_balance_complete` is false. +Retain the process exit status and terminal context as well as the artifact; an older snapshot cannot report a later +failure. Comparisons require matching backend/phase/layout, schema/methodology and component identities, model/phase geometry, -fixed or automatic work policy, frozen work-plan identity, software, hardware, applicable worker counts, and -sufficiently similar thermal/power/load conditions. A comparative consumer must also require complete, -position-balanced schema state, every planned measurement to be measured, and the selected metric to be non-null; -process success alone is insufficient. Paged cohorts additionally require matching block size, physical/padding/table -geometry, and permutation identity/hash; contiguous and paged samples are not interchangeable. +fixed or automatic work policy, frozen work-plan identity, seeds, software, hardware, applicable worker counts, +conditioning, and output/checkpoint cadence, with sufficiently similar thermal/power/load conditions. Require accepted +runs and non-null selected metrics; inspect sample count, variability, duration, and environment separately. A +position-balanced comparison additionally requires `scenario_order_balance_complete`. Paged cohorts require matching +block size, physical/padding/table geometry, and permutation identity/hash; contiguous and paged samples are not +interchangeable. See the [LLM Memory Profile Whitepaper](LLM_MEMORY_PROFILE_WHITEPAPER.md) for formulas, timing, +validation boundaries, and interpretation. ## Memory and Cache Latency @@ -208,7 +242,7 @@ See the [Core-to-Core Whitepaper](CORE_TO_CORE_WHITEPAPER.md) for the assembly p Built-in sweeps execute supported parameter lists without shell orchestration. Standard and pattern modes can sweep buffer size and thread count. Standard latency also supports cache size, stride, locality, and chain-mode sweeps; TLB mode supports its stride, chain-mode, and density controls; core-to-core mode supports loop count and sample depth. -Multiple sweep options form a Cartesian product, and combined output requires `--output`. GPU schema 1 and LLM schema 1 +Multiple sweep options form a Cartesian product, and combined output requires `--output`. GPU schema 1 and LLM schema 2 do not support sweeps. JSON is designed as auditable evidence, not merely a list of numbers. It preserves the resolved configuration, work and seed identity, measurements, statistics, status, and enough completion information to distinguish complete, partial, interrupted, and unavailable results. Missing measurements are nullable rather than represented by numeric zero. @@ -217,8 +251,9 @@ Every result-producing direct mode and the CPU modes' supported sweeps reserve t machine-readable output: stdout contains one final JSON document and the post-parse human transcript is routed to stderr. An empty output value disables JSON for a direct command but is missing/invalid for a sweep. Every other non-empty value is a file target, including `./-` and flag-shaped names such as `-G`. File output is atomic; standard -commands, sweeps, GPU, and LLM retain their mode-specific checkpoints, while stdout remains final-only. GPU and LLM -stdout execute their logical checkpoint transitions and stop observations without intermediate serialization. The +commands, sweeps, GPU, and LLM retain their mode-specific checkpoints, while stdout remains final-only. GPU stdout +retains logical checkpoint transitions without intermediate serialization; LLM stdout skips intermediate snapshot +preparation while preserving task-boundary stop observations. The current support matrix and process acceptance procedure are in the [Machine-Readable CLI API](API.md). Current standard results use schema 3, which requires `configuration.mode: "benchmark"`, a string diff --git a/documents/GPU_BANDWIDTH_WHITEPAPER.md b/documents/GPU_BANDWIDTH_WHITEPAPER.md index 0037ab3..218c5b9 100644 --- a/documents/GPU_BANDWIDTH_WHITEPAPER.md +++ b/documents/GPU_BANDWIDTH_WHITEPAPER.md @@ -135,6 +135,9 @@ The Objective-C++ backend is auto-discovered, compiled with `-fobjc-arc`, and li No third-party production dependency is added. CommonCrypto SHA-256 comes from the platform/libSystem API and needs no additional framework link. Coverage includes production `.mm` beside `.cpp`. +Python 3 is required for both release and test builds, including ordinary `make`: the Makefile runs +`build-support/generate_provenance.py` to generate build provenance before compiling objects. + Runtime admission requires: 1. `MTLCreateSystemDefaultDevice()` returns a device. @@ -667,8 +670,9 @@ make test-all ./memory_benchmark --gpu-bandwidth --help ``` -The aggregate `make test-all` gate requires Python 3; it runs the focused script-example entry test after all GTest -cases pass. `jq` is not required by this gate. +The aggregate `make test-all` gate runs all GTest cases, followed by `make test-script-examples` and +`make test-llm-verifier`. Both Python test drivers require Python 3; the latter checks the independent LLM schema-2 +artifact verifier. `jq` is not required by this gate. Real Metal tests may skip on an unsupported/no-device execution environment. That skip does not replace deterministic unsupported-path tests and does not create a performance-validation claim. diff --git a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md index 87a4e82..93588b0 100644 --- a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md +++ b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md @@ -58,7 +58,7 @@ They support MHA, GQA, and MQA geometry through explicit query- and KV-head coun count, and 1-, 2-, or 4-byte KV elements. Paged KV adds timed table indirection, deterministic physical scatter, block-granular ownership, and full-block suffix padding. -Schema-v1 vocabulary includes `cpu|metal`, `decode|prefill`, `contiguous|paged`, +Schema-2 vocabulary includes `cpu|metal`, `decode|prefill`, `contiguous|paged`, `decode_step|prefill_operation`, and `none|current_token_append|full_prompt_population`. Contiguous and paged are public for both phases on both backends. The four Metal profiles never receive hidden fallback. @@ -1078,10 +1078,14 @@ Indexes, bounded small inputs and work units are JSON integers below 2^53, never seeds and checksums are canonical unsigned decimal strings (`0` or `[1-9][0-9]*`), with the relevant uint64/uint32 bounds. Known zero is not null. Unavailable/inapplicable values retain null plus applicability/status/reason semantics. -`build_manifest` reserves `manifest_version: 1`, `status: "unavailable"`, -`reason_code: "build-provenance-not-provided"`, and null `binary_sha256`, `git_commit`, `git_dirty`, `compiler`, -`compile_flags`, `link_flags`, `target_arch`, `sdk`, `min_os`. Optional CPU raw-tick evidence is not emitted in this -revision. Neither this reservation nor software/MSL version identity is independent build attestation. +`build_manifest` version 1 records build-time Git provenance, compiler, compile/link flags, target architecture, +SDK and deployment target. The command captures `binary_sha256` once before tasks. Missing build fields or an +unavailable binary hash produce partial evidence; a caller supplying no manifest retains `status: "unavailable"`, +`reason_code: "build-provenance-not-provided"`, and null provenance fields. +CPU measurement and excluded-attempt `execution.timing.cpu_raw` retain the original Mach `start_ticks`, `stop_ticks`, +`delta_ticks` and timebase numerator/denominator. Missing snapshots and Metal CPU-timing evidence are null; older +schema-2 artifacts may omit this optional field. See the [API contract](API.md#result-schemas-and-completion) for +availability and numeric rules. Neither manifest/file binding nor software/MSL identity is signed execution attestation. Consumer acceptance requires successful process outcome, exact mode/schema2/backend/phase/layout/v2 methodology, `run_policy_version: "llm-run-policy-bounded-loop-snapshots-v1"`, complete status, `results_complete: true`, diff --git a/documents/MANUAL.md b/documents/MANUAL.md index 01f653e..b0d40c1 100644 --- a/documents/MANUAL.md +++ b/documents/MANUAL.md @@ -1870,7 +1870,8 @@ head dimension 8, one-byte KV elements, context 32, block size 16, seed 42, one reference targets but displays only measurement ID 1 and selected nested fields. Geometry, plan identities, expected worker vectors, other measurements/calibration and full statistics are omitted. It is **not a complete acceptable artifact** and must not be passed to a result-acceptance predicate. The displayed checksum states are observed fixture -data; they do not independently prove the implementation. +data; they do not independently prove the implementation. The displayed unavailable `build_manifest` illustrates +the fallback for a caller supplying no manifest; normal CLI runs populate the available build and binary provenance. ```json { @@ -2056,8 +2057,15 @@ Schema 2 ownership and interpretation: and positive-mean CV are zero. Classification is `insufficient-samples` for n<3, `undefined` for undefined CV, `above-threshold` for payload CV strictly over 5%, otherwise `below-threshold`; equality is below-threshold. These descriptive percentiles are not confidence levels. Default console shows n/median/min/max/CV/MAD. -- `build_manifest` is currently an unavailable reservation with the exact keys shown above. Optional CPU raw ticks are - not emitted; software/version or MSL provenance alone is not build attestation or independent verification. +- `build_manifest` records build-time Git state, compiler, compile/link flags, target architecture, SDK and deployment + target. The command captures the executable's `binary_sha256` once before tasks. Missing build fields or a failed + binary read leave partial provenance with explicit reasons; a caller supplying no manifest retains the unavailable + fallback shown above. These fields bind available artifacts, not signed execution attestation. +- CPU measurements and excluded calibration attempts retain original Mach boundaries in `execution.timing.cpu_raw`: + `start_ticks`, `stop_ticks` and `delta_ticks` are unsigned decimal strings; `timebase_numer` and `timebase_denom` are + integer JSON numbers. A missing snapshot is null; older schema-2 artifacts may omit this optional field. Metal retains + GPU timestamps and null CPU raw evidence. See the [API contract](API.md#result-schemas-and-completion) for duration + reconstruction and provenance availability rules. For N planned loops, files snapshot every `K=max(1,ceil(N/8))` completed loops, at most eight progress snapshots plus terminal: maximum 4/7/9 normal snapshots for N=3/12/48. Abrupt termination can lose up to `3K` truly completed attempts, diff --git a/documents/PARAMETER_MATRIX.md b/documents/PARAMETER_MATRIX.md index d0d0671..d603dd8 100644 --- a/documents/PARAMETER_MATRIX.md +++ b/documents/PARAMETER_MATRIX.md @@ -17,14 +17,14 @@ Runtime platform: macOS 26 or later on Apple Silicon (ARM64). | — | `--llm-memory-backend` | `cpu\|metal` | LLM execution backend; default `cpu`; CPU and capability-gated Metal support decode/prefill with contiguous or paged KV | | — | `--weight-size-mb` | `` | Required positive active weight size for LLM-memory mode | | — | `--layers` | `` | Required positive LLM layer count | -| — | `--query-heads` | `` | Required positive query-head count; at least the KV-head count and divisible by it | +| — | `--query-heads` | `` | Required positive query-head count; at least the KV-head count and divisible by it; defines theoretical attention context | | — | `--kv-heads` | `` | Required positive physical KV-head count | | — | `--head-dim` | `` | Required positive K/V head-vector element count | | — | `--kv-element-bytes` | `1\|2\|4` | LLM KV element width; default `2` | | — | `--phase` | `decode\|prefill` | LLM phase; default `decode` | | — | `--context-tokens` | `` | Required only for decode; fixed visible context including the current synthetic token | | — | `--prompt-tokens` | `

` | Required only for prefill; positive full-prompt token count | -| — | `--attention-query-tile-tokens` | `` | Required only for prefill; query-tile size in `1..P` | +| — | `--attention-query-tile-tokens` | `` | Required only for prefill; query-tile size in `1..P` controls synthetic prefix rereads | | — | `--batch-size` | `` | Positive LLM batch-sequence count; default `1` | | — | `--kv-layout` | `contiguous\|paged` | LLM KV layout; default `contiguous` | | — | `--kv-block-tokens` | `` | Required only for paged KV: a positive power of two no greater than `UINT32_MAX`; may exceed the active phase length | @@ -42,7 +42,7 @@ Runtime platform: macOS 26 or later on Apple Silicon (ARM64). | `-W` | `--only-bandwidth` | — | Run only standard benchmark bandwidth tests; requires `--benchmark` | | `-L` | `--only-latency` | — | Run only standard benchmark latency tests; requires `--benchmark` | | `-u` | `--non-cacheable` | — | Apply best-effort cache-discouraging allocation hints; does not create truly uncached memory | -| `-o` | `--output` | `` | Result target syntax. Exact `-` selects one final stdout document for result-producing modes. An empty direct value disables JSON; every other non-empty value is a file. LLM file output checkpoints each terminal scenario measurement and command terminal | +| `-o` | `--output` | `` | Result target syntax. Exact `-` selects one final stdout document for result-producing modes. An empty direct value disables JSON; every other non-empty value is a file. LLM file output snapshots every `K=max(1,ceil(count/8))` completed loops and at command terminal | | `-S` | `--sweep` | `` | Add a Cartesian sweep parameter; repeat once per distinct key and use with `--output` | | `-X` | `--sweep-max-runs` | `` | Positive generated-run limit; default `256`, or `16` with `--analyze-tlb`; effective only with `--sweep` | | `-h` | `--help` | — | Show help; the standalone `--analyze-tlb` whitelist is the exception and rejects this combination | @@ -183,20 +183,21 @@ Decode requires exactly one `--context-tokens`; prefill requires exactly one `-- defaults to `contiguous`. Paged layout requires exactly one `--kv-block-tokens `; contiguous layout rejects that option. `G` must be a positive power of two no greater than `UINT32_MAX`, and `G` may exceed the active phase length. These rules are order-independent. After checked configuration -and memory-budget preflight, the command allocates the layout-specific resources and runs the three schema-1 scenarios. +and memory-budget preflight, the command allocates the layout-specific resources and runs the three scenarios. +LLM output uses schema 2 and `llm-memory-v2---` methodology identifiers. | Modifier | Compatible | Notes | |----------|------------|-------| | `--llm-memory-backend ` | ✅ | Default `cpu`. Both backends support decode/prefill with contiguous or paged KV. Metal performs no CPU fallback, reports capability absence as terminal `unsupported`, and reports runtime compiler/pipeline/resource/task failure as terminal `failed`/`invalid` evidence | | `--weight-size-mb ` | ✅ required | Positive active weight size; checked MiB-to-byte conversion | | `--layers ` | ✅ required | Positive layer count | -| `--query-heads ` | ✅ required | Positive; must be at least and evenly divisible by KV heads | +| `--query-heads ` | ✅ required | Positive; must be at least and evenly divisible by KV heads. Defines theoretical attention context; no attention computation is executed | | `--kv-heads ` | ✅ required | Positive physical KV-head count | | `--head-dim ` | ✅ required | Positive K/V head-vector element count | | `--phase ` | ✅ | Default `decode`; selects a versioned work-unit contract | | `--context-tokens ` | ✅ decode only | Positive fixed visible context including the current synthetic token | | `--prompt-tokens

` | ✅ prefill only | Positive full-prompt length; no default | -| `--attention-query-tile-tokens ` | ✅ prefill only | Required with prefill; `1 <= Q <= P`, with no default | +| `--attention-query-tile-tokens ` | ✅ prefill only | Required with prefill; `1 <= Q <= P`, with no default. Controls executed synthetic prefix rereads; it does not specify inference-kernel tiling | | `--kv-element-bytes <1\|2\|4>` | ✅ | Default `2`; every other width is rejected | | `--batch-size ` | ✅ | Positive; default `1` | | `--kv-layout ` | ✅ | Default `contiguous`; both layouts are executable for decode and prefill on CPU and Metal | @@ -205,12 +206,19 @@ and memory-budget preflight, the command allocates the layout-specific resources | `-i, --iterations ` | ✅ | Positive exact work units per scenario; omission selects excluded per-scenario calibration toward 150 ms. CPU values fit the common work/task guardrails; Metal additionally caps one dispatch at 65,536 work units. Metal prefill caps lane-local serial range-helper visits at 1,048,576 per task, including `T*L` for paged `weights_only`; paged Metal profiles also enforce semantic-lookup, owner-ordinal, threadgroup, and per-visit vector-iteration caps | | `-r, --count ` | ✅ | Positive cyclic loop count; default `3` | | `--seed ` | ✅ | Exact base seed including zero; a non-zero seed is generated once when omitted | -| `-o, --output ` | ✅ | Empty disables JSON; exact `-` emits one final schema 1 document; `./-`, flag-shaped values, and every other non-empty non-sentinel value are atomic file targets. A non-empty target adds the conservative JSON output peak to memory admission | +| `-o, --output ` | ✅ | Empty disables JSON; exact `-` emits one final schema 2 document; `./-`, flag-shaped values, and every other non-empty non-sentinel value are atomic file targets. A non-empty target adds the conservative JSON output peak to memory admission | | `-h, --help` | ✅ | Prints dedicated help before enforcing required inputs or resolving worker/seed/session/QoS state; malformed or duplicate tokens still fail | -| `--sweep`, `--sweep-max-runs`, `--non-cacheable` | ❌ | Outside the frozen v1 whitelist | +| `--sweep`, `--sweep-max-runs`, `--non-cacheable` | ❌ | Outside the standalone whitelist | | Buffer/cache/latency/TLB/pattern/core-to-core/GPU modifiers | ❌ | Outside the standalone whitelist | | Any other primary mode | ❌ | Primary modes are mutually exclusive | +File output writes an atomic progress snapshot after every `K=max(1,ceil(count/8))` fully completed loops +(at most eight progress snapshots), plus a command-terminal snapshot on success, graceful interruption, or representable +failure. An abrupt exit can lose up to `3K` completed measurement attempts since the last successful snapshot. A failed +checkpoint stops the command without retry; a late command exception after successful terminal persistence may write +one corrective failure snapshot. Stdout builds only the final document, and disabled output builds no JSON document. +See [API output checkpoints](API.md#checkpoints-and-final-snapshots) for the full persistence contract. + The parser rejects checked weight/KV geometry that overflows or makes even one scenario work unit exceed the 64 GiB task-accounted-byte ceiling. For paged KV, task-accounted bytes include logical model bytes plus timed block-table lookup traffic, while throughput remains model bytes divided by timed seconds. Before allocation, the planner separately admits @@ -232,6 +240,10 @@ publish no threadgroup-cost evidence. Full-prompt write samples and applicable p KV-active task. This is a cyclic assignment, not a weighted balance. The command does not fall back to another backend, phase, or layout. +Prefill theoretical causal-pair, attention-pair, and FMA counts are descriptive model context. Overflow in those +calculations produces JSON `null` with reason `arithmetic-overflow` and does not by itself reject an otherwise valid +workload. Actual byte, prefix-read, lookup, and allocation arithmetic must still pass checked validation. + ### Sweep Compatibility `--sweep` runs a Cartesian product over one or more parameter lists. It always requires a non-empty diff --git a/documents/PROJECT_STRUCTURE.md b/documents/PROJECT_STRUCTURE.md index a7777e2..ae7bbe4 100644 --- a/documents/PROJECT_STRUCTURE.md +++ b/documents/PROJECT_STRUCTURE.md @@ -45,6 +45,7 @@ This document describes the layout of project files, organized by purpose. It is | `Makefile` | Primary build system; discovers C++/Objective-C++/assembly, targets sources and final links at macOS 26.0, accepts a configurable `GTEST_DIR`, compiles `.mm` with ARC, links Metal/Foundation, and produces release/test binaries | | `coverage.sh` | Runs isolated LLVM unit/all-test C++/Objective-C++ source coverage builds under `/tmp` without replacing normal workspace binaries | | `.clang-format` | Clang-Format style baseline for C++ sources | +| `build-support/generate_provenance.py` | Python 3 build-time generator for `.build-provenance.h`; freezes Git, compiler, flags, target, SDK, and deployment-target metadata and updates the generated header only when inputs change | ### Root-level documentation @@ -70,14 +71,15 @@ This document describes the layout of project files, organized by purpose. It is | `documents/LATENCY_WHITEPAPER.md` | Whitepaper: cache and memory latency measurement methodology | | `documents/CORE_TO_CORE_WHITEPAPER.md` | Whitepaper: calibrated two-thread token-handoff methodology, audit schema, and interpretation limits | | `documents/GPU_BANDWIDTH_WHITEPAPER.md` | Whitepaper: Metal compute bandwidth methodology, GPU schema 1, validation, capability limits, and maintenance policy | -| `documents/LLM_MEMORY_PROFILE_WHITEPAPER.md` | Whitepaper: CPU and capability-gated Metal decode/prefill contiguous/paged methodologies, with exact payload formulas, resources, validation, schema 1, and interpretation limits | +| `documents/LLM_MEMORY_PROFILE_WHITEPAPER.md` | Whitepaper: CPU and capability-gated Metal decode/prefill contiguous/paged methodologies, with exact payload formulas, resources, validation, schema 2, checksum evidence limits, and interpretation limits | | `documents/PROJECT_STRUCTURE.md` | This file | --- ## 2. src/ — Source code -All production C++, Objective-C++, and ARM64 assembly lives under `src/`. Headers use include paths relative to `src/` +Production C++, Objective-C++, and ARM64 assembly lives under `src/`, except for the root entry point `main.cpp`. +Headers use include paths relative to `src/` (e.g., `#include "core/config/config.h"`). Metal/Objective-C types are confined to private `.mm` platform boundaries. --- @@ -207,7 +209,7 @@ Core infrastructure for configuration, memory management, macOS system introspec | File | Purpose | |---|---| -| `timer.h` / `.cpp` | High-resolution Mach timer with exact tick conversion plus a deterministic clock/timebase provider seam | +| `timer.h` / `.cpp` | High-resolution Mach timer with tick/timebase conversion, retained original start/stop/delta snapshots, and a deterministic clock/timebase provider seam | --- @@ -276,7 +278,7 @@ Objective-C++ Metal backend so deterministic unit tests do not require GPU work. ### 2.6 src/llm_memory/ — Synthetic LLM memory profile -Standalone generic schema-1 vocabulary with eight active backend/phase/layout profiles: four CPU profiles and four +Standalone schema-2 implementation with eight active backend/phase/layout profiles: four CPU profiles and four capability-gated Metal profiles spanning decode/prefill and contiguous/paged KV. Pure logical phase planning, deterministic paged geometry/permutation, backend-specific execution planning/evidence, the Objective-C-free backend contract, CPU mapping and ARM64 execution, the Objective-C++ Metal boundary, environment capture, console composition, @@ -295,10 +297,11 @@ hot kernels. Capability failures remain explicit, and no unsupported Metal reque | `llm_metal_backend.mm` | Objective-C++ Metal boundary for capability probing, selected-profile runtime compilation, Tier 2 argument encoding, private/shared allocation, table upload/validation, exact-tail dispatch, paged-prefill cyclic ownership, GPU timestamps, layout-aware dual-mod32 checksum, full-prompt K/V-write and padding validation, diagnostics, and idempotent cleanup | | `llm_metal_kernels_source.h` | Canonical embedded MSL 2.3 source with profile-specific decode/prefill contiguous/paged parameter ABIs, common foundation entrypoints, scenario-specialized workload entrypoints, named-lane volatile paged-table publication, and phase-specific K/V-write/padding validation; exact selected source bytes are hashed at runtime | | `llm_executor.h` / `.cpp` | CPU-specific full-size mappings, paged-table preparation, deterministic initialization/pre-touch, phase/layout descriptor materialization, independent checksum oracles, padding canaries, synchronized worker team, timer boundary, and ARM64 adapters retained behind `LlmCpuBackend` | -| `llm_runner.h` / `.cpp` | Backend-independent lifecycle, per-scenario automatic calibration or exact-work planning, generic task acceptance, frozen plans, cyclic loop order, status/counters, task-boundary interruption, aggregates, warnings, and logical checkpoints; resources are released before the command-terminal checkpoint | -| `llm_json.h` / `.cpp` | Ordered generic LLM schema-1 builder plus conservative output-peak estimator, with backend/phase/layout identity, resolved Metal segmentation, capability/resource/task evidence, decimal-string exact integers, nullable non-applicability, traffic diagnostics, environment evidence, and interpretation contract | +| `llm_runner.h` / `.cpp` | Backend-independent lifecycle, per-scenario calibration or exact-work planning, canonical/frozen plans, named-check task acceptance, cyclic position balance, status/counters, interruption, accepted measurement populations, snapshot-time statistics, and bounded loop/terminal checkpoints; correctness acceptance is separate from comparison quality, and resources are released before the command-terminal checkpoint | +| `llm_json.h` / `.cpp` | Ordered LLM schema-2 builder and conservative output-peak estimator; projects canonical plans and document-local references, expected versus observed checksums, named validation checks, accepted measurement IDs, raw timing, build provenance, backend/resource evidence, nullable states, and checkpoint writer observations | +| `llm_validation.h` | Fixed, allocation-free structure/write/padding observation slots with independent applicability, evaluation, validity, and stable reasons for existing cold validation walks | | `llm_output.h` / `.cpp` | Human-readable work-unit/model-payload/accounted geometry, Metal device/resource/task evidence, scenario headlines, interpretation limits, and evidence-backed quality warnings through centralized message helpers | -| `llm_environment.h` / `.mm` | Objective-C-free snapshot type plus macOS thermal-state and Low Power Mode capture through Foundation | +| `llm_environment.h` / `.mm` | Objective-C-free host snapshot, macOS thermal-state/Low Power Mode and physical-memory capture, and streaming SHA-256 of the executable file on the cold command path | --- @@ -346,7 +349,7 @@ bandwidth warm-up covers the full target buffer, and latency warm-up page-touche | `json_utils.h` / `.cpp` | JSON helper functions shared between the TLB, core-to-core, and standard output serializers | | `cyclic_order.h` / `.cpp` | Shared deterministic cyclic ordering used by CPU and GPU planners | | `seed_utils.h` / `.cpp` | Shared SplitMix64 derivation and generate-once seed helper with deterministic provider seam | -| `hash_utils.h` / `.cpp` | CommonCrypto-based SHA-256 helper used for exact embedded MSL source provenance | +| `hash_utils.h` / `.cpp` | CommonCrypto-based one-shot, incremental, and allocation-free SHA-256 helpers for MSL source, layout/identity evidence, and executable-file provenance | | `numeric_utils.h` / `.cpp` | Overflow-safe size arithmetic plus bounded pilot-count and duration-calibration helpers | | `descriptive_statistics.h` / `.cpp` | Canonical average, percentile, sample-deviation, coefficient-of-variation, and median-absolute-deviation calculations | @@ -365,7 +368,7 @@ bandwidth warm-up covers the full target buffer, and latency warm-up page-touche GoogleTest-based unit and integration tests are supplemented by Python example and LLM verifier tests. `make test-script-examples` exercises the current JSON-reading examples directly. `make test-llm-verifier` checks the independent schema-2 LLM reader, numeric goldens and semantic mutations. `make test-all` runs both after all GTest cases pass. Python 3 is required; `jq` is optional, and only its dedicated test branch is skipped when it is not -installed. All `.cpp` files are picked up automatically by the Makefile. Tests named `*Integration*` are excluded from +installed. All test `.cpp` and `.mm` files are picked up automatically by the Makefile. Tests named `*Integration*` are excluded from `make test` (unit-only) and run through the integration or all-test targets. | File | Suite name | Coverage focus | @@ -381,7 +384,8 @@ installed. All `.cpp` files are picked up automatically by the Makefile. Tests n | `test_llm_memory_work_plan.cpp` | `LlmMemoryWorkPlanTest` | Production checked decode/prefill geometry and payloads, physical/padding/table/lookup/accounted math including paged-prefill `N+2*M`, deterministic permutation/hash, CPU token/block ownership, Metal cyclic block-owner scheduling and component identities, memory budget, descriptor/range/layout invariants, worker reduction, scenario caps/calibration, and cyclic order | | `test_llm_memory_executor.cpp` | `LlmMemoryExecutorTest` | CPU phase/layout dispatch, atomic mapping/table-preparation failure seams, contiguous and paged decode/prefill descriptor materialization, independent checksums, padding canaries, synchronized timing, QoS, cancellation, and fake-kernel validation | | `test_llm_memory_runner.cpp` | `LlmMemoryRunnerTest` | Fake-backend lifecycle and generic decode/prefill task seams, lifecycle unsupported/failure handling, common identity/timing/completion/validation acceptance, exact calibration/single-unit/freeze/frozen-warmup order, cyclic measurements, status/counter/aggregate semantics, interruption, release/checkpoint precedence, auxiliary budgeting, and runner exception boundaries | -| `test_llm_memory_json.cpp` | `LlmMemoryJsonTest` | Schema-1 identity for all eight CPU/Metal profiles, exact nullable geometry/workers, paged layout/permutation, prefill geometry, CPU ownership, Metal cyclic owner/per-threadgroup cost evidence, segmentation, capability/resource/task K/V-write/lookup/padding evidence, output-peak, status, interpretation, environment, and checkpoints | +| `test_llm_memory_json.cpp` | `LlmMemoryJsonTest` | Schema-2 identities for all eight profiles, canonical plan references, expected/actual checksums, named validation, accepted populations, raw Mach timing, build manifest, nullable geometry/workers, paged and Metal resource/grid evidence, output-peak estimates, acceptance, environment, and bounded checkpoint observations | +| `test_llm_metal_checksum.mm` | `LlmMetalChecksumHelperIntegrationTest` | Real-device checks of shared MSL affine checksum collision boundaries and six-component modulo reduction across partial SIMD/threadgroups | | `test_llm_memory_output.cpp` | `LlmMemoryOutputTest` | Exact decode/prefill, contiguous/paged, and Metal device/resource/table/task K/V-write/lookup/padding plus cyclic owner/per-threadgroup cost formatting, interpretation text, and deduplicated warnings | | `test_llm_memory_kernels.cpp` | `LlmMemoryKernelIntegrationTest` | Real contiguous/paged decode/prefill ARM64 descriptor/kernel scenarios, paged tails/lookups, full-prompt writes, exact partial tiled scans, checksum, padding, worker-count, and AAPCS64 coverage | | `test_llm_metal_backend.cpp` | `LlmMetalBackendTest`, `LlmMetalBackendIntegrationTest`, `LlmMetalBackendFailureInjectionIntegrationTest` | Pure capability, contiguous/paged segmentation, argument-buffer, decode/prefill ABIs, source-hash and prefill loop-order audit, phase-aware checksum/reason mapping, paged-prefill `N+2*M`, cyclic no-duplication ownership and per-threadgroup accounted-byte evidence, plus real-device selected-profile compilation, Tier-2 slots, private table/K/V initialization, exact-tail decode, full-prompt/tiled-prefix prefill, final-ordinal write/padding detection, interruption/failure cleanup, and idempotent release | @@ -414,7 +418,7 @@ installed. All `.cpp` files are picked up automatically by the Makefile. Tests n | `test_statistics.cpp` | `StatisticsTest` | Standard multi-loop summary composition, mode filtering, loop/sample population separation, and rendered values | | `test_descriptive_statistics.cpp` | `DescriptiveStatisticsTest` | Canonical shared percentiles, deviation, CV, and MAD contracts | | `test_statistics_renderer.cpp` | `StatisticsRendererTest` | Shared console-summary ordering, precision, indentation, and diagnostics | -| `test_timer.cpp` | `HighResTimerTest` | Deterministic Mach-time conversion, creation failures, seconds/nanoseconds results, and tick wraparound | +| `test_timer.cpp` | `HighResTimerTest` | Deterministic Mach-time conversion, original boundary snapshots, creation failures, seconds/nanoseconds results, and tick wraparound | | `test_system_info.cpp` | `BenchmarkQosTest`, `SystemInfoTest`, `SystemInfoIntegrationTest` | Deterministic QoS and system-information success/failure contracts plus one real-provider integration smoke | | `test_tlb_chain.cpp` | `TlbChainTest` | Spread/packed planning, every traversal policy, explicit corruption statuses, and one ASM smoke | | `test_tlb_measurement_scheduler.cpp` | `TlbMeasurementSchedulerTest` | Seeded balance, stop/error boundaries, callback contracts, convergence, and exact pass accounting | @@ -422,13 +426,14 @@ installed. All `.cpp` files are picked up automatically by the Makefile. Tests n | `test_tlb_sweep_planner.cpp` | `TlbSweepPlannerTest` | Page-aligned base/refinement planning, stride bounds, deduplication, and source tracking | | `test_utils.cpp` | `SeedUtilsTest`, `ProgressSpinnerTest`, `UtilsTest` | Seed-provider behavior, TTY-gated spinner rendering/cleanup, and worker-thread joining | -### tests/fixtures/ — Script-example input fixtures +### tests/fixtures/ — Script-example and LLM verifier input fixtures | File | Purpose | |---|---| | `standard-schema-v3-complete-current.json` | Minimized structurally faithful current full standard result for the two plotter JSON entry paths | | `standard-schema-v3-custom-complete-current.json` | Minimized structurally faithful current custom-cache standard result for shell jq/Python extraction and stride/TLB summaries | -| `README.md` | Records source HEAD, capture commands, reduction policy, example ownership, and the current-only fixture refresh policy | +| `llm-schema-v2/` | Eight captured CPU/Metal × decode/prefill × contiguous/paged schema-2 profiles plus `unsupported.json` for independent verifier coverage | +| `README.md` | Records source HEAD, capture commands, reduction policy, example/verifier ownership, and fixture refresh policy | Volatile source/test counts and the authoritative generated inventory are maintained in `DRY_CHECK.md`. @@ -538,3 +543,4 @@ policy documented in their own row below. |---|---| | `.github/pull_request_template.md` | Default pull request description template | | `.github/ISSUE_TEMPLATE/bug_report.md` | Structured bug report template | +| `.github/workflows/pr-tests.yml` | Runs `make test-all` on `macos-26` after installing GoogleTest, for pushes to `development` and pull requests targeting `main` or `development` | diff --git a/documents/TECHNICAL_SPECIFICATION.md b/documents/TECHNICAL_SPECIFICATION.md index 1505b4a..e30bf4c 100644 --- a/documents/TECHNICAL_SPECIFICATION.md +++ b/documents/TECHNICAL_SPECIFICATION.md @@ -22,7 +22,7 @@ Out of scope: - `--analyze-core2core` methodology details (see [CORE_TO_CORE_WHITEPAPER.md](CORE_TO_CORE_WHITEPAPER.md)). - Detailed `--gpu-bandwidth` methodology and schema field catalog (see [GPU_BANDWIDTH_WHITEPAPER.md](GPU_BANDWIDTH_WHITEPAPER.md)). -- Detailed `--llm-memory` generic schema-v1 vocabulary, active decode/prefill methodologies, and field catalog (see +- Detailed `--llm-memory` schema-2 vocabulary, active decode/prefill methodologies, and field catalog (see [LLM_MEMORY_PROFILE_WHITEPAPER.md](LLM_MEMORY_PROFILE_WHITEPAPER.md)). - Historical behavior from older releases. @@ -32,8 +32,11 @@ Out of scope: - Target CPU architecture: ARM64 Apple Silicon. - Language: C++17 with ARM64 Apple Silicon assembly kernels, two Objective-C++ Metal backends, and one Objective-C++ environment-capture unit. -- Build: `Makefile` (`clang++`, `as`). -- Test framework: GoogleTest (`test_runner`). +- Build: `Makefile` (`clang++`, `as`), with Python 3 required to generate embedded build provenance. +- Test frameworks: GoogleTest (`test_runner`) and Python `unittest` for script examples and the independent LLM + artifact verifier. Objective-C++ test sources are compiled with test flags and ARC. +- Build provenance records Git revision/dirty state, compiler, target, SDK, deployment target, and build/link flags. + Make regenerates the manifest and invalidates objects when these inputs change; unavailable Git metadata remains null. - Deployment target: macOS 26.0 for production/test sources, assembly, and final links. External static archives must not require a newer OS. This is the oldest runtime family exercised by the current repeatable release gate. - First-party link dependencies: `-framework Metal -framework Foundation`; no new third-party production dependency. @@ -140,8 +143,10 @@ or deterministic paged KV: 5. Create the selected pure-C++ `LlmBackend` factory product. Build a preliminary pointer-free logical work plan and ask the backend and runner for their exact auxiliary backing, then add the conservative JSON DOM/serialization peak for a non-empty output target. Its checked variable-string term includes component/layout identities and all prefill - execution/scenario/scope identities. Frozen identities are charged once, while scenario-plan identities scale with - the maximum retained calibration attempts and planned measurement loops. Preliminary and finalized plans use the + execution/scenario/scope identities. Canonical scenario identities and expected checksums are retained once per + distinct plan; runtime evidence scales with planned measurements. Admission includes canonical-plan storage, expected reconstruction transients, accepted-ID + maps, exact-statistics workspaces, and the simultaneous DOM/serialization peak. Bounded snapshot frequency does not + reduce the single-snapshot peak allowance. Preliminary and finalized plans use the same canonical identity-size formula. Rebuild and admit the final plan with the full page-rounded weight/physical-K/physical-V mappings, paged table and validation transient when applicable, descriptor/planner storage, checksum storage, and @@ -162,11 +167,11 @@ or deterministic paged KV: backend calls. Each call owns reset, timed work, correctness validation, and generic task evidence. A Metal call uses one reset command buffer, one timed command buffer/serial encoder/workload dispatch with its T loop in-kernel, authoritative `GPUStartTime`/`GPUEndTime`, dual-mod32 checksum comparison, and excluded phase-neutral K/V-write - validation. Each terminal - measurement is offered to the logical checkpoint hook while resources remain live. + validation. Stop remains observable at each task boundary. File progress snapshots are offered only after every Kth + completed loop, with `K=max(1,ceil(count/8))`; exact aggregate statistics are prepared at snapshots and terminal finalization. 8. Release backend resources exactly once before offering the command-terminal checkpoint, then capture final environment state, render the centralized console report, and either retain the runner-owned atomic file cadence or - emit one final schema-1 stdout document. Execution status and output status remain separate. + emit one final schema-2 stdout document. Execution status and output status remain separate. Pre-run failures before runner-result initialization leave stdout empty. Once initialized, partial/interrupted/unsupported/failed evidence is representable; a file checkpoint failure is terminal and is not @@ -716,10 +721,10 @@ CPU adapter, an Objective-C++ Metal capability/resource/task boundary, phase/lay ordered schema builder, and a Foundation-backed environment snapshot. Generic enums cover backend `cpu|metal`, phase `decode|prefill`, and KV layout `contiguous|paged`. All eight backend/phase/layout combinations are active. The CPU identities are -`llm-memory-v1-cpu-decode-contiguous`, `llm-memory-v1-cpu-decode-paged`, -`llm-memory-v1-cpu-prefill-contiguous`, and `llm-memory-v1-cpu-prefill-paged`. The Metal identities are -`llm-memory-v1-metal-decode-contiguous`, `llm-memory-v1-metal-decode-paged`, -`llm-memory-v1-metal-prefill-contiguous`, and `llm-memory-v1-metal-prefill-paged`. Capability failure remains explicit; +`llm-memory-v2-cpu-decode-contiguous`, `llm-memory-v2-cpu-decode-paged`, +`llm-memory-v2-cpu-prefill-contiguous`, and `llm-memory-v2-cpu-prefill-paged`. The Metal identities are +`llm-memory-v2-metal-decode-contiguous`, `llm-memory-v2-metal-decode-paged`, +`llm-memory-v2-metal-prefill-contiguous`, and `llm-memory-v2-metal-prefill-paged`. Capability failure remains explicit; no Metal request receives fallback CPU execution. `LlmMemoryWorkPlan` keeps logical geometry, resources, accounting, seeds, and identities common while @@ -810,7 +815,10 @@ backend auxiliary estimate includes its checksum and orchestration backing in co The prefill planner resolves prompt length `P` and query-tile length `Q`. It computes `C = ceil(P/Q)`, prefix visits `S = Q*triangular(P/Q) + (P%Q != 0 ? P : 0)`, causal pairs, logical -attention audit counts, one weight pass, full-prompt K/V writes, tiled prefix reads, and checked per-scenario payloads. +attention context counts, one weight pass, full-prompt K/V writes, tiled prefix reads, and checked per-scenario payloads. +The theoretical causal-pair/attention/FMA context is nullable on arithmetic overflow; actual byte, prefix, lookup, and +allocation overflow still rejects the workload. Query heads affect theoretical context, while query tile Q affects +executed prefix reads. No attention or FMA computation is performed. For paged geometry it computes prefix block visits and the exact `N + 2*M` semantic lookup count with a logarithmic floor-sum. Its bounded semantic oracle freezes owner-local ascending-token writes with K then V for each token, followed for every tile by the complete owned K prefix and then the complete owned V prefix. Its affine64 checksum @@ -953,9 +961,11 @@ Omitted iterations calibrate the three scenarios independently outside measureme warmed once; correction candidates do not receive general warmups, while the first irreducible one-work-unit candidate receives one confirmation warmup when that shape has not already been warmed. Explicit iterations are exact. All three plans freeze atomically before canonical same-shape frozen warmups, and loop order then cyclically rotates weights-only, -KV-only, and mixed. No measured task begins until every frozen warmup succeeds. Only measured and checksum-valid results -enter aggregates. Stop is checked between atomic backend tasks. File output checkpoints each terminal scenario; -resources are released before the command-terminal checkpoint. Stdout emits one final schema-1 object. See +KV-only, and mixed. No measured task begins until every frozen warmup succeeds. Only measured results with accepted timing, checksum, +and required cold-check evidence enter aggregates. Stop is checked between atomic backend tasks. File progress snapshots occur after every Kth complete +loop, where `K=max(1,ceil(count/8))`; resources are released before the command-terminal snapshot. Stdout emits one +final schema-2 object. Scenario position balance is separate from run acceptance and does not balance directed +predecessor pairs across loop or repeated-block boundaries. Frozen warmups run once before the measured loops. See [LLM_MEMORY_PROFILE_WHITEPAPER.md](LLM_MEMORY_PROFILE_WHITEPAPER.md) for the complete formulas, ABI, schema, and interpretation contract. @@ -1130,7 +1140,7 @@ measurement schema. [API.md](API.md) is the process-integration contract and sup - GPU schema 1: top-level mode/schema/methodology/status, exact counters and completeness, effective/copy/DRAM semantics, config/argv, environment, backend device/compile/allocation, memory budget, frozen plans, excluded calibration, status-bearing measurements/loop records, aggregates, and warnings. -- LLM schema 1: top-level backend/phase/layout/methodology/status, `configuration`, generic `resolved_plan`, tagged +- LLM schema 2: top-level backend/phase/layout/methodology/status, `configuration`, generic `resolved_plan`, tagged `backend_evidence`, memory-budget/calibration evidence, status-bearing generic work-unit measurements, measured-only aggregates, and interpretation, plus additive exact traffic, checksum, loop/checkpoint, environment, and warning evidence. @@ -1175,10 +1185,12 @@ can coexist with command completeness without being consumable as a measured val also requires `affinity_hint_comparison_interpretable == true`. GPU requires `status == "complete" && results_complete == true && conclusions_valid == true`; position-balanced comparisons additionally require `operation_order_balance_complete == true`. -LLM requires `mode == "llm_memory" && schema_version == 1`, top-level backend/phase/layout equal to the request, -methodology equal to `llm-memory-v1---`, `status == "complete"`, `results_complete == true`, -`conclusions_valid == true`, and every planned measurement to be `measured`. A selected LLM metric additionally -requires non-null/checksum-accepted evidence; comparative validity already includes complete scenario-order balance. +LLM requires `mode == "llm_memory" && schema_version == 2`, top-level backend/phase/layout equal to the request, +methodology equal to `llm-memory-v2---`, `status == "complete"`, `results_complete == true`, +`run_accepted == true`, the run-policy identity `llm-run-policy-bounded-loop-snapshots-v1`, and every planned +measurement to be `measured`. A selected metric additionally requires accepted timing, checksum, and required +post-validation evidence. Run acceptance is independent of position balance, sample count, CV, duration, and environment; +comparison policy must assess those separately and match frozen work, seeds, component identities, and output cadence. ### 18.1 Configuration keys @@ -1259,24 +1271,26 @@ an exact software version; schema 2 and unversioned historical standard JSON are separate `timed_accumulator_algorithm` and `final_checksum_algorithm` identities plus expected/actual checksums. - See [GPU_BANDWIDTH_WHITEPAPER.md](GPU_BANDWIDTH_WHITEPAPER.md) for the complete consumer/maintenance contract. -### 18.6 LLM schema 1 +### 18.6 LLM schema 2 -- The prior CPU/step-specific schema-1 shape was unpublished. Generic v1 replaces it without compatibility aliases, - fallback reading, or a schema-version increment. +- Schema 2 replaces the unpublished schema-1 contract without compatibility aliases or fallback reading. - Required top-level fields are `schema_version`, `mode`, `backend`, `phase`, `kv_layout`, `methodology_version`, `software`, `configuration`, `resolved_plan`, `backend_evidence`, `memory_budget`, `calibration`, `measurements`, - `aggregates`, `status`, `reason_code`, `results_complete`, `conclusions_valid`, and `interpretation`. -- Methodology is exactly `llm-memory-v1---`. Active identities are - `llm-memory-v1-cpu-decode-contiguous`, `llm-memory-v1-cpu-decode-paged`, - `llm-memory-v1-cpu-prefill-contiguous`, `llm-memory-v1-cpu-prefill-paged`, - `llm-memory-v1-metal-decode-contiguous`, `llm-memory-v1-metal-decode-paged`, - `llm-memory-v1-metal-prefill-contiguous`, and `llm-memory-v1-metal-prefill-paged`. + `aggregates`, `status`, `reason_code`, `results_complete`, `run_accepted`, `interpretation`, `diagnostic`, + `interruption_requested`, `scenario_order_balance_complete`, `seeds`, `counters`, `checkpoint_lifecycle`, + `loop_records`, `environment`, `quality_warnings`, and `build_manifest`. +- Methodology is exactly `llm-memory-v2---`. Active identities are + `llm-memory-v2-cpu-decode-contiguous`, `llm-memory-v2-cpu-decode-paged`, + `llm-memory-v2-cpu-prefill-contiguous`, `llm-memory-v2-cpu-prefill-paged`, + `llm-memory-v2-metal-decode-contiguous`, `llm-memory-v2-metal-decode-paged`, + `llm-memory-v2-metal-prefill-contiguous`, and `llm-memory-v2-metal-prefill-paged`. - `configuration` retains exact `argv` and `resolved_sources`. `resolved_plan` owns phase-applicable `geometry`, layout-applicable `layout`, immutable logical/physical `resources`, and exact `component_identities`. Exactly one of `geometry.decode` and `.prefill` is populated. Prefill records integer P/Q and decimal-string - C/prefix-visit/causal-pair/logical-attention/FMA counts; decode-only crossover and weight/KV-read ratio fields are - null. Admitted paged plans populate block geometry; runtime permutation and combined layout-identity fields remain - null when Metal terminates before table preparation. Complete paged results populate them. Contiguous results use + C/prefix-visit counts. Theoretical causal-pair/logical-attention/FMA counts live in `model_context.prefill` as + decimal strings with individual `*_reason_code: "valid"`, or null with `arithmetic-overflow`. That object is null + for decode; decode-only crossover and weight/KV-read ratio fields are null for prefill. Admitted paged plans populate + block geometry; runtime permutation and combined layout-identity fields remain null when Metal terminates before table preparation. Complete paged results populate them. Contiguous results use null for paged-only values. Component identities include logical profile, KV layout, optional permutation, backend executor, resource ABI, schedule, timer, buffer, write, checksum, and nullable MSL identities under the canonical `llm-memory-components-v1` fixed-order serialization. @@ -1305,13 +1319,30 @@ an exact software version; schema 2 and unversioned historical standard JSON are table bytes. Metal additionally records 256 MiB segmentation, exact segment lengths/tails, Tier-2 argument-buffer slots/encoded length/alignment, status/staging sizes, and resource-plan admission, while allocation/preparation/admission peaks live in `memory_budget`. Pointer/range values are not exposed. -- `measurements[]` records frozen work identity, `work_unit_kind`, integer planned/completed work units, - `kv_write_kind`, per-work-unit and planned/completed model payload, layout metadata, accounted bytes, nullable - `synthetic_work_unit_latency_seconds`, nullable `synthetic_memory_work_units_per_second`, nullable - `effective_model_payload_gb_s`, working set, backend-specific lifecycle, and expected/actual checksum evidence. Metal - task evidence adds pipeline/grid, raw GPU timestamps and host envelope, command/encoder/dispatch counts, - dual-mod32 W/K/V values, and excluded phase-neutral validation fields `kv_write_evaluated` and `kv_write_valid`. Only - measured/checksum-valid records populate `aggregates.scenarios.weights_only`, `.kv_only`, and `.mixed`. +- `resolved_plan.scenario_plans[]` owns each canonical scenario/work-policy/work-unit plan, its payload/accounting, + exact identity, and sole expected checksum witness. `resolved_plan.frozen_plan_refs` selects the frozen plans. + Measurements have explicit zero-based `measurement_id` and `plan_ref`; excluded calibration attempts also use + `plan_ref`. Runtime records retain actual observations rather than duplicate plan identities and expected checksums. +- `measurements[]` retains status, order, completed work, working set, actual checksum, and backend execution evidence. + Accepted records expose `synthetic_work_unit_latency_seconds`, `synthetic_memory_work_units_per_second`, and + `effective_model_payload_gb_s`; invalid or unavailable metrics are null. Metal evidence retains pipeline/grid, + GPU timestamps, host envelope, and command/encoder/dispatch counts. +- Measurement and calibration `execution.validation.checks[]` records named checks with independent `applicable`, + `evaluated`, `valid`, and `reason_code` fields. Names are `post-validation-structure`, `kv-append-final`, + `kv-prefill-final-samples`, `kv-append-unchanged`, and `kv-padding-canary`, selected by backend/phase/layout/scenario. + CPU decode validates the final append state outside timing. Inapplicable checks have null evaluated/valid values; + missing required observations cannot become successful checks. Prefill content validation remains sampled. + Checksum agreement is not exhaustive proof of contents, addresses, or visit multiplicity; a checksum-valid attempt + with a failed required cold check remains invalid and excluded from aggregates. +- Each scenario aggregate owns `accepted_measurement_ids` shared by all three metrics. Rates are derived per + measurement before exact statistics; raw metric vectors are not duplicated in aggregates. Exact median/MAD/percentiles + are prepared only at snapshots or terminal, using reusable workspaces. `observed_cv_classification` is + `insufficient-samples` for n<3, `undefined` for undefined CV, `above-threshold` for payload CV strictly above 5%, + otherwise `below-threshold`. This is observed sample quality, not an acceptance or reproducibility guarantee. +- `build_manifest` binds reported build inputs and a once-per-command executable SHA-256; missing evidence remains + null with an explicit unavailable/partial status. It is provenance, not signed runtime attestation. + CPU measurement and excluded-attempt `execution.timing.cpu_raw` retain original start/stop/delta ticks as decimal + strings and timebase numerator/denominator as integers. Metal has null CPU raw evidence and retains GPU timestamps. - For paged KV, `kv_block_tokens` is an integer input, while potentially large block/table/lookup/byte counts remain canonical decimal strings. `permutation_seed_uint64_decimal` is a decimal string and `permutation_sha256` is exactly 64 lowercase hexadecimal characters. Decode KV-only and mixed report `L*B*(2*N+1)` lookups; prefill KV-only and mixed @@ -1324,10 +1355,11 @@ an exact software version; schema 2 and unversioned historical standard JSON are `weight_payload_dominant`, `near_crossover`, and `kv_read_payload_dominant`. Near means exact equality, and `classification_is_payload_only` prevents a hardware-bottleneck interpretation. - The complete-result predicate also requires backend/phase/layout to match the request, exact derived methodology, - `status == "complete"`, `results_complete == true`, `conclusions_valid == true`, and every planned measurement to be - measured. Paged comparison/acceptance additionally matches `G`, `N`, tail, logical/physical/padding/table resources, - permutation version/domain/seed/hash, lookup/accounting, schedule, timer, and checksum identities. A count-one result - may be complete yet fail conclusions because scenario positions are not balanced. + `status == "complete"`, `results_complete == true`, `run_accepted == true`, the exact run policy, and every planned + measurement to be measured. Paged comparison/acceptance additionally matches `G`, `N`, tail, logical/physical/padding/table + resources, permutation version/domain/seed/hash, lookup/accounting, schedule, timer, and checksum identities. + A correct count-one result may be accepted while `scenario_order_balance_complete` is false. Position balance requires complete loops and equal + nonzero first/middle/last counts for every scenario; it does not establish predecessor-pair balance. - See [LLM_MEMORY_PROFILE_WHITEPAPER.md](LLM_MEMORY_PROFILE_WHITEPAPER.md) for the field groups and consumer boundary. ### 18.7 Command-output transport boundary @@ -1352,11 +1384,16 @@ before sweep execution, the runtime banner, and worker creation. GPU selects it before its banner, QoS, signal scope, and backend factory. LLM selects it before banner, QoS, signal scope, work-plan admission, mappings, and workers. Direct pattern, TLB, and core-to-core files receive one atomic final write; standard and sweep files retain their existing intermediate checkpoints. GPU files retain terminal-measurement/failure -checkpoints and their post-release replacement. LLM files checkpoint every terminal scenario measurement and, unless a -measurement checkpoint itself fails, one post-release command terminal; failure is terminal and is not retried. Stdout -boundaries perform lazy no-op persistence at every logical checkpoint and emit one terminal payload after orchestration; -GPU and LLM still perform their checkpoint-boundary stop reads. The supported process contract, including help and -pre-result-failure exceptions, is defined in [API.md](API.md). +checkpoints and their post-release replacement. LLM files snapshot every Kth fully completed loop, +where `K=max(1,ceil(planned_loops/8))`, at most eight progress writes plus one post-release command terminal. An abrupt +exit can lose up to `3K` completed attempts, including while replacement is in progress; no file may exist before the +first successful snapshot. Graceful interruption retains the finished prefix when terminal persistence succeeds. +A late command exception after successful terminal persistence may produce one corrective failure snapshot. A failed +checkpoint is terminal and is never retried. Atomic rename does not promise power-loss durability. +LLM `checkpoint_lifecycle` reports prior actual file-writer attempts/successes before current snapshot preparation; +`current_persistence_success` remains null because the snapshot cannot attest to its own future write. Stdout emits one +terminal DOM, and disabled output builds none. Stop checks remain at task boundaries even when progress is skipped. +The supported process contract, including help and pre-result-failure exceptions, is defined in [API.md](API.md). ### 18.8 Path behavior @@ -1386,9 +1423,8 @@ This codebase uses boundary-aware mixed error handling: terminal `failed`/`invalid` schema evidence. Neither path falls back to another backend. The runner contains backend-task and checkpoint exceptions as stable status/reason evidence, finalizes untouched slots deterministically, gives real failures precedence over interruption, and keeps graceful task-boundary interruption separate from - conclusion validity. Backend release is attempted once before the command-terminal checkpoint; release failure - becomes terminal failure evidence. A file measurement-checkpoint failure is terminal, releases resources, and is not - retried. + run acceptance. Backend release is attempted once before the command-terminal checkpoint; release failure + becomes terminal failure evidence. A file progress-snapshot failure is terminal, releases resources, and is not retried. Principle: no uncaught exceptions should escape to `main()` control flow. @@ -1439,6 +1475,7 @@ Recommended validation commands: - Build: `make` - Unit tests (non-integration): `make test` - Script-example JSON entry paths: `make test-script-examples` +- Independent LLM schema-2 artifact contract and mutation tests: `make test-llm-verifier` - Integration-only: `make test-integration` - Full test set: `make test-all` - CLI help smoke check: `./memory_benchmark -h` @@ -1457,16 +1494,28 @@ Recommended validation commands: - Deterministic GPU-focused tests: `./test_runner --gtest_filter='GpuBandwidthParserTest.*:GpuMemoryBudgetTest.*:GpuRunnerTest.*:GpuJsonTest.*:GpuWorkPlanTest.*:GpuTimedAccumulatorOracleTest.*:ModeSelectorTest.*:HashUtilsTest.*'` - Real Metal contract: `./test_runner '--gtest_filter=GpuMetalBackendIntegrationTest.*'`; unsupported hardware may skip - this integration suite but does not replace deterministic unsupported-path coverage. + this integration suite but does not replace deterministic unsupported-path coverage. On a supported host, sandbox + `metal-device-unavailable` skips require narrowly scoped unsandboxed probes and applicable real-device gates, as + specified in `AGENTS.md`; sandbox skips do not establish hardware unavailability. For narrow changes, prefer targeted `gtest` filters via `./test_runner --gtest_filter=...`. The aggregate `make test-all` gate requires Python 3; after all GTest cases pass, it runs the focused script-example -entry test. `jq` is not required by this gate. +entry test and independent LLM verifier tests. `jq` is not required by this gate. + +`python3 script-examples/verify_llm_result.py result.json` independently reconstructs the eight supported profiles +without invoking producer helpers. It checks geometry, plan/checksum identities, timing, named validation, accepted +populations, and exact statistics. Verdicts separate `artifact_consistent` from `run_accepted`: exit 0 means consistent +and accepted, exit 1 means inconsistent or consistently unaccepted, and exit 2 means unsupported or bounded evidence +is unavailable. `--require-raw-timing` requires original CPU timing; `--binary PATH` checks the manifest hash without +running that file. Verification does not prove original process success, physical traffic, or runtime authenticity. ## 23. Source Map (Primary Entry Points) - Program entry: `main.cpp` +- Build provenance: `build-support/generate_provenance.py` +- Independent LLM verifier: `script-examples/verify_llm_result.py`, `script-examples/llm_verify_profiles.py`, + and `script-examples/llm_verify_oracles.py`. - LLM memory profile with all four CPU and all four Metal phase/layout profiles: - `src/llm_memory/llm_memory.cpp` - `src/llm_memory/llm_work_plan.cpp` diff --git a/src/core/config/version.h b/src/core/config/version.h index b3a1585..2323156 100644 --- a/src/core/config/version.h +++ b/src/core/config/version.h @@ -29,6 +29,6 @@ * @def SOFTVERSION * @brief Software version number (semantic versioning format as string) */ -#define SOFTVERSION "0.63.1" +#define SOFTVERSION "0.64.0" #endif // VERSION_H From 0129deb624b730079065b23efd5a71cc50a86e06 Mon Sep 17 00:00:00 2001 From: timoheimonen Date: Sun, 6 Sep 2026 11:48:45 +0300 Subject: [PATCH 15/16] Fix out-of-bounds checkpoint assertions in LLM runner tests --- tests/test_llm_memory_runner.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_llm_memory_runner.cpp b/tests/test_llm_memory_runner.cpp index f9acf5d..5b79a79 100644 --- a/tests/test_llm_memory_runner.cpp +++ b/tests/test_llm_memory_runner.cpp @@ -2117,7 +2117,9 @@ TEST(LlmMemoryRunnerTest, StopAfterLastSuccessfulTaskRetainsCompleteAcceptedEvid EXPECT_EQ(result.counters.measured_measurements, 9u); EXPECT_EQ(result.logical_checkpoint_attempts, 4u); ASSERT_EQ(checkpoints.size(), 4u); - EXPECT_FALSE(checkpoints[8].interruption_requested); + EXPECT_EQ(checkpoints[2].kind, LlmCheckpointKind::MeasurementTerminal); + EXPECT_EQ(checkpoints[2].measured_measurements, 9u); + EXPECT_FALSE(checkpoints[2].interruption_requested); EXPECT_TRUE(checkpoints.back().interruption_requested); } @@ -2153,7 +2155,9 @@ TEST(LlmMemoryRunnerTest, StopHookExceptionAfterLastMeasurementInvalidatesOtherw EXPECT_FALSE(result.run_accepted); EXPECT_EQ(result.counters.measured_measurements, 9u); ASSERT_EQ(checkpoints.size(), 4u); - EXPECT_EQ(checkpoints[8].kind, LlmCheckpointKind::MeasurementTerminal); + EXPECT_EQ(checkpoints[2].kind, LlmCheckpointKind::MeasurementTerminal); + EXPECT_EQ(checkpoints[2].measured_measurements, 9u); + EXPECT_EQ(checkpoints[2].status, LlmRunStatus::Complete); EXPECT_EQ(checkpoints.back().kind, LlmCheckpointKind::CommandTerminal); EXPECT_EQ(checkpoints.back().status, LlmRunStatus::Failed); } From 0d0697b413961040a01477d1f7298ec844998d48 Mon Sep 17 00:00:00 2001 From: timoheimonen Date: Sun, 6 Sep 2026 12:12:51 +0300 Subject: [PATCH 16/16] Consolidate redundant tests and strengthen warmup coverage --- tests/test_core_to_core_cli.cpp | 9 - tests/test_json_schema.cpp | 86 +----- tests/test_llm_memory_contract.cpp | 428 ----------------------------- tests/test_llm_memory_json.cpp | 20 -- tests/test_llm_memory_runner.cpp | 13 +- tests/test_pattern_benchmark.cpp | 100 +++++-- 6 files changed, 93 insertions(+), 563 deletions(-) diff --git a/tests/test_core_to_core_cli.cpp b/tests/test_core_to_core_cli.cpp index 5d040b9..b19ac9e 100644 --- a/tests/test_core_to_core_cli.cpp +++ b/tests/test_core_to_core_cli.cpp @@ -257,15 +257,6 @@ TEST(CoreToCoreCliTest, RejectsUnknownOptionsInStandaloneMode) { EXPECT_EQ(parse_result, EXIT_FAILURE); } -TEST(CoreToCoreCliTest, RejectsInvalidCountValues) { - // Count must be a positive integer. - CoreToCoreLatencyConfig config; - const int parse_result = - parse_with_args({"memory_benchmark", "--analyze-core2core", "--count", "0"}, config); - - EXPECT_EQ(parse_result, EXIT_FAILURE); -} - TEST(CoreToCoreCliTest, ParsesStrictPositiveIntegerBoundaries) { CoreToCoreLatencyConfig config; const std::string int_max = diff --git a/tests/test_json_schema.cpp b/tests/test_json_schema.cpp index 8abc40b..150ea89 100644 --- a/tests/test_json_schema.cpp +++ b/tests/test_json_schema.cpp @@ -1047,82 +1047,6 @@ TEST(JsonSchemaTest, TlbAnalysisBuilderCoversStatusesAndExporterHandlesUnavailab std::string::npos); } -TEST(JsonSchemaTest, CoreToCorePayloadPersistsThroughSharedWriter) { - const TemporaryJsonFile output_file("core2core_schema"); - CoreToCoreLatencyConfig config; - config.loop_count = 1; - config.latency_sample_count = 2; - - const std::string cpu_name = "test-cpu"; - const std::vector scenarios = { - { - Constants::CORE_TO_CORE_SCENARIO_NO_AFFINITY, - {10.0, 11.0}, - {10.2, 10.4}, - {}, - {}, - }, - }; - - const CoreToCoreLatencyJsonContext context = { - config, - cpu_name, - 4, - 6, - 100, - 200, - 20, - scenarios, - 4.5, - }; - - const nlohmann::ordered_json payload = build_core_to_core_latency_json(context); - ASSERT_EQ(write_json_to_file(output_file.path(), payload, false), EXIT_SUCCESS); - const nlohmann::json output_json = read_json_file(output_file.path()); - - EXPECT_EQ(output_json[JsonKeys::CONFIGURATION][JsonKeys::MODE], Constants::CORE_TO_CORE_JSON_MODE_NAME); - ASSERT_TRUE(output_json["core_to_core_latency"]["scenarios"][0].contains(JsonKeys::SAMPLES_NS)); - const nlohmann::json samples_json = output_json["core_to_core_latency"]["scenarios"][0][JsonKeys::SAMPLES_NS]; - EXPECT_TRUE(samples_json.contains(JsonKeys::VALUES)); - EXPECT_TRUE(samples_json.contains(JsonKeys::STATISTICS)); -} - -TEST(JsonSchemaTest, CoreToCoreJsonBuilderReturnsInMemoryPayload) { - CoreToCoreLatencyConfig config; - config.output_file.clear(); - config.loop_count = 1; - config.latency_sample_count = 2; - - const std::string cpu_name = "test-cpu"; - const std::vector scenarios = { - { - Constants::CORE_TO_CORE_SCENARIO_NO_AFFINITY, - {10.0}, - {10.2, 10.4}, - {}, - {}, - }, - }; - - const CoreToCoreLatencyJsonContext context = { - config, - cpu_name, - 4, - 6, - 100, - 200, - 20, - scenarios, - 4.5, - }; - - const nlohmann::json output_json = build_core_to_core_latency_json(context); - EXPECT_EQ(output_json[JsonKeys::CONFIGURATION][JsonKeys::MODE], Constants::CORE_TO_CORE_JSON_MODE_NAME); - EXPECT_TRUE(output_json.contains("core_to_core_latency")); - EXPECT_EQ(output_json["core_to_core_latency"]["scenarios"][0]["name"], - Constants::CORE_TO_CORE_SCENARIO_NO_AFFINITY); -} - TEST(JsonSchemaTest, CoreToCoreBuilderRetainsFailedAndInterruptedIncompleteState) { struct StatusCase { @@ -1280,7 +1204,17 @@ TEST(JsonSchemaTest, CoreToCoreBuilderSerializesOneWayValuesAndThreadHints) { const nlohmann::json output_json = build_core_to_core_latency_json(context); + EXPECT_EQ(output_json[JsonKeys::CONFIGURATION][JsonKeys::MODE], Constants::CORE_TO_CORE_JSON_MODE_NAME); + ASSERT_TRUE(output_json["core_to_core_latency"].is_object()); + ASSERT_EQ(output_json["core_to_core_latency"]["scenarios"].size(), 1u); const nlohmann::json scenario_json = output_json["core_to_core_latency"]["scenarios"][0]; + EXPECT_EQ(scenario_json["name"], Constants::CORE_TO_CORE_SCENARIO_SAME_AFFINITY); + const nlohmann::json samples_json = scenario_json[JsonKeys::SAMPLES_NS]; + EXPECT_EQ(samples_json[JsonKeys::VALUES], nlohmann::json::array({15.0, 17.0})); + ASSERT_TRUE(samples_json[JsonKeys::STATISTICS].is_object()); + EXPECT_DOUBLE_EQ(samples_json[JsonKeys::STATISTICS]["average"].get(), 16.0); + EXPECT_DOUBLE_EQ(samples_json[JsonKeys::STATISTICS]["median"].get(), 16.0); + ASSERT_EQ(scenario_json["one_way_estimate_ns"][JsonKeys::VALUES].size(), 2u); EXPECT_DOUBLE_EQ(scenario_json["one_way_estimate_ns"][JsonKeys::VALUES][0].get(), 6.0); EXPECT_DOUBLE_EQ(scenario_json["one_way_estimate_ns"][JsonKeys::VALUES][1].get(), 9.0); diff --git a/tests/test_llm_memory_contract.cpp b/tests/test_llm_memory_contract.cpp index 5e9e9c0..8ff22bd 100644 --- a/tests/test_llm_memory_contract.cpp +++ b/tests/test_llm_memory_contract.cpp @@ -26,7 +26,6 @@ #include #include #include -#include #include #include @@ -53,55 +52,17 @@ constexpr size_t kCanonicalSegmentSlotsPerPool = 256; constexpr uint64_t kCanonicalPoolCapacityBytes = kCanonicalSegmentCapacityBytes * kCanonicalSegmentSlotsPerPool; -enum class ContractBackend { - Cpu, - Metal, -}; - -enum class ContractPhase { - Decode, - Prefill, -}; - enum class ContractKvLayout { Contiguous, Paged, }; -enum class ContractWorkUnitKind { - DecodeStep, - PrefillOperation, -}; - -enum class ContractKvWriteKind { - None, - CurrentTokenAppend, - FullPromptPopulation, -}; - enum class ContractScenario { WeightsOnly, KvOnly, Mixed, }; -enum class ContractRunStatus { - NotStarted, - Complete, - Partial, - Interrupted, - Unsupported, - Failed, -}; - -enum class ContractMeasurementStatus { - NotRun, - Measured, - Interrupted, - Invalid, - Failed, -}; - struct PrefillPayloadContract { std::vector tile_ends; uint64_t attention_prefix_token_visits = 0; @@ -160,14 +121,6 @@ struct ComponentIdentityContract { std::optional msl_source_sha256; }; -struct ProfileActivationContract { - ContractBackend backend; - ContractPhase phase; - ContractKvLayout layout; - std::optional activation_phase; - bool preexisting_at_phase_zero; -}; - struct ScenarioAccountingContract { uint64_t model_payload_bytes = 0; uint64_t layout_metadata_lookups = 0; @@ -252,128 +205,6 @@ PayloadContract resolve_payload_contract(uint64_t weight_bytes, return result; } -std::string_view contract_token(ContractBackend backend) { - switch (backend) { - case ContractBackend::Cpu: - return "cpu"; - case ContractBackend::Metal: - return "metal"; - } - return {}; -} - -std::string_view contract_token(ContractPhase phase) { - switch (phase) { - case ContractPhase::Decode: - return "decode"; - case ContractPhase::Prefill: - return "prefill"; - } - return {}; -} - -std::string_view contract_token(ContractKvLayout layout) { - switch (layout) { - case ContractKvLayout::Contiguous: - return "contiguous"; - case ContractKvLayout::Paged: - return "paged"; - } - return {}; -} - -std::string_view contract_token(ContractWorkUnitKind kind) { - switch (kind) { - case ContractWorkUnitKind::DecodeStep: - return "decode_step"; - case ContractWorkUnitKind::PrefillOperation: - return "prefill_operation"; - } - return {}; -} - -std::string_view contract_token(ContractKvWriteKind kind) { - switch (kind) { - case ContractKvWriteKind::None: - return "none"; - case ContractKvWriteKind::CurrentTokenAppend: - return "current_token_append"; - case ContractKvWriteKind::FullPromptPopulation: - return "full_prompt_population"; - } - return {}; -} - -std::string_view contract_token(ContractScenario scenario) { - switch (scenario) { - case ContractScenario::WeightsOnly: - return "weights_only"; - case ContractScenario::KvOnly: - return "kv_only"; - case ContractScenario::Mixed: - return "mixed"; - } - return {}; -} - -std::string_view contract_token(ContractRunStatus status) { - switch (status) { - case ContractRunStatus::NotStarted: - return "not_started"; - case ContractRunStatus::Complete: - return "complete"; - case ContractRunStatus::Partial: - return "partial"; - case ContractRunStatus::Interrupted: - return "interrupted"; - case ContractRunStatus::Unsupported: - return "unsupported"; - case ContractRunStatus::Failed: - return "failed"; - } - return {}; -} - -std::string_view contract_token(ContractMeasurementStatus status) { - switch (status) { - case ContractMeasurementStatus::NotRun: - return "not_run"; - case ContractMeasurementStatus::Measured: - return "measured"; - case ContractMeasurementStatus::Interrupted: - return "interrupted"; - case ContractMeasurementStatus::Invalid: - return "invalid"; - case ContractMeasurementStatus::Failed: - return "failed"; - } - return {}; -} - -std::string methodology_token(ContractBackend backend, - ContractPhase phase, - ContractKvLayout layout) { - return "llm-memory-v2-" + std::string(contract_token(backend)) + "-" + - std::string(contract_token(phase)) + "-" + - std::string(contract_token(layout)); -} - -ContractWorkUnitKind work_unit_kind_for_phase(ContractPhase phase) { - return phase == ContractPhase::Decode - ? ContractWorkUnitKind::DecodeStep - : ContractWorkUnitKind::PrefillOperation; -} - -ContractKvWriteKind kv_write_kind_for(ContractPhase phase, - ContractScenario scenario) { - if (scenario == ContractScenario::WeightsOnly) { - return ContractKvWriteKind::None; - } - return phase == ContractPhase::Decode - ? ContractKvWriteKind::CurrentTokenAppend - : ContractKvWriteKind::FullPromptPopulation; -} - uint64_t ceil_divide_small(uint64_t value, uint64_t divisor) { return value / divisor + (value % divisor != 0 ? 1 : 0); } @@ -896,28 +727,6 @@ RunChecksum fold_components( return run; } -struct alignas(16) LayerDescriptorAbiV1 { - const uint8_t* weight_ptr; - uint64_t weight_bytes; - uint64_t first_sequence_index; - uint64_t sequence_count; - uint64_t layer_index; - uint64_t reserved_zero; -}; - -struct alignas(16) KvSequenceDescriptorAbiV1 { - const uint8_t* k_visible_ptr; - uint64_t k_visible_bytes; - const uint8_t* v_visible_ptr; - uint64_t v_visible_bytes; - uint8_t* k_append_ptr; - uint64_t k_append_bytes; - uint8_t* v_append_ptr; - uint64_t v_append_bytes; - uint64_t batch_sequence_index; - uint64_t append_record_byte_offset; -}; - bool accepted_generic_result(const GenericResultIdentity& identity) { const bool backend_is_known = identity.backend == "cpu" || identity.backend == "metal"; @@ -1115,35 +924,6 @@ TEST(LlmMemoryContractTest, EXPECT_NE(wrong_order.state_b, folded.state_b); } -TEST(LlmMemoryContractTest, DescriptorAbiLayoutGolden) { - static_assert(sizeof(void*) == 8, "LLM descriptor ABI requires ARM64 pointers"); - static_assert(std::is_standard_layout_v); - static_assert(std::is_standard_layout_v); - - EXPECT_EQ(alignof(LayerDescriptorAbiV1), 16u); - EXPECT_EQ(sizeof(LayerDescriptorAbiV1), 48u); - EXPECT_EQ(offsetof(LayerDescriptorAbiV1, weight_ptr), 0u); - EXPECT_EQ(offsetof(LayerDescriptorAbiV1, weight_bytes), 8u); - EXPECT_EQ(offsetof(LayerDescriptorAbiV1, first_sequence_index), 16u); - EXPECT_EQ(offsetof(LayerDescriptorAbiV1, sequence_count), 24u); - EXPECT_EQ(offsetof(LayerDescriptorAbiV1, layer_index), 32u); - EXPECT_EQ(offsetof(LayerDescriptorAbiV1, reserved_zero), 40u); - - EXPECT_EQ(alignof(KvSequenceDescriptorAbiV1), 16u); - EXPECT_EQ(sizeof(KvSequenceDescriptorAbiV1), 80u); - EXPECT_EQ(offsetof(KvSequenceDescriptorAbiV1, k_visible_ptr), 0u); - EXPECT_EQ(offsetof(KvSequenceDescriptorAbiV1, k_visible_bytes), 8u); - EXPECT_EQ(offsetof(KvSequenceDescriptorAbiV1, v_visible_ptr), 16u); - EXPECT_EQ(offsetof(KvSequenceDescriptorAbiV1, v_visible_bytes), 24u); - EXPECT_EQ(offsetof(KvSequenceDescriptorAbiV1, k_append_ptr), 32u); - EXPECT_EQ(offsetof(KvSequenceDescriptorAbiV1, k_append_bytes), 40u); - EXPECT_EQ(offsetof(KvSequenceDescriptorAbiV1, v_append_ptr), 48u); - EXPECT_EQ(offsetof(KvSequenceDescriptorAbiV1, v_append_bytes), 56u); - EXPECT_EQ(offsetof(KvSequenceDescriptorAbiV1, batch_sequence_index), 64u); - EXPECT_EQ( - offsetof(KvSequenceDescriptorAbiV1, append_record_byte_offset), 72u); -} - TEST(LlmMemoryContractTest, GenericV2AcceptancePredicateIsExact) { const GenericResultIdentity accepted = { "llm_memory", @@ -1210,117 +990,6 @@ TEST(LlmMemoryContractTest, GenericV2AcceptancePredicateIsExact) { EXPECT_FALSE(accepted_generic_result(candidate)); } -TEST(LlmMemoryContractTest, - GenericVocabularyAndMethodologyTokensAreCanonical) { - EXPECT_EQ( - (std::array{ - contract_token(ContractBackend::Cpu), - contract_token(ContractBackend::Metal), - }), - (std::array{"cpu", "metal"})); - EXPECT_EQ( - (std::array{ - contract_token(ContractPhase::Decode), - contract_token(ContractPhase::Prefill), - }), - (std::array{"decode", "prefill"})); - EXPECT_EQ( - (std::array{ - contract_token(ContractKvLayout::Contiguous), - contract_token(ContractKvLayout::Paged), - }), - (std::array{"contiguous", "paged"})); - EXPECT_EQ( - (std::array{ - contract_token(ContractWorkUnitKind::DecodeStep), - contract_token(ContractWorkUnitKind::PrefillOperation), - }), - (std::array{"decode_step", - "prefill_operation"})); - EXPECT_EQ( - (std::array{ - contract_token(ContractKvWriteKind::None), - contract_token(ContractKvWriteKind::CurrentTokenAppend), - contract_token(ContractKvWriteKind::FullPromptPopulation), - }), - (std::array{"none", "current_token_append", - "full_prompt_population"})); - EXPECT_EQ( - (std::array{ - contract_token(ContractScenario::WeightsOnly), - contract_token(ContractScenario::KvOnly), - contract_token(ContractScenario::Mixed), - }), - (std::array{"weights_only", "kv_only", - "mixed"})); - EXPECT_EQ( - (std::array{ - contract_token(ContractRunStatus::NotStarted), - contract_token(ContractRunStatus::Complete), - contract_token(ContractRunStatus::Partial), - contract_token(ContractRunStatus::Interrupted), - contract_token(ContractRunStatus::Unsupported), - contract_token(ContractRunStatus::Failed), - }), - (std::array{ - "not_started", "complete", "partial", "interrupted", - "unsupported", "failed"})); - EXPECT_EQ( - (std::array{ - contract_token(ContractMeasurementStatus::NotRun), - contract_token(ContractMeasurementStatus::Measured), - contract_token(ContractMeasurementStatus::Interrupted), - contract_token(ContractMeasurementStatus::Invalid), - contract_token(ContractMeasurementStatus::Failed), - }), - (std::array{ - "not_run", "measured", "interrupted", "invalid", "failed"})); - - EXPECT_EQ(work_unit_kind_for_phase(ContractPhase::Decode), - ContractWorkUnitKind::DecodeStep); - EXPECT_EQ(work_unit_kind_for_phase(ContractPhase::Prefill), - ContractWorkUnitKind::PrefillOperation); - for (ContractPhase phase : - {ContractPhase::Decode, ContractPhase::Prefill}) { - EXPECT_EQ(kv_write_kind_for(phase, ContractScenario::WeightsOnly), - ContractKvWriteKind::None); - } - for (ContractScenario scenario : - {ContractScenario::KvOnly, ContractScenario::Mixed}) { - EXPECT_EQ(kv_write_kind_for(ContractPhase::Decode, scenario), - ContractKvWriteKind::CurrentTokenAppend); - EXPECT_EQ(kv_write_kind_for(ContractPhase::Prefill, scenario), - ContractKvWriteKind::FullPromptPopulation); - } - - const std::array backends = { - ContractBackend::Cpu, ContractBackend::Metal}; - const std::array phases = { - ContractPhase::Decode, ContractPhase::Prefill}; - const std::array layouts = { - ContractKvLayout::Contiguous, ContractKvLayout::Paged}; - const std::array expected_methodologies = { - "llm-memory-v2-cpu-decode-contiguous", - "llm-memory-v2-cpu-decode-paged", - "llm-memory-v2-cpu-prefill-contiguous", - "llm-memory-v2-cpu-prefill-paged", - "llm-memory-v2-metal-decode-contiguous", - "llm-memory-v2-metal-decode-paged", - "llm-memory-v2-metal-prefill-contiguous", - "llm-memory-v2-metal-prefill-paged", - }; - size_t methodology_index = 0; - for (ContractBackend backend : backends) { - for (ContractPhase phase : phases) { - for (ContractKvLayout layout : layouts) { - EXPECT_EQ(methodology_token(backend, phase, layout), - expected_methodologies[methodology_index]); - ++methodology_index; - } - } - } -} - TEST(LlmMemoryContractTest, PrefillPayloadGoldenCoversTilesBoundsAndSingleWeightPass) { const PrefillPayloadContract golden = @@ -1850,103 +1519,6 @@ TEST(LlmMemoryContractTest, "|msl_revision=null|msl_source_sha256=null"); } -TEST(LlmMemoryContractTest, - FinalSupportMatrixAndPublicActivationOrderAreFrozen) { - constexpr std::array profiles = {{ - {ContractBackend::Cpu, ContractPhase::Decode, - ContractKvLayout::Contiguous, std::nullopt, true}, - {ContractBackend::Cpu, ContractPhase::Decode, - ContractKvLayout::Paged, 4, false}, - {ContractBackend::Cpu, ContractPhase::Prefill, - ContractKvLayout::Contiguous, 6, false}, - {ContractBackend::Cpu, ContractPhase::Prefill, - ContractKvLayout::Paged, 7, false}, - {ContractBackend::Metal, ContractPhase::Decode, - ContractKvLayout::Contiguous, 9, false}, - {ContractBackend::Metal, ContractPhase::Decode, - ContractKvLayout::Paged, 10, false}, - {ContractBackend::Metal, ContractPhase::Prefill, - ContractKvLayout::Contiguous, 11, false}, - {ContractBackend::Metal, ContractPhase::Prefill, - ContractKvLayout::Paged, 12, false}, - }}; - constexpr std::array expected_profiles = { - "cpu/decode/contiguous", "cpu/decode/paged", - "cpu/prefill/contiguous", "cpu/prefill/paged", - "metal/decode/contiguous", "metal/decode/paged", - "metal/prefill/contiguous", "metal/prefill/paged", - }; - constexpr std::array, 8> expected_activation_phases = { - std::nullopt, 4, 6, 7, 9, 10, 11, 12}; - - size_t scenario_profile_count = 0; - for (size_t index = 0; index < profiles.size(); ++index) { - const ProfileActivationContract& profile = profiles[index]; - const std::string serialized = - std::string(contract_token(profile.backend)) + "/" + - std::string(contract_token(profile.phase)) + "/" + - std::string(contract_token(profile.layout)); - EXPECT_EQ(serialized, expected_profiles[index]); - EXPECT_EQ(profile.activation_phase, expected_activation_phases[index]); - EXPECT_EQ(profile.preexisting_at_phase_zero, index == 0); - scenario_profile_count += 3; - for (size_t other = index + 1; other < profiles.size(); ++other) { - EXPECT_FALSE(profile.backend == profiles[other].backend && - profile.phase == profiles[other].phase && - profile.layout == profiles[other].layout); - } - } - EXPECT_EQ(scenario_profile_count, 24u); - - constexpr std::array, 7> - expected_new_profile_activation_order = {{ - {4, "cpu/decode/paged"}, - {6, "cpu/prefill/contiguous"}, - {7, "cpu/prefill/paged"}, - {9, "metal/decode/contiguous"}, - {10, "metal/decode/paged"}, - {11, "metal/prefill/contiguous"}, - {12, "metal/prefill/paged"}, - }}; - size_t activation_index = 0; - for (const ProfileActivationContract& profile : profiles) { - if (!profile.activation_phase.has_value()) { - continue; - } - ASSERT_LT(activation_index, - expected_new_profile_activation_order.size()); - const std::string profile_name = - std::string(contract_token(profile.backend)) + "/" + - std::string(contract_token(profile.phase)) + "/" + - std::string(contract_token(profile.layout)); - EXPECT_EQ(*profile.activation_phase, - expected_new_profile_activation_order[activation_index].first); - EXPECT_EQ(profile_name, - expected_new_profile_activation_order[activation_index].second); - ++activation_index; - } - EXPECT_EQ(activation_index, expected_new_profile_activation_order.size()); - - constexpr int generic_schema_rename_phase = 1; - constexpr int capability_validation_pending_phase = 12; - constexpr int capability_production_supported_phase = 13; - EXPECT_EQ(generic_schema_rename_phase, 1); - EXPECT_EQ(capability_validation_pending_phase, 12); - EXPECT_EQ(capability_production_supported_phase, 13); - EXPECT_LT(capability_validation_pending_phase, - capability_production_supported_phase); - - constexpr std::array phases_without_new_profile_activation = { - 0, 1, 2, 3, 5, 8, 13}; - for (int phase : phases_without_new_profile_activation) { - EXPECT_TRUE(std::none_of( - profiles.begin(), profiles.end(), [phase](const auto& profile) { - return profile.activation_phase.has_value() && - *profile.activation_phase == phase; - })); - } -} - namespace { constexpr uint64_t kPrefillContractPhaseDomain = diff --git a/tests/test_llm_memory_json.cpp b/tests/test_llm_memory_json.cpp index f0d6f9e..ed8754e 100644 --- a/tests/test_llm_memory_json.cpp +++ b/tests/test_llm_memory_json.cpp @@ -1666,26 +1666,6 @@ TEST(LlmMemoryJsonTest, UnknownAndContradictorySnapshotReferencesAreRejected) { EXPECT_THROW(build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result), std::invalid_argument); } -TEST(LlmMemoryJsonTest, CanonicalInsertionDeduplicatesAndRejectsContradictoryWork) { - const auto config = explicit_config(1); - const auto plan = admitted_plan(config); - ASSERT_TRUE(plan.valid); - FakeLlmBackend backend; - LlmMemoryResult result; - const auto mixed = build_llm_scenario_work_plan(plan, LlmScenario::Mixed, 4, true); - const auto handle = register_llm_scenario_plan(result, plan, mixed, backend); - EXPECT_EQ(register_llm_scenario_plan(result, plan, mixed, backend), handle); - auto changed = mixed; - ++changed.effective_model_payload_bytes; - EXPECT_THROW(register_llm_scenario_plan(result, plan, changed, backend), std::invalid_argument); - const auto warm = build_llm_scenario_work_plan(plan, LlmScenario::WeightsOnly, 1, false); - const auto warm_handle = register_llm_scenario_plan(result, plan, warm, backend); - prepare_llm_result_snapshot(result); - EXPECT_EQ(result.snapshot_plan_refs[warm_handle], 0u); - EXPECT_EQ(result.snapshot_plan_refs[handle], 1u); - EXPECT_EQ(result.scenario_plans.size(), 2u); -} - TEST(LlmMemoryJsonTest, OneLoopAcceptanceIsIndependentOfBalanceAndObservedCvQuality) { const auto config = explicit_config(1); const auto plan = admitted_plan(config); diff --git a/tests/test_llm_memory_runner.cpp b/tests/test_llm_memory_runner.cpp index 5b79a79..e2d8a69 100644 --- a/tests/test_llm_memory_runner.cpp +++ b/tests/test_llm_memory_runner.cpp @@ -3327,10 +3327,21 @@ TEST(LlmMemoryRunnerTest, CanonicalInsertionReusesExactPlanAndSnapshotRemapsStab const auto eight = build_llm_scenario_work_plan(model, LlmScenario::Mixed, 8, true); const auto two = build_llm_scenario_work_plan(model, LlmScenario::Mixed, 2, true); const auto first = register_llm_scenario_plan(result, model, eight, backend); - const auto second = register_llm_scenario_plan(result, model, two, backend); EXPECT_EQ(register_llm_scenario_plan(result, model, eight, backend), first); + auto changed_payload = eight; + ++changed_payload.effective_model_payload_bytes; + EXPECT_THROW(register_llm_scenario_plan(result, model, changed_payload, backend), std::invalid_argument); + EXPECT_EQ(backend.expected_calls, 1u); + const auto warm = build_llm_scenario_work_plan(model, LlmScenario::WeightsOnly, 1, false); + const auto warm_handle = register_llm_scenario_plan(result, model, warm, backend); + prepare_llm_result_snapshot(result); + EXPECT_EQ(result.snapshot_plan_refs.at(warm_handle), 0u); + EXPECT_EQ(result.snapshot_plan_refs.at(first), 1u); EXPECT_EQ(backend.expected_calls, 2u); EXPECT_EQ(result.scenario_plans.size(), 2u); + const auto second = register_llm_scenario_plan(result, model, two, backend); + EXPECT_EQ(backend.expected_calls, 3u); + EXPECT_EQ(result.scenario_plans.size(), 3u); result.frozen_plan_handles[2] = first; for (size_t t = 3; t < 20; ++t) { register_llm_scenario_plan(result, model, diff --git a/tests/test_pattern_benchmark.cpp b/tests/test_pattern_benchmark.cpp index 6aacfa8..9116c47 100644 --- a/tests/test_pattern_benchmark.cpp +++ b/tests/test_pattern_benchmark.cpp @@ -37,8 +37,14 @@ #include "pattern_benchmark/pattern_benchmark.h" #include "pattern_benchmark/pattern_work_plan.h" #include "test_config_helpers.h" +#include "test_timer_system_calls.h" #include "utils/benchmark.h" // Declares system_info functions #include "warmup/warmup.h" + +// Internal strided entry point, also used by pattern_coordinator.cpp. +int run_strided_pattern_benchmarks(const PatternBuffers& buffers, const BenchmarkConfig& config, + size_t stride, PatternResults& results, HighResTimer& timer); + extern "C" uint64_t verify_pattern_callee_saved_registers_asm(uintptr_t function_address, uintptr_t arg0, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, uintptr_t arg5); @@ -183,6 +189,12 @@ __asm__(R"ASM( namespace { +std::atomic strided_preflight_timer_reads{0}; + +uint64_t strided_preflight_timer_ticks() { + return 100 * (strided_preflight_timer_reads.fetch_add(1, std::memory_order_relaxed) + 1); +} + BenchmarkConfig make_pattern_config(size_t buffer_size, int iterations, int num_threads = 1) { BenchmarkConfig config; config.buffer_size = buffer_size; @@ -268,12 +280,12 @@ void expect_core_pattern_bandwidths_positive(const PatternResults& results) { } } -void expect_2mb_pattern_bandwidths_zero(const PatternResults& results) { +void expect_2mb_pattern_measurements_skipped(const PatternResults& results, const std::string& expected_reason) { for (PatternOperation operation : {PatternOperation::Read, PatternOperation::Write, PatternOperation::Copy}) { const PatternMeasurement& measurement = get_pattern_measurement(results, PatternKind::Strided2MiB, operation); EXPECT_EQ(measurement.status, PatternMeasurementStatus::Skipped); EXPECT_FALSE(measurement.bandwidth_gb_s.has_value()); - EXPECT_EQ(measurement.status_reason, Messages::pattern_reason_stride_transition_unavailable()); + EXPECT_EQ(measurement.status_reason, expected_reason); } } @@ -677,18 +689,55 @@ TEST(PatternBenchmarkTest, ExecutionOrderIsDeterministicRotatingAndBalancedAcros } } -// Integration test: Test that the representative pattern benchmark run produces every core pattern result. -// NOTE: This is an integration test that performs actual system operations. -// It runs real pattern benchmarks which may be slower and can fail on slow systems or under load. -// Use 'make test-integration' to run integration tests, or 'make test' for unit tests only. -TEST(PatternBenchmarkTest, RunPatternBenchmarksCorePatternsIntegration) { - BenchmarkConfig config = make_pattern_config(512 * 1024, 1); +TEST(PatternBenchmarkTest, Strided2MiBPreflightSkipsUnavailableTransitionsWithoutMeasurement) { + // Exercise only the production preflight: empty buffers and a fake clock keep + // both validator and planner rejection paths free of kernels and measurements. + strided_preflight_timer_reads.store(0, std::memory_order_relaxed); + const test_timer_system_calls::ScopedTimerSystemCalls timer_system_calls; + auto timer = HighResTimer::create(); + ASSERT_TRUE(timer.has_value()); + const PatternBuffers buffers; + struct PreflightCase { + size_t buffer_size; + std::string expected_reason; + }; + const std::array cases = {{ + {512 * 1024, Messages::pattern_reason_stride_transition_unavailable()}, + {Constants::PATTERN_STRIDE_SUPERPAGE_2MB, Messages::pattern_reason_buffer_lacks_two_strided_accesses()}, + }}; + for (const PreflightCase& test_case : cases) { + SCOPED_TRACE(test_case.buffer_size); + BenchmarkConfig config; + config.buffer_size = test_case.buffer_size; + config.num_threads = 1; + PatternResults results = make_complete_pattern_loop(); + + EXPECT_EQ(run_strided_pattern_benchmarks(buffers, config, Constants::PATTERN_STRIDE_SUPERPAGE_2MB, + results, *timer), EXIT_SUCCESS); + expect_2mb_pattern_measurements_skipped(results, test_case.expected_reason); + for (PatternOperation operation : {PatternOperation::Read, PatternOperation::Write, PatternOperation::Copy}) { + const PatternMeasurement& measurement = get_pattern_measurement(results, PatternKind::Strided2MiB, operation); + EXPECT_EQ(measurement.stride_bytes, Constants::PATTERN_STRIDE_SUPERPAGE_2MB); + EXPECT_EQ(measurement.requested_threads, 1); + EXPECT_EQ(measurement.effective_threads, 0); + EXPECT_EQ(measurement.passes, 0u); + EXPECT_EQ(measurement.total_accesses, 0u); + EXPECT_EQ(measurement.total_payload_bytes, 0u); + EXPECT_DOUBLE_EQ(measurement.elapsed_seconds, 0.0); + } + } + EXPECT_EQ(strided_preflight_timer_reads.load(std::memory_order_relaxed), 0u); +} + +// One real-work smoke run covers coordinator dispatch for all seven patterns. +TEST(PatternBenchmarkTest, RunPatternBenchmarksAllPatternsIntegration) { + BenchmarkConfig config = make_pattern_config(8 * 1024 * 1024, 1); PatternResults results; ASSERT_TRUE(run_pattern_benchmarks_with_fresh_buffers(config, results)); expect_core_pattern_bandwidths_positive(results); - expect_2mb_pattern_bandwidths_zero(results); + expect_2mb_pattern_bandwidths_positive(results); const PatternMeasurement& forward_read = get_pattern_measurement(results, PatternKind::SequentialForward, PatternOperation::Read); EXPECT_EQ(forward_read.status, PatternMeasurementStatus::Measured); @@ -700,15 +749,6 @@ TEST(PatternBenchmarkTest, RunPatternBenchmarksCorePatternsIntegration) { EXPECT_EQ(random_read.seed, config.pattern_seed); } -TEST(PatternBenchmarkTest, Strided2MiBRouteAndKernelIntegration) { - BenchmarkConfig config = make_pattern_config(8 * 1024 * 1024, 1); - - PatternResults results; - ASSERT_TRUE(run_pattern_benchmarks_with_fresh_buffers(config, results)); - - expect_2mb_pattern_bandwidths_positive(results); -} - TEST(PatternBenchmarkTest, PhasedStridedKernelsRespectAccessBoundariesIntegration) { const std::vector strides = { Constants::PATTERN_STRIDE_CACHE_LINE, @@ -805,19 +845,21 @@ TEST(PatternBenchmarkTest, FinalizedPatternPlansDriveWarmupKernelsIntegration) { EXPECT_EQ(checksum.load(std::memory_order_acquire), expected_checksum); } -TEST(PatternBenchmarkTest, CacheReadWarmupUsesCacheKernelIntegration) { - constexpr size_t buffer_size = 1024; - std::vector storage(buffer_size + Constants::CACHE_LINE_SIZE_BYTES); - unsigned char* source = align_to_cache_line(storage.data()); - for (size_t offset = 0; offset < buffer_size; ++offset) { - source[offset] = static_cast((offset * 37 + 11) & 0xff); - } +TEST(PatternBenchmarkTest, CacheReadWarmupAccumulatesEveryWorkerChunkIntegration) { + alignas(Constants::CACHE_LINE_SIZE_BYTES) std::array source{}; + constexpr uint64_t initial_checksum = 0x123456789abcdef0ULL; + // Four aligned 256-byte chunks contribute distinct bits. The byte positions + // exercise upper vector lanes and chunk ends without cancelling to zero. + source[8] = 0x01; + source[256 + 31] = 0x02; + source[512 + 255] = 0x04; + source[768] = 0x08; + constexpr uint64_t expected_xor = 0x0600000000000009ULL; + std::atomic checksum{initial_checksum}; - const uint64_t expected_checksum = memory_read_cache_loop_asm(source, buffer_size); - std::atomic checksum{0}; - warmup_cache_read(source, buffer_size, 1, checksum); + warmup_cache_read(source.data(), source.size(), 4, checksum); - EXPECT_EQ(checksum.load(std::memory_order_acquire), expected_checksum); + EXPECT_EQ(checksum.load(std::memory_order_acquire), initial_checksum ^ expected_xor); } TEST(PatternBenchmarkTest, RandomWarmupUsesChunkRelativeWorkerOffsetsIntegration) {