|
| 1 | +"""Bounded native benchmark measurement and immutable evidence validation.""" |
| 2 | +from bisect import bisect_right |
| 3 | +import hashlib |
| 4 | +import json |
| 5 | +import math |
| 6 | +from pathlib import Path |
| 7 | + |
| 8 | +UNITS = dict(timestamp='ns', duration='s', latency='ms', throughput='requests/s') |
| 9 | +DEFAULTS = dict(warmup=256, requests=4096, concurrency=8, payload=1024, workers=4, h3_concurrency=4, deadline=300) |
| 10 | + |
| 11 | + |
| 12 | +def require(value, message): |
| 13 | + if not value: |
| 14 | + raise ValueError(message) |
| 15 | + |
| 16 | + |
| 17 | +def parse_json(text): |
| 18 | + def pairs(items): |
| 19 | + result = {} |
| 20 | + for key, value in items: |
| 21 | + require(key not in result, 'duplicate JSON key: ' + key) |
| 22 | + result[key] = value |
| 23 | + return result |
| 24 | + def invalid(value): |
| 25 | + raise ValueError('nonfinite JSON number: ' + value) |
| 26 | + return json.loads(text, object_pairs_hook=pairs, parse_constant=invalid) |
| 27 | + |
| 28 | + |
| 29 | +def required_cells(tls): |
| 30 | + require(tls in ('yes', 'no'), 'unknown TLS mode') |
| 31 | + protocols = ('h1', 'h1-tls', 'h2', 'h3') if tls == 'yes' else ('h1',) |
| 32 | + return [(linkage, loop, protocol) for linkage in ('shared', 'static') |
| 33 | + for loop in ('managed', 'external') for protocol in protocols] |
| 34 | + |
| 35 | + |
| 36 | +def effective_concurrency(protocol, workload): |
| 37 | + return min(workload['concurrency'], workload['h3_concurrency']) if protocol == 'h3' else workload['concurrency'] |
| 38 | + |
| 39 | + |
| 40 | +def epoch_workload(protocol, requests, concurrency, warmup): |
| 41 | + if protocol not in ('h2', 'h3'): return 1, warmup |
| 42 | + epochs = math.ceil(requests / (64 if protocol == 'h2' else concurrency)) |
| 43 | + return epochs, epochs * concurrency |
| 44 | + |
| 45 | + |
| 46 | +def validate_cells(cells, tls): |
| 47 | + require(sorted(cells) == sorted(required_cells(tls)), 'missing, duplicate or unknown benchmark cells') |
| 48 | + |
| 49 | + |
| 50 | +def positive_integer(value): |
| 51 | + return type(value) is int and value > 0 |
| 52 | + |
| 53 | + |
| 54 | +def percentile(values, rank): |
| 55 | + require(bool(values), 'no measured samples') |
| 56 | + return sorted(values)[math.ceil(len(values) * rank / 100) - 1] |
| 57 | + |
| 58 | + |
| 59 | +def statistics(samples, epochs): |
| 60 | + duration = sum(epoch['end_ns'] - epoch['start_ns'] for epoch in epochs) / 1e9 |
| 61 | + require(duration > 0 and math.isfinite(duration), 'invalid measured duration') |
| 62 | + latencies = [(sample['end_ns'] - sample['start_ns']) / 1e6 for sample in samples] |
| 63 | + return dict(duration_s=duration, throughput_rps=len(samples) / duration, |
| 64 | + **{'p%d_ms' % rank: percentile(latencies, rank) for rank in (50, 95, 99)}) |
| 65 | + |
| 66 | + |
| 67 | +def validate_observation(value, run_id, protocol): |
| 68 | + require(value['run_id'] == run_id and value['protocol'] == protocol, 'stale run/protocol') |
| 69 | + require(value['units'] == UNITS, 'wrong units') |
| 70 | + require(value['clock']['name'] == 'monotonic_ns' and positive_integer(value['clock']['resolution_ns']), 'wrong clock') |
| 71 | + count = value['requested'] |
| 72 | + require(positive_integer(count) and count <= 65536, 'request bound') |
| 73 | + require(value['attempted'] == count == value['completed'] and value['errors'] == 0, 'incomplete or failed requests') |
| 74 | + samples, warmup, epochs = value['samples'], value['warmup'], value['epochs'] |
| 75 | + require(len(samples) == count and {row['id'] for row in samples} == set(range(count)), 'sample membership') |
| 76 | + require(positive_integer(value['warmup_requested']) and value['warmup_requested'] <= 65536, 'warmup bound') |
| 77 | + require(value['warmup_requested'] == value['warmup_completed'] == len(warmup), 'incomplete warmup') |
| 78 | + require({row['id'] for row in warmup} == set(range(len(warmup))), 'warmup membership') |
| 79 | + require(epochs and len(epochs) <= count, 'missing measured epochs') |
| 80 | + previous = 0 |
| 81 | + for epoch in epochs: |
| 82 | + require(positive_integer(epoch['start_ns']) and positive_integer(epoch['end_ns']) and |
| 83 | + previous < epoch['start_ns'] < epoch['end_ns'], 'invalid epoch clock') |
| 84 | + previous = epoch['end_ns'] |
| 85 | + members_by_epoch = [[] for _ in epochs] |
| 86 | + for sample in samples: |
| 87 | + require(type(sample['id']) is int, 'invalid request ID') |
| 88 | + require(type(sample['epoch']) is int and 0 <= sample['epoch'] < len(epochs), 'unknown sample epoch') |
| 89 | + members_by_epoch[sample['epoch']].append(sample) |
| 90 | + epoch = epochs[sample['epoch']] |
| 91 | + require(positive_integer(sample['start_ns']) and positive_integer(sample['end_ns']) and |
| 92 | + epoch['start_ns'] <= sample['start_ns'] < sample['end_ns'] <= epoch['end_ns'], 'invalid request clock') |
| 93 | + require(sample['status'] == 200 and sample['valid'] is True and sample['protocol'] == protocol, 'invalid response') |
| 94 | + for index, epoch in enumerate(epochs): |
| 95 | + members = members_by_epoch[index] |
| 96 | + require(members and max(row['end_ns'] for row in members) == epoch['end_ns'], 'epoch completion boundary') |
| 97 | + epoch_ends = [epoch['end_ns'] for epoch in epochs] |
| 98 | + for row in warmup: |
| 99 | + require(positive_integer(row['start_ns']) and positive_integer(row['end_ns']) and row['start_ns'] < row['end_ns'], 'warmup clock') |
| 100 | + following = bisect_right(epoch_ends, row['start_ns']) |
| 101 | + require(following == len(epochs) or row['end_ns'] <= epochs[following]['start_ns'], 'warmup contamination') |
| 102 | + expected = statistics(samples, epochs) |
| 103 | + require(set(value['statistics']) == set(expected), 'missing statistics or units') |
| 104 | + for key, result in expected.items(): |
| 105 | + actual = value['statistics'][key] |
| 106 | + require(type(actual) in (int, float) and math.isfinite(actual) and actual > 0 and |
| 107 | + math.isclose(actual, result, rel_tol=1e-12), 'statistics disagreement: ' + key) |
| 108 | + return expected |
| 109 | + |
| 110 | + |
| 111 | +def artifact(path): |
| 112 | + path = Path(path).resolve() |
| 113 | + return dict(path=str(path), bytes=path.stat().st_size, sha256=hashlib.sha256(path.read_bytes()).hexdigest()) |
| 114 | + |
| 115 | + |
| 116 | +def verify_artifact(record): |
| 117 | + try: |
| 118 | + require(artifact(record['path']) == record, 'changed artifact: ' + record['path']) |
| 119 | + except OSError as error: |
| 120 | + raise ValueError('missing artifact: ' + record['path']) from error |
| 121 | + |
| 122 | + |
| 123 | +def validate_server(server, observation): |
| 124 | + require(server.get('exit') == 0 and 'error' not in server, 'server exit/cleanup failure') |
| 125 | + lines = [line[7:] for line in server['stdout'].splitlines() if line.startswith('COUNTS ')] |
| 126 | + require(len(lines) == 1, 'missing or duplicate server observation') |
| 127 | + counts = parse_json(lines[0]) |
| 128 | + require(counts == dict(run_id=observation['run_id'], warmup=observation['warmup_completed'], |
| 129 | + measured=observation['completed'], errors=0), 'server/client counts differ') |
| 130 | + |
| 131 | + |
| 132 | +def validate_receipt(receipt, expected_identity, expected_run_id=None): |
| 133 | + require(receipt['schema_version'] == 1 and receipt['status'] == 'passed', 'failed or unknown receipt') |
| 134 | + require(receipt['identity'] == expected_identity, 'stale source/package/configuration identity') |
| 135 | + require('workload' not in receipt or receipt['workload'] == expected_identity.get('workload'), 'stale workload') |
| 136 | + require(len(receipt['run_id']) == 32 and all(c in '0123456789abcdef' for c in receipt['run_id']), 'invalid run identifier') |
| 137 | + if expected_run_id is not None: |
| 138 | + require(receipt['run_id'] == expected_run_id, 'reused run identifier') |
| 139 | + validate_cells([tuple(cell[k] for k in ('linkage', 'loop', 'protocol')) for cell in receipt['cells']], receipt['tls']) |
| 140 | + require(receipt['artifacts'], 'missing immutable artifacts') |
| 141 | + sample_paths = [cell['observation']['path'] for cell in receipt['cells']] |
| 142 | + require(len(sample_paths) == len(set(sample_paths)), 'reused samples across cells') |
| 143 | + for record in receipt['artifacts']: verify_artifact(record) |
| 144 | + for cell in receipt['cells']: |
| 145 | + require(cell['client_exit'] == 0 and cell['fresh_execution'] is True, 'missing live client execution') |
| 146 | + for key in ('observation', 'server'): |
| 147 | + require(cell[key] in receipt['artifacts'], 'unbound observation artifact') |
| 148 | + observation = parse_json(Path(cell['observation']['path']).read_text()) |
| 149 | + workload = expected_identity.get('workload') |
| 150 | + if workload is not None: |
| 151 | + require(observation['requested'] == workload['requests'], 'incomplete workload') |
| 152 | + concurrency = effective_concurrency(cell['protocol'], workload) |
| 153 | + require(cell['concurrency'] == concurrency and observation['concurrency'] == concurrency, 'wrong effective concurrency') |
| 154 | + expected_epochs, expected_warmup = epoch_workload(cell['protocol'], workload['requests'], concurrency, workload['warmup']) |
| 155 | + require(observation['warmup_requested'] == expected_warmup, 'wrong warmup workload') |
| 156 | + require(len(observation['epochs']) == expected_epochs, 'wrong epoch count') |
| 157 | + validate_observation(observation, receipt['run_id'], cell['protocol']) |
| 158 | + validate_server(parse_json(Path(cell['server']['path']).read_text()), observation) |
| 159 | + return receipt |
0 commit comments