diff --git a/cmd/autocar/bench.go b/cmd/autocar/bench.go index 6806cbe..3dc303d 100644 --- a/cmd/autocar/bench.go +++ b/cmd/autocar/bench.go @@ -256,10 +256,19 @@ func populateAccelerationMetadata(output *benchOutput, mode byte, dialer transpo } } +// percentile expects sorted, nonnegative measurements. The median averages +// both central values for even sample counts; tail percentiles retain the +// nearest-order-statistic convention instead of interpolating observations. func percentile(sorted []float64, fraction float64) float64 { if len(sorted) == 0 { return 0 } + if fraction == 0.5 && len(sorted)%2 == 0 { + middle := len(sorted) / 2 + lower, upper := sorted[middle-1], sorted[middle] + // Avoid overflowing the sum for large, finite measurements. + return lower + (upper-lower)/2 + } index := int(float64(len(sorted)-1)*fraction + 0.5) if index < 0 { index = 0 diff --git a/cmd/autocar/bench_median_test.go b/cmd/autocar/bench_median_test.go new file mode 100644 index 0000000..0261dd2 --- /dev/null +++ b/cmd/autocar/bench_median_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "math" + "testing" +) + +func TestPercentileMedian(t *testing.T) { + for _, tc := range []struct { + name string + sorted []float64 + want float64 + }{ + {"empty", nil, 0}, + {"single", []float64{3}, 3}, + {"two", []float64{2, 10}, 6}, + {"four", []float64{1, 2, 8, 100}, 5}, + {"odd", []float64{1, 3, 100}, 3}, + {"repeated", []float64{4, 4, 4, 4}, 4}, + {"large", []float64{math.MaxFloat64, math.MaxFloat64}, math.MaxFloat64}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := percentile(tc.sorted, 0.5); got != tc.want { + t.Fatalf("median(%v) = %v, want %v", tc.sorted, got, tc.want) + } + }) + } +} + +func TestPercentileTailConventionUnchanged(t *testing.T) { + values := []float64{2, 10} + if got := percentile(values, 0.05); got != 2 { + t.Fatalf("p05 = %v, want 2", got) + } + if got := percentile(values, 0.95); got != 10 { + t.Fatalf("p95 = %v, want 10", got) + } +} diff --git a/docs/BENCHMARK.md b/docs/BENCHMARK.md index c7d515c..9c14bcd 100644 --- a/docs/BENCHMARK.md +++ b/docs/BENCHMARK.md @@ -30,6 +30,12 @@ Measure direct and tunneled paths with the same payload, warmup and iterations: JSON contains raw `results_mbps` and `durations_ms`, plus `median_mbps`, `p05_mbps`, `p95_mbps`, `median_duration_ms` and `p95_duration_ms`. The throughput p95 is the upper tail; use throughput p05 or duration p95 to discuss slow runs. +Both medians use the middle observation for an odd sample count and the average +of the two middle observations for an even count. Tail percentiles retain the +nearest order statistic at index `round((n-1)*p)` (zero-based); they do not +interpolate and are descriptive summaries, not confidence bounds. Older builds +used that rounded index for medians too, selecting the upper middle observation +for even counts. Recompute from raw samples when comparing those reports. The selected transport and sender metadata must remain identical across all measured iterations or the command fails. `transport` is the requested policy (`auto`, `quic`, `tls` or `direct`); `selected_transport` records the actual