From 5075c3eb8d4acc341e9f4270549e96ef15734a9c Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman <91019033+matt-dz@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:09:17 -0400 Subject: [PATCH] ci(fuzz): tolerate only Go's bare -fuzztime timeout, not real findings The fuzz jobs treated any failing run as a coordinator boundary timeout unless some line looked like an indented `_test.go:N:` reference. A panic reports through `testing.go:N:` and its stack frames end in `+0x..`, not a colon, so real crashes (and build errors) passed CI. Reproduced locally: a fuzz target that writes to a nil map fails with "Failing input written to testdata/fuzz/..." and the old check exits 0. Port the stricter check from datadog-agent's tasks/fuzz.py (DataDog/datadog-agent#58502): a failing run passes only when it printed "context deadline exceeded" and Go wrote no failing input. Both fuzz jobs now share scripts/fuzz-run.sh instead of two copies of the inline function. Environment: Datadog workspace Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/fuzz.yml | 64 ++------------------------------------ scripts/fuzz-run.sh | 43 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 62 deletions(-) create mode 100644 scripts/fuzz-run.sh diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index ed1a059d0..fae08dd3d 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -171,39 +171,9 @@ jobs: echo "No fuzz targets found in ${{ matrix.pkg }}, skipping" exit 0 fi - # fuzz_run wraps a single fuzz invocation and treats "context deadline exceeded" - # at the fuzz time boundary as success. When -fuzztime=30s expires, Go's internal - # fuzz coordinator cancels any in-flight iteration and emits "context deadline - # exceeded" with no file:line reference — this is not a real test failure. - # Real failures always include a file:line reference (e.g. foo_test.go:42:). - # - # We use `tee` rather than output=$(...) so that: - # 1. Progress lines appear in the Actions log in real time (output=$(...) buffers - # everything and bash -e exits the function before we ever reach echo "$output"). - # 2. PIPESTATUS[0] captures go test's exit code even when the pipe itself exits 0. - fuzz_run() { - local func="$1" - local tmpfile exit_code - tmpfile=$(mktemp) - go test -fuzz="^${func}$" -fuzztime=30s ${{ matrix.pkg }} -timeout 90s 2>&1 | tee "$tmpfile" || true - exit_code=${PIPESTATUS[0]} - if [ $exit_code -ne 0 ]; then - # Check whether any line has a file:line reference (real assertion failure). - # Coordinator boundary timeouts produce " context deadline exceeded" with - # no file:line, so they will NOT match this pattern. - if grep -qE '[[:space:]]+[^[:space:]]+_test\.go:[0-9]+:' "$tmpfile"; then - rm -f "$tmpfile" - echo "FAIL: $func — test assertion failure detected" >&2 - return $exit_code - fi - echo "NOTE: $func — fuzz coordinator boundary timeout (expected at fuzz time limit, not a failure)" - fi - rm -f "$tmpfile" - return 0 - } for FUNC in $FUZZ_FUNCS; do echo "Fuzzing $FUNC..." - fuzz_run "$FUNC" + bash scripts/fuzz-run.sh ${{ matrix.pkg }} "$FUNC" done # Save corpus @@ -255,39 +225,9 @@ jobs: echo "No differential fuzz targets found in ${{ matrix.pkg }}, skipping" exit 0 fi - # fuzz_run wraps a single fuzz invocation and treats "context deadline exceeded" - # at the fuzz time boundary as success. When -fuzztime=30s expires, Go's internal - # fuzz coordinator cancels any in-flight iteration and emits "context deadline - # exceeded" with no file:line reference — this is not a real test failure. - # Real failures always include a file:line reference (e.g. foo_test.go:42:). - # - # We use `tee` rather than output=$(...) so that: - # 1. Progress lines appear in the Actions log in real time (output=$(...) buffers - # everything and bash -e exits the function before we ever reach echo "$output"). - # 2. PIPESTATUS[0] captures go test's exit code even when the pipe itself exits 0. - fuzz_run() { - local func="$1" - local tmpfile exit_code - tmpfile=$(mktemp) - go test -fuzz="^${func}$" -fuzztime=30s ${{ matrix.pkg }} -timeout 90s 2>&1 | tee "$tmpfile" || true - exit_code=${PIPESTATUS[0]} - if [ $exit_code -ne 0 ]; then - # Check whether any line has a file:line reference (real assertion failure). - # Coordinator boundary timeouts produce " context deadline exceeded" with - # no file:line, so they will NOT match this pattern. - if grep -qE '[[:space:]]+[^[:space:]]+_test\.go:[0-9]+:' "$tmpfile"; then - rm -f "$tmpfile" - echo "FAIL: $func — test assertion failure detected" >&2 - return $exit_code - fi - echo "NOTE: $func — fuzz coordinator boundary timeout (expected at fuzz time limit, not a failure)" - fi - rm -f "$tmpfile" - return 0 - } for FUNC in $FUZZ_FUNCS; do echo "Fuzzing $FUNC..." - fuzz_run "$FUNC" + bash scripts/fuzz-run.sh ${{ matrix.pkg }} "$FUNC" done - name: Save fuzz corpus diff --git a/scripts/fuzz-run.sh b/scripts/fuzz-run.sh new file mode 100644 index 000000000..093ef98e1 --- /dev/null +++ b/scripts/fuzz-run.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Run one Go fuzz target for a fixed duration, treating only Go's -fuzztime +# boundary timeout as a pass. +# +# Usage: bash scripts/fuzz-run.sh [fuzztime] [timeout] +# +# When -fuzztime expires, Go's fuzz coordinator cancels in-flight inputs and +# can fail the run with a bare "context deadline exceeded" and no failing +# input. That is not a finding. A real finding (a panic, t.Fatal, a hang the +# worker reports, ...) always makes Go write the input to testdata/fuzz and +# print "Failing input written to", so any failure that printed it, or that +# lacks the bare timeout message, still fails. +set -uo pipefail + +if [ $# -lt 2 ] || [ $# -gt 4 ]; then + echo "usage: $0 [fuzztime] [timeout]" >&2 + exit 2 +fi + +pkg="$1" +func="$2" +fuzztime="${3:-30s}" +timeout="${4:-90s}" + +tmpfile=$(mktemp) +trap 'rm -f "$tmpfile"' EXIT + +# tee streams progress to the CI log in real time; PIPESTATUS keeps go test's +# exit code rather than tee's. +go test -fuzz="^${func}\$" -fuzztime="$fuzztime" "$pkg" -timeout "$timeout" 2>&1 | tee "$tmpfile" +exit_code=${PIPESTATUS[0]} + +if [ "$exit_code" -eq 0 ]; then + exit 0 +fi + +if grep -q 'context deadline exceeded' "$tmpfile" && ! grep -q 'Failing input written to' "$tmpfile"; then + echo "NOTE: $func hit Go's fuzz coordinator timeout at the -fuzztime limit; not a failure" + exit 0 +fi + +echo "FAIL: $func" >&2 +exit "$exit_code"