From d4f27ac64b7a193e1772731624e8e9ce0be7a8d7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 01:23:43 +0900 Subject: [PATCH 01/21] test: prove pg-erd shared logs exclude sensitive request material --- tests/pg_erd_payload_free_observability.rs | 199 +++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 tests/pg_erd_payload_free_observability.rs diff --git a/tests/pg_erd_payload_free_observability.rs b/tests/pg_erd_payload_free_observability.rs new file mode 100644 index 00000000..5e1d31df --- /dev/null +++ b/tests/pg_erd_payload_free_observability.rs @@ -0,0 +1,199 @@ +//! Real-listener payload-free logging acceptance for the dedicated pg-erd migration binary. +//! +//! The shared observability bounded context promises that request paths, query strings, headers, +//! cookies, credentials, customer payloads, and product identifiers never enter its access-log +//! vocabulary. This contract proves that boundary through the compiled migration process while +//! sensitive request material is actually present on the proxied request path. + +use std::io::{Read, Write}; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::process::{Child, Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; + +use tempfile::NamedTempFile; + +struct GatewayProcess(Option); + +impl GatewayProcess { + fn capture_stderr(mut self) -> String { + let mut child = self.0.take().expect("gateway child should still be owned"); + child.kill().expect("gateway should be terminable after traffic"); + let output = child + .wait_with_output() + .expect("gateway output should be collectable after termination"); + String::from_utf8(output.stderr).expect("gateway log output should be UTF-8") + } +} + +impl Drop for GatewayProcess { + fn drop(&mut self) { + if let Some(child) = self.0.as_mut() { + let _ = child.kill(); + let _ = child.wait(); + } + } +} + +fn reserve_loopback() -> SocketAddr { + TcpListener::bind("127.0.0.1:0") + .expect("loopback port should be reservable") + .local_addr() + .expect("reservation should expose an address") +} + +fn write_config( + listener: SocketAddr, + metrics_listener: SocketAddr, + backend: SocketAddr, + frontend: SocketAddr, +) -> NamedTempFile { + let mut file = NamedTempFile::new().expect("temporary config should be writable"); + writeln!( + file, + "version: 1\nlistener: {listener}\nmetrics_listener: {metrics_listener}\nmax_request_body_bytes: 8\nmax_in_flight_requests: 8\nupstream_keepalive_pool_size: 4\nupstreams:\n - name: backend\n address: {backend}\n tls: false\n timeouts:\n connection_ms: 200\n total_connection_ms: 400\n read_ms: 1000\n write_ms: 1000\n idle_ms: 5000\n - name: frontend\n address: {frontend}\n tls: false\n timeouts:\n connection_ms: 200\n total_connection_ms: 400\n read_ms: 1000\n write_ms: 1000\n idle_ms: 5000" + ) + .expect("migration config should be written"); + file +} + +fn wait_until_listening(address: SocketAddr, process: &mut Child) { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + if let Some(status) = process + .try_wait() + .expect("gateway process state should be readable") + { + panic!("gateway exited before accepting traffic: {status}"); + } + if TcpStream::connect_timeout(&address, Duration::from_millis(100)).is_ok() { + return; + } + assert!(Instant::now() < deadline, "gateway did not start within 10s"); + thread::sleep(Duration::from_millis(25)); + } +} + +fn start_gateway( + config: &NamedTempFile, + gateway_address: SocketAddr, + metrics_address: SocketAddr, +) -> GatewayProcess { + let mut child = Command::new(env!("CARGO_BIN_EXE_cwl-pingora-pg-erd-migration")) + .args(["--config", config.path().to_str().expect("UTF-8 temp path")]) + .env("RUST_LOG", "cwl_pingora_gateway::observability=info") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .expect("compiled pg-erd migration binary should start"); + wait_until_listening(gateway_address, &mut child); + wait_until_listening(metrics_address, &mut child); + GatewayProcess(Some(child)) +} + +fn raw_request(address: SocketAddr, request: &[u8]) -> String { + let mut downstream = TcpStream::connect(address).expect("gateway should accept traffic"); + downstream + .set_read_timeout(Some(Duration::from_secs(5))) + .expect("downstream timeout should be configurable"); + downstream + .write_all(request) + .expect("downstream request should be writable"); + let mut response = String::new(); + downstream + .read_to_string(&mut response) + .expect("gateway response should be readable"); + response +} + +fn read_request_headers(stream: &mut TcpStream) -> String { + let mut bytes = Vec::new(); + let mut buffer = [0_u8; 1024]; + loop { + let read = stream.read(&mut buffer).expect("origin request should be readable"); + assert!(read > 0, "gateway closed origin request before headers completed"); + bytes.extend_from_slice(&buffer[..read]); + if bytes.windows(4).any(|window| window == b"\r\n\r\n") { + return String::from_utf8_lossy(&bytes).into_owned(); + } + } +} + +#[test] +fn compiled_pg_erd_shared_access_log_excludes_request_sensitive_material() { + let backend = TcpListener::bind("127.0.0.1:0").expect("backend fixture should bind"); + let backend_address = backend.local_addr().expect("backend address should exist"); + let backend_origin = thread::spawn(move || { + let (mut stream, _) = backend + .accept() + .expect("routed request should reach the characterized backend authority"); + let request = read_request_headers(&mut stream); + let lower = request.to_ascii_lowercase(); + assert!(lower.starts_with( + "get /api/log-contract?customer=query-secret http/1.1\r\n" + )); + assert!(lower.contains("host: tenant-secret.example:8080\r\n")); + assert!(lower.contains("authorization: bearer authorization-secret\r\n")); + assert!(lower.contains("cookie: session=cookie-secret\r\n")); + assert!(lower.contains("x-product-context: product-secret\r\n")); + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") + .expect("backend response should be writable"); + }); + + let frontend = TcpListener::bind("127.0.0.1:0").expect("frontend fixture should bind"); + let frontend_address = frontend.local_addr().expect("frontend address should exist"); + + let gateway_address = reserve_loopback(); + let metrics_address = reserve_loopback(); + let config = write_config( + gateway_address, + metrics_address, + backend_address, + frontend_address, + ); + let process = start_gateway(&config, gateway_address, metrics_address); + + let response = raw_request( + gateway_address, + b"GET /api/log-contract?customer=query-secret HTTP/1.1\r\nHost: tenant-secret.example:8080\r\nAuthorization: Bearer authorization-secret\r\nCookie: session=cookie-secret\r\nX-Product-Context: product-secret\r\nConnection: close\r\n\r\n", + ); + assert!( + response.starts_with("HTTP/1.1 200"), + "sensitive-material fixture request should proxy successfully: {response:?}" + ); + backend_origin + .join() + .expect("backend sensitive-material fixture should complete"); + + let stderr = process.capture_stderr(); + let request_logs: Vec<_> = stderr + .lines() + .filter(|line| line.contains("gateway_request")) + .collect(); + assert_eq!( + request_logs.len(), + 1, + "the shared observability target should emit one completion record: {stderr:?}" + ); + let access_log = request_logs[0]; + assert!( + access_log.contains("gateway_request status=200 outcome=ok request_body_bytes=0"), + "shared access logging should contain only bounded transport facts: {access_log:?}" + ); + + for forbidden in [ + "/api/log-contract", + "query-secret", + "tenant-secret.example", + "authorization-secret", + "cookie-secret", + "product-secret", + ] { + assert!( + !access_log.contains(forbidden), + "shared access logging leaked request-sensitive material {forbidden:?}: {access_log:?}" + ); + } +} From 8fcd0400c94aade84c97cd3ef573b985aa452ef3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 01:24:15 +0900 Subject: [PATCH 02/21] docs: record pg-erd payload-free log acceptance --- TEST_STRATEGY.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TEST_STRATEGY.md b/TEST_STRATEGY.md index fd3f46b2..1d70ab9e 100644 --- a/TEST_STRATEGY.md +++ b/TEST_STRATEGY.md @@ -8,7 +8,7 @@ Migration characterization is executable before transport activation. `tests/pg_ The bounded Admin Config transition has its own executable contract. `tests/pg_erd_admin_config_contract.rs` rejects unknown/future configuration, listener collision, zero runtime/keepalive budgets, missing/extra/duplicate/renamed authority, and invalid concrete transport configuration; it also proves only `backend` and `frontend` can bind the compiled route profile. `tests/pg_erd_binary_startup.rs` exercises the dedicated compiled process and requires fail-closed behavior for omitted configuration, unreadable configuration, invalid Admin Config, and custom TLS trust material that cannot be materialized before listener activation. -`tests/pg_erd_production_path.rs` is the first dedicated compiled-listener traffic contract. It starts real loopback `backend` and `frontend` origins plus `cwl-pingora-pg-erd-migration`, requires `/livez` and `/readyz` to remain gateway-local, requires consumer `/healthz` and raw `/apiary` to reach `backend`, requires fallback product traffic to reach `frontend`, requires request-controlled forwarding identity to be discarded and rebuilt from accepted loopback transport/Host data, requires upstream `X-Frame-Options: SAMEORIGIN` to be replaced by the characterized `DENY` policy together with the other three captured response fields, and requires an over-limit declared body to fail with 413 before origin delivery. `tests/pg_erd_runtime_isolation_traffic.rs` extends that compiled-process boundary with a chunked request whose streamed body crosses `max_request_body_bytes`, requiring HTTP 413 followed by healthy `/readyz`, and with a one-request in-flight budget that holds one routed backend request open, requires the next routed request to fail fast with 503, keeps `/readyz` and `/metrics` available, observes exactly one backpressure rejection, then proves capacity is released for a later routed request. `tests/pg_erd_upstream_failure_traffic.rs` adds the first dedicated origin-failure contract: the characterized backend address is made actively unavailable after its concrete port is captured, `/api` traffic must fail as HTTP 502 within the configured connection budget, `/readyz` must remain healthy, the low-cardinality request-error counter must record the failure, and an independent fallback request must still reach the characterized `frontend` authority successfully. `tests/pg_erd_read_stall_traffic.rs` covers the next transport phase: the characterized backend accepts the HTTP/1.1 request but emits no response bytes, so the configured Pingora per-read `read_ms` budget must fail the request as HTTP 502 within a conservative outer bound, preserve `/readyz`, record the low-cardinality error, and leave the independent fallback route usable. The fixture intentionally sends no partial response, so it does not transfer evidence to reset, partial-response, streaming-failure, slow-drip, or total-response-lifetime behavior. `tests/pg_erd_partial_response_traffic.rs` covers a distinct post-header failure phase: the characterized backend commits HTTP 200 plus `Content-Length: 20`, sends only `partial`, then closes. The downstream must retain the committed 200/framing and terminate before the declared body completes rather than receiving an invented second status or silent failover; `/readyz`, low-cardinality error telemetry, and an independent `frontend` route must remain usable afterward. The fixture uses an orderly origin close and therefore does not transfer evidence to an explicit TCP reset, upgraded/WebSocket failure, generic streaming failure, or slow-drip/whole-response lifetime. On Unix, `tests/pg_erd_graceful_shutdown.rs` adds consumer-root drain evidence: a routed `/api/held` request is held at the characterized backend, SIGTERM is delivered only after the backend has received it, the response is released within the shared grace period, the downstream must still receive HTTP 200, and the migration process must exit successfully inside the external termination budget. These source contracts are not GREEN evidence by themselves: they must compile and pass with formatting, clippy, rustdoc and 100% owned-production line/region coverage on the same exact head before listener/runtime/failure-recovery/drain parity is credited. +`tests/pg_erd_production_path.rs` is the first dedicated compiled-listener traffic contract. It starts real loopback `backend` and `frontend` origins plus `cwl-pingora-pg-erd-migration`, requires `/livez` and `/readyz` to remain gateway-local, requires consumer `/healthz` and raw `/apiary` to reach `backend`, requires fallback product traffic to reach `frontend`, requires request-controlled forwarding identity to be discarded and rebuilt from accepted loopback transport/Host data, requires upstream `X-Frame-Options: SAMEORIGIN` to be replaced by the characterized `DENY` policy together with the other three captured response fields, and requires an over-limit declared body to fail with 413 before origin delivery. `tests/pg_erd_runtime_isolation_traffic.rs` extends that compiled-process boundary with a chunked request whose streamed body crosses `max_request_body_bytes`, requiring HTTP 413 followed by healthy `/readyz`, and with a one-request in-flight budget that holds one routed backend request open, requires the next routed request to fail fast with 503, keeps `/readyz` and `/metrics` available, observes exactly one backpressure rejection, then proves capacity is released for a later routed request. `tests/pg_erd_upstream_failure_traffic.rs` adds the first dedicated origin-failure contract: the characterized backend address is made actively unavailable after its concrete port is captured, `/api` traffic must fail as HTTP 502 within the configured connection budget, `/readyz` must remain healthy, the low-cardinality request-error counter must record the failure, and an independent fallback request must still reach the characterized `frontend` authority successfully. `tests/pg_erd_read_stall_traffic.rs` covers the next transport phase: the characterized backend accepts the HTTP/1.1 request but emits no response bytes, so the configured Pingora per-read `read_ms` budget must fail the request as HTTP 502 within a conservative outer bound, preserve `/readyz`, record the low-cardinality error, and leave the independent fallback route usable. The fixture intentionally sends no partial response, so it does not transfer evidence to reset, partial-response, streaming-failure, slow-drip, or total-response-lifetime behavior. `tests/pg_erd_partial_response_traffic.rs` covers a distinct post-header failure phase: the characterized backend commits HTTP 200 plus `Content-Length: 20`, sends only `partial`, then closes. The downstream must retain the committed 200/framing and terminate before the declared body completes rather than receiving an invented second status or silent failover; `/readyz`, low-cardinality error telemetry, and an independent `frontend` route must remain usable afterward. The fixture uses an orderly origin close and therefore does not transfer evidence to an explicit TCP reset, upgraded/WebSocket failure, generic streaming failure, or slow-drip/whole-response lifetime. `tests/pg_erd_payload_free_observability.rs` exercises the compiled migration binary with a real routed request whose URI/query, Host, Authorization header, Cookie, and product-context header all carry unique sentinel values. The backend must receive those sensitive request values so the fixture is non-vacuous, while the shared `cwl_pingora_gateway::observability` log target must emit exactly the bounded `status`/`outcome`/`request_body_bytes` completion record and none of the sentinels. This proves the shared gateway access-log vocabulary remains payload-free without claiming that unrelated product-owned logs or third-party logger configuration are governed by this bounded context. On Unix, `tests/pg_erd_graceful_shutdown.rs` adds consumer-root drain evidence: a routed `/api/held` request is held at the characterized backend, SIGTERM is delivered only after the backend has received it, the response is released within the shared grace period, the downstream must still receive HTTP 200, and the migration process must exit successfully inside the external termination budget. These source contracts are not GREEN evidence by themselves: they must compile and pass with formatting, clippy, rustdoc and 100% owned-production line/region coverage on the same exact head before listener/runtime/failure-recovery/drain/observability parity is credited. The exact OCI gate builds one digest-pinned distroless image and exercises both composition roots under uid/gid 65532, a read-only root filesystem, all Linux capabilities dropped, `no-new-privileges`, and read-only configuration mounts. Generic v1 remains the image default entrypoint. The pg-erd migration binary is selected explicitly, uses only the fixed `backend`/`frontend` authority profile, and must expose both its process-health and metrics listeners. This is source acceptance until the unchanged exact-head OCI job executes terminal-success; it is not deployment, canary, or cutover evidence. @@ -16,4 +16,4 @@ The exact OCI gate builds one digest-pinned distroless image and exercises both Every behavioral migration should begin with characterization against the old owned edge behavior, then add equivalent production-path evidence for Pingora. Static-serving consumers must cover route precedence, SPA fallback, MIME, ETag/cache, Range/HEAD/304/416, redirects/security headers and compression as applicable. Proxy consumers must cover Host/SNI/TLS, forwarding trust, WebSocket/upgrade, limits, timeout/retry behavior, streaming/uploads, saturation/backpressure, errors, health/readiness and drain. Characterized response policies must additionally be exercised through the compiled proxy path before any parity or canary claim. -Release-quality gaps remain: no property/fuzz tests yet; no downstream TLS listener contract; no HTTP/2 or HTTP/3 parity evidence; no tracing evidence; no immutable published registry digest/provenance and rehearsed rollback; and no benchmark against replaced Nginx/Traefik traffic. For the pg-erd candidate specifically, explicit TCP-reset and broader streaming-failure recovery, slow-drip/whole-response lifetime control, origin-capacity/representative deployment measurement, shadow/canary and rollback still lack terminal executable evidence. Refused-backend, connected-silent-backend, partial-response, routed graceful drain, dedicated OCI invocation, and routed loopback k6 now have source acceptance, but they remain uncredited until terminal exact-head execution. Generic OCI hardening, owned-production 100% line/region coverage, public rustdoc, SBOM/image vulnerability, local-CA upstream TLS, explicit backpressure/recovery, and the generic minimal k6 loopback path already have gates or tests, but each must be terminal-success on the exact release candidate and cannot be transferred to a changed or consumer-specific head. Production performance claims remain forbidden until representative deployment measurements exist. \ No newline at end of file +Release-quality gaps remain: no property/fuzz tests yet; no downstream TLS listener contract; no HTTP/2 or HTTP/3 parity evidence; no tracing evidence; no immutable published registry digest/provenance and rehearsed rollback; and no benchmark against replaced Nginx/Traefik traffic. For the pg-erd candidate specifically, explicit TCP-reset and broader streaming-failure recovery, slow-drip/whole-response lifetime control, origin-capacity/representative deployment measurement, shadow/canary and rollback still lack terminal executable evidence. Refused-backend, connected-silent-backend, partial-response, payload-free shared access logging, routed graceful drain, dedicated OCI invocation, and routed loopback k6 now have source acceptance, but they remain uncredited until terminal exact-head execution. Generic OCI hardening, owned-production 100% line/region coverage, public rustdoc, SBOM/image vulnerability, local-CA upstream TLS, explicit backpressure/recovery, and the generic minimal k6 loopback path already have gates or tests, but each must be terminal-success on the exact release candidate and cannot be transferred to a changed or consumer-specific head. Production performance claims remain forbidden until representative deployment measurements exist. From d45c57d94414095e7d871dd3709b512f3cde5258 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 01:24:55 +0900 Subject: [PATCH 03/21] docs: record pg-erd payload-free log contract --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cdf0f6f3..dd4ba1a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ All notable changes are tracked here. No release has been published yet. - Kept migration admin parsing side-effect free for custom TLS trust material: exact transport-authority and upstream-contract validation happens during parse, while peer/trust-bundle materialization occurs once during `build_proxy` before listener creation. This removes an avoidable validate-then-reload trust-file window. - Added a separate Ingress forwarding-policy boundary for the pg-erd migration. Request-controlled `Forwarded`, `X-Forwarded-*` and `X-Real-IP` values are removed, then the compatibility `X-Forwarded-For`, `X-Real-IP`, `X-Forwarded-Host`, `X-Forwarded-Port` and `X-Forwarded-Proto` fields are rebuilt from accepted transport metadata. The current characterized Traefik `web` entryPoint remains explicitly HTTP; TLS-derived scheme behavior is not claimed before a TLS listener contract exists. - Added a shared `observability` bounded context so both `GatewayProxy` and `MigrationGatewayProxy` use the same low-cardinality request/error/body/backpressure counters and coarse access-log shape instead of duplicating telemetry. The public observation vocabulary contains only response status, `ok`/`error`, and observed request-body bytes; paths, query strings, headers/cookies, credentials, customer payloads and product identifiers stay out of the shared telemetry contract. +- Added dedicated compiled pg-erd payload-free access-log acceptance. A routed request carries unique URI/query, Host, Authorization, Cookie, and product-context sentinel values and the backend must actually receive them, while the shared `cwl_pingora_gateway::observability` target must emit only `status`, `outcome`, and `request_body_bytes` without any sentinel. This proves the shared gateway log vocabulary rather than asserting control over unrelated product-owned or third-party logs. - Added mandatory positive `max_in_flight_requests` and `upstream_keepalive_pool_size` capacity budgets; Pingora's framework keepalive default is overridden from the validated edge contract. - Added process-local fail-fast backpressure: non-health requests above the in-flight budget receive HTTP 503, health remains observable, rejection telemetry increments, and capacity is released after request completion or failure. - Added dedicated compiled pg-erd runtime-isolation traffic acceptance: chunked bodies that cross `max_request_body_bytes` must return 413 without poisoning `/readyz`; a held routed request at an in-flight budget of one must force the next routed request to 503 while `/readyz` and metrics remain available, increment the backpressure counter, and release capacity for a subsequent routed request. @@ -40,4 +41,4 @@ All notable changes are tracked here. No release has been published yet. - Added missing-public-rustdoc enforcement and documentation builds with warnings denied. - Added DDD, product, technical, security, threat, test, operability, configuration, migration-gap, and primary-source traceability documentation. -Release remains blocked on the organization decision for the exact Pingora release versus `RUSTSEC-2026-0253` and the separate time-bounded disposition of unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388` (`ContextualWisdomLab/.github#1605`), restoration of authoritative public non-fork Dependency Review evidence (`ContextualWisdomLab/.github#810`), terminal exact-current-head CI/supply-chain/security/review evidence, representative compiled-binary pg-erd explicit TCP-reset/streaming-failure/slow-drip/origin-capacity and representative deployment/network benchmark evidence, an immutable registry digest with provenance and rehearsed rollback, and protected-branch integration. Refused-backend, connected-silent-backend, partial-response, routed graceful drain, dedicated pg-erd OCI invocation, and routed loopback latency now have source acceptance but remain uncredited until terminal exact-head execution. No consumer migration, canary, cutover, or legacy removal is claimed before those release and traffic-contract gates are satisfied. \ No newline at end of file +Release remains blocked on the organization decision for the exact Pingora release versus `RUSTSEC-2026-0253` and the separate time-bounded disposition of unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388` (`ContextualWisdomLab/.github#1605`), restoration of authoritative public non-fork Dependency Review evidence (`ContextualWisdomLab/.github#810`), terminal exact-current-head CI/supply-chain/security/review evidence, representative compiled-binary pg-erd explicit TCP-reset/streaming-failure/slow-drip/origin-capacity and representative deployment/network benchmark evidence, an immutable registry digest with provenance and rehearsed rollback, and protected-branch integration. Refused-backend, connected-silent-backend, partial-response, payload-free shared access logging, routed graceful drain, dedicated pg-erd OCI invocation, and routed loopback latency now have source acceptance but remain uncredited until terminal exact-head execution. No consumer migration, canary, cutover, or legacy removal is claimed before those release and traffic-contract gates are satisfied. From 48d45ade0d77edc52e36e13624391e141e52cca5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 01:27:01 +0900 Subject: [PATCH 04/21] docs: advance pg-erd observability gap baseline --- docs/product-technical-gap-baseline.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 15d4eaad..3a71aa94 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -23,10 +23,10 @@ This baseline is code-current for the Pingora migration stack. Exact source head | Failure recovery | Refused-origin, connected-silent-origin, and post-header partial-response source acceptance added; hosted GREEN pending | The generic compiled loopback contract requires a refused origin connection to return HTTP 502 within configured connection budget and proves `/readyz` remains healthy afterward. PR #17 adds the dedicated multi-route refused-backend equivalent with bounded 502 failure, readiness, low-cardinality error telemetry, and independent `frontend` recovery. PR #18 adds the next pre-header transport phase: the characterized backend accepts the request but emits no response bytes, so Pingora's configured per-read `read_ms` inactivity budget must produce HTTP 502, preserve readiness/error telemetry, and leave the independent route usable. PR #21 adds the distinct post-header phase: `backend` commits HTTP 200 with `Content-Length: 20`, sends only `partial`, then closes; the downstream must retain the committed status/framing and terminate before body completion, while readiness, error telemetry, and an independent `frontend` request recover. This matches Pingora's phase boundary: once the response header is downstream, a later failure cannot be rewritten into a second error response or failover. Explicit TCP reset, broader streaming/upgraded failure, and slow-drip/whole-response-lifetime cases remain gaps. No source contract is credited until exact-head terminal execution | | Health | Shared process boundary implemented; dedicated exact-head execution pending | `/livez` and `/readyz` are served locally through a shared internal Pingora process-health boundary in both adapters and bypass migration fallback routing. Consumer `/healthz` remains characterized application traffic to `backend`. The dedicated process contract explicitly distinguishes them; PR #16 additionally requires readiness to remain available after streamed-body rejection and while routed application capacity is saturated, PR #17 after refused upstream transport, PR #18 after connected read-timeout failure, and PR #21 after a post-header truncated response. Terminal exact-head execution is still required | | Graceful drain | Dedicated routed source acceptance added; hosted GREEN pending | Both composition roots use shared explicit Pingora server policy: 5 s grace plus 10 s runtime shutdown timeout inside a 30 s external termination budget. Generic drain is tested. PR #20 adds the consumer-root equivalent: a characterized `/api` request must reach `backend` and remain in flight before SIGTERM, then complete HTTP 200 when released during the grace period, after which the migration process must exit successfully inside the external termination budget. Generic evidence is not transferred; exact-head execution remains mandatory | -| Logs / metrics / traces | Listener-capable candidate | Shared `observability` owns low-cardinality request/error/body/backpressure counters and credential/cookie-safe coarse logs. Both adapters delegate to it; paths, query strings, headers/cookies, credentials, customer payloads and product identifiers are excluded. PR #16 requires dedicated saturation to advance the backpressure counter while metrics remain reachable; PRs #17, #18, and #21 require the request-error counter to advance after refused, connected-silent, and post-header truncated upstream failures. Dedicated payload-free log assertions and tracing remain gaps | +| Logs / metrics / traces | Dedicated payload-free access-log source acceptance added; hosted GREEN and tracing pending | Shared `observability` owns low-cardinality request/error/body/backpressure counters and credential/cookie-safe coarse logs. Both adapters delegate to it; paths, query strings, headers/cookies, credentials, customer payloads and product identifiers are excluded. PR #16 requires dedicated saturation to advance the backpressure counter while metrics remain reachable; PRs #17, #18, and #21 require the request-error counter to advance after refused, connected-silent, and post-header truncated upstream failures. PR #23 now adds a compiled pg-erd request carrying unique URI/query, Host, Authorization, Cookie, and product-context sentinels; the backend must receive them while the shared observability target must emit only `status`/`outcome`/`request_body_bytes` and none of those sentinels. That assertion is scoped to the canonical shared gateway log target; tracing and unrelated product/third-party log policy remain separate gaps | | OCI isolation | Dedicated image invocation source acceptance added; hosted GREEN pending | Runtime remains uid/gid 65532, read-only-root compatible, capability-free and `no-new-privileges`; base images are digest-pinned. PR #19 packages both Rust composition roots in the same exact image while retaining generic v1 as the default entrypoint, then invokes `cwl-pingora-pg-erd-migration` explicitly with a read-only fixed-profile config under the same rootless/read-only/capability-free boundary and requires its process-health plus metrics listeners to become reachable. Source presence is not deployment evidence until the exact OCI job executes terminal-success | | Dependency policy | Release-blocked | `.github#1605` owns the exact-release Pingora vs patched-`lru` decision and disposition of unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388`; `.github#810` independently owns the public non-fork Dependency Review compare-API HTTP 403 incident. Known-unsound downgrade, blanket advisory waiver, fail-open 403 handling, or substitute-scanner promotion is prohibited | -| Coverage / public API docs | Gates implemented, current head pending | Owned production line/region coverage is required at 100%; `#![deny(missing_docs)]` and warning-denied rustdoc cover public APIs. Fixed-profile/callback construction was modeled as infallible after validated boundaries instead of keeping structurally impossible error branches merely to evade coverage. PR #14 includes decision-path coverage for shared wildcard/equality listener authority plus generic zero-port traffic/metrics/upstream admission; the forwarding-trust successor adds an executable generic regression for the full client-controlled proxy-identity set. PRs #16-#18, #20, and #21 add integration-only traffic acceptance, while PR #19 adds OCI packaging/invocation acceptance and PR #22 adds CI/load-fixture acceptance; none changes owned-production coverage denominators. Current exact head still must pass the unchanged 100% line/region gate | +| Coverage / public API docs | Gates implemented, current head pending | Owned production line/region coverage is required at 100%; `#![deny(missing_docs)]` and warning-denied rustdoc cover public APIs. Fixed-profile/callback construction was modeled as infallible after validated boundaries instead of keeping structurally impossible error branches merely to evade coverage. PR #14 includes decision-path coverage for shared wildcard/equality listener authority plus generic zero-port traffic/metrics/upstream admission; the forwarding-trust successor adds an executable generic regression for the full client-controlled proxy-identity set. PRs #16-#18, #20, #21, and #23 add integration-only traffic/observability acceptance, while PR #19 adds OCI packaging/invocation acceptance and PR #22 adds CI/load-fixture acceptance; none changes owned-production coverage denominators. Current exact head still must pass the unchanged 100% line/region gate | | Load / 20 ms p95 | Generic and bounded pg-erd local regression source acceptance; exact-head GREEN pending | Checksum-pinned k6 2.2.0 keeps the generic 400-request/four-VU release-binary loopback gate unchanged. PR #22 adds a separate exact-release `cwl-pingora-pg-erd-migration` contract with distinct backend/frontend origins and 400 requests across four VUs alternating characterized backend and fallback routes; every response must preserve the route-specific body, request failures must remain zero, and loopback `http_req_duration` p95 must remain <20 ms. Fixture and process startup probes do not pre-exercise application routes. This is not production SLO evidence: larger origin-capacity, TLS/network/multi-hop and representative deployment measurements remain required | | Rollback | Documented, not rehearsed | Rehearsal requires an immutable protected release artifact/digest and a real consumer traffic transition | @@ -37,7 +37,7 @@ Fresh organization code evidence finds no OpenResty usage. Responsibility class, | Repository / evidence | Classification | Migration consequence | | --- | --- | --- | | `linux-cluster-ops/docs/architecture/nginx-routing-inventory.md` plus Nginx/Certbot recovery evidence | ACTIVE_RUNTIME / CURRENT_OPERATOR_DOC | True shared-edge candidate, but current multi-vhost routing, static/PHP-FPM and certificate-adjacent operations exceed Pingora v1. Split authority and freeze executable traffic/TLS contracts first | -| `pg-erd-cloud/deploy/traefik/dynamic.yaml` at protected `main@8dc746920c12988f082e914879d95e13c9693535` | ACTIVE_DEPLOYMENT / PLAUSIBLE_CONSUMER | Fresh 2026-09-02 read confirms ordered exact `/healthz -> backend`, raw-prefix `/api -> backend`, fallback `/ -> frontend` plus four response-security fields are unchanged. Consumer code also conditionally trusts sanitized `X-Forwarded-For`, so forwarding identity is migration behavior rather than cosmetic metadata. PRs #5/#6/#7/#10 characterize route/header/authority/peer binding, PR #11 composes them in Pingora callbacks, PR #12 adds bounded Admin Config, dedicated process startup tests, shared process-health separation and a real loopback backend/frontend traffic contract, PR #14 closes shared wildcard-collision plus generic port-zero network-authority gaps before activation, PR #15 closes the residual generic forwarding-identity spoofing set, PR #16 adds dedicated streamed-body and saturation/recovery traffic acceptance, PR #17 adds refused-backend recovery acceptance, PR #18 adds connected-silent-origin read-timeout acceptance, PR #19 packages/invokes the bounded migration root under the same hardened OCI boundary, PR #20 adds routed SIGTERM drain acceptance, PR #21 adds post-header partial-response failure/recovery acceptance, and PR #22 adds a routed multi-authority k6 regression. No terminal exact-head pg-erd parity, production performance, shadow/canary or cutover is claimed yet | +| `pg-erd-cloud/deploy/traefik/dynamic.yaml` at protected `main@8dc746920c12988f082e914879d95e13c9693535` | ACTIVE_DEPLOYMENT / PLAUSIBLE_CONSUMER | Fresh 2026-09-03 read confirms ordered exact `/healthz -> backend`, raw-prefix `/api -> backend`, fallback `/ -> frontend` plus four response-security fields are unchanged. Consumer code also conditionally trusts sanitized `X-Forwarded-For`, so forwarding identity is migration behavior rather than cosmetic metadata. PRs #5/#6/#7/#10 characterize route/header/authority/peer binding, PR #11 composes them in Pingora callbacks, PR #12 adds bounded Admin Config, dedicated process startup tests, shared process-health separation and a real loopback backend/frontend traffic contract, PR #14 closes shared wildcard-collision plus generic port-zero network-authority gaps before activation, PR #15 closes the residual generic forwarding-identity spoofing set, PR #16 adds dedicated streamed-body and saturation/recovery traffic acceptance, PR #17 adds refused-backend recovery acceptance, PR #18 adds connected-silent-origin read-timeout acceptance, PR #19 packages/invokes the bounded migration root under the same hardened OCI boundary, PR #20 adds routed SIGTERM drain acceptance, PR #21 adds post-header partial-response failure/recovery acceptance, PR #22 adds a routed multi-authority k6 regression, and PR #23 adds payload-free shared access-log acceptance. No terminal exact-head pg-erd parity, production performance, shadow/canary or cutover is claimed yet | | `naruon` NGINX ingress/live-E2E plus Traefik evaluation | ACTIVE_DEPLOYMENT / TEST_RUNTIME | Its Nginx proxy contract includes HTTP/1.1, long read/send timeouts, WebSocket Upgrade/Connection and forwarded identity semantics. Keycloak/authentication stays outside Pingora; only transport/edge policy can migrate after explicit parity evidence | | `scopeweave`, `LineageWeave`, `inkspan` Nginx static-serving images/config | ACTIVE_STATIC_RUNTIME | Static hosting is not automatically a shared-edge migration; prove gateway responsibility before queueing | | `life-os` ClusterIP-only base manifests with separately managed edge namespace | DELEGATED EDGE | Repository base manifests do not prove an embedded legacy edge to migrate | @@ -56,10 +56,10 @@ The EA owner path must consume that released contract and project each approved 1. Reacquire exact-current-head CI, 100% owned production line/region coverage, rustdoc, k6, OCI, SAST and supply-chain evidence after every source or documentation movement; repair only evidence-backed repository defects. Organization runner acquisition is separately tracked in `.github#712`; queued jobs are not source GREEN. 2. Keep `.github#1605` and `.github#810` fail-closed until their respective policy and GitHub Dependency Review availability owner paths are resolved; do not suppress `RUSTSEC-2024-0388` generically. -3. Keep PRs #5, #6, #7, #10, #11, #12, #14, #15, #16, #17, #18, #19, #20, #21 and #22 pre-traffic until exact-head quality/security evidence and coherent dependency ancestry are terminal. No child evidence repairs a parent release blocker. -4. PR #12 contains the bounded startup/Admin Config transition plus a dedicated compiled-process loopback traffic contract; PR #14 tightens shared generic/migration network-authority admission; PR #15 closes residual generic forwarding-identity spoofing; PR #16 adds dedicated streamed-body and in-flight saturation/readiness/telemetry/recovery traffic; PR #17 adds refused-backend failure/recovery; PR #18 adds connected-silent-origin per-read timeout/recovery while documenting that `read_ms` is not a whole-response budget; PR #19 packages and explicitly invokes the bounded migration binary under the same hardened OCI image/runtime boundary; PR #20 adds routed in-flight SIGTERM drain acceptance; PR #21 adds the committed-header/truncated-body failure phase without inventing post-commit retry semantics; PR #22 adds route-correct four-VU/400-request local k6 evidence for the bounded binary. The immediate gate remains terminal exact-head execution of these stacked contracts together with fmt/compile/clippy/rustdoc/100% coverage and applicable security/supply-chain checks. Any deterministic failure must be repaired causally; source test or packaging presence alone is not parity. -5. After the basic listener/runtime-isolation/refused-origin/read-stall/partial-response/drain/routed-load contracts are GREEN, extend dedicated pg-erd acceptance with explicit TCP reset, broader streaming/upgraded failure and slow-drip/whole-response-lifetime handling, payload-free log assertions, larger origin-capacity stress, and representative TLS/network/deployment measurements before adopting a production 20 ms p95 objective. A future HTTPS listener must separately prove TLS-derived forwarded scheme instead of reusing the current clear-text `web` assumption. +3. Keep PRs #5, #6, #7, #10, #11, #12, #14, #15, #16, #17, #18, #19, #20, #21, #22 and #23 pre-traffic until exact-head quality/security evidence and coherent dependency ancestry are terminal. No child evidence repairs a parent release blocker. +4. PR #12 contains the bounded startup/Admin Config transition plus a dedicated compiled-process loopback traffic contract; PR #14 tightens shared generic/migration network-authority admission; PR #15 closes residual generic forwarding-identity spoofing; PR #16 adds dedicated streamed-body and in-flight saturation/readiness/telemetry/recovery traffic; PR #17 adds refused-backend failure/recovery; PR #18 adds connected-silent-origin per-read timeout/recovery while documenting that `read_ms` is not a whole-response budget; PR #19 packages and explicitly invokes the bounded migration binary under the same hardened OCI image/runtime boundary; PR #20 adds routed in-flight SIGTERM drain acceptance; PR #21 adds the committed-header/truncated-body failure phase without inventing post-commit retry semantics; PR #22 adds route-correct four-VU/400-request local k6 evidence for the bounded binary; PR #23 adds compiled shared-log redaction evidence with non-vacuous request-sensitive sentinels. The immediate gate remains terminal exact-head execution of these stacked contracts together with fmt/compile/clippy/rustdoc/100% coverage and applicable security/supply-chain checks. Any deterministic failure must be repaired causally; source test or packaging presence alone is not parity. +5. After the basic listener/runtime-isolation/refused-origin/read-stall/partial-response/drain/routed-load/log-redaction contracts are GREEN, extend dedicated pg-erd acceptance with explicit TCP reset, broader streaming/upgraded failure and slow-drip/whole-response-lifetime handling, larger origin-capacity stress, tracing, and representative TLS/network/deployment measurements before adopting a production 20 ms p95 objective. A future HTTPS listener must separately prove TLS-derived forwarded scheme instead of reusing the current clear-text `web` assumption. 6. Once the dedicated OCI invocation is terminal GREEN, add a protected release path publishing an immutable image digest with SBOM/provenance/reproducibility evidence and rehearse rollback against that exact digest. 7. Satisfy then-live protected-branch review/governance without self-approval, bot-as-human claims, stale evidence transfer, or routine administrator bypass. 8. Wait for an immutable released Context Graph bundle with source-bound consumer-verifiable release evidence and a coherent compatible GREEN EA admission path before asserting authoritative architecture execution state. -9. Only then move `pg-erd-cloud` through explicit parity -> shadow/canary -> cutover -> rollback -> legacy removal. Other Nginx/Traefik surfaces remain separate responsibility-bound migration candidates and are not inherited automatically from this consumer profile. \ No newline at end of file +9. Only then move `pg-erd-cloud` through explicit parity -> shadow/canary -> cutover -> rollback -> legacy removal. Other Nginx/Traefik surfaces remain separate responsibility-bound migration candidates and are not inherited automatically from this consumer profile. From 5cbe2d0a65af0d72fdef64034e4a038ab5790fb3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 01:28:12 +0900 Subject: [PATCH 05/21] test: synchronize pg-erd log capture on completion metrics --- tests/pg_erd_payload_free_observability.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/pg_erd_payload_free_observability.rs b/tests/pg_erd_payload_free_observability.rs index 5e1d31df..01402ba2 100644 --- a/tests/pg_erd_payload_free_observability.rs +++ b/tests/pg_erd_payload_free_observability.rs @@ -167,6 +167,15 @@ fn compiled_pg_erd_shared_access_log_excludes_request_sensitive_material() { .join() .expect("backend sensitive-material fixture should complete"); + let metrics = raw_request( + metrics_address, + b"GET /metrics HTTP/1.1\r\nHost: metrics\r\nConnection: close\r\n\r\n", + ); + assert!( + metrics.contains("cwl_pingora_gateway_requests_total 1"), + "metrics scrape should prove the proxied request reached shared completion recording before log capture: {metrics:?}" + ); + let stderr = process.capture_stderr(); let request_logs: Vec<_> = stderr .lines() From a85763984a72a7d47b0ded0192de5a56a7c886f3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 01:37:12 +0900 Subject: [PATCH 06/21] test: assert sensitive material is absent from all shared log output --- tests/pg_erd_payload_free_observability.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pg_erd_payload_free_observability.rs b/tests/pg_erd_payload_free_observability.rs index 01402ba2..d1a892b5 100644 --- a/tests/pg_erd_payload_free_observability.rs +++ b/tests/pg_erd_payload_free_observability.rs @@ -201,8 +201,8 @@ fn compiled_pg_erd_shared_access_log_excludes_request_sensitive_material() { "product-secret", ] { assert!( - !access_log.contains(forbidden), - "shared access logging leaked request-sensitive material {forbidden:?}: {access_log:?}" + !stderr.contains(forbidden), + "shared observability target leaked request-sensitive material {forbidden:?}: {stderr:?}" ); } } From d81a0efbd10e6287348b6bc4475ae6652102eb49 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 01:41:47 +0900 Subject: [PATCH 07/21] test: wait for pg-erd completion log before shutdown --- tests/pg_erd_payload_free_observability.rs | 64 ++++++++++++++++++---- 1 file changed, 53 insertions(+), 11 deletions(-) diff --git a/tests/pg_erd_payload_free_observability.rs b/tests/pg_erd_payload_free_observability.rs index d1a892b5..84265088 100644 --- a/tests/pg_erd_payload_free_observability.rs +++ b/tests/pg_erd_payload_free_observability.rs @@ -5,6 +5,7 @@ //! vocabulary. This contract proves that boundary through the compiled migration process while //! sensitive request material is actually present on the proxied request path. +use std::fs; use std::io::{Read, Write}; use std::net::{SocketAddr, TcpListener, TcpStream}; use std::process::{Child, Command, Stdio}; @@ -13,22 +14,53 @@ use std::time::{Duration, Instant}; use tempfile::NamedTempFile; -struct GatewayProcess(Option); +struct GatewayProcess { + child: Option, + stderr: NamedTempFile, +} impl GatewayProcess { + fn wait_until_stderr_contains(&mut self, needle: &str) { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let captured = fs::read_to_string(self.stderr.path()) + .expect("gateway stderr capture should remain readable"); + if captured.contains(needle) { + return; + } + if let Some(status) = self + .child + .as_mut() + .expect("gateway child should still be owned") + .try_wait() + .expect("gateway process state should be readable") + { + panic!("gateway exited before expected log {needle:?}: {status}; stderr={captured:?}"); + } + assert!( + Instant::now() < deadline, + "gateway did not emit expected log {needle:?} within 10s; stderr={captured:?}" + ); + thread::sleep(Duration::from_millis(10)); + } + } + fn capture_stderr(mut self) -> String { - let mut child = self.0.take().expect("gateway child should still be owned"); + let mut child = self + .child + .take() + .expect("gateway child should still be owned"); child.kill().expect("gateway should be terminable after traffic"); - let output = child - .wait_with_output() - .expect("gateway output should be collectable after termination"); - String::from_utf8(output.stderr).expect("gateway log output should be UTF-8") + child + .wait() + .expect("gateway should terminate after traffic capture"); + fs::read_to_string(self.stderr.path()).expect("gateway log output should be UTF-8") } } impl Drop for GatewayProcess { fn drop(&mut self) { - if let Some(child) = self.0.as_mut() { + if let Some(child) = self.child.as_mut() { let _ = child.kill(); let _ = child.wait(); } @@ -79,17 +111,24 @@ fn start_gateway( gateway_address: SocketAddr, metrics_address: SocketAddr, ) -> GatewayProcess { + let stderr = NamedTempFile::new().expect("gateway stderr capture should be writable"); + let stderr_writer = stderr + .reopen() + .expect("gateway stderr capture should be reopenable for the child"); let mut child = Command::new(env!("CARGO_BIN_EXE_cwl-pingora-pg-erd-migration")) .args(["--config", config.path().to_str().expect("UTF-8 temp path")]) .env("RUST_LOG", "cwl_pingora_gateway::observability=info") .stdin(Stdio::null()) .stdout(Stdio::null()) - .stderr(Stdio::piped()) + .stderr(Stdio::from(stderr_writer)) .spawn() .expect("compiled pg-erd migration binary should start"); wait_until_listening(gateway_address, &mut child); wait_until_listening(metrics_address, &mut child); - GatewayProcess(Some(child)) + GatewayProcess { + child: Some(child), + stderr, + } } fn raw_request(address: SocketAddr, request: &[u8]) -> String { @@ -153,7 +192,7 @@ fn compiled_pg_erd_shared_access_log_excludes_request_sensitive_material() { backend_address, frontend_address, ); - let process = start_gateway(&config, gateway_address, metrics_address); + let mut process = start_gateway(&config, gateway_address, metrics_address); let response = raw_request( gateway_address, @@ -173,7 +212,10 @@ fn compiled_pg_erd_shared_access_log_excludes_request_sensitive_material() { ); assert!( metrics.contains("cwl_pingora_gateway_requests_total 1"), - "metrics scrape should prove the proxied request reached shared completion recording before log capture: {metrics:?}" + "metrics scrape should prove the proxied request reached shared completion recording: {metrics:?}" + ); + process.wait_until_stderr_contains( + "gateway_request status=200 outcome=ok request_body_bytes=0", ); let stderr = process.capture_stderr(); From bd66dd81c96060211731c18929a9eec9ff3162a4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 07:44:08 +0900 Subject: [PATCH 08/21] style(test): apply rustfmt to pg-erd observability fixture --- tests/pg_erd_payload_free_observability.rs | 35 ++++++++++++++-------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/tests/pg_erd_payload_free_observability.rs b/tests/pg_erd_payload_free_observability.rs index 84265088..7d46d041 100644 --- a/tests/pg_erd_payload_free_observability.rs +++ b/tests/pg_erd_payload_free_observability.rs @@ -35,7 +35,9 @@ impl GatewayProcess { .try_wait() .expect("gateway process state should be readable") { - panic!("gateway exited before expected log {needle:?}: {status}; stderr={captured:?}"); + panic!( + "gateway exited before expected log {needle:?}: {status}; stderr={captured:?}" + ); } assert!( Instant::now() < deadline, @@ -50,7 +52,9 @@ impl GatewayProcess { .child .take() .expect("gateway child should still be owned"); - child.kill().expect("gateway should be terminable after traffic"); + child + .kill() + .expect("gateway should be terminable after traffic"); child .wait() .expect("gateway should terminate after traffic capture"); @@ -101,7 +105,10 @@ fn wait_until_listening(address: SocketAddr, process: &mut Child) { if TcpStream::connect_timeout(&address, Duration::from_millis(100)).is_ok() { return; } - assert!(Instant::now() < deadline, "gateway did not start within 10s"); + assert!( + Instant::now() < deadline, + "gateway did not start within 10s" + ); thread::sleep(Duration::from_millis(25)); } } @@ -150,8 +157,13 @@ fn read_request_headers(stream: &mut TcpStream) -> String { let mut bytes = Vec::new(); let mut buffer = [0_u8; 1024]; loop { - let read = stream.read(&mut buffer).expect("origin request should be readable"); - assert!(read > 0, "gateway closed origin request before headers completed"); + let read = stream + .read(&mut buffer) + .expect("origin request should be readable"); + assert!( + read > 0, + "gateway closed origin request before headers completed" + ); bytes.extend_from_slice(&buffer[..read]); if bytes.windows(4).any(|window| window == b"\r\n\r\n") { return String::from_utf8_lossy(&bytes).into_owned(); @@ -169,9 +181,7 @@ fn compiled_pg_erd_shared_access_log_excludes_request_sensitive_material() { .expect("routed request should reach the characterized backend authority"); let request = read_request_headers(&mut stream); let lower = request.to_ascii_lowercase(); - assert!(lower.starts_with( - "get /api/log-contract?customer=query-secret http/1.1\r\n" - )); + assert!(lower.starts_with("get /api/log-contract?customer=query-secret http/1.1\r\n")); assert!(lower.contains("host: tenant-secret.example:8080\r\n")); assert!(lower.contains("authorization: bearer authorization-secret\r\n")); assert!(lower.contains("cookie: session=cookie-secret\r\n")); @@ -182,7 +192,9 @@ fn compiled_pg_erd_shared_access_log_excludes_request_sensitive_material() { }); let frontend = TcpListener::bind("127.0.0.1:0").expect("frontend fixture should bind"); - let frontend_address = frontend.local_addr().expect("frontend address should exist"); + let frontend_address = frontend + .local_addr() + .expect("frontend address should exist"); let gateway_address = reserve_loopback(); let metrics_address = reserve_loopback(); @@ -214,9 +226,8 @@ fn compiled_pg_erd_shared_access_log_excludes_request_sensitive_material() { metrics.contains("cwl_pingora_gateway_requests_total 1"), "metrics scrape should prove the proxied request reached shared completion recording: {metrics:?}" ); - process.wait_until_stderr_contains( - "gateway_request status=200 outcome=ok request_body_bytes=0", - ); + process + .wait_until_stderr_contains("gateway_request status=200 outcome=ok request_body_bytes=0"); let stderr = process.capture_stderr(); let request_logs: Vec<_> = stderr From fe6362616d72f0e0151288ff82136d11f4a2da82 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 14:39:11 +0900 Subject: [PATCH 09/21] test: hold observability listener reservations together --- tests/pg_erd_payload_free_observability.rs | 25 ++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tests/pg_erd_payload_free_observability.rs b/tests/pg_erd_payload_free_observability.rs index 7d46d041..2ed82735 100644 --- a/tests/pg_erd_payload_free_observability.rs +++ b/tests/pg_erd_payload_free_observability.rs @@ -71,11 +71,22 @@ impl Drop for GatewayProcess { } } -fn reserve_loopback() -> SocketAddr { - TcpListener::bind("127.0.0.1:0") - .expect("loopback port should be reservable") +/// Reserves both process listeners at once so sequential bind-and-drop cannot +/// reuse the first ephemeral port and manufacture an Admin Config collision. +fn reserve_gateway_addresses() -> (TcpListener, TcpListener, SocketAddr, SocketAddr) { + let gateway = TcpListener::bind("127.0.0.1:0").expect("gateway port should be reservable"); + let metrics = TcpListener::bind("127.0.0.1:0").expect("metrics port should be reservable"); + let gateway_address = gateway .local_addr() - .expect("reservation should expose an address") + .expect("gateway reservation should expose an address"); + let metrics_address = metrics + .local_addr() + .expect("metrics reservation should expose an address"); + assert_ne!( + gateway_address, metrics_address, + "traffic and metrics reservations must remain distinct" + ); + (gateway, metrics, gateway_address, metrics_address) } fn write_config( @@ -196,14 +207,16 @@ fn compiled_pg_erd_shared_access_log_excludes_request_sensitive_material() { .local_addr() .expect("frontend address should exist"); - let gateway_address = reserve_loopback(); - let metrics_address = reserve_loopback(); + let (gateway_reservation, metrics_reservation, gateway_address, metrics_address) = + reserve_gateway_addresses(); let config = write_config( gateway_address, metrics_address, backend_address, frontend_address, ); + drop(gateway_reservation); + drop(metrics_reservation); let mut process = start_gateway(&config, gateway_address, metrics_address); let response = raw_request( From ef3b30c86dda033db9b3122977ff358f2dc9bd1d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 14:52:58 +0900 Subject: [PATCH 10/21] test: bound pg-erd observability origin reads --- tests/pg_erd_payload_free_observability.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/pg_erd_payload_free_observability.rs b/tests/pg_erd_payload_free_observability.rs index 2ed82735..b465653c 100644 --- a/tests/pg_erd_payload_free_observability.rs +++ b/tests/pg_erd_payload_free_observability.rs @@ -14,6 +14,8 @@ use std::time::{Duration, Instant}; use tempfile::NamedTempFile; +const MAX_ORIGIN_REQUEST_HEADER_BYTES: usize = 64 * 1024; + struct GatewayProcess { child: Option, stderr: NamedTempFile, @@ -164,18 +166,27 @@ fn raw_request(address: SocketAddr, request: &[u8]) -> String { response } +/// Reads one origin-side request header block under a finite timeout and byte +/// budget so a broken forwarding path fails deterministically instead of hanging CI. fn read_request_headers(stream: &mut TcpStream) -> String { + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .expect("origin request timeout should be configurable"); let mut bytes = Vec::new(); let mut buffer = [0_u8; 1024]; loop { let read = stream .read(&mut buffer) - .expect("origin request should be readable"); + .expect("origin request should be readable before the fixture deadline"); assert!( read > 0, "gateway closed origin request before headers completed" ); bytes.extend_from_slice(&buffer[..read]); + assert!( + bytes.len() <= MAX_ORIGIN_REQUEST_HEADER_BYTES, + "gateway origin request headers exceeded the fixture bound" + ); if bytes.windows(4).any(|window| window == b"\r\n\r\n") { return String::from_utf8_lossy(&bytes).into_owned(); } From caa1dfb644cde568f9a1519290ca4e56dfd23ca6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 15:26:40 +0900 Subject: [PATCH 11/21] test: match pg-erd observability headers semantically --- tests/pg_erd_payload_free_observability.rs | 49 +++++++++++++++++++--- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/tests/pg_erd_payload_free_observability.rs b/tests/pg_erd_payload_free_observability.rs index b465653c..1f8fe208 100644 --- a/tests/pg_erd_payload_free_observability.rs +++ b/tests/pg_erd_payload_free_observability.rs @@ -193,6 +193,28 @@ fn read_request_headers(stream: &mut TcpStream) -> String { } } +/// Selects exact HTTP field names case-insensitively while rejecting lookalike +/// fields such as `X-Forwarded-Host` that contain `Host` only as a suffix. +fn header_values<'a>(request: &'a str, name: &str) -> Vec<&'a str> { + request + .split("\r\n") + .skip(1) + .take_while(|line| !line.is_empty()) + .filter_map(|line| line.split_once(':')) + .filter_map(|(field_name, value)| { + field_name + .eq_ignore_ascii_case(name) + .then_some(value.trim()) + }) + .collect() +} + +#[test] +fn exact_header_matching_rejects_forwarded_host_lookalikes() { + let request = "GET / HTTP/1.1\r\nX-Forwarded-Host: tenant-secret.example:8080\r\nhOsT: expected.example\r\n\r\n"; + assert_eq!(header_values(request, "Host"), vec!["expected.example"]); +} + #[test] fn compiled_pg_erd_shared_access_log_excludes_request_sensitive_material() { let backend = TcpListener::bind("127.0.0.1:0").expect("backend fixture should bind"); @@ -202,12 +224,27 @@ fn compiled_pg_erd_shared_access_log_excludes_request_sensitive_material() { .accept() .expect("routed request should reach the characterized backend authority"); let request = read_request_headers(&mut stream); - let lower = request.to_ascii_lowercase(); - assert!(lower.starts_with("get /api/log-contract?customer=query-secret http/1.1\r\n")); - assert!(lower.contains("host: tenant-secret.example:8080\r\n")); - assert!(lower.contains("authorization: bearer authorization-secret\r\n")); - assert!(lower.contains("cookie: session=cookie-secret\r\n")); - assert!(lower.contains("x-product-context: product-secret\r\n")); + assert!( + request + .to_ascii_lowercase() + .starts_with("get /api/log-contract?customer=query-secret http/1.1\r\n") + ); + assert_eq!( + header_values(&request, "Host"), + vec!["tenant-secret.example:8080"] + ); + assert_eq!( + header_values(&request, "Authorization"), + vec!["Bearer authorization-secret"] + ); + assert_eq!( + header_values(&request, "Cookie"), + vec!["session=cookie-secret"] + ); + assert_eq!( + header_values(&request, "X-Product-Context"), + vec!["product-secret"] + ); stream .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") .expect("backend response should be writable"); From 08c0bd40ec4958ba2ec6e43046ecae7b8af967f3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 17:22:52 +0900 Subject: [PATCH 12/21] test: require exact observability metric sample --- tests/pg_erd_payload_free_observability.rs | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/pg_erd_payload_free_observability.rs b/tests/pg_erd_payload_free_observability.rs index 1f8fe208..2bc40c4e 100644 --- a/tests/pg_erd_payload_free_observability.rs +++ b/tests/pg_erd_payload_free_observability.rs @@ -209,12 +209,29 @@ fn header_values<'a>(request: &'a str, name: &str) -> Vec<&'a str> { .collect() } +/// Requires a complete Prometheus sample line so a value such as `10` cannot +/// satisfy an oracle that expects the exact counter value `1`. +fn contains_exact_metric_sample(metrics: &str, sample: &str) -> bool { + metrics + .lines() + .any(|line| line.trim_end_matches('\r') == sample) +} + #[test] fn exact_header_matching_rejects_forwarded_host_lookalikes() { let request = "GET / HTTP/1.1\r\nX-Forwarded-Host: tenant-secret.example:8080\r\nhOsT: expected.example\r\n\r\n"; assert_eq!(header_values(request, "Host"), vec!["expected.example"]); } +#[test] +fn exact_metric_sample_rejects_numeric_prefix_lookalikes() { + let metrics = "# TYPE cwl_pingora_gateway_requests_total counter\ncwl_pingora_gateway_requests_total 10\n"; + assert!(!contains_exact_metric_sample( + metrics, + "cwl_pingora_gateway_requests_total 1" + )); +} + #[test] fn compiled_pg_erd_shared_access_log_excludes_request_sensitive_material() { let backend = TcpListener::bind("127.0.0.1:0").expect("backend fixture should bind"); @@ -284,8 +301,8 @@ fn compiled_pg_erd_shared_access_log_excludes_request_sensitive_material() { b"GET /metrics HTTP/1.1\r\nHost: metrics\r\nConnection: close\r\n\r\n", ); assert!( - metrics.contains("cwl_pingora_gateway_requests_total 1"), - "metrics scrape should prove the proxied request reached shared completion recording: {metrics:?}" + contains_exact_metric_sample(&metrics, "cwl_pingora_gateway_requests_total 1"), + "metrics scrape should prove exactly one proxied request reached shared completion recording: {metrics:?}" ); process .wait_until_stderr_contains("gateway_request status=200 outcome=ok request_body_bytes=0"); From b3b045e14b6c4011202a9e9a8c93e640baae0b3f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 17:25:58 +0900 Subject: [PATCH 13/21] docs: carry payload-free observability through restack --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 445cfdeb..8b0e4b39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ All notable changes are tracked here. No release has been published yet. - Kept migration admin parsing side-effect free for custom TLS trust material: exact transport-authority and upstream-contract validation happens during parse, while peer/trust-bundle materialization occurs once during `build_proxy` before listener creation. This removes an avoidable validate-then-reload trust-file window. - Added a separate Ingress forwarding-policy boundary for the pg-erd migration. Request-controlled `Forwarded`, `X-Forwarded-*` and `X-Real-IP` values are removed, then the compatibility `X-Forwarded-For`, `X-Real-IP`, `X-Forwarded-Host`, `X-Forwarded-Port` and `X-Forwarded-Proto` fields are rebuilt from accepted transport metadata. The current characterized Traefik `web` entryPoint remains explicitly HTTP; TLS-derived scheme behavior is not claimed before a TLS listener contract exists. - Added a shared `observability` bounded context so both `GatewayProxy` and `MigrationGatewayProxy` use the same low-cardinality request/error/body/backpressure counters and coarse access-log shape instead of duplicating telemetry. The public observation vocabulary contains only response status, `ok`/`error`, and observed request-body bytes; paths, query strings, headers/cookies, credentials, customer payloads and product identifiers stay out of the shared telemetry contract. +- Added dedicated compiled pg-erd payload-free access-log acceptance. A routed request carries unique URI/query, Host, Authorization, Cookie and product-context sentinels; the backend must receive those exact fields so the fixture is non-vacuous, while shared gateway stderr must emit only the bounded completion vocabulary and none of the sentinels. Test oracles use exact case-insensitive HTTP field matching and exact Prometheus sample-line matching so `X-Forwarded-Host` cannot satisfy `Host` and counter value `10` cannot satisfy the expected value `1`. - Added mandatory positive `max_in_flight_requests` and `upstream_keepalive_pool_size` capacity budgets; Pingora's framework keepalive default is overridden from the validated edge contract. - Added process-local fail-fast backpressure: non-health requests above the in-flight budget receive HTTP 503, health remains observable, rejection telemetry increments, and capacity is released after request completion or failure. - Added dedicated compiled pg-erd traffic acceptance for streamed/chunked body overflow and routed in-flight saturation/recovery: the migration process must return 413 above the shared body budget, return 503 in less than one second above the in-flight budget, keep `/readyz` observable, expose the exact single-rejection Prometheus sample, and admit a later routed request after capacity is released. @@ -40,4 +41,4 @@ All notable changes are tracked here. No release has been published yet. - Added missing-public-rustdoc enforcement and documentation builds with warnings denied. - Added DDD, product, technical, security, threat, test, operability, configuration, migration-gap, and primary-source traceability documentation. -Release remains blocked on the exact Pingora supplier disposition, including unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388`, restoration of authoritative dependency-review evidence, terminal exact-current-head CI/supply-chain/security/review evidence, representative pg-erd TLS/protocol/failure/concurrency performance, immutable registry/package identity with release-bound SBOM/provenance/reproducibility, rollback rehearsal, and protected-branch integration. #21 post-header partial-response acceptance is terminal exact-head hosted GREEN at `51f1242663ccbf164efc50ca2ac74c4d0a1c7126`; the changed #22 routed-load candidate must independently acquire unchanged exact-head CI/Supply Chain and review evidence before routed p95 credit is granted. No consumer migration, canary, cutover, rollback, or legacy removal is claimed before those release and traffic-contract gates are satisfied. +Release remains blocked on the exact Pingora supplier disposition, including unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388`, restoration of authoritative dependency-review evidence, terminal exact-current-head CI/supply-chain/security/review evidence, representative pg-erd TLS/protocol/failure/concurrency performance, immutable registry/package identity with release-bound SBOM/provenance/reproducibility, rollback rehearsal, and protected-branch integration. Routed-load #22 has terminal unchanged-head hosted CI/Supply Chain and exact changed-range technical review; the current #23 payload-free observability child must independently acquire exact-head hosted and review evidence after its ordinary/non-force parent adoption. No consumer migration, canary, cutover, rollback, or legacy removal is claimed before those release and traffic-contract gates are satisfied. From 68fff099bc923e29f991be880cc5e7855d496737 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 17:28:08 +0900 Subject: [PATCH 14/21] docs: define payload-free observability acceptance --- TEST_STRATEGY.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TEST_STRATEGY.md b/TEST_STRATEGY.md index 503b5d8a..7c4e3f8a 100644 --- a/TEST_STRATEGY.md +++ b/TEST_STRATEGY.md @@ -20,6 +20,8 @@ Runtime and failure traffic are split by causal phase. `tests/pg_erd_runtime_iso `tests/pg_erd_partial_response_traffic.rs` covers the post-header phase. The backend sends HTTP 200 with `Content-Length: 20` and only the seven-byte `partial` prefix, then waits until the downstream has observed the complete header block plus that exact prefix before closing. Acceptance requires the committed status/framing to remain visible without a fabricated second status or silent failover, `/readyz` to stay 200, exact `cwl_pingora_gateway_request_errors_total 1`, and an independent frontend recovery request. The framing oracle parses header lines, matches `Content-Length` field identity case-insensitively, trims field-value whitespace, requires exactly one value equal to `20`, and rejects lookalike or duplicate/conflicting fields. Exact #21 `51f1242663ccbf164efc50ca2ac74c4d0a1c7126` has terminal CI `34185538078` and Supply Chain `34185538063` GREEN; descendants must revalidate rather than transfer that receipt. +`tests/pg_erd_payload_free_observability.rs` covers the shared gateway observability boundary through the compiled migration process. A real routed request carries unique URI/query, Host, Authorization, Cookie and product-context sentinels; the backend must receive those values so the test cannot pass vacuously, while the `cwl_pingora_gateway::observability` target may emit only the bounded completion vocabulary and none of the sentinels. Traffic and metrics listener reservations are held concurrently until process start, origin header reads are bounded by five seconds and 64 KiB, HTTP field identity is matched case-insensitively by exact field name with OWS trimming, and the Prometheus oracle requires a complete exact sample line so a counter value such as `10` cannot satisfy an expected value of `1`. This proves only the shared gateway observability contract; it does not claim authority over product-owned logging, tracing, identity or third-party logger configuration. + ## Routed performance acceptance #22 adds the first dedicated routed pg-erd concurrency/latency gate on the current #21 ancestry. `tests/load/load_origin.rs` is the only measured origin implementation: bounded std-only Rust, finite worker/queue capacity, 64 KiB header bound, deterministic Content-Length framing and direct parser/framing tests. The CI load lane compiles both admitted gateway binaries, runs `rustfmt` and `rustc -D warnings --test` on the origin, then builds the optimized fixture used by measured traffic. From 7006f45958699c0ca97c2d6aaa8accdab780a44c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 17:29:00 +0900 Subject: [PATCH 15/21] docs: advance migration baseline through payload-free observability --- docs/product-technical-gap-baseline.md | 45 +++++++++++++------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 05515701..11155394 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,6 +1,6 @@ # Product / Technical Gap Baseline -This file is the code-current migration baseline for `ContextualWisdomLab/pingora-gateway`. Live protected-branch, PR, review, workflow, release and supplier metadata remain the exact authority. Historical RED/GREEN detail belongs in PRs, commits, workflow receipts, ADRs and `CHANGELOG.md`; this snapshot keeps only evidence needed to understand the current migration graph and remaining buyer-visible gaps. Mutable current-head SHAs and in-flight run IDs are deliberately not copied here because embedding them in a source file makes every evidence refresh create a new head and immediately stale its own statement; the live PR/issue authority records those exact values instead. +This file is the code-current migration baseline for `ContextualWisdomLab/pingora-gateway`. Live protected-branch, PR, review, workflow, release and supplier metadata remain the exact authority. Historical RED/GREEN detail belongs in PRs, commits, workflow receipts, ADRs and `CHANGELOG.md`; this snapshot keeps durable architecture, acceptance and remaining buyer-visible gaps. Mutable current-head SHAs and in-flight run IDs are deliberately not copied here because an evidence-refresh commit would immediately stale its own statement. ## Authority and bounded contexts @@ -10,51 +10,50 @@ Generic v1 remains a one-upstream Rust/Pingora process. The bounded `cwl-pingora ## Dependency and promotion root -Foundation #1 remains `0da81a93f93e869c15bb7d34c55fc87479d16522`. Compiler prerequisite #56 remains exact `18fb38b1ba70c4bf222642ef347f3d57a98379a2` with terminal CI/Supply Chain GREEN, but protected promotion still requires an independent `APPROVED` review; owner or bot technical comments are not substituted for that gate. +Foundation #1 and compiler prerequisite #56 remain the earlier promotion root. #56 has terminal CI/Supply Chain evidence but protected promotion still requires an independent `APPROVED` review; owner or bot technical comments are not substituted for that gate. -Supplier-intake #54 remains exact `50b0516a9249c4066e3a0f305dbf2759eae3ae06` with intentional hosted RED because committed `Cargo.lock` still contains `derivative 2.2.0`. Audit ignores, lock deletion, mutable supplier pins, scanner suppression and muted regressions are not admissible repairs. Supplier-semantics #62 remains exact `32e0aeedac7b0fe6234d476245f37994b1b9168f` with exact hosted GREEN and the required `PeerOptions` Debug plus `Backend` equality/hash/order semantics. +Supplier-intake #54 remains intentional hosted RED because the committed gateway dependency graph still contains unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388`. Audit ignores, lock deletion, mutable supplier pins, scanner suppression and muted regressions are not admissible repairs. Supplier-semantics #62 remains exact hosted/technical GREEN for the required `PeerOptions` Debug surface, `Backend` equality/hash/order semantics and bounded Rust-origin load contract. -The supplier promotion path remains: maintainer-integrated, release-qualified Pingora repair removing `derivative` from the relevant workspace/core/load-balancing manifests and regenerated lock while preserving #62 semantics → exact gateway supplier bump and committed lock regeneration → unchanged #54 GREEN and preserved #62 GREEN → #56 independent approval/governance → protected foundation promotion. +The supplier promotion path remains: maintainer-integrated, release-qualified Pingora repair removing `derivative` from the relevant workspace/core/load-balancing manifests and regenerated lock while preserving #62 semantics → exact gateway supplier bump and committed lock regeneration → unchanged #54 absence regression GREEN and preserved #62 GREEN → #56 independent approval/governance → protected foundation promotion. ## Current pg-erd stack -The parent chain through #20 is ordinary/non-force and has exact hosted evidence at each current head. Relevant current heads are #12 `69f22265cd88881b8e14cedb32defa0a91180aa2`, #14 `8937364909b82f50fd911aa001a8d073b517f5d9`, #15 `bb65f2810b178bda46ed7eb5c6a62aae9fe36403`, #16 `356f3f250043d71c9bb1c0655481cd6984f3ae57`, #17 `7a7e1f1ca4c8310220b7ff2fb96e01027a7e89f3`, #18 `9749d01ae0e9aae027d7fce1a2c15e6a8358acd9`, #19 `86a6eb1b8fd5777b578cdbce49f40d52e916cc9b`, and #20 `d4d4565854cc924a2214de2b67a966d2f253da3e`. Their retained contracts cover bounded Admin Config/socket authority, forwarding-trust reconstruction, body/in-flight isolation, refused and connected-silent origins, post-header inactivity, OCI metrics identity and routed SIGTERM drain. Predecessor receipts are not transferred to changed descendants. +The parent chain through #20 is ordinary/non-force and has exact hosted evidence at each retained current head. Those contracts cover bounded Admin Config/socket authority, forwarding-trust reconstruction, body/in-flight isolation, refused and connected-silent origins, post-header inactivity, OCI metrics identity and routed SIGTERM drain. Predecessor receipts are never transferred to changed descendants. -#21 owns the post-header partial-response phase. Ordinary two-parent commit `bff40ec74448a6e63bac8c408962aad7f7309d4e` adopted exact #20 while retaining the valid child test delta. The final exact head is `51f1242663ccbf164efc50ca2ac74c4d0a1c7126`. Its backend declares `Content-Length: 20`, commits only the seven-byte `partial` prefix, waits until that prefix has been observed downstream, then closes. Acceptance preserves the committed 200/framing rather than inventing a second status or failover, requires `/readyz` 200, exact `cwl_pingora_gateway_request_errors_total 1`, and an independent `frontend` recovery request. The prior substring-based Content-Length oracle was replaced by semantic field parsing that matches the field name case-insensitively, trims field-value whitespace, requires exactly `vec!["20"]`, and rejects lookalike or duplicate/conflicting fields. +#21 owns the post-header partial-response phase. Its characterized backend declares `Content-Length: 20`, commits only the seven-byte `partial` prefix, waits until that prefix has been observed downstream, then closes. Acceptance preserves the committed 200/framing rather than inventing a second status or failover, requires `/readyz` 200, an exact single request-error Prometheus sample, and an independent `frontend` recovery request. The framing oracle parses field lines semantically, matches `Content-Length` case-insensitively, trims field-value whitespace, requires exactly one value equal to `20`, and rejects lookalike or duplicate/conflicting fields. #21 has terminal exact-head CI/Supply Chain evidence; descendants must independently revalidate. -Exact #21 CI `34185538078` is terminal GREEN for `test`, `load-contract` and dual-profile `oci-runtime`; Supply Chain `34185538063` is terminal GREEN on the same exact SHA. The test lane passed exact checkout, Rust 1.98.0 formatting, locked compile/test, strict Clippy, warning-denied public rustdoc, complete owned-production line/region coverage enforcement and resolved-lock verification. Existing owner and CodeRabbit comments are technical evidence only; there is no independent human `APPROVED` review credit. +#22 owns routed concurrency/latency acceptance and has now closed its unchanged exact-head hosted and changed-range technical-review gates. Its measured origin is Rust-only: `tests/load/load_origin.rs` is a bounded std-only HTTP/1.1 loopback origin with finite worker and queue budgets, a 64 KiB request-header cap, deterministic Content-Length framing, startup validation and direct fixture tests. The load lane compiles that fixture with `rustc -D warnings`, runs its tests, builds an optimized origin, and contains no Python invocation. -#22 owns the next routed concurrency/latency acceptance. Ordinary two-parent commit `bc6d50fce4e7a94b928dc91dce4e299e558a4c93` keeps historical #22 `c71bd51fec222508cbf98931e76a98f0830c312b` as first parent and exact final #21 `51f1242663ccbf164efc50ca2ac74c4d0a1c7126` as second parent while using the exact #21 tree as the resolution tree. Valid child semantics were then reapplied without replaying stale parent source/docs/workflow blobs. The live PR compare remains the ancestry authority. +Routed k6 acceptance uses four VUs and 400 total iterations, alternates characterized `/api/load-contract` and `/load-contract` requests, tags every request with `backend` or `frontend`, requires exact 200/body identity and zero HTTP failures, and independently gates aggregate, backend and frontend `http_req_duration` at p95 `<20 ms`. Each route must contribute at least 198 measured samples. The workflow starts distinct bounded Rust origins for the characterized `backend` and `frontend` authorities and the compiled migration process with only admitted transport/runtime configuration; it does not add an operator route DSL. Current exact hosted traffic meets this controlled-loopback contract, but those measurements are not representative deployment, TLS, multi-hop or origin-capacity SLO evidence. -The #22 measured path is Rust-only. `tests/load/load_origin.rs` is a bounded std-only HTTP/1.1 loopback origin with finite worker and queue budgets, a 64 KiB request-header cap, deterministic Content-Length framing, startup validation and direct fixture tests. The measured `load-contract` compiles that fixture with `rustc -D warnings`, runs its tests, builds an optimized origin, and contains no Python invocation. The historical Python origin file is removed from this child. +#23 owns payload-free shared-observability acceptance and has been ordinarily/non-force restacked on the final #22 tree. The restack preserves the historical child as first parent and final #22 as second parent while using the final #22 tree as the resolution basis, then reapplies only the valid child contract rather than stale parent source or documentation. -Routed k6 acceptance uses 4 VUs and 400 total iterations, alternates characterized `/api/load-contract` and `/load-contract` requests, tags every request with `backend` or `frontend`, requires exact 200/body identity and zero HTTP failures, and independently gates aggregate, backend and frontend `http_req_duration` at p95 `<20 ms`. Each route must contribute at least 198 measured samples. The workflow starts distinct bounded Rust origins for the characterized `backend` and `frontend` authorities, starts the compiled `cwl-pingora-pg-erd-migration` binary with only its admitted transport/runtime configuration, then records `k6-pg-erd-summary.json`. Route selection remains compiled into the pg-erd migration plan; workflow configuration does not add a route DSL. +`tests/pg_erd_payload_free_observability.rs` drives a real routed request carrying unique URI/query, Host, Authorization, Cookie and product-context sentinels through `cwl-pingora-pg-erd-migration`. The backend must receive those values, proving the fixture is non-vacuous, while the shared `cwl_pingora_gateway::observability` target may emit only bounded transport completion facts and none of the sentinels. The test-only reliability repairs retained on this child are: -A fresh source check caught an invalid intermediate workflow draft that attempted to add `routes:` to the strict Admin Config. Because `PgErdMigrationConfig` uses `deny_unknown_fields` and obtains its routing plan from `pg_erd_migration_plan()`, that shape would fail startup rather than prove routed performance. The current workflow has removed the invalid field and tests the actual product boundary. The Rust-origin workflow regression also normalizes shell line continuations before checking exact command semantics so formatting cannot cause a false RED while Python remains forbidden from the measured job. +- traffic and metrics listener reservations remain held concurrently until immediately before process start so ephemeral-port reuse cannot manufacture an Admin Config collision; +- origin request-header reads are bounded by a five-second socket timeout and 64 KiB byte cap so forwarding failure becomes finite RED instead of a hanging suite; +- HTTP field lookup uses case-insensitive exact field-name parsing plus OWS trimming so `X-Forwarded-Host` cannot satisfy `Host`; +- Prometheus validation uses exact complete sample-line matching and a focused regression so a counter value such as `10` cannot satisfy an expected value of `1`. -The predecessor routed-load head `ef1833d7234433454252d9bc49beb60bf8d6d856` completed enough hosted execution to isolate a deterministic formatter defect. Supply Chain `34191744912`, routed `load-contract 101951050926`, and dual-profile `oci-runtime 101951051109` were GREEN, while `test 101951051107` failed immediately at Rust 1.98.0 `cargo fmt --all -- --check` because two long assertions in `tests/rust_load_origin_workflow_contract.rs` needed formatter-prescribed line wrapping. Its routed artifact `10042883451` (`sha256:9e9193bbbc65c6b5e785372adb7c0208d671c4621a29e276273a92bd626d2b9e`) recorded 400 requests, 800/800 checks, zero HTTP failures, aggregate p95 `0.5510101 ms`, backend p95 `0.61089945 ms` over 200 requests, and frontend p95 `0.5004503 ms` over 200 requests. These controlled-loopback numbers are predecessor lane evidence, not production SLO proof and not transferable exact-head GREEN. - -Commit `648db6b25c8227baf1f4cad7170790637b70a515` applies only that deterministic rustfmt repair. It changes no workflow commands, traffic volume, sample floor, thresholds, routing, measured-origin behavior, production Rust, OCI/security policy or authority boundary. The live #22 PR records the current exact head, current CI/Supply Chain receipts and fresh review state so this baseline can stay code-current without self-referential SHA churn. No predecessor GREEN, protected merge, release, canary or cutover credit transfers to a changed head. - -Immediate child #23 remains on historical #22 ancestry until final #22 closes its unchanged exact hosted/review gates. Its valid payload-free observability delta now carries three test-evidence repairs that must survive later ordinary/non-force succession: `fe6362616d72f0e0151288ff82136d11f4a2da82` holds traffic and metrics listener reservations concurrently so the OS cannot reuse one ephemeral port and manufacture an Admin Config collision; `ef3b30c86dda033db9b3122977ff358f2dc9bd1d` bounds origin header reads with a five-second socket timeout and 64 KiB cap so forwarding failure becomes finite RED rather than a hanging suite; and `caa1dfb644cde568f9a1519290ca4e56dfd23ca6` replaces false-positive substring header matching with case-insensitive exact field-name parsing plus OWS trimming, including a focused regression proving `X-Forwarded-Host` cannot satisfy the required `Host` sentinel. These are test-only evidence repairs and do not change production observability, routing, auth/TLS or product authority. +These repairs change no production Rust, observability vocabulary, routing, authentication/TLS behavior or product authority. The current #23 source/docs must independently acquire exact-head hosted CI/Supply Chain and current-range review evidence after the restack before observability parity is credited. ## Capability state and buyer-visible gaps | Area | Current state | Remaining acceptance | | --- | --- | --- | | Admin Config / network authority | Characterized pg-erd transport binding is fail closed; routes remain compiled rather than operator-configurable | Revalidate on every descendant restack | -| Generic forwarding trust | #15 invariant inherited through the current parent stack | Preserve exact sanitizer/reconstruction invariant; no client-IP/trusted-proxy claim until separately characterized | -| Runtime isolation / recovery | #16–#21 cover body/in-flight rejection, refused origin, read stall, partial response and graceful drain on current ancestry | Explicit reset, broader streaming/upgraded failure, slow-drip/whole-response lifetime and rollback traffic remain unproven | +| Generic forwarding trust | Sanitizer/reconstruction invariant is inherited through the current parent stack | No client-IP/trusted-proxy claim until separately characterized | +| Runtime isolation / recovery | Body/in-flight rejection, refused origin, read stall, partial response and graceful drain are covered on current ancestry | Explicit TCP reset, broader streaming/upgraded failure, slow-drip/whole-response lifetime and rollback traffic remain unproven | | Upstream TLS | Generic local-CA/SNI verification and pg-erd fail-closed trust activation exist | Successful pg-erd TLS origin path plus representative TLS performance remain unproven | | Protocols | HTTP/1.1 migration path exists | Downstream TLS/H2, H2→H1 Cookie behavior, WebSocket/Extended CONNECT and explicit H3/QUIC disposition require separate supplier-capable contracts | -| OCI / supply chain | #21 exact dual-profile OCI and Supply Chain evidence GREEN; #22 predecessor isolated a formatting-only test-lane RED after routed load/OCI/Supply Chain passed | Obtain unchanged current #22 exact closure; immutable registry digest, signing/attestation/provenance, release-bound SBOM, reproducibility receipt and rollback rehearsal remain gaps | -| Performance | #22 contains routed Rust-origin aggregate/per-route/sample-floor acceptance; predecessor hosted traffic passed all thresholds | Obtain unchanged current-head hosted k6 evidence; controlled loopback is not representative production SLO proof; TLS/multi-hop/origin-capacity deployment measurements remain required | -| Observability | Low-cardinality counters/logging and separate metrics listener are covered by parent traffic tests; #23 has repaired payload-free fixture evidence awaiting parent adoption | Preserve exact telemetry semantics through final #22/#23 and later failure/protocol children | -| Documentation / review | Baseline, Test Strategy and Changelog reflect final #21 and current #22 semantics; mutable exact head/run state stays in live PR/issue authority | Obtain exact #22 hosted closure and fresh exact-range review; bot/static review is not independent human approval | +| OCI / supply chain | #22 exact dual-profile OCI and Supply Chain evidence are GREEN | #23 and every changed descendant must independently revalidate; immutable registry digest, signing/attestation/provenance, release-bound SBOM, reproducibility receipt and rollback rehearsal remain gaps | +| Performance | #22 exact routed Rust-origin traffic passes aggregate/per-route/sample-floor acceptance | Controlled loopback is not production SLO proof; TLS/multi-hop/container scheduling/origin-capacity deployment measurements remain required | +| Observability | Low-cardinality shared counters/logging exist; #23 now contains non-vacuous payload-free compiled-process acceptance with exact header and metric oracles | Obtain unchanged #23 exact hosted/review closure and preserve semantics through later failure/protocol children | +| Documentation / review | Changelog, Test Strategy and this baseline are aligned to the #22→#23 succession and avoid self-staling run IDs | Bot/static technical review is not independent human approval; later changed heads need their own review evidence | | Release / migration | No protected release or consumer cutover credit | Immutable release → parity → shadow/canary → rollback rehearsal → cutover → verified Nginx/OpenResty/legacy removal | ## Execution order -The current dependency order is `#54 derivative RED + #62 exact semantics/load GREEN → maintainer-integrated immutable supplier repair → gateway supplier bump and committed lock regeneration → #54 GREEN + preserved #62 GREEN → #56 independent APPROVED/governance → protected foundation integration → #12 → #14 → #15 → #16 → #17 → #18 → #19 → #20 → #21 exact hosted GREEN → #22 exact hosted/review closure → #23 and later descendants parent-first ordinary/non-force → remaining TLS/failure/protocol acceptance → immutable gateway release/SBOM/provenance/reproducibility/rollback → shadow/canary → cutover → verified legacy removal`. +The current dependency order is `#54 derivative RED + #62 exact semantics/load GREEN → maintainer-integrated release-qualified supplier repair → gateway supplier bump and committed lock regeneration → #54 GREEN + preserved #62 GREEN → #56 independent APPROVED/governance → protected foundation integration → #12 → #14 → #15 → #16 → #17 → #18 → #19 → #20 → #21 exact hosted GREEN → #22 exact hosted/technical-review GREEN → #23 exact hosted/review closure → later descendants parent-first ordinary/non-force → remaining TLS/failure/protocol acceptance → immutable gateway release/SBOM/provenance/reproducibility/rollback → shadow/canary → cutover → verified legacy removal`. No Draft state, predecessor receipt, bot review, local image ID, mutable supplier PR, queue state or controlled loopback measurement is treated as protected merge, release, canary, cutover, rollback or legacy-removal evidence. From 89aa2940ad8f6705610f050f9a6dbd3f6b0b1c45 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 18:27:07 +0900 Subject: [PATCH 16/21] style: apply hosted rustfmt for payload-free observability --- tests/pg_erd_payload_free_observability.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/pg_erd_payload_free_observability.rs b/tests/pg_erd_payload_free_observability.rs index 2bc40c4e..86006571 100644 --- a/tests/pg_erd_payload_free_observability.rs +++ b/tests/pg_erd_payload_free_observability.rs @@ -241,11 +241,9 @@ fn compiled_pg_erd_shared_access_log_excludes_request_sensitive_material() { .accept() .expect("routed request should reach the characterized backend authority"); let request = read_request_headers(&mut stream); - assert!( - request - .to_ascii_lowercase() - .starts_with("get /api/log-contract?customer=query-secret http/1.1\r\n") - ); + assert!(request + .to_ascii_lowercase() + .starts_with("get /api/log-contract?customer=query-secret http/1.1\r\n")); assert_eq!( header_values(&request, "Host"), vec!["tenant-secret.example:8080"] From 369528831929ceae4b6892188f0d28e27b447592 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 19:06:20 +0900 Subject: [PATCH 17/21] test: preserve case-sensitive observability request target --- tests/pg_erd_payload_free_observability.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/pg_erd_payload_free_observability.rs b/tests/pg_erd_payload_free_observability.rs index 86006571..81fc8298 100644 --- a/tests/pg_erd_payload_free_observability.rs +++ b/tests/pg_erd_payload_free_observability.rs @@ -241,9 +241,15 @@ fn compiled_pg_erd_shared_access_log_excludes_request_sensitive_material() { .accept() .expect("routed request should reach the characterized backend authority"); let request = read_request_headers(&mut stream); - assert!(request - .to_ascii_lowercase() - .starts_with("get /api/log-contract?customer=query-secret http/1.1\r\n")); + let request_line = request + .split("\r\n") + .next() + .expect("origin request should contain a request line"); + assert_eq!( + request_line, + "GET /api/log-contract?customer=query-secret HTTP/1.1", + "request target and query sentinel must be preserved exactly" + ); assert_eq!( header_values(&request, "Host"), vec!["tenant-secret.example:8080"] From 0c2cfce661c4b2d9b40407842fa5d6ee8df5cc8d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 19:20:35 +0900 Subject: [PATCH 18/21] test: apply rustfmt to observability request-target assertion --- tests/pg_erd_payload_free_observability.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/pg_erd_payload_free_observability.rs b/tests/pg_erd_payload_free_observability.rs index 81fc8298..9369175d 100644 --- a/tests/pg_erd_payload_free_observability.rs +++ b/tests/pg_erd_payload_free_observability.rs @@ -246,8 +246,7 @@ fn compiled_pg_erd_shared_access_log_excludes_request_sensitive_material() { .next() .expect("origin request should contain a request line"); assert_eq!( - request_line, - "GET /api/log-contract?customer=query-secret HTTP/1.1", + request_line, "GET /api/log-contract?customer=query-secret HTTP/1.1", "request target and query sentinel must be preserved exactly" ); assert_eq!( From 7c8f27222152a62b21d0b19a303cd2b0c5cebd08 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 19:23:30 +0900 Subject: [PATCH 19/21] test: close observability review false-greens --- TEST_STRATEGY.md | 2 +- docs/product-technical-gap-baseline.md | 12 +++++++++--- tests/pg_erd_payload_free_observability.rs | 9 +++++++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/TEST_STRATEGY.md b/TEST_STRATEGY.md index 7c4e3f8a..a752c8b6 100644 --- a/TEST_STRATEGY.md +++ b/TEST_STRATEGY.md @@ -24,7 +24,7 @@ Runtime and failure traffic are split by causal phase. `tests/pg_erd_runtime_iso ## Routed performance acceptance -#22 adds the first dedicated routed pg-erd concurrency/latency gate on the current #21 ancestry. `tests/load/load_origin.rs` is the only measured origin implementation: bounded std-only Rust, finite worker/queue capacity, 64 KiB header bound, deterministic Content-Length framing and direct parser/framing tests. The CI load lane compiles both admitted gateway binaries, runs `rustfmt` and `rustc -D warnings --test` on the origin, then builds the optimized fixture used by measured traffic. +PR `#22` adds the first dedicated routed pg-erd concurrency/latency gate on the current #21 ancestry. `tests/load/load_origin.rs` is the only measured origin implementation: bounded std-only Rust, finite worker/queue capacity, 64 KiB header bound, deterministic Content-Length framing and direct parser/framing tests. The CI load lane compiles both admitted gateway binaries, runs `rustfmt` and `rustc -D warnings --test` on the origin, then builds the optimized fixture used by measured traffic. `tests/load/pg_erd_gateway_smoke.js` runs four VUs for 400 total iterations. It alternates `/api/load-contract` and `/load-contract`, tags each request `backend` or `frontend`, requires exact 200/body identity and zero HTTP failures, gates aggregate plus each route independently at p95 `<20 ms`, and requires at least 198 measured requests per route. `tests/pg_erd_routed_latency_contract.rs` freezes the route/path/body/tag binding and per-route thresholds/sample floors. `tests/rust_load_origin_workflow_contract.rs` prevents regression to interpreted measured-origin execution by inspecting only the `load-contract` job and requiring the bounded Rust build/test/start commands with no Python invocation. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 11155394..0d2071f2 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -20,13 +20,19 @@ The supplier promotion path remains: maintainer-integrated, release-qualified Pi The parent chain through #20 is ordinary/non-force and has exact hosted evidence at each retained current head. Those contracts cover bounded Admin Config/socket authority, forwarding-trust reconstruction, body/in-flight isolation, refused and connected-silent origins, post-header inactivity, OCI metrics identity and routed SIGTERM drain. Predecessor receipts are never transferred to changed descendants. -#21 owns the post-header partial-response phase. Its characterized backend declares `Content-Length: 20`, commits only the seven-byte `partial` prefix, waits until that prefix has been observed downstream, then closes. Acceptance preserves the committed 200/framing rather than inventing a second status or failover, requires `/readyz` 200, an exact single request-error Prometheus sample, and an independent `frontend` recovery request. The framing oracle parses field lines semantically, matches `Content-Length` case-insensitively, trims field-value whitespace, requires exactly one value equal to `20`, and rejects lookalike or duplicate/conflicting fields. #21 has terminal exact-head CI/Supply Chain evidence; descendants must independently revalidate. +### `#21` post-header partial-response phase -#22 owns routed concurrency/latency acceptance and has now closed its unchanged exact-head hosted and changed-range technical-review gates. Its measured origin is Rust-only: `tests/load/load_origin.rs` is a bounded std-only HTTP/1.1 loopback origin with finite worker and queue budgets, a 64 KiB request-header cap, deterministic Content-Length framing, startup validation and direct fixture tests. The load lane compiles that fixture with `rustc -D warnings`, runs its tests, builds an optimized origin, and contains no Python invocation. +This phase owns the characterized backend contract that declares `Content-Length: 20`, commits only the seven-byte `partial` prefix, waits until that prefix has been observed downstream, then closes. Acceptance preserves the committed 200/framing rather than inventing a second status or failover, requires `/readyz` 200, an exact single request-error Prometheus sample, and an independent `frontend` recovery request. The framing oracle parses field lines semantically, matches `Content-Length` case-insensitively, trims field-value whitespace, requires exactly one value equal to `20`, and rejects lookalike or duplicate/conflicting fields. #21 has terminal exact-head CI/Supply Chain evidence; descendants must independently revalidate. + +### `#22` routed concurrency/latency acceptance + +This phase has closed its unchanged exact-head hosted and changed-range technical-review gates. Its measured origin is Rust-only: `tests/load/load_origin.rs` is a bounded std-only HTTP/1.1 loopback origin with finite worker and queue budgets, a 64 KiB request-header cap, deterministic Content-Length framing, startup validation and direct fixture tests. The load lane compiles that fixture with `rustc -D warnings`, runs its tests, builds an optimized origin, and contains no Python invocation. Routed k6 acceptance uses four VUs and 400 total iterations, alternates characterized `/api/load-contract` and `/load-contract` requests, tags every request with `backend` or `frontend`, requires exact 200/body identity and zero HTTP failures, and independently gates aggregate, backend and frontend `http_req_duration` at p95 `<20 ms`. Each route must contribute at least 198 measured samples. The workflow starts distinct bounded Rust origins for the characterized `backend` and `frontend` authorities and the compiled migration process with only admitted transport/runtime configuration; it does not add an operator route DSL. Current exact hosted traffic meets this controlled-loopback contract, but those measurements are not representative deployment, TLS, multi-hop or origin-capacity SLO evidence. -#23 owns payload-free shared-observability acceptance and has been ordinarily/non-force restacked on the final #22 tree. The restack preserves the historical child as first parent and final #22 as second parent while using the final #22 tree as the resolution basis, then reapplies only the valid child contract rather than stale parent source or documentation. +### `#23` payload-free shared-observability acceptance + +This phase has been ordinarily/non-force restacked on the final #22 tree. The restack preserves the historical child as first parent and final #22 as second parent while using the final #22 tree as the resolution basis, then reapplies only the valid child contract rather than stale parent source or documentation. `tests/pg_erd_payload_free_observability.rs` drives a real routed request carrying unique URI/query, Host, Authorization, Cookie and product-context sentinels through `cwl-pingora-pg-erd-migration`. The backend must receive those values, proving the fixture is non-vacuous, while the shared `cwl_pingora_gateway::observability` target may emit only bounded transport completion facts and none of the sentinels. The test-only reliability repairs retained on this child are: diff --git a/tests/pg_erd_payload_free_observability.rs b/tests/pg_erd_payload_free_observability.rs index 9369175d..760de38a 100644 --- a/tests/pg_erd_payload_free_observability.rs +++ b/tests/pg_erd_payload_free_observability.rs @@ -321,8 +321,13 @@ fn compiled_pg_erd_shared_access_log_excludes_request_sensitive_material() { "the shared observability target should emit one completion record: {stderr:?}" ); let access_log = request_logs[0]; - assert!( - access_log.contains("gateway_request status=200 outcome=ok request_body_bytes=0"), + let completion = access_log + .split_once("gateway_request ") + .expect("shared access log should contain the completion message") + .1; + assert_eq!( + completion, + "status=200 outcome=ok request_body_bytes=0", "shared access logging should contain only bounded transport facts: {access_log:?}" ); From 5b58bcf982f286495d5beda19fa3da902e6378a4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 19:29:56 +0900 Subject: [PATCH 20/21] test: apply rustfmt to exact completion oracle --- tests/pg_erd_payload_free_observability.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/pg_erd_payload_free_observability.rs b/tests/pg_erd_payload_free_observability.rs index 760de38a..2e31a4b4 100644 --- a/tests/pg_erd_payload_free_observability.rs +++ b/tests/pg_erd_payload_free_observability.rs @@ -326,8 +326,7 @@ fn compiled_pg_erd_shared_access_log_excludes_request_sensitive_material() { .expect("shared access log should contain the completion message") .1; assert_eq!( - completion, - "status=200 outcome=ok request_body_bytes=0", + completion, "status=200 outcome=ok request_body_bytes=0", "shared access logging should contain only bounded transport facts: {access_log:?}" ); From a5fb0004fec89a178bc3478d64bedd2b6577afa7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 12 Sep 2026 23:07:15 +0900 Subject: [PATCH 21/21] docs: bind payload-free observability child to repaired parent evidence --- CHANGELOG.md | 2 +- TEST_STRATEGY.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b0e4b39..13cc91f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,4 +41,4 @@ All notable changes are tracked here. No release has been published yet. - Added missing-public-rustdoc enforcement and documentation builds with warnings denied. - Added DDD, product, technical, security, threat, test, operability, configuration, migration-gap, and primary-source traceability documentation. -Release remains blocked on the exact Pingora supplier disposition, including unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388`, restoration of authoritative dependency-review evidence, terminal exact-current-head CI/supply-chain/security/review evidence, representative pg-erd TLS/protocol/failure/concurrency performance, immutable registry/package identity with release-bound SBOM/provenance/reproducibility, rollback rehearsal, and protected-branch integration. Routed-load #22 has terminal unchanged-head hosted CI/Supply Chain and exact changed-range technical review; the current #23 payload-free observability child must independently acquire exact-head hosted and review evidence after its ordinary/non-force parent adoption. No consumer migration, canary, cutover, rollback, or legacy removal is claimed before those release and traffic-contract gates are satisfied. +Release remains blocked on the exact Pingora supplier disposition, including unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388`, restoration of authoritative dependency-review evidence, terminal exact-current-head CI/supply-chain/security/review evidence, representative pg-erd TLS/protocol/failure/concurrency performance, immutable registry/package identity with release-bound SBOM/provenance/reproducibility, rollback rehearsal, and protected-branch integration. Current parent #22 is `d74207828f3889393f8099948101e033721f7d7c` after ordinary ancestry/single-writer repair and must independently reacquire exact-head evidence; the current #23 payload-free observability child must independently acquire exact-head hosted and review evidence as well. Historical `3db4fe0...` and `5b58bcf...` GREEN receipts do not transfer. No consumer migration, canary, cutover, rollback, or legacy removal is claimed before those release and traffic-contract gates are satisfied. diff --git a/TEST_STRATEGY.md b/TEST_STRATEGY.md index a752c8b6..a0c0950b 100644 --- a/TEST_STRATEGY.md +++ b/TEST_STRATEGY.md @@ -18,7 +18,7 @@ The generic `load-contract` is deliberately separate from functional production- Runtime and failure traffic are split by causal phase. `tests/pg_erd_runtime_isolation_traffic.rs` covers streamed body overflow plus in-flight saturation/recovery and exact rejection telemetry. `tests/pg_erd_upstream_failure_traffic.rs` creates deterministic `ECONNREFUSED`, requires bounded 502 recovery, exact request-error telemetry and later frontend success. `tests/pg_erd_read_stall_traffic.rs` keeps an accepted origin connection silent and open so read inactivity cannot be confused with origin closure. `tests/pg_erd_graceful_shutdown.rs` holds a routed request in flight, fixes one SIGTERM-relative external termination deadline, releases the response during the shared grace period, requires downstream 200 and clean process exit before that same deadline. -`tests/pg_erd_partial_response_traffic.rs` covers the post-header phase. The backend sends HTTP 200 with `Content-Length: 20` and only the seven-byte `partial` prefix, then waits until the downstream has observed the complete header block plus that exact prefix before closing. Acceptance requires the committed status/framing to remain visible without a fabricated second status or silent failover, `/readyz` to stay 200, exact `cwl_pingora_gateway_request_errors_total 1`, and an independent frontend recovery request. The framing oracle parses header lines, matches `Content-Length` field identity case-insensitively, trims field-value whitespace, requires exactly one value equal to `20`, and rejects lookalike or duplicate/conflicting fields. Exact #21 `51f1242663ccbf164efc50ca2ac74c4d0a1c7126` has terminal CI `34185538078` and Supply Chain `34185538063` GREEN; descendants must revalidate rather than transfer that receipt. +`tests/pg_erd_partial_response_traffic.rs` covers the post-header phase. The backend sends HTTP 200 with `Content-Length: 20` and only the seven-byte `partial` prefix, then waits until the downstream has observed the complete header block plus that exact prefix before closing. Acceptance requires the committed status/framing to remain visible without a fabricated second status or silent failover, `/readyz` to stay 200, exact `cwl_pingora_gateway_request_errors_total 1`, and an independent frontend recovery request. The framing oracle parses header lines, matches `Content-Length` field identity case-insensitively, trims field-value whitespace, requires exactly one value equal to `20`, and rejects lookalike or duplicate/conflicting fields. Current #21 exact `4d3cf712b89a0b607db1e5f80db2d60ed29f6e1c` must independently reacquire exact-head CI/Supply Chain/review evidence after its ordinary repair; no historical GREEN transfers. `tests/pg_erd_payload_free_observability.rs` covers the shared gateway observability boundary through the compiled migration process. A real routed request carries unique URI/query, Host, Authorization, Cookie and product-context sentinels; the backend must receive those values so the test cannot pass vacuously, while the `cwl_pingora_gateway::observability` target may emit only the bounded completion vocabulary and none of the sentinels. Traffic and metrics listener reservations are held concurrently until process start, origin header reads are bounded by five seconds and 64 KiB, HTTP field identity is matched case-insensitively by exact field name with OWS trimming, and the Prometheus oracle requires a complete exact sample line so a counter value such as `10` cannot satisfy an expected value of `1`. This proves only the shared gateway observability contract; it does not claim authority over product-owned logging, tracing, identity or third-party logger configuration. @@ -40,4 +40,4 @@ Supply-chain evidence must remain exact-source-bound and include dependency audi ## Remaining gaps -Open acceptance still includes explicit TCP reset, broader streaming and WebSocket/Upgrade failure behavior, slow-drip/whole-response lifetime, downstream TLS/H2, H2→H1 Cookie handling, Extended CONNECT, explicit H3/QUIC disposition, tracing, property/fuzz testing, representative routed TLS/origin-capacity load, shadow/canary and rollback. Nginx/OpenResty/legacy removal is permitted only after parity, immutable release, canary/cutover and rollback evidence are all current on protected ancestry. +Open acceptance still includes explicit TCP reset, broader streaming and WebSocket/Upgrade failure behavior, slow-drip/whole-response lifetime, downstream TLS/H2, H2→H1 Cookie handling, Extended CONNECT, explicit H3/QUIC disposition, tracing, property/fuzz testing, representative routed TLS/origin-capacity load, shadow/canary and rollback. Current #22 and #23 exact heads must independently reacquire hosted/review evidence after ordinary restack; no historical receipt transfers. Nginx/OpenResty/legacy removal is permitted only after parity, immutable release, canary/cutover and rollback evidence are all current on protected ancestry.