From ffa52952df54ec0cd6af2c785b78b7e9b42f9bf0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 00:25:30 +0900 Subject: [PATCH 01/43] test: characterize pg-erd partial upstream response --- tests/pg_erd_partial_response_traffic.rs | 253 +++++++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 tests/pg_erd_partial_response_traffic.rs diff --git a/tests/pg_erd_partial_response_traffic.rs b/tests/pg_erd_partial_response_traffic.rs new file mode 100644 index 00000000..de0e129e --- /dev/null +++ b/tests/pg_erd_partial_response_traffic.rs @@ -0,0 +1,253 @@ +//! Real-listener partial upstream response acceptance for the dedicated pg-erd migration binary. +//! +//! This contract distinguishes a response that fails after its status/header block has already +//! been received from failures that occur before downstream response commitment. It proves that a +//! truncated characterized origin response is not rewritten into an invented retry/failover, +//! leaves process health observable, records the transport failure, and does not poison an +//! independent characterized route. + +use std::io::{ErrorKind, Read, Write}; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::process::{Child, Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; + +use tempfile::NamedTempFile; + +struct GatewayProcess(Child); + +impl Drop for GatewayProcess { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DownstreamTermination { + Eof, + ConnectionReset, +} + +fn reserve_loopback() -> SocketAddr { + TcpListener::bind("127.0.0.1:0") + .expect("loopback port should be reservable") + .local_addr() + .expect("reservation should expose an address") +} + +fn write_config( + listener: SocketAddr, + metrics_listener: SocketAddr, + backend: SocketAddr, + frontend: SocketAddr, +) -> NamedTempFile { + let mut file = NamedTempFile::new().expect("temporary config should be writable"); + writeln!( + file, + "version: 1\nlistener: {listener}\nmetrics_listener: {metrics_listener}\nmax_request_body_bytes: 8\nmax_in_flight_requests: 8\nupstream_keepalive_pool_size: 4\nupstreams:\n - name: backend\n address: {backend}\n tls: false\n timeouts:\n connection_ms: 200\n total_connection_ms: 400\n read_ms: 500\n write_ms: 1000\n idle_ms: 5000\n - name: frontend\n address: {frontend}\n tls: false\n timeouts:\n connection_ms: 200\n total_connection_ms: 400\n read_ms: 1000\n write_ms: 1000\n idle_ms: 5000" + ) + .expect("migration config should be written"); + file +} + +fn wait_until_listening(address: SocketAddr, process: &mut Child) { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + if let Some(status) = process + .try_wait() + .expect("gateway process state should be readable") + { + panic!("gateway exited before accepting traffic: {status}"); + } + if TcpStream::connect_timeout(&address, Duration::from_millis(100)).is_ok() { + return; + } + assert!(Instant::now() < deadline, "gateway did not start within 10s"); + thread::sleep(Duration::from_millis(25)); + } +} + +fn start_gateway( + config: &NamedTempFile, + gateway_address: SocketAddr, + metrics_address: SocketAddr, +) -> GatewayProcess { + let mut child = Command::new(env!("CARGO_BIN_EXE_cwl-pingora-pg-erd-migration")) + .args(["--config", config.path().to_str().expect("UTF-8 temp path")]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("compiled pg-erd migration binary should start"); + wait_until_listening(gateway_address, &mut child); + wait_until_listening(metrics_address, &mut child); + GatewayProcess(child) +} + +fn raw_request(address: SocketAddr, request: &[u8]) -> String { + let mut downstream = TcpStream::connect(address).expect("gateway should accept traffic"); + downstream + .set_read_timeout(Some(Duration::from_secs(5))) + .expect("downstream timeout should be configurable"); + downstream + .write_all(request) + .expect("downstream request should be writable"); + let mut response = String::new(); + downstream + .read_to_string(&mut response) + .expect("gateway response should be readable"); + response +} + +fn raw_request_until_terminal( + address: SocketAddr, + request: &[u8], +) -> (Vec, DownstreamTermination) { + let mut downstream = TcpStream::connect(address).expect("gateway should accept traffic"); + downstream + .set_read_timeout(Some(Duration::from_secs(5))) + .expect("downstream timeout should be configurable"); + downstream + .write_all(request) + .expect("downstream request should be writable"); + + let mut response = Vec::new(); + let mut buffer = [0_u8; 1024]; + loop { + match downstream.read(&mut buffer) { + Ok(0) => return (response, DownstreamTermination::Eof), + Ok(read) => response.extend_from_slice(&buffer[..read]), + Err(error) if error.kind() == ErrorKind::ConnectionReset => { + return (response, DownstreamTermination::ConnectionReset); + } + Err(error) => panic!("partial downstream response should terminate, not stall: {error}"), + } + } +} + +fn get(address: SocketAddr, path: &str) -> String { + raw_request( + address, + format!("GET {path} HTTP/1.1\r\nHost: app.example:8080\r\nConnection: close\r\n\r\n") + .as_bytes(), + ) +} + +fn read_request_headers(stream: &mut TcpStream) -> String { + let mut bytes = Vec::new(); + let mut buffer = [0_u8; 1024]; + loop { + let read = stream.read(&mut buffer).expect("origin request should be readable"); + assert!(read > 0, "gateway closed origin request before headers completed"); + bytes.extend_from_slice(&buffer[..read]); + if bytes.windows(4).any(|window| window == b"\r\n\r\n") { + return String::from_utf8_lossy(&bytes).into_owned(); + } + } +} + +#[test] +fn compiled_pg_erd_truncated_response_stays_committed_and_preserves_independent_routing() { + let backend = TcpListener::bind("127.0.0.1:0").expect("backend fixture should bind"); + let backend_address = backend.local_addr().expect("backend address should exist"); + let backend_origin = thread::spawn(move || { + let (mut stream, _) = backend + .accept() + .expect("routed request should reach the characterized backend authority"); + let request = read_request_headers(&mut stream); + assert!(request.starts_with("GET /api/partial-response HTTP/1.1\r\n")); + + // Commit a valid response header and only part of the declared payload, then close. Once + // that status/header block has been forwarded, a later upstream framing failure cannot be + // replaced with a second HTTP status or silently failed over to another origin. + stream + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 20\r\nConnection: close\r\n\r\npartial", + ) + .expect("partial backend response should be writable"); + }); + + let frontend = TcpListener::bind("127.0.0.1:0").expect("frontend fixture should bind"); + let frontend_address = frontend.local_addr().expect("frontend address should exist"); + let frontend_origin = thread::spawn(move || { + let (mut stream, _) = frontend + .accept() + .expect("fallback request should reach the independent frontend authority"); + let request = read_request_headers(&mut stream); + assert!(request.starts_with("GET /after-partial-response HTTP/1.1\r\n")); + stream + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 9\r\nConnection: close\r\n\r\nrecovered", + ) + .expect("frontend recovery response should be writable"); + }); + + let gateway_address = reserve_loopback(); + let metrics_address = reserve_loopback(); + let config = write_config( + gateway_address, + metrics_address, + backend_address, + frontend_address, + ); + let _process = start_gateway(&config, gateway_address, metrics_address); + + let (partial, termination) = raw_request_until_terminal( + gateway_address, + b"GET /api/partial-response HTTP/1.1\r\nHost: app.example:8080\r\nConnection: close\r\n\r\n", + ); + assert!( + matches!( + termination, + DownstreamTermination::Eof | DownstreamTermination::ConnectionReset + ), + "a committed truncated response must terminate the downstream connection" + ); + let header_end = partial + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|position| position + 4) + .expect("committed partial response must contain a complete header block"); + let headers = String::from_utf8_lossy(&partial[..header_end]).to_ascii_lowercase(); + assert!( + headers.starts_with("http/1.1 200"), + "a post-header upstream failure cannot be rewritten as a new status: {headers:?}" + ); + assert!( + headers.contains("content-length: 20"), + "the committed response must retain its declared framing for this fixture: {headers:?}" + ); + let body = &partial[header_end..]; + assert_eq!(body, b"partial"); + assert!( + body.len() < 20, + "fixture must terminate before its declared response body completes" + ); + + let readiness = get(gateway_address, "/readyz"); + assert!( + readiness.starts_with("HTTP/1.1 200"), + "one truncated upstream response must not poison process readiness: {readiness:?}" + ); + + let metrics = get(metrics_address, "/metrics"); + assert!( + metrics.contains("cwl_pingora_gateway_request_errors_total 1"), + "the post-header upstream framing failure must remain visible through low-cardinality error telemetry: {metrics:?}" + ); + + let recovered = get(gateway_address, "/after-partial-response"); + assert!( + recovered.starts_with("HTTP/1.1 200"), + "an independent characterized route must remain usable after a truncated response: {recovered:?}" + ); + assert!(recovered.ends_with("\r\n\r\nrecovered")); + + frontend_origin + .join() + .expect("frontend recovery fixture should complete"); + backend_origin + .join() + .expect("partial backend fixture should complete"); +} From 90a3183de4f80be97c7a45dff45173789dded1b5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 00:25:56 +0900 Subject: [PATCH 02/43] docs: align technical requirements with migration evidence --- TRD.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/TRD.md b/TRD.md index 382a2598..330fd6c8 100644 --- a/TRD.md +++ b/TRD.md @@ -28,6 +28,8 @@ Requests with a parseable `Content-Length` larger than `max_request_body_bytes` The version-1 process policy allows one total upstream attempt and therefore no automatic gateway retry. It overrides Pingora's keepalive-pool default from validated configuration and uses a 5-second grace period plus 10-second Pingora runtime shutdown timeout inside a 30-second external termination budget. Product idempotency/replay policy remains outside the gateway. +Failure semantics are phase-aware. Before an upstream response header is committed downstream, transport failure may be represented as the gateway's fail-closed error response under the configured attempt policy. After a valid response header has already been committed, a later upstream framing/body failure cannot be rewritten into a second HTTP status or silently failed over; the downstream response is terminated, low-cardinality error telemetry records the failed request, process readiness remains available, and independent routes must remain usable. This is a transport invariant, not product retry policy. + ## Packaging and release evidence -The OCI runtime executes as uid/gid `65532`, is compatible with read-only-root and capability-free operation, and relies on digest-pinned base images. The generic binary has executable OCI acceptance; the dedicated pg-erd binary still requires equivalent OCI invocation evidence. A committed lockfile, exact-head test/coverage/rustdoc/load/security/supply-chain evidence, immutable image digest, SBOM/provenance/reproducibility, rollback rehearsal, and protected integration remain release gates. Source-level migration capability is not parity, canary, cutover, rollback, or legacy-removal evidence. +The OCI runtime executes as uid/gid `65532`, is compatible with read-only-root and capability-free operation, and relies on digest-pinned base images. The exact image now packages both the generic and bounded pg-erd composition roots while retaining the generic binary as the default entrypoint; dedicated pg-erd invocation under uid/gid 65532, read-only root, dropped capabilities, `no-new-privileges`, read-only configuration, process-health, and metrics constraints has source acceptance but still requires terminal exact-head OCI execution. A committed lockfile, exact-head test/coverage/rustdoc/load/security/supply-chain evidence, immutable image digest, SBOM/provenance/reproducibility, rollback rehearsal, and protected integration remain release gates. Source-level migration capability is not parity, canary, cutover, rollback, or legacy-removal evidence. \ No newline at end of file From 9d0dd2dc8265518395b1a2d2069f6877426c7f88 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 00:26:24 +0900 Subject: [PATCH 03/43] docs: record partial-response traffic contract --- TEST_STRATEGY.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TEST_STRATEGY.md b/TEST_STRATEGY.md index df3bb21c..e00b55df 100644 --- a/TEST_STRATEGY.md +++ b/TEST_STRATEGY.md @@ -8,7 +8,7 @@ Migration characterization is executable before transport activation. `tests/pg_ The bounded Admin Config transition has its own executable contract. `tests/pg_erd_admin_config_contract.rs` rejects unknown/future configuration, listener collision, zero runtime/keepalive budgets, missing/extra/duplicate/renamed authority, and invalid concrete transport configuration; it also proves only `backend` and `frontend` can bind the compiled route profile. `tests/pg_erd_binary_startup.rs` exercises the dedicated compiled process and requires fail-closed behavior for omitted configuration, unreadable configuration, invalid Admin Config, and custom TLS trust material that cannot be materialized before listener activation. -`tests/pg_erd_production_path.rs` is the first dedicated compiled-listener traffic contract. It starts real loopback `backend` and `frontend` origins plus `cwl-pingora-pg-erd-migration`, requires `/livez` and `/readyz` to remain gateway-local, requires consumer `/healthz` and raw `/apiary` to reach `backend`, requires fallback product traffic to reach `frontend`, requires request-controlled forwarding identity to be discarded and rebuilt from accepted loopback transport/Host data, requires upstream `X-Frame-Options: SAMEORIGIN` to be replaced by the characterized `DENY` policy together with the other three captured response fields, and requires an over-limit declared body to fail with 413 before origin delivery. `tests/pg_erd_runtime_isolation_traffic.rs` extends that compiled-process boundary with a chunked request whose streamed body crosses `max_request_body_bytes`, requiring HTTP 413 followed by healthy `/readyz`, and with a one-request in-flight budget that holds one routed backend request open, requires the next routed request to fail fast with 503, keeps `/readyz` and `/metrics` available, observes exactly one backpressure rejection, then proves capacity is released for a later routed request. `tests/pg_erd_upstream_failure_traffic.rs` adds the first dedicated origin-failure contract: the characterized backend address is made actively unavailable after its concrete port is captured, `/api` traffic must fail as HTTP 502 within the configured connection budget, `/readyz` must remain healthy, the low-cardinality request-error counter must record the failure, and an independent fallback request must still reach the characterized `frontend` authority successfully. `tests/pg_erd_read_stall_traffic.rs` covers the next transport phase: the characterized backend accepts the HTTP/1.1 request but emits no response bytes, so the configured Pingora per-read `read_ms` budget must fail the request as HTTP 502 within a conservative outer bound, preserve `/readyz`, record the low-cardinality error, and leave the independent fallback route usable. The fixture intentionally sends no partial response, so it does not transfer evidence to reset, partial-response, streaming-failure, slow-drip, or total-response-lifetime behavior. On Unix, `tests/pg_erd_graceful_shutdown.rs` adds consumer-root drain evidence: a routed `/api/held` request is held at the characterized backend, SIGTERM is delivered only after the backend has received it, the response is released within the shared grace period, the downstream must still receive HTTP 200, and the migration process must exit successfully inside the external termination budget. These source contracts are not GREEN evidence by themselves: they must compile and pass with formatting, clippy, rustdoc and 100% owned-production line/region coverage on the same exact head before listener/runtime/failure-recovery/drain parity is credited. +`tests/pg_erd_production_path.rs` is the first dedicated compiled-listener traffic contract. It starts real loopback `backend` and `frontend` origins plus `cwl-pingora-pg-erd-migration`, requires `/livez` and `/readyz` to remain gateway-local, requires consumer `/healthz` and raw `/apiary` to reach `backend`, requires fallback product traffic to reach `frontend`, requires request-controlled forwarding identity to be discarded and rebuilt from accepted loopback transport/Host data, requires upstream `X-Frame-Options: SAMEORIGIN` to be replaced by the characterized `DENY` policy together with the other three captured response fields, and requires an over-limit declared body to fail with 413 before origin delivery. `tests/pg_erd_runtime_isolation_traffic.rs` extends that compiled-process boundary with a chunked request whose streamed body crosses `max_request_body_bytes`, requiring HTTP 413 followed by healthy `/readyz`, and with a one-request in-flight budget that holds one routed backend request open, requires the next routed request to fail fast with 503, keeps `/readyz` and `/metrics` available, observes exactly one backpressure rejection, then proves capacity is released for a later routed request. `tests/pg_erd_upstream_failure_traffic.rs` adds the first dedicated origin-failure contract: the characterized backend address is made actively unavailable after its concrete port is captured, `/api` traffic must fail as HTTP 502 within the configured connection budget, `/readyz` must remain healthy, the low-cardinality request-error counter must record the failure, and an independent fallback request must still reach the characterized `frontend` authority successfully. `tests/pg_erd_read_stall_traffic.rs` covers the next transport phase: the characterized backend accepts the HTTP/1.1 request but emits no response bytes, so the configured Pingora per-read `read_ms` budget must fail the request as HTTP 502 within a conservative outer bound, preserve `/readyz`, record the low-cardinality error, and leave the independent fallback route usable. The fixture intentionally sends no partial response, so it does not transfer evidence to reset, partial-response, streaming-failure, slow-drip, or total-response-lifetime behavior. `tests/pg_erd_partial_response_traffic.rs` covers a distinct post-header failure phase: the characterized backend commits HTTP 200 plus `Content-Length: 20`, sends only `partial`, then closes. The downstream must retain the committed 200/framing and terminate before the declared body completes rather than receiving an invented second status or silent failover; `/readyz`, low-cardinality error telemetry, and an independent `frontend` route must remain usable afterward. The fixture uses an orderly origin close and therefore does not transfer evidence to an explicit TCP reset, upgraded/WebSocket failure, generic streaming failure, or slow-drip/whole-response lifetime. On Unix, `tests/pg_erd_graceful_shutdown.rs` adds consumer-root drain evidence: a routed `/api/held` request is held at the characterized backend, SIGTERM is delivered only after the backend has received it, the response is released within the shared grace period, the downstream must still receive HTTP 200, and the migration process must exit successfully inside the external termination budget. These source contracts are not GREEN evidence by themselves: they must compile and pass with formatting, clippy, rustdoc and 100% owned-production line/region coverage on the same exact head before listener/runtime/failure-recovery/drain parity is credited. The exact OCI gate builds one digest-pinned distroless image and exercises both composition roots under uid/gid 65532, a read-only root filesystem, all Linux capabilities dropped, `no-new-privileges`, and read-only configuration mounts. Generic v1 remains the image default entrypoint. The pg-erd migration binary is selected explicitly, uses only the fixed `backend`/`frontend` authority profile, and must expose both its process-health and metrics listeners. This is source acceptance until the unchanged exact-head OCI job executes terminal-success; it is not deployment, canary, or cutover evidence. @@ -16,4 +16,4 @@ The exact OCI gate builds one digest-pinned distroless image and exercises both Every behavioral migration should begin with characterization against the old owned edge behavior, then add equivalent production-path evidence for Pingora. Static-serving consumers must cover route precedence, SPA fallback, MIME, ETag/cache, Range/HEAD/304/416, redirects/security headers and compression as applicable. Proxy consumers must cover Host/SNI/TLS, forwarding trust, WebSocket/upgrade, limits, timeout/retry behavior, streaming/uploads, saturation/backpressure, errors, health/readiness and drain. Characterized response policies must additionally be exercised through the compiled proxy path before any parity or canary claim. -Release-quality gaps remain: no property/fuzz tests yet; no downstream TLS listener contract; no HTTP/2 or HTTP/3 parity evidence; no tracing evidence; no immutable published registry digest/provenance and rehearsed rollback; and no benchmark against replaced Nginx/Traefik traffic. For the pg-erd candidate specifically, reset, partial-response and streaming-failure recovery, slow-drip/whole-response lifetime control, origin-capacity/routed-load measurement, shadow/canary and rollback still lack terminal executable evidence. Refused-backend, connected-silent-backend, routed graceful drain, and dedicated OCI invocation now have source acceptance, but they remain uncredited until terminal exact-head execution. Generic OCI hardening, owned-production 100% line/region coverage, public rustdoc, SBOM/image vulnerability, local-CA upstream TLS, explicit backpressure/recovery, and the minimal k6 loopback path already have gates or tests, but each must be terminal-success on the exact release candidate and cannot be transferred to a changed or consumer-specific head. Production performance claims remain forbidden until representative deployment measurements exist. +Release-quality gaps remain: no property/fuzz tests yet; no downstream TLS listener contract; no HTTP/2 or HTTP/3 parity evidence; no tracing evidence; no immutable published registry digest/provenance and rehearsed rollback; and no benchmark against replaced Nginx/Traefik traffic. For the pg-erd candidate specifically, explicit TCP-reset and broader streaming-failure recovery, slow-drip/whole-response lifetime control, origin-capacity/routed-load measurement, shadow/canary and rollback still lack terminal executable evidence. Refused-backend, connected-silent-backend, partial-response, routed graceful drain, and dedicated OCI invocation now have source acceptance, but they remain uncredited until terminal exact-head execution. Generic OCI hardening, owned-production 100% line/region coverage, public rustdoc, SBOM/image vulnerability, local-CA upstream TLS, explicit backpressure/recovery, and the minimal k6 loopback path already have gates or tests, but each must be terminal-success on the exact release candidate and cannot be transferred to a changed or consumer-specific head. Production performance claims remain forbidden until representative deployment measurements exist. \ No newline at end of file From b2bf4d57e8737557061ca8db0cedc846d1caad27 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 00:26:52 +0900 Subject: [PATCH 04/43] docs: advance migration changelog after partial-response characterization --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5f0b421..e37ed105 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ All notable changes are tracked here. No release has been published yet. - Added dedicated compiled pg-erd runtime-isolation traffic acceptance: chunked bodies that cross `max_request_body_bytes` must return 413 without poisoning `/readyz`; a held routed request at an in-flight budget of one must force the next routed request to 503 while `/readyz` and metrics remain available, increment the backpressure counter, and release capacity for a subsequent routed request. - Added dedicated compiled pg-erd refused-origin recovery acceptance: a characterized `backend` connection refusal must return HTTP 502 within the configured connection budget, preserve `/readyz`, increment low-cardinality request-error telemetry, and leave the independent fallback `frontend` route able to complete successfully. This is source acceptance pending terminal exact-head execution, not a parity or cutover claim. - Added dedicated compiled pg-erd connected-but-silent origin acceptance: the backend accepts the routed request but emits no response bytes, so the configured Pingora per-read `read_ms` budget must produce HTTP 502, preserve `/readyz`, record low-cardinality request-error telemetry, and leave the independent fallback route usable. The configuration contract now explicitly states that Pingora resets this timer after each successful upstream read; slow-drip/whole-response lifetime remains a separate open isolation requirement. +- Added dedicated compiled pg-erd partial-response failure acceptance: the backend commits HTTP 200 with a declared 20-byte body, sends only `partial`, then closes. The downstream must retain the already-committed status/framing and terminate before body completion rather than receiving an invented second status or silent failover; `/readyz`, low-cardinality error telemetry, and the independent `frontend` route must remain usable. Explicit TCP reset, upgraded/WebSocket failure, broader streaming failure, and slow-drip/whole-response lifetime remain separate contracts. - Added dedicated routed pg-erd graceful-drain acceptance: a characterized `/api` backend request is held open, SIGTERM is sent only after the backend has accepted it, the response is released within the shared grace period, the downstream must still receive HTTP 200, and the migration process must exit successfully inside the external termination budget. Generic drain evidence is not transferred to this composition root. - Packaged the bounded `cwl-pingora-pg-erd-migration` composition root in the same digest-pinned distroless OCI image as generic v1 while keeping `cwl-pingora-gateway` as the default entrypoint. Exact-image CI now invokes both composition roots as uid/gid 65532 with a read-only root filesystem, all Linux capabilities dropped, `no-new-privileges`, and read-only versioned configuration; the pg-erd invocation must expose its process-health and metrics listeners before the OCI gate passes. - Added optional per-upstream absolute PEM trust-bundle consumption without taking ownership of certificate issuance/rotation; trust material is loaded fail-closed before listeners open. @@ -38,4 +39,4 @@ All notable changes are tracked here. No release has been published yet. - Added missing-public-rustdoc enforcement and documentation builds with warnings denied. - Added DDD, product, technical, security, threat, test, operability, configuration, migration-gap, and primary-source traceability documentation. -Release remains blocked on the organization decision for the exact Pingora release versus `RUSTSEC-2026-0253` and the separate time-bounded disposition of unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388` (`ContextualWisdomLab/.github#1605`), restoration of authoritative public non-fork Dependency Review evidence (`ContextualWisdomLab/.github#810`), terminal exact-current-head CI/supply-chain/security/review evidence, representative compiled-binary pg-erd reset/partial-response/streaming-failure/slow-drip/origin-capacity and benchmark evidence, an immutable registry digest with provenance and rehearsed rollback, and protected-branch integration. Refused-backend, connected-silent-backend, routed graceful drain, and dedicated pg-erd OCI invocation now have source acceptance but remain uncredited until terminal exact-head execution. No consumer migration, canary, cutover, or legacy removal is claimed before those release and traffic-contract gates are satisfied. +Release remains blocked on the organization decision for the exact Pingora release versus `RUSTSEC-2026-0253` and the separate time-bounded disposition of unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388` (`ContextualWisdomLab/.github#1605`), restoration of authoritative public non-fork Dependency Review evidence (`ContextualWisdomLab/.github#810`), terminal exact-current-head CI/supply-chain/security/review evidence, representative compiled-binary pg-erd explicit TCP-reset/streaming-failure/slow-drip/origin-capacity and benchmark evidence, an immutable registry digest with provenance and rehearsed rollback, and protected-branch integration. Refused-backend, connected-silent-backend, partial-response, routed graceful drain, and dedicated pg-erd OCI invocation now have source acceptance but remain uncredited until terminal exact-head execution. No consumer migration, canary, cutover, or legacy removal is claimed before those release and traffic-contract gates are satisfied. \ No newline at end of file From 3255f3f004ca20a64476facca1c6112bf7500578 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 00:28:02 +0900 Subject: [PATCH 05/43] docs: advance pg-erd failure-recovery gap baseline --- docs/product-technical-gap-baseline.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 9f1eaffc..b55dd150 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -20,13 +20,13 @@ This baseline is code-current for the Pingora migration stack. Exact source head | Hop-by-hop / forwarding trust | Generic trust repair + dedicated migration contract, exact-head hosted GREEN pending | Generic v1 strips request-controlled `Forwarded`, `X-Forwarded-For`, `X-Forwarded-Host`, `X-Forwarded-Port`, `X-Forwarded-Proto`, `X-Forwarded-Server`, and `X-Real-IP` before emitting only gateway-owned `Forwarded: proto=http`; it does not assert client identity. The dedicated `forwarding_policy` separately removes those request-controlled fields and rebuilds characterized `X-Forwarded-For`, `X-Real-IP`, `X-Forwarded-Host`, `X-Forwarded-Port` and `X-Forwarded-Proto` from accepted downstream transport/request authority. The dedicated loopback contract supplies hostile proxy identity and requires direct loopback/Host-derived values with no attacker identity. The characterized Traefik entryPoint is clear-text `web`, so `http` is explicit; HTTPS needs its own TLS-derived scheme contract | | Retry policy | Implemented, intentionally minimal | `max_retries=1` means one total upstream attempt and zero generic automatic retries; domain idempotency/replay policy stays with the product owner | | Request limits | Partial | Declared and streamed/chunked body size plus process-wide in-flight backpressure are bounded. Configurable header, connection, per-route and origin-capacity budgets remain gaps | -| Failure recovery | Refused-origin and connected-silent-origin source acceptance added; hosted GREEN pending | The generic compiled loopback contract requires a refused origin connection to return HTTP 502 within configured connection budget and proves `/readyz` remains healthy afterward. PR #17 adds the dedicated multi-route refused-backend equivalent with bounded 502 failure, readiness, low-cardinality error telemetry, and independent `frontend` recovery. PR #18 adds the next transport phase: the characterized backend accepts the request but emits no response bytes, so Pingora's configured per-read `read_ms` inactivity budget must produce HTTP 502, preserve readiness/error telemetry, and leave the independent route usable. The contract explicitly does not reinterpret `read_ms` as a whole-response lifetime because Pingora resets it after successful reads. Reset, partial-response, streaming-failure and slow-drip/whole-response-lifetime cases remain gaps, and no source contract is credited until exact-head terminal execution | -| Health | Shared process boundary implemented; dedicated exact-head execution pending | `/livez` and `/readyz` are served locally through a shared internal Pingora process-health boundary in both adapters and bypass migration fallback routing. Consumer `/healthz` remains characterized application traffic to `backend`. The dedicated process contract explicitly distinguishes them; PR #16 additionally requires readiness to remain available after streamed-body rejection and while routed application capacity is saturated, PR #17 requires readiness after refused upstream transport, and PR #18 requires readiness after connected read-timeout failure. Terminal exact-head execution is still required | +| Failure recovery | Refused-origin, connected-silent-origin, and post-header partial-response source acceptance added; hosted GREEN pending | The generic compiled loopback contract requires a refused origin connection to return HTTP 502 within configured connection budget and proves `/readyz` remains healthy afterward. PR #17 adds the dedicated multi-route refused-backend equivalent with bounded 502 failure, readiness, low-cardinality error telemetry, and independent `frontend` recovery. PR #18 adds the next pre-header transport phase: the characterized backend accepts the request but emits no response bytes, so Pingora's configured per-read `read_ms` inactivity budget must produce HTTP 502, preserve readiness/error telemetry, and leave the independent route usable. PR #21 adds the distinct post-header phase: `backend` commits HTTP 200 with `Content-Length: 20`, sends only `partial`, then closes; the downstream must retain the committed status/framing and terminate before body completion, while readiness, error telemetry, and an independent `frontend` request recover. This matches Pingora's phase boundary: once the response header is downstream, a later failure cannot be rewritten into a second error response or failover. Explicit TCP reset, broader streaming/upgraded failure, and slow-drip/whole-response-lifetime cases remain gaps. No source contract is credited until exact-head terminal execution | +| Health | Shared process boundary implemented; dedicated exact-head execution pending | `/livez` and `/readyz` are served locally through a shared internal Pingora process-health boundary in both adapters and bypass migration fallback routing. Consumer `/healthz` remains characterized application traffic to `backend`. The dedicated process contract explicitly distinguishes them; PR #16 additionally requires readiness to remain available after streamed-body rejection and while routed application capacity is saturated, PR #17 after refused upstream transport, PR #18 after connected read-timeout failure, and PR #21 after a post-header truncated response. Terminal exact-head execution is still required | | Graceful drain | Dedicated routed source acceptance added; hosted GREEN pending | Both composition roots use shared explicit Pingora server policy: 5 s grace plus 10 s runtime shutdown timeout inside a 30 s external termination budget. Generic drain is tested. PR #20 adds the consumer-root equivalent: a characterized `/api` request must reach `backend` and remain in flight before SIGTERM, then complete HTTP 200 when released during the grace period, after which the migration process must exit successfully inside the external termination budget. Generic evidence is not transferred; exact-head execution remains mandatory | -| Logs / metrics / traces | Listener-capable candidate | Shared `observability` owns low-cardinality request/error/body/backpressure counters and credential/cookie-safe coarse logs. Both adapters delegate to it; paths, query strings, headers/cookies, credentials, customer payloads and product identifiers are excluded. PR #16 requires dedicated saturation to advance the backpressure counter while metrics remain reachable; PRs #17 and #18 require the request-error counter to advance after refused and connected-silent upstream failures. Dedicated payload-free log assertions and tracing remain gaps | +| Logs / metrics / traces | Listener-capable candidate | Shared `observability` owns low-cardinality request/error/body/backpressure counters and credential/cookie-safe coarse logs. Both adapters delegate to it; paths, query strings, headers/cookies, credentials, customer payloads and product identifiers are excluded. PR #16 requires dedicated saturation to advance the backpressure counter while metrics remain reachable; PRs #17, #18, and #21 require the request-error counter to advance after refused, connected-silent, and post-header truncated upstream failures. Dedicated payload-free log assertions and tracing remain gaps | | OCI isolation | Dedicated image invocation source acceptance added; hosted GREEN pending | Runtime remains uid/gid 65532, read-only-root compatible, capability-free and `no-new-privileges`; base images are digest-pinned. PR #19 packages both Rust composition roots in the same exact image while retaining generic v1 as the default entrypoint, then invokes `cwl-pingora-pg-erd-migration` explicitly with a read-only fixed-profile config under the same rootless/read-only/capability-free boundary and requires its process-health plus metrics listeners to become reachable. Source presence is not deployment evidence until the exact OCI job executes terminal-success | | Dependency policy | Release-blocked | `.github#1605` owns the exact-release Pingora vs patched-`lru` decision and disposition of unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388`; `.github#810` independently owns the public non-fork Dependency Review compare-API HTTP 403 incident. Known-unsound downgrade, blanket advisory waiver, fail-open 403 handling, or substitute-scanner promotion is prohibited | -| Coverage / public API docs | Gates implemented, current head pending | Owned production line/region coverage is required at 100%; `#![deny(missing_docs)]` and warning-denied rustdoc cover public APIs. Fixed-profile/callback construction was modeled as infallible after validated boundaries instead of keeping structurally impossible error branches merely to evade coverage. PR #14 includes decision-path coverage for shared wildcard/equality listener authority plus generic zero-port traffic/metrics/upstream admission; the forwarding-trust successor adds an executable generic regression for the full client-controlled proxy-identity set. PRs #16-#18 and #20 add integration-only traffic acceptance, while PR #19 adds OCI packaging/invocation acceptance; none changes owned-production coverage denominators. Current exact head still must pass the unchanged 100% line/region gate | +| Coverage / public API docs | Gates implemented, current head pending | Owned production line/region coverage is required at 100%; `#![deny(missing_docs)]` and warning-denied rustdoc cover public APIs. Fixed-profile/callback construction was modeled as infallible after validated boundaries instead of keeping structurally impossible error branches merely to evade coverage. PR #14 includes decision-path coverage for shared wildcard/equality listener authority plus generic zero-port traffic/metrics/upstream admission; the forwarding-trust successor adds an executable generic regression for the full client-controlled proxy-identity set. PRs #16-#18, #20, and #21 add integration-only traffic acceptance, while PR #19 adds OCI packaging/invocation acceptance; none changes owned-production coverage denominators. Current exact head still must pass the unchanged 100% line/region gate | | Load / 20 ms p95 | Executable generic candidate only | Checksum-pinned k6 2.2.0 exercises 400 release-mode generic loopback requests across four VUs and gates p95 <20 ms with zero failures. This does not transfer to pg-erd multi-route serving. Representative routed concurrency, origin capacity and deployment/network measurements are required before 20 ms p95 becomes a pg-erd objective | | Rollback | Documented, not rehearsed | Rehearsal requires an immutable protected release artifact/digest and a real consumer traffic transition | @@ -37,7 +37,7 @@ Fresh organization code evidence finds no OpenResty usage. Responsibility class, | Repository / evidence | Classification | Migration consequence | | --- | --- | --- | | `linux-cluster-ops/docs/architecture/nginx-routing-inventory.md` plus Nginx/Certbot recovery evidence | ACTIVE_RUNTIME / CURRENT_OPERATOR_DOC | True shared-edge candidate, but current multi-vhost routing, static/PHP-FPM and certificate-adjacent operations exceed Pingora v1. Split authority and freeze executable traffic/TLS contracts first | -| `pg-erd-cloud/deploy/traefik/dynamic.yaml` at protected `main@8dc746920c12988f082e914879d95e13c9693535` | ACTIVE_DEPLOYMENT / PLAUSIBLE_CONSUMER | Fresh 2026-09-02 read confirms ordered exact `/healthz -> backend`, raw-prefix `/api -> backend`, fallback `/ -> frontend` plus four response-security fields are unchanged. Consumer code also conditionally trusts sanitized `X-Forwarded-For`, so forwarding identity is migration behavior rather than cosmetic metadata. PRs #5/#6/#7/#10 characterize route/header/authority/peer binding, PR #11 composes them in Pingora callbacks, PR #12 adds bounded Admin Config, dedicated process startup tests, shared process-health separation and a real loopback backend/frontend traffic contract, PR #14 closes shared wildcard-collision plus generic port-zero network-authority gaps before activation, PR #15 closes the residual generic forwarding-identity spoofing set, PR #16 adds dedicated streamed-body and saturation/recovery traffic acceptance, PR #17 adds refused-backend recovery acceptance, PR #18 adds connected-silent-origin read-timeout acceptance, PR #19 packages/invokes the bounded migration root under the same hardened OCI boundary, and PR #20 adds routed SIGTERM drain acceptance. No terminal exact-head pg-erd parity, shadow/canary or cutover is claimed yet | +| `pg-erd-cloud/deploy/traefik/dynamic.yaml` at protected `main@8dc746920c12988f082e914879d95e13c9693535` | ACTIVE_DEPLOYMENT / PLAUSIBLE_CONSUMER | Fresh 2026-09-02 read confirms ordered exact `/healthz -> backend`, raw-prefix `/api -> backend`, fallback `/ -> frontend` plus four response-security fields are unchanged. Consumer code also conditionally trusts sanitized `X-Forwarded-For`, so forwarding identity is migration behavior rather than cosmetic metadata. PRs #5/#6/#7/#10 characterize route/header/authority/peer binding, PR #11 composes them in Pingora callbacks, PR #12 adds bounded Admin Config, dedicated process startup tests, shared process-health separation and a real loopback backend/frontend traffic contract, PR #14 closes shared wildcard-collision plus generic port-zero network-authority gaps before activation, PR #15 closes the residual generic forwarding-identity spoofing set, PR #16 adds dedicated streamed-body and saturation/recovery traffic acceptance, PR #17 adds refused-backend recovery acceptance, PR #18 adds connected-silent-origin read-timeout acceptance, PR #19 packages/invokes the bounded migration root under the same hardened OCI boundary, PR #20 adds routed SIGTERM drain acceptance, and PR #21 adds post-header partial-response failure/recovery acceptance. No terminal exact-head pg-erd parity, shadow/canary or cutover is claimed yet | | `naruon` NGINX ingress/live-E2E plus Traefik evaluation | ACTIVE_DEPLOYMENT / TEST_RUNTIME | Its Nginx proxy contract includes HTTP/1.1, long read/send timeouts, WebSocket Upgrade/Connection and forwarded identity semantics. Keycloak/authentication stays outside Pingora; only transport/edge policy can migrate after explicit parity evidence | | `scopeweave`, `LineageWeave`, `inkspan` Nginx static-serving images/config | ACTIVE_STATIC_RUNTIME | Static hosting is not automatically a shared-edge migration; prove gateway responsibility before queueing | | `life-os` ClusterIP-only base manifests with separately managed edge namespace | DELEGATED EDGE | Repository base manifests do not prove an embedded legacy edge to migrate | @@ -56,10 +56,10 @@ The EA owner path must consume that released contract and project each approved 1. Reacquire exact-current-head CI, 100% owned production line/region coverage, rustdoc, k6, OCI, SAST and supply-chain evidence after every source or documentation movement; repair only evidence-backed repository defects. Organization runner acquisition is separately tracked in `.github#712`; queued jobs are not source GREEN. 2. Keep `.github#1605` and `.github#810` fail-closed until their respective policy and GitHub Dependency Review availability owner paths are resolved; do not suppress `RUSTSEC-2024-0388` generically. -3. Keep PRs #5, #6, #7, #10, #11, #12, #14, #15, #16, #17, #18, #19 and #20 pre-traffic until exact-head quality/security evidence and coherent dependency ancestry are terminal. No child evidence repairs a parent release blocker. -4. PR #12 contains the bounded startup/Admin Config transition plus a dedicated compiled-process loopback traffic contract; PR #14 tightens shared generic/migration network-authority admission; PR #15 closes residual generic forwarding-identity spoofing; PR #16 adds dedicated streamed-body and in-flight saturation/readiness/telemetry/recovery traffic; PR #17 adds refused-backend failure/recovery; PR #18 adds connected-silent-origin per-read timeout/recovery while documenting that `read_ms` is not a whole-response budget; PR #19 packages and explicitly invokes the bounded migration binary under the same hardened OCI image/runtime boundary; PR #20 adds routed in-flight SIGTERM drain acceptance. The immediate gate remains terminal exact-head execution of these stacked contracts together with fmt/compile/clippy/rustdoc/100% coverage and applicable security/supply-chain checks. Any deterministic failure must be repaired causally; source test or packaging presence alone is not parity. -5. After the basic listener/runtime-isolation/refused-origin/read-stall/drain contracts are GREEN, extend dedicated pg-erd acceptance with reset, partial-response, streaming-failure and slow-drip/whole-response-lifetime handling, payload-free log assertions, and representative routed concurrency/origin-capacity measurements before adopting a production 20 ms p95 objective. A future HTTPS listener must separately prove TLS-derived forwarded scheme instead of reusing the current clear-text `web` assumption. +3. Keep PRs #5, #6, #7, #10, #11, #12, #14, #15, #16, #17, #18, #19, #20 and #21 pre-traffic until exact-head quality/security evidence and coherent dependency ancestry are terminal. No child evidence repairs a parent release blocker. +4. PR #12 contains the bounded startup/Admin Config transition plus a dedicated compiled-process loopback traffic contract; PR #14 tightens shared generic/migration network-authority admission; PR #15 closes residual generic forwarding-identity spoofing; PR #16 adds dedicated streamed-body and in-flight saturation/readiness/telemetry/recovery traffic; PR #17 adds refused-backend failure/recovery; PR #18 adds connected-silent-origin per-read timeout/recovery while documenting that `read_ms` is not a whole-response budget; PR #19 packages and explicitly invokes the bounded migration binary under the same hardened OCI image/runtime boundary; PR #20 adds routed in-flight SIGTERM drain acceptance; PR #21 adds the committed-header/truncated-body failure phase without inventing post-commit retry semantics. The immediate gate remains terminal exact-head execution of these stacked contracts together with fmt/compile/clippy/rustdoc/100% coverage and applicable security/supply-chain checks. Any deterministic failure must be repaired causally; source test or packaging presence alone is not parity. +5. After the basic listener/runtime-isolation/refused-origin/read-stall/partial-response/drain contracts are GREEN, extend dedicated pg-erd acceptance with explicit TCP reset, broader streaming/upgraded failure and slow-drip/whole-response-lifetime handling, payload-free log assertions, and representative routed concurrency/origin-capacity measurements before adopting a production 20 ms p95 objective. A future HTTPS listener must separately prove TLS-derived forwarded scheme instead of reusing the current clear-text `web` assumption. 6. Once the dedicated OCI invocation is terminal GREEN, add a protected release path publishing an immutable image digest with SBOM/provenance/reproducibility evidence and rehearse rollback against that exact digest. 7. Satisfy then-live protected-branch review/governance without self-approval, bot-as-human claims, stale evidence transfer, or routine administrator bypass. 8. Wait for an immutable released Context Graph bundle with source-bound consumer-verifiable release evidence and a coherent compatible GREEN EA admission path before asserting authoritative architecture execution state. -9. Only then move `pg-erd-cloud` through explicit parity -> shadow/canary -> cutover -> rollback -> legacy removal. Other Nginx/Traefik surfaces remain separate responsibility-bound migration candidates and are not inherited automatically from this consumer profile. +9. Only then move `pg-erd-cloud` through explicit parity -> shadow/canary -> cutover -> rollback -> legacy removal. Other Nginx/Traefik surfaces remain separate responsibility-bound migration candidates and are not inherited automatically from this consumer profile. \ No newline at end of file From 38562b1f840e3573aa6e462df0f70287381a6dcd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 3 Sep 2026 00:28:47 +0900 Subject: [PATCH 06/43] docs: trace post-header failure semantics to pinned Pingora --- docs/doctoring/TRACEABILITY.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/doctoring/TRACEABILITY.md b/docs/doctoring/TRACEABILITY.md index 4f065600..240b62cb 100644 --- a/docs/doctoring/TRACEABILITY.md +++ b/docs/doctoring/TRACEABILITY.md @@ -10,6 +10,8 @@ This file links material technical/security claims to primary standards or upstr | Graceful SIGTERM uses `grace_period_seconds` and `graceful_shutdown_timeout_seconds`, with framework fallbacks when unset | `pingora-core/src/server/mod.rs` and `pingora-core/src/server/configuration/mod.rs` at the pinned commit; CWL v1 sets 5 s grace and 10 s per-runtime graceful timeout explicitly inside a 30 s external termination budget | | Standard upstream request policy supports hop-by-hop/connection-nominated stripping and normalized WebSocket-only HTTP/1 upgrade forwarding | Cloudflare Pingora `HttpUpstreamRequestPolicy` / peer implementation at pinned commit `09696b51bc59315353d96686355861604d0bb48c` | | Pingora `read_timeout` is a per-individual-read inactivity budget and resets after each successful upstream `read()`; it is not a total-response lifetime bound | Cloudflare Pingora `docs/user_guide/peer.md` and `pingora-proxy/src/proxy_h1.rs` at pinned commit `09696b51bc59315353d96686355861604d0bb48c`; the pg-erd read-stall acceptance therefore characterizes a connected origin that sends no response bytes and deliberately does not claim slow-drip/whole-response bounding | +| A proxy failure after the upstream response header has already been sent downstream cannot be replaced with a new error response or failover; Pingora logs/surfaces the error and gives up that request | Cloudflare Pingora `docs/user_guide/failover.md` and `pingora-proxy/src/proxy_h1.rs` at pinned commit `09696b51bc59315353d96686355861604d0bb48c`; this phase boundary is the basis of the dedicated pg-erd partial-response traffic contract | +| Pingora HTTP/1 body framing treats a body that ends before its declared `Content-Length` as `PREMATURE_BODY_END`, while upstream read failures are propagated as failed proxy tasks | `pingora-core/src/protocols/http/v1/body.rs`, `pingora-core/src/protocols/http/v1/client.rs`, and `pingora-proxy/src/proxy_h1.rs` at pinned commit `09696b51bc59315353d96686355861604d0bb48c`; RFC 9112 defines HTTP/1.1 message framing requirements | | Pingora OpenSSL peers support a per-peer CA store; when configured it replaces the verification store for that peer while certificate and hostname verification remain separately enabled | `pingora-core/src/upstreams/peer.rs`, `pingora-core/src/connectors/tls/boringssl_openssl/mod.rs`, and `pingora-core/src/protocols/tls/boringssl_openssl/mod.rs` at pinned commit `09696b51bc59315353d96686355861604d0bb48c` | | Traefik normally adds `X-Forwarded-For`, `X-Real-Ip`, `X-Forwarded-Host`, `X-Forwarded-Port`, `X-Forwarded-Proto`, and `X-Forwarded-Server` when proxying HTTP | Traefik official Getting Started FAQ, current documentation revalidated 2026-09-02 | | Incoming Traefik `X-Forwarded-*` identity is trusted only when an EntryPoint explicitly configures trusted IPs or insecure trust; insecure mode is not recommended for production | Traefik official EntryPoints documentation, current documentation revalidated 2026-09-02 | @@ -36,8 +38,14 @@ Cloudflare. (n.d.). *Pingora upstream peer options* [Source code, commit 09696b5 Cloudflare. (n.d.). *Peer: how to connect to upstream* [Documentation, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/docs/user_guide/peer.md +Cloudflare. (n.d.). *Handling failures and failover* [Documentation, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/docs/user_guide/failover.md + Cloudflare. (n.d.). *Pingora HTTP/1 proxy implementation* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-proxy/src/proxy_h1.rs +Cloudflare. (n.d.). *Pingora HTTP/1 client session* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/protocols/http/v1/client.rs + +Cloudflare. (n.d.). *Pingora HTTP/1 body framing* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/protocols/http/v1/body.rs + Cloudflare. (n.d.). *Pingora OpenSSL upstream TLS connector* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/connectors/tls/boringssl_openssl/mod.rs Cloudflare. (n.d.). *Pingora downstream HTTP session* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/protocols/http/server.rs @@ -78,4 +86,4 @@ Open Container Initiative. (2025, November 4). *OCI runtime-spec v1.3.0 release Rust Release Team. (2026, August 20). *Announcing Rust 1.98.0*. Rust Blog. https://blog.rust-lang.org/2026/08/20/Rust-1.98.0/ -Rust Secure Code Working Group. (2026, August 11). *RUSTSEC-2026-0253: lru—memory safety issue under panic*. RustSec Advisory Database. https://rustsec.org/advisories/RUSTSEC-2026-0253.html +Rust Secure Code Working Group. (2026, August 11). *RUSTSEC-2026-0253: lru—memory safety issue under panic*. RustSec Advisory Database. https://rustsec.org/advisories/RUSTSEC-2026-0253.html \ No newline at end of file From f1d54688c2a62756bacd953a091d506505328d96 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 19:00:36 +0900 Subject: [PATCH 07/43] test(pg-erd): restore partial-response traffic contract --- tests/pg_erd_partial_response_traffic.rs | 261 +++++++++++++++++++++++ 1 file changed, 261 insertions(+) create mode 100644 tests/pg_erd_partial_response_traffic.rs diff --git a/tests/pg_erd_partial_response_traffic.rs b/tests/pg_erd_partial_response_traffic.rs new file mode 100644 index 00000000..3f306d7c --- /dev/null +++ b/tests/pg_erd_partial_response_traffic.rs @@ -0,0 +1,261 @@ +//! Real-listener partial upstream response acceptance for the dedicated pg-erd migration binary. +//! +//! This contract distinguishes a response that fails after its status/header block has already +//! been received from failures that occur before downstream response commitment. It proves that a +//! truncated characterized origin response is not rewritten into an invented retry/failover, +//! leaves process health observable, records the transport failure, and does not poison an +//! independent characterized route. + +use std::io::{ErrorKind, Read, Write}; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::process::{Child, Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; + +use tempfile::NamedTempFile; + +struct GatewayProcess(Child); + +impl Drop for GatewayProcess { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DownstreamTermination { + Eof, + ConnectionReset, +} + +fn reserve_distinct_loopback_addresses() -> (SocketAddr, SocketAddr) { + // Hold both ephemeral reservations at once so listener and metrics authority cannot + // accidentally collapse to the same port before the migration process binds them. + let traffic = TcpListener::bind("127.0.0.1:0").expect("traffic port should be reservable"); + let metrics = TcpListener::bind("127.0.0.1:0").expect("metrics port should be reservable"); + let addresses = ( + traffic + .local_addr() + .expect("traffic reservation should expose an address"), + metrics + .local_addr() + .expect("metrics reservation should expose an address"), + ); + assert_ne!(addresses.0, addresses.1); + addresses +} + +fn write_config( + listener: SocketAddr, + metrics_listener: SocketAddr, + backend: SocketAddr, + frontend: SocketAddr, +) -> NamedTempFile { + let mut file = NamedTempFile::new().expect("temporary config should be writable"); + writeln!( + file, + "version: 1\nlistener: {listener}\nmetrics_listener: {metrics_listener}\nmax_request_body_bytes: 8\nmax_in_flight_requests: 8\nupstream_keepalive_pool_size: 4\nupstreams:\n - name: backend\n address: {backend}\n tls: false\n timeouts:\n connection_ms: 200\n total_connection_ms: 400\n read_ms: 500\n write_ms: 1000\n idle_ms: 5000\n - name: frontend\n address: {frontend}\n tls: false\n timeouts:\n connection_ms: 200\n total_connection_ms: 400\n read_ms: 1000\n write_ms: 1000\n idle_ms: 5000" + ) + .expect("migration config should be written"); + file +} + +fn wait_until_listening(address: SocketAddr, process: &mut Child) { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + if let Some(status) = process + .try_wait() + .expect("gateway process state should be readable") + { + panic!("gateway exited before accepting traffic: {status}"); + } + if TcpStream::connect_timeout(&address, Duration::from_millis(100)).is_ok() { + return; + } + assert!(Instant::now() < deadline, "gateway did not start within 10s"); + thread::sleep(Duration::from_millis(25)); + } +} + +fn start_gateway( + config: &NamedTempFile, + gateway_address: SocketAddr, + metrics_address: SocketAddr, +) -> GatewayProcess { + let mut child = Command::new(env!("CARGO_BIN_EXE_cwl-pingora-pg-erd-migration")) + .args(["--config", config.path().to_str().expect("UTF-8 temp path")]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("compiled pg-erd migration binary should start"); + wait_until_listening(gateway_address, &mut child); + wait_until_listening(metrics_address, &mut child); + GatewayProcess(child) +} + +fn raw_request(address: SocketAddr, request: &[u8]) -> String { + let mut downstream = TcpStream::connect(address).expect("gateway should accept traffic"); + downstream + .set_read_timeout(Some(Duration::from_secs(5))) + .expect("downstream timeout should be configurable"); + downstream + .write_all(request) + .expect("downstream request should be writable"); + let mut response = String::new(); + downstream + .read_to_string(&mut response) + .expect("gateway response should be readable"); + response +} + +fn raw_request_until_terminal( + address: SocketAddr, + request: &[u8], +) -> (Vec, DownstreamTermination) { + let mut downstream = TcpStream::connect(address).expect("gateway should accept traffic"); + downstream + .set_read_timeout(Some(Duration::from_secs(5))) + .expect("downstream timeout should be configurable"); + downstream + .write_all(request) + .expect("downstream request should be writable"); + + let mut response = Vec::new(); + let mut buffer = [0_u8; 1024]; + loop { + match downstream.read(&mut buffer) { + Ok(0) => return (response, DownstreamTermination::Eof), + Ok(read) => response.extend_from_slice(&buffer[..read]), + Err(error) if error.kind() == ErrorKind::ConnectionReset => { + return (response, DownstreamTermination::ConnectionReset); + } + Err(error) => panic!("partial downstream response should terminate, not stall: {error}"), + } + } +} + +fn get(address: SocketAddr, path: &str) -> String { + raw_request( + address, + format!("GET {path} HTTP/1.1\r\nHost: app.example:8080\r\nConnection: close\r\n\r\n") + .as_bytes(), + ) +} + +fn read_request_headers(stream: &mut TcpStream) -> String { + let mut bytes = Vec::new(); + let mut buffer = [0_u8; 1024]; + loop { + let read = stream.read(&mut buffer).expect("origin request should be readable"); + assert!(read > 0, "gateway closed origin request before headers completed"); + bytes.extend_from_slice(&buffer[..read]); + if bytes.windows(4).any(|window| window == b"\r\n\r\n") { + return String::from_utf8_lossy(&bytes).into_owned(); + } + } +} + +#[test] +fn compiled_pg_erd_truncated_response_stays_committed_and_preserves_independent_routing() { + let backend = TcpListener::bind("127.0.0.1:0").expect("backend fixture should bind"); + let backend_address = backend.local_addr().expect("backend address should exist"); + let backend_origin = thread::spawn(move || { + let (mut stream, _) = backend + .accept() + .expect("routed request should reach the characterized backend authority"); + let request = read_request_headers(&mut stream); + assert!(request.starts_with("GET /api/partial-response HTTP/1.1\r\n")); + + // Once this status/header block is forwarded, a later framing failure cannot be replaced + // with a second HTTP status or silently failed over to another characterized origin. + stream + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 20\r\nConnection: close\r\n\r\npartial", + ) + .expect("partial backend response should be writable"); + }); + + let frontend = TcpListener::bind("127.0.0.1:0").expect("frontend fixture should bind"); + let frontend_address = frontend.local_addr().expect("frontend address should exist"); + let frontend_origin = thread::spawn(move || { + let (mut stream, _) = frontend + .accept() + .expect("fallback request should reach the independent frontend authority"); + let request = read_request_headers(&mut stream); + assert!(request.starts_with("GET /after-partial-response HTTP/1.1\r\n")); + stream + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 9\r\nConnection: close\r\n\r\nrecovered", + ) + .expect("frontend recovery response should be writable"); + }); + + let (gateway_address, metrics_address) = reserve_distinct_loopback_addresses(); + let config = write_config( + gateway_address, + metrics_address, + backend_address, + frontend_address, + ); + let _process = start_gateway(&config, gateway_address, metrics_address); + + let (partial, termination) = raw_request_until_terminal( + gateway_address, + b"GET /api/partial-response HTTP/1.1\r\nHost: app.example:8080\r\nConnection: close\r\n\r\n", + ); + assert!( + matches!( + termination, + DownstreamTermination::Eof | DownstreamTermination::ConnectionReset + ), + "a committed truncated response must terminate the downstream connection" + ); + let header_end = partial + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|position| position + 4) + .expect("committed partial response must contain a complete header block"); + let headers = String::from_utf8_lossy(&partial[..header_end]).to_ascii_lowercase(); + assert!( + headers.starts_with("http/1.1 200"), + "a post-header upstream failure cannot be rewritten as a new status: {headers:?}" + ); + assert!( + headers.contains("content-length: 20"), + "the committed response must retain its declared framing for this fixture: {headers:?}" + ); + let body = &partial[header_end..]; + assert_eq!(body, b"partial"); + assert!( + body.len() < 20, + "fixture must terminate before its declared response body completes" + ); + + let readiness = get(gateway_address, "/readyz"); + assert!( + readiness.starts_with("HTTP/1.1 200"), + "one truncated upstream response must not poison process readiness: {readiness:?}" + ); + + let metrics = get(metrics_address, "/metrics"); + assert!( + metrics.contains("cwl_pingora_gateway_request_errors_total 1"), + "the post-header upstream framing failure must remain visible through low-cardinality error telemetry: {metrics:?}" + ); + + let recovered = get(gateway_address, "/after-partial-response"); + assert!( + recovered.starts_with("HTTP/1.1 200"), + "an independent characterized route must remain usable after a truncated response: {recovered:?}" + ); + assert!(recovered.ends_with("\r\n\r\nrecovered")); + + frontend_origin + .join() + .expect("frontend recovery fixture should complete"); + backend_origin + .join() + .expect("partial backend fixture should complete"); +} From 53ed7e5a834c9a4fa2ae7e8e35f1b2be6c6cdcd2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 19:03:53 +0900 Subject: [PATCH 08/43] docs: restack partial-response migration gap baseline --- docs/product-technical-gap-baseline.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 3cb94cd1..32266e65 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -20,10 +20,10 @@ This baseline is code-current for the Pingora migration stack. Exact source head | Hop-by-hop / forwarding trust | Generic sanitizer restacked on current #14; pg-erd compiled-listener contract present; hosted GREEN pending | Generic v1 removes request-controlled `Forwarded`, `X-Forwarded-For`, `X-Forwarded-Host`, `X-Forwarded-Port`, `X-Forwarded-Proto`, `X-Forwarded-Server`, and `X-Real-IP`, emits only gateway-owned `Forwarded: proto=http`, and makes no client-IP claim. The pg-erd `forwarding_policy` separately removes hostile forwarding identity and rebuilds only characterized `X-Forwarded-For`, `X-Real-IP`, `X-Forwarded-Host`, `X-Forwarded-Port`, and `X-Forwarded-Proto` from accepted downstream transport/request authority. Product auth/identity authority is not moved into either adapter | | Retry policy | Implemented, intentionally minimal | `max_retries=1` means one total upstream attempt and zero generic automatic retries; domain idempotency/replay policy stays with the product owner | | Request limits | Partial | Declared and streamed/chunked body size plus process-wide in-flight backpressure are bounded. Dedicated pg-erd source acceptance now covers streamed overflow and routed capacity saturation/recovery. A connected read inactivity budget is also wired to Pingora, but configurable header, connection-count, per-route, whole-response-lifetime and origin-capacity budgets remain gaps | -| Failure recovery | Refusal + connected read-stall source acceptance; exact hosted GREEN pending | Current #17 requires a real loopback backend connection refusal to produce 502 inside a conservative one-second envelope around configured 200/400 ms connection budgets, increment request-error telemetry, preserve readiness and leave an independent `frontend` route usable. Current #18 adds the next distinct phase: the backend accepts the routed request and remains connected without sending response bytes until after the gateway has failed it; with `read_ms=100`, downstream must receive 502 inside a conservative one-second envelope while readiness/error telemetry/independent-route recovery remain intact. TCP reset, post-commit truncation, slow-drip/whole-response lifetime, retry/failover and streaming-network failure remain distinct gaps | -| Health | Shared process boundary implemented; dedicated exact-head execution pending | `/livez` and `/readyz` are served locally through a shared internal Pingora process-health boundary in both adapters and bypass migration fallback routing. Consumer `/healthz` remains characterized application traffic to `backend`. Dedicated source acceptance preserves readiness under body/backpressure rejection, refused-origin failure and connected read-stall failure; terminal exact-head execution is still required | -| Graceful drain | Dedicated routed source acceptance; exact hosted GREEN pending | Both composition roots use shared explicit Pingora server policy: 5 s grace plus 10 s runtime shutdown timeout inside a 30 s external termination budget. Generic drain is tested. Restacked Draft #20 now adds consumer-root acceptance that holds a characterized `/api` backend request in flight, sends SIGTERM only after backend receipt, releases the response during the grace period, requires downstream HTTP 200, and requires successful process exit inside the termination budget. This is source acceptance only until unchanged exact-head execution is terminal GREEN | -| Logs / metrics / traces | Prometheus service-identity OCI acceptance added | Shared `observability` owns low-cardinality request/error/body/backpressure counters and credential/cookie-safe coarse logs. Both adapters delegate to it; paths, query strings, headers/cookies, credentials, customer payloads and product identifiers are excluded. Dedicated source acceptance observes the bounded backpressure counter in #16 and request-error counter in #17/#18. Restacked Draft #19 requires the dedicated metrics listener to answer from the same rootless/read-only-root OCI profile as `/livez` and identifies Pingora's Prometheus service through its `text/plain` response media type, without requiring a metric family before application traffic has emitted one. Broader payload-free runtime assertions and tracing remain gaps | +| Failure recovery | Refusal + read-stall + post-header truncation source acceptance; exact hosted GREEN pending | Current #17 requires a real loopback backend connection refusal to produce 502 inside a conservative one-second envelope around configured 200/400 ms connection budgets, increment request-error telemetry, preserve readiness and leave an independent `frontend` route usable. Current #18 adds the next pre-header phase: the backend accepts the routed request and remains connected without sending response bytes until after the gateway has failed it; with `read_ms=100`, downstream must receive 502 inside a conservative one-second envelope while readiness/error telemetry/independent-route recovery remain intact. Restacked Draft #21 adds a distinct post-header phase: `backend` commits HTTP 200 with `Content-Length: 20`, sends only `partial`, then closes; the downstream must retain the committed status/framing and terminate before body completion while readiness, low-cardinality error telemetry and an independent `frontend` request recover. Explicit TCP reset, broader streaming/upgraded failure, slow-drip/whole-response lifetime and retry/failover remain distinct gaps | +| Health | Shared process boundary implemented; dedicated exact-head execution pending | `/livez` and `/readyz` are served locally through a shared internal Pingora process-health boundary in both adapters and bypass migration fallback routing. Consumer `/healthz` remains characterized application traffic to `backend`. Dedicated source acceptance preserves readiness under body/backpressure rejection, refused-origin failure, connected read-stall failure and the #21 post-header truncated-response failure; terminal exact-head execution is still required | +| Graceful drain | Dedicated routed source acceptance; exact hosted GREEN pending | Both composition roots use shared explicit Pingora server policy: 5 s grace plus 10 s runtime shutdown timeout inside a 30 s external termination budget. Generic drain is tested. Restacked Draft #20 adds consumer-root acceptance that holds a characterized `/api` backend request in flight, sends SIGTERM only after backend receipt, releases the response during the grace period, requires downstream HTTP 200, and requires successful process exit inside the termination budget. Its current fixture reserves traffic/metrics ports simultaneously so listener-authority validation cannot be bypassed by ephemeral-port reuse. This is source acceptance only until unchanged exact-head execution is terminal GREEN | +| Logs / metrics / traces | Prometheus service-identity OCI acceptance added | Shared `observability` owns low-cardinality request/error/body/backpressure counters and credential/cookie-safe coarse logs. Both adapters delegate to it; paths, query strings, headers/cookies, credentials, customer payloads and product identifiers are excluded. Dedicated source acceptance observes the bounded backpressure counter in #16 and request-error counter in #17/#18/#21. Restacked Draft #19 requires the dedicated metrics listener to answer from the same rootless/read-only-root OCI profile as `/livez` and identifies Pingora's Prometheus service through its `text/plain` response media type, without requiring a metric family before application traffic has emitted one. Broader payload-free runtime assertions and tracing remain gaps | | OCI isolation | Dedicated candidate source extended through #19; exact hosted GREEN pending | Digest-pinned builder/runtime images and uid/gid 65532 remain shared. The Dockerfile fail-closes `CWL_GATEWAY_BIN` to exactly the generic or bounded pg-erd process and copies only the selected executable to one fixed distroless runtime path. OCI CI builds both admitted image profiles and requires each to run read-only with all capabilities dropped and `no-new-privileges`; the pg-erd profile mounts only `examples/pg-erd-migration.yaml` read-only. Current #19 preserves that one-binary-per-image design and requires `/livez` on the traffic listener plus a separately published `/metrics` endpoint whose media type matches Pingora Prometheus `text/plain*`. Supply-chain CI builds and HIGH/CRITICAL-scans both image profiles and binds both local image IDs/per-image scan receipts to the exact source SHA. None of this is hosted GREEN until the unchanged current head reaches terminal success, and it is not routed pg-erd parity or immutable release evidence | | Dependency policy | Release-blocked | `.github#1605` owns the exact-release Pingora vs patched-`lru` decision and disposition of unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388`; `.github#810` independently owns the public non-fork Dependency Review compare-API HTTP 403 incident. Known-unsound downgrade, blanket advisory waiver, fail-open 403 handling, or substitute-scanner promotion is prohibited | | Coverage / public API docs | Gates implemented, current head pending | Owned production line/region coverage is required at 100%; `#![deny(missing_docs)]` and warning-denied rustdoc cover public APIs. Fixed-profile/callback construction was modeled as infallible after validated boundaries instead of keeping structurally impossible error branches merely to evade coverage. Current exact head still must pass the unchanged 100% line/region gate | @@ -37,7 +37,7 @@ Fresh organization code evidence finds no OpenResty usage. Responsibility class, | Repository / evidence | Classification | Migration consequence | | --- | --- | --- | | `linux-cluster-ops/docs/architecture/nginx-routing-inventory.md` plus Nginx/Certbot recovery evidence | ACTIVE_RUNTIME / CURRENT_OPERATOR_DOC | True shared-edge candidate, but current multi-vhost routing, static/PHP-FPM and certificate-adjacent operations exceed Pingora v1. Split authority and freeze executable traffic/TLS contracts first | -| `pg-erd-cloud/deploy/traefik/dynamic.yaml` at source commit `8dc746920c12988f082e914879d95e13c9693535` | ACTIVE_DEPLOYMENT / PLAUSIBLE_CONSUMER | Ordered exact `/healthz -> backend`, raw-prefix `/api -> backend`, fallback `/ -> frontend` plus four response-security fields. Consumer code also conditionally trusts sanitized `X-Forwarded-For`, so forwarding identity is migration behavior rather than cosmetic metadata. PRs #5/#6/#7/#10 characterize route/header/authority/peer binding, PR #11 composes them in Pingora callbacks, PR #12 adds bounded Admin Config, dedicated process startup tests, shared process-health separation and a real loopback backend/frontend traffic contract, #14 defines the dedicated rootless/read-only-root OCI profile and per-image security scan lane, #15 repairs generic forwarding distrust without absorbing product identity, #16 adds dedicated runtime-isolation traffic acceptance, #17 adds dedicated refused-origin recovery acceptance, #18 adds connected read-stall acceptance without inventing whole-response semantics, #19 proves one-binary-per-image least-privilege OCI startup plus Prometheus metrics-service identity, and #20 adds routed SIGTERM drain source acceptance. No terminal exact-head pg-erd parity, immutable artifact, shadow/canary or cutover is claimed yet | +| `pg-erd-cloud/deploy/traefik/dynamic.yaml` at source commit `8dc746920c12988f082e914879d95e13c9693535` | ACTIVE_DEPLOYMENT / PLAUSIBLE_CONSUMER | Ordered exact `/healthz -> backend`, raw-prefix `/api -> backend`, fallback `/ -> frontend` plus four response-security fields. Consumer code also conditionally trusts sanitized `X-Forwarded-For`, so forwarding identity is migration behavior rather than cosmetic metadata. PRs #5/#6/#7/#10 characterize route/header/authority/peer binding, PR #11 composes them in Pingora callbacks, PR #12 adds bounded Admin Config, dedicated process startup tests, shared process-health separation and a real loopback backend/frontend traffic contract, #14 defines the dedicated rootless/read-only-root OCI profile and per-image security scan lane, #15 repairs generic forwarding distrust without absorbing product identity, #16 adds dedicated runtime-isolation traffic acceptance, #17 adds dedicated refused-origin recovery acceptance, #18 adds connected read-stall acceptance without inventing whole-response semantics, #19 proves one-binary-per-image least-privilege OCI startup plus Prometheus metrics-service identity, #20 adds routed SIGTERM drain source acceptance, and #21 adds post-header partial-response failure/recovery source acceptance. No terminal exact-head pg-erd parity, immutable artifact, shadow/canary or cutover is claimed yet | | `naruon` NGINX ingress/live-E2E plus Traefik evaluation | ACTIVE_DEPLOYMENT / TEST_RUNTIME | Its Nginx proxy contract includes HTTP/1.1, long read/send timeouts, WebSocket Upgrade/Connection and forwarded identity semantics. Keycloak/authentication stays outside Pingora; only transport/edge policy can migrate after explicit parity evidence | | `scopeweave`, `LineageWeave`, `inkspan` Nginx static-serving images/config | ACTIVE_STATIC_RUNTIME | Static hosting is not automatically a shared-edge migration; prove gateway responsibility before queueing | | `life-os` ClusterIP-only base manifests with separately managed edge namespace | DELEGATED EDGE | Repository base manifests do not prove an embedded legacy edge to migrate | @@ -63,9 +63,10 @@ The EA owner path must consume that released contract and project each approved 7. Draft #17 originally depended on historical #16 `9d5a108b355030e5d13571399dff93332ac019c3` and was 82 commits behind current #16 `49d3fb5d89bf15543c7df8b433809acb8fac88eb`. Ordinary two-parent commit `edceb2c948231aebfd9a1987c1fae6e8e220b84e` preserved historical #17 as first parent while adopting exact current #16 as the second parent and resolution tree; `93cbb28cc3ccde54380e5c1287c2e1b74a1a71e7` reapplied only the still-valid refused-origin traffic contract on the current stack. The contract requires 502, a conservative one-second loopback envelope around configured 200/400 ms connection budgets, readiness, low-cardinality request-error telemetry and independent `frontend` recovery. It is not connected read-timeout, reset, partial-response, streaming or retry/failover evidence. Exact current-head hosted and review evidence must be reacquired. 8. Draft #18 originally depended on historical #17 `5a522b1ce6e58cb879e4dfe33282748a63554473` and was 87 commits behind current #17 before repair. Ordinary two-parent commit `dba4ce7921d2a87a4029043b78ec4a6a6135a729` preserves the historical #18 lineage while adopting exact current #17 `ce2b3032580fa851f9ccd34462801920be947a09` as the resolution tree. The replayed read-stall contract was strengthened: the accepted backend now stays connected and silent until the gateway has already returned, so fixture closure cannot fake `read_ms` enforcement. With `read_ms=100`, the source requires 502 inside a conservative one-second envelope, readiness, request-error telemetry and independent `frontend` recovery. The API contract and primary-source traceability state explicitly that Pingora's read timeout is per read, not a whole-response deadline. Exact current-head hosted/review evidence must be reacquired. 9. Draft #19 was 94 commits behind current #18 and carried an older packaging approach that copied both binaries into one image. Ordinary two-parent succession preserves historical #19 while adopting exact #18 as its resolution tree instead of replaying that superseded packaging design. The current one-binary-per-image pg-erd profile runs as uid/gid 65532 with read-only root, all capabilities dropped, `no-new-privileges`, and only the versioned pg-erd config mounted read-only. Its OCI acceptance now requires `/livez` plus the separately published `/metrics` endpoint to identify Pingora's Prometheus service through `text/plain*`; a bare HTTP 200 is intentionally insufficient, while a pre-traffic metric family remains unnecessary. Exact current-head hosted/review evidence must be reacquired. -10. Draft #20 originally depended on historical #19 `5c80f35fdef12775a59d82ff3e8861f308173e07` and diverged after #19 succession. Ordinary two-parent commit `bbb9070b0d36efe8023bf01c3e6a83eda3e85a0f` preserves historical #20 as first parent while adopting exact current #19 `e0ab23d43d3ef4ae77aaf8475154084d72ff4a95` as second parent and resolution tree, then reapplies the still-valid routed SIGTERM drain test and code-current changelog/test/baseline documentation. Generic drain evidence is not transferred. Exact current-head fmt/compile/test/clippy/rustdoc/100% coverage, applicable OCI/supply-chain and fresh review must execute before routed drain is credited. -11. After the listener/forwarding/runtime-isolation/refusal/read-stall/OCI/drain stack is GREEN, extend dedicated pg-erd acceptance with TCP reset, post-commit partial-response/streaming-network failure recovery, slow-drip/whole-response lifetime, payload-free observability assertions, and representative routed concurrency/origin-capacity measurements before adopting a production 20 ms p95 objective. A future HTTPS listener must separately prove TLS-derived forwarded scheme instead of reusing the current clear-text `web` assumption. -12. Exact-head supply-chain must build/scan both admitted image profiles and bind their receipts to source. Only after terminal GREEN may the release path publish registry-bound immutable image digests with release SBOM/provenance/reproducibility evidence and rehearse rollback against those exact digests. -13. Satisfy then-live protected-branch review/governance without self-approval, bot-as-human claims, stale evidence transfer, or routine administrator bypass. -14. Wait for an immutable released Context Graph bundle with source-bound consumer-verifiable release evidence and a coherent compatible GREEN EA admission path before asserting authoritative architecture execution state. -15. Only then move `pg-erd-cloud` through explicit parity -> shadow/canary -> cutover -> rollback -> legacy removal. Other Nginx/Traefik surfaces remain separate responsibility-bound migration candidates and are not inherited automatically from this consumer profile. +10. Draft #20 originally depended on historical #19 `5c80f35fdef12775a59d82ff3e8861f308173e07` and diverged after #19 succession. Ordinary two-parent commit `bbb9070b0d36efe8023bf01c3e6a83eda3e85a0f` preserves historical #20 as first parent while adopting exact current #19 `e0ab23d43d3ef4ae77aaf8475154084d72ff4a95` as second parent and resolution tree, then reapplies the still-valid routed SIGTERM drain contract. Fresh review found a distinct-port fixture race in two sequential bind-and-drop reservations; current repair `df0d7c03a395dda9c4caebd2a70f3eff4ff27db1` holds traffic and metrics reservations simultaneously before releasing them. Generic drain evidence is not transferred. Exact current-head fmt/compile/test/clippy/rustdoc/100% coverage, applicable OCI/supply-chain and fresh review must execute before routed drain is credited. +11. Draft #21 was still based on historical #20 `d81bed16cfa6d8ea8d16064d195a1f8c4c5c7108` after #20 advanced, leaving it 110 commits behind current #20. Ordinary two-parent commit `fe76050e12bda9c27d466c6b8cf38be67210c83e` preserves the historical #21 lineage while adopting exact current #20 `df0d7c03a395dda9c4caebd2a70f3eff4ff27db1` as the resolution tree. The still-valid post-header partial-response traffic contract is replayed on top; its own traffic/metrics ephemeral reservations use the same simultaneous distinct-port discipline rather than reintroducing the #20 fixture race. Historical #21 packaging wording that said both binaries shared one image is deliberately not replayed because current #19/#20 preserve stronger one-binary-per-image packaging. Exact current-head hosted/review evidence must be reacquired. +12. After the listener/forwarding/runtime-isolation/refusal/read-stall/OCI/drain/partial-response stack is GREEN, extend dedicated pg-erd acceptance with explicit TCP reset, broader streaming/upgraded failure, slow-drip/whole-response lifetime, payload-free observability assertions, and representative routed concurrency/origin-capacity measurements before adopting a production 20 ms p95 objective. A future HTTPS listener must separately prove TLS-derived forwarded scheme instead of reusing the current clear-text `web` assumption. +13. Exact-head supply-chain must build/scan both admitted image profiles and bind their receipts to source. Only after terminal GREEN may the release path publish registry-bound immutable image digests with release SBOM/provenance/reproducibility evidence and rehearse rollback against those exact digests. +14. Satisfy then-live protected-branch review/governance without self-approval, bot-as-human claims, stale evidence transfer, or routine administrator bypass. +15. Wait for an immutable released Context Graph bundle with source-bound consumer-verifiable release evidence and a coherent compatible GREEN EA admission path before asserting authoritative architecture execution state. +16. Only then move `pg-erd-cloud` through explicit parity -> shadow/canary -> cutover -> rollback -> legacy removal. Other Nginx/Traefik surfaces remain separate responsibility-bound migration candidates and are not inherited automatically from this consumer profile. From 490aab1871bf4af748adcc5f37faa2fef2749a4a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 19:04:34 +0900 Subject: [PATCH 09/43] docs: restore partial-response test strategy --- TEST_STRATEGY.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/TEST_STRATEGY.md b/TEST_STRATEGY.md index bd9f1aad..93e1f1bf 100644 --- a/TEST_STRATEGY.md +++ b/TEST_STRATEGY.md @@ -16,7 +16,9 @@ The bounded Admin Config transition has its own executable contract. `tests/pg_e `tests/pg_erd_read_stall_traffic.rs` separates connected upstream inactivity from refusal. The characterized backend accepts `/api/read-stall`, reads the request headers, then remains connected and sends no response bytes until the test explicitly releases it after the gateway has already returned. With `read_ms=100`, the gateway must fail as HTTP 502 inside a conservative one-second outer envelope, preserve `/readyz`, increment the shared request-error counter, and leave an independent `frontend` route usable. Keeping the fixture connection open prevents origin closure from masquerading as the timeout. Because Pingora's `read_timeout` is per successful `read()` rather than a whole-response lifetime, reset, post-commit partial response, slow-drip and whole-response deadline behavior remain separate contracts. -On Unix, `tests/pg_erd_graceful_shutdown.rs` proves the bounded pg-erd composition root consumes the shared drain policy instead of borrowing generic-binary evidence. The test routes `/api/held` to the characterized backend, holds that response open until the backend has confirmed the in-flight request, then delivers SIGTERM and releases the response during the shared grace period. The downstream must still receive HTTP 200 and the migration process must exit successfully inside the external termination budget. This is source-defined drain acceptance until the unchanged exact head executes it to terminal GREEN. +`tests/pg_erd_partial_response_traffic.rs` covers the distinct post-header failure phase. The characterized backend commits HTTP 200 with `Content-Length: 20`, writes only `partial`, then closes normally. Acceptance requires the downstream to retain that committed status/framing and terminate before all 20 bytes arrive rather than receiving an invented second status or silent route failover. `/readyz` must stay HTTP 200, the shared request-error counter must record the framing failure, and an independent `frontend` request must still complete successfully. Traffic and metrics ports are reserved simultaneously before the process starts so ephemeral-port reuse cannot manufacture an invalid listener collision. This orderly-close contract does not transfer evidence to explicit TCP reset, WebSocket/Upgrade, broader streaming failure, or slow-drip/whole-response lifetime. + +On Unix, `tests/pg_erd_graceful_shutdown.rs` proves the bounded pg-erd composition root consumes the shared drain policy instead of borrowing generic-binary evidence. The test routes `/api/held` to the characterized backend, holds that response open until the backend has confirmed the in-flight request, then delivers SIGTERM and releases the response during the shared grace period. The downstream must still receive HTTP 200 and the migration process must exit successfully inside the external termination budget. The traffic and metrics reservation sockets are held simultaneously before startup to prevent ephemeral-port reuse from producing a false listener-collision failure. This is source-defined drain acceptance until the unchanged exact head executes it to terminal GREEN. The `oci-runtime` job separately validates artifact composition instead of inferring it from compiled-process tests. The Dockerfile admits only `cwl-pingora-gateway` and `cwl-pingora-pg-erd-migration` as build-time process identities, normalizes the selected executable to one fixed distroless runtime path, and CI builds both image profiles. Each exact candidate must declare uid/gid `65532`, start under a read-only root filesystem with all capabilities dropped and `no-new-privileges`, and consume only a read-only configuration mount. The generic profile must expose local `/livez`. The pg-erd profile must expose local `/livez` on the traffic listener and its separately published `/metrics` listener must identify the Pingora Prometheus service by a `text/plain` response media type before the container is accepted. `tests/pg_erd_oci_metrics_workflow_contract.rs` prevents a bare HTTP 200 from false-greening a mistakenly bound proxy service while deliberately avoiding a metric-family requirement before routed application traffic has emitted one. `examples/pg-erd-migration.yaml` is deliberately an origin-independent OCI smoke fixture; this gate proves the dedicated binary is actually packaged and both process/observability listeners start under the required container isolation but does not claim routed pg-erd parity, origin health, or performance. The supply-chain job additionally builds and vulnerability-scans both image profiles and binds both local image IDs plus per-image scan outputs to the exact source SHA. These workflow contracts count only after terminal success on the unchanged exact head. @@ -24,4 +26,4 @@ The `oci-runtime` job separately validates artifact composition instead of infer Every behavioral migration should begin with characterization against the old owned edge behavior, then add equivalent production-path evidence for Pingora. Static-serving consumers must cover route precedence, SPA fallback, MIME, ETag/cache, Range/HEAD/304/416, redirects/security headers and compression as applicable. Proxy consumers must cover Host/SNI/TLS, forwarding trust, WebSocket/upgrade, limits, timeout/retry behavior, streaming/uploads, saturation/backpressure, errors, health/readiness and drain. Characterized response policies must additionally be exercised through the compiled proxy path before any parity or canary claim. -Release-quality gaps remain: no property/fuzz tests yet; no downstream TLS listener contract; no HTTP/2 or HTTP/3 parity evidence; no tracing evidence; no immutable published registry digest/provenance and rehearsed rollback; and no benchmark against replaced Nginx/Traefik traffic. For the pg-erd candidate specifically, the dedicated source now covers streamed body overflow, in-flight saturation/recovery, refused-origin recovery, connected silent-origin read timeout, routed graceful drain, and OCI process/Prometheus-listener identity, but TCP reset, post-commit partial-response/streaming failure, slow-drip/whole-response lifetime, representative routed origin-capacity load, terminal current-head traffic/OCI/supply-chain execution, shadow/canary and rollback still lack evidence. OCI non-root/read-only-root source acceptance covers both admitted process images, while owned-production 100% line/region coverage, public rustdoc, SBOM/image vulnerability, local-CA upstream TLS, explicit backpressure/recovery, and the minimal k6 loopback path remain exact-head gates. None can be transferred to a changed or consumer-specific head. Production performance claims remain forbidden until representative deployment measurements exist. +Release-quality gaps remain: no property/fuzz tests yet; no downstream TLS listener contract; no HTTP/2 or HTTP/3 parity evidence; no tracing evidence; no immutable published registry digest/provenance and rehearsed rollback; and no benchmark against replaced Nginx/Traefik traffic. For the pg-erd candidate specifically, the dedicated source now covers streamed body overflow, in-flight saturation/recovery, refused-origin recovery, connected silent-origin read timeout, orderly post-header truncation, routed graceful drain, and OCI process/Prometheus-listener identity, but explicit TCP reset, broader streaming/upgraded failure, slow-drip/whole-response lifetime, representative routed origin-capacity load, terminal current-head traffic/OCI/supply-chain execution, shadow/canary and rollback still lack evidence. OCI non-root/read-only-root source acceptance covers both admitted process images, while owned-production 100% line/region coverage, public rustdoc, SBOM/image vulnerability, local-CA upstream TLS, explicit backpressure/recovery, and the minimal k6 loopback path remain exact-head gates. None can be transferred to a changed or consumer-specific head. Production performance claims remain forbidden until representative deployment measurements exist. From 34aafa25c31ba7cb6a4b9b5e23e8d583aa52aa42 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 19:05:06 +0900 Subject: [PATCH 10/43] docs: restore phase-aware partial-response contract --- TRD.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TRD.md b/TRD.md index d460550f..1e7dfa45 100644 --- a/TRD.md +++ b/TRD.md @@ -28,6 +28,8 @@ Non-health requests acquire the process `max_in_flight_requests` budget before u Generic v1 makes one prevalidated upstream peer available per request. The pg-erd migration adapter selects only peers already bound by `MigrationDeliveryPlan`; neither path performs request-controlled service discovery. Domain retries, failover, and idempotency policy are not invented by this runtime. +Failure handling is phase-aware. Before an upstream response header is committed downstream, transport failure may still be represented by the gateway's fail-closed error response under the one-attempt policy. After a valid response header has been committed, a later upstream framing/body failure cannot be rewritten into a second HTTP status or silently failed over: the incomplete downstream response terminates, low-cardinality error telemetry records the failed request, process readiness remains available, and independent routes must remain usable. This is an edge transport invariant, not product retry authority. + ## Health and observability `GET /livez` and `/readyz` return HTTP 200 with an empty, non-cacheable response through the process-local Pingora health boundary. Readiness proves validated configuration plus an active serving path, not product dependency health. In the pg-erd migration profile, consumer `/healthz` remains ordinary routed application traffic and is not confused with process liveness/readiness. From 3e86675f084ac51667eee919ff97cbea4d175632 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 19:05:59 +0900 Subject: [PATCH 11/43] docs: restore partial-response primary-source traceability --- docs/doctoring/TRACEABILITY.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/doctoring/TRACEABILITY.md b/docs/doctoring/TRACEABILITY.md index 32eb248c..36d08ed7 100644 --- a/docs/doctoring/TRACEABILITY.md +++ b/docs/doctoring/TRACEABILITY.md @@ -10,6 +10,8 @@ This file links material technical/security claims to primary standards or upstr | Graceful SIGTERM uses `grace_period_seconds` and `graceful_shutdown_timeout_seconds`, with framework fallbacks when unset | `pingora-core/src/server/mod.rs` and `pingora-core/src/server/configuration/mod.rs` at the pinned commit; CWL v1 sets 5 s grace and 10 s per-runtime graceful timeout explicitly inside a 30 s external termination budget | | Standard upstream request policy supports hop-by-hop/connection-nominated stripping and normalized WebSocket-only HTTP/1 upgrade forwarding | Cloudflare Pingora `HttpUpstreamRequestPolicy` / peer implementation at pinned commit `09696b51bc59315353d96686355861604d0bb48c` | | Pingora `read_timeout` is a per-individual-read inactivity budget and resets after each successful upstream `read()`; it is not a total-response lifetime bound | Cloudflare Pingora `docs/user_guide/peer.md` and `pingora-proxy/src/proxy_h1.rs` at pinned commit `09696b51bc59315353d96686355861604d0bb48c`; the pg-erd read-stall acceptance therefore keeps a connected origin silent without closing its socket and deliberately does not claim slow-drip/whole-response bounding | +| A proxy failure after the upstream response header has already been sent downstream cannot be replaced with a new error response or failover | Cloudflare Pingora `docs/user_guide/failover.md` and `pingora-proxy/src/proxy_h1.rs` at pinned commit `09696b51bc59315353d96686355861604d0bb48c`; this phase boundary is the basis of the dedicated pg-erd partial-response traffic contract | +| Pingora HTTP/1 body framing treats a body that ends before its declared `Content-Length` as a premature body-end failure, while upstream read failures propagate through the proxy task stream | `pingora-core/src/protocols/http/v1/body.rs`, `pingora-core/src/protocols/http/v1/client.rs`, and `pingora-proxy/src/proxy_h1.rs` at pinned commit `09696b51bc59315353d96686355861604d0bb48c`; RFC 9112 defines HTTP/1.1 message framing requirements | | Pingora OpenSSL peers support a per-peer CA store; when configured it replaces the verification store for that peer while certificate and hostname verification remain separately enabled | `pingora-core/src/upstreams/peer.rs`, `pingora-core/src/connectors/tls/boringssl_openssl/mod.rs`, and `pingora-core/src/protocols/tls/boringssl_openssl/mod.rs` at pinned commit `09696b51bc59315353d96686355861604d0bb48c` | | IPv4-mapped IPv6 addresses represent IPv4 nodes in IPv6 form; Rust's `Ipv6Addr::to_ipv4_mapped` performs the bounded mapped-only canonicalization used by the socket-authority invariant. Linux IPv6 sockets can expose IPv4 peers as mapped IPv6 addresses, so mapped/native authority must not be compared only as unrelated textual address families | RFC 4291 §2.5.5.2; Rust `std::net::Ipv6Addr` documentation; Linux `ipv6(7)` | | Traefik normally adds `X-Forwarded-For`, `X-Real-Ip`, `X-Forwarded-Host`, `X-Forwarded-Port`, `X-Forwarded-Proto`, and `X-Forwarded-Server` when proxying HTTP | Traefik official Getting Started FAQ, current documentation revalidated 2026-09-02 | @@ -38,8 +40,14 @@ Cloudflare. (n.d.). *Pingora upstream peer options* [Source code, commit 09696b5 Cloudflare. (n.d.). *Peer: how to connect to upstream* [Documentation, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/docs/user_guide/peer.md +Cloudflare. (n.d.). *Handling failures and failover* [Documentation, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/docs/user_guide/failover.md + Cloudflare. (n.d.). *Pingora HTTP/1 proxy implementation* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-proxy/src/proxy_h1.rs +Cloudflare. (n.d.). *Pingora HTTP/1 client session* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/protocols/http/v1/client.rs + +Cloudflare. (n.d.). *Pingora HTTP/1 body framing* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/protocols/http/v1/body.rs + Cloudflare. (n.d.). *Pingora OpenSSL upstream TLS connector* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/connectors/tls/boringssl_openssl/mod.rs Cloudflare. (n.d.). *Pingora downstream HTTP session* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/protocols/http/server.rs From 90099f10127feaf9897c973601fa8d38278c0139 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 19:06:33 +0900 Subject: [PATCH 12/43] docs: restore partial-response changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42486de2..385bd79b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ All notable changes are tracked here. No release has been published yet. - Added dedicated compiled pg-erd traffic acceptance for streamed/chunked body overflow and routed in-flight saturation/recovery: the migration process must return 413 above the shared body budget, 503 above the in-flight budget, keep `/readyz` and backpressure telemetry observable, and admit a later routed request after capacity is released. This remains source-defined acceptance until the unchanged exact head reaches hosted GREEN. - Added dedicated compiled pg-erd refused-origin recovery acceptance: a characterized backend connection refusal must return 502 within a conservative one-second envelope around the configured 200/400 ms connection budgets, keep `/readyz` and request-error telemetry observable, and allow a later independent frontend route to recover. Connected read stall, TCP reset, partial-response/streaming failure, retry and failover behavior remain separate gaps. - Added dedicated compiled pg-erd connected read-stall acceptance with `read_ms=100`: the backend accepts the routed request and remains open without response bytes until the gateway has already failed it, preventing fixture closure from faking timeout behavior. The contract requires 502 inside a conservative one-second envelope, preserved `/readyz`, request-error telemetry, and independent frontend recovery. Pingora `read_timeout` remains a per-read inactivity budget, not a whole-response lifetime; reset, partial-response and slow-drip cases remain open. +- Added dedicated compiled pg-erd post-header partial-response acceptance: the backend commits HTTP 200 with `Content-Length: 20`, writes only `partial`, and closes. The downstream must retain the committed status/framing and terminate before body completion instead of receiving an invented second status or silent failover; `/readyz`, low-cardinality request-error telemetry, and an independent `frontend` route must remain usable. Explicit TCP reset, broader streaming/upgraded failure, and slow-drip/whole-response lifetime remain separate gaps. - Added dedicated routed pg-erd graceful-drain acceptance: a characterized `/api` backend request is held open, SIGTERM is sent only after the backend has accepted it, the response is released within the shared grace period, the downstream must still receive HTTP 200, and the migration process must exit successfully inside the external termination budget. Generic drain evidence is not transferred to this composition root; the contract remains source-defined until the exact head executes terminal GREEN. - Added optional per-upstream absolute PEM trust-bundle consumption without taking ownership of certificate issuance/rotation; trust material is loaded fail-closed before listeners open. - Added an executable local-CA TLS test through the compiled gateway that holds CA trust constant and proves SNI/hostname mismatch is rejected. @@ -38,4 +39,4 @@ All notable changes are tracked here. No release has been published yet. - Added missing-public-rustdoc enforcement and documentation builds with warnings denied. - Added DDD, product, technical, security, threat, test, operability, configuration, migration-gap, and primary-source traceability documentation. -Release remains blocked on the organization decision for the exact Pingora release versus `RUSTSEC-2026-0253` and the separate time-bounded disposition of unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388` (`ContextualWisdomLab/.github#1605`), restoration of authoritative public non-fork Dependency Review evidence (`ContextualWisdomLab/.github#810`), terminal exact-current-head CI/supply-chain/security/review evidence, representative compiled-binary pg-erd route/header/forwarding/body/backpressure/failure/concurrency/drain and benchmark evidence, an immutable registry digest with provenance and rehearsed rollback, and protected-branch integration. Routed graceful drain now has source acceptance but remains uncredited until terminal exact-head execution. No consumer migration, canary, cutover, or legacy removal is claimed before those release and traffic-contract gates are satisfied. +Release remains blocked on the organization decision for the exact Pingora release versus `RUSTSEC-2026-0253` and the separate time-bounded disposition of unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388` (`ContextualWisdomLab/.github#1605`), restoration of authoritative public non-fork Dependency Review evidence (`ContextualWisdomLab/.github#810`), terminal exact-current-head CI/supply-chain/security/review evidence, representative compiled-binary pg-erd route/header/forwarding/body/backpressure/failure/concurrency/drain and benchmark evidence, an immutable registry digest with provenance and rehearsed rollback, and protected-branch integration. Orderly post-header truncation and routed graceful drain now have source acceptance but remain uncredited until terminal exact-head execution; explicit TCP reset, broader streaming/upgraded failure, slow-drip/whole-response lifetime and representative routed load remain open. No consumer migration, canary, cutover, or legacy removal is claimed before those release and traffic-contract gates are satisfied. From 68d7d6b55b3a45037c896b66d8a2f31b8df90a16 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 19:12:23 +0900 Subject: [PATCH 13/43] docs: project current drain repair into partial-response baseline --- docs/product-technical-gap-baseline.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 32266e65..c9d9db3b 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -22,7 +22,7 @@ This baseline is code-current for the Pingora migration stack. Exact source head | Request limits | Partial | Declared and streamed/chunked body size plus process-wide in-flight backpressure are bounded. Dedicated pg-erd source acceptance now covers streamed overflow and routed capacity saturation/recovery. A connected read inactivity budget is also wired to Pingora, but configurable header, connection-count, per-route, whole-response-lifetime and origin-capacity budgets remain gaps | | Failure recovery | Refusal + read-stall + post-header truncation source acceptance; exact hosted GREEN pending | Current #17 requires a real loopback backend connection refusal to produce 502 inside a conservative one-second envelope around configured 200/400 ms connection budgets, increment request-error telemetry, preserve readiness and leave an independent `frontend` route usable. Current #18 adds the next pre-header phase: the backend accepts the routed request and remains connected without sending response bytes until after the gateway has failed it; with `read_ms=100`, downstream must receive 502 inside a conservative one-second envelope while readiness/error telemetry/independent-route recovery remain intact. Restacked Draft #21 adds a distinct post-header phase: `backend` commits HTTP 200 with `Content-Length: 20`, sends only `partial`, then closes; the downstream must retain the committed status/framing and terminate before body completion while readiness, low-cardinality error telemetry and an independent `frontend` request recover. Explicit TCP reset, broader streaming/upgraded failure, slow-drip/whole-response lifetime and retry/failover remain distinct gaps | | Health | Shared process boundary implemented; dedicated exact-head execution pending | `/livez` and `/readyz` are served locally through a shared internal Pingora process-health boundary in both adapters and bypass migration fallback routing. Consumer `/healthz` remains characterized application traffic to `backend`. Dedicated source acceptance preserves readiness under body/backpressure rejection, refused-origin failure, connected read-stall failure and the #21 post-header truncated-response failure; terminal exact-head execution is still required | -| Graceful drain | Dedicated routed source acceptance; exact hosted GREEN pending | Both composition roots use shared explicit Pingora server policy: 5 s grace plus 10 s runtime shutdown timeout inside a 30 s external termination budget. Generic drain is tested. Restacked Draft #20 adds consumer-root acceptance that holds a characterized `/api` backend request in flight, sends SIGTERM only after backend receipt, releases the response during the grace period, requires downstream HTTP 200, and requires successful process exit inside the termination budget. Its current fixture reserves traffic/metrics ports simultaneously so listener-authority validation cannot be bypassed by ephemeral-port reuse. This is source acceptance only until unchanged exact-head execution is terminal GREEN | +| Graceful drain | Dedicated routed source acceptance; exact hosted GREEN pending | Both composition roots use shared explicit Pingora server policy: 5 s grace plus 10 s runtime shutdown timeout inside a 30 s external termination budget. Generic drain is tested. Restacked Draft #20 holds a characterized `/api` backend request in flight, sends SIGTERM only after backend receipt, anchors the absolute external termination deadline to that signal instant, releases the response during the grace period, requires downstream HTTP 200, and requires successful process exit before that absolute deadline. Its fixture reserves traffic/metrics ports simultaneously so listener-authority validation cannot be bypassed by ephemeral-port reuse. This is source acceptance only until unchanged exact-head execution is terminal GREEN | | Logs / metrics / traces | Prometheus service-identity OCI acceptance added | Shared `observability` owns low-cardinality request/error/body/backpressure counters and credential/cookie-safe coarse logs. Both adapters delegate to it; paths, query strings, headers/cookies, credentials, customer payloads and product identifiers are excluded. Dedicated source acceptance observes the bounded backpressure counter in #16 and request-error counter in #17/#18/#21. Restacked Draft #19 requires the dedicated metrics listener to answer from the same rootless/read-only-root OCI profile as `/livez` and identifies Pingora's Prometheus service through its `text/plain` response media type, without requiring a metric family before application traffic has emitted one. Broader payload-free runtime assertions and tracing remain gaps | | OCI isolation | Dedicated candidate source extended through #19; exact hosted GREEN pending | Digest-pinned builder/runtime images and uid/gid 65532 remain shared. The Dockerfile fail-closes `CWL_GATEWAY_BIN` to exactly the generic or bounded pg-erd process and copies only the selected executable to one fixed distroless runtime path. OCI CI builds both admitted image profiles and requires each to run read-only with all capabilities dropped and `no-new-privileges`; the pg-erd profile mounts only `examples/pg-erd-migration.yaml` read-only. Current #19 preserves that one-binary-per-image design and requires `/livez` on the traffic listener plus a separately published `/metrics` endpoint whose media type matches Pingora Prometheus `text/plain*`. Supply-chain CI builds and HIGH/CRITICAL-scans both image profiles and binds both local image IDs/per-image scan receipts to the exact source SHA. None of this is hosted GREEN until the unchanged current head reaches terminal success, and it is not routed pg-erd parity or immutable release evidence | | Dependency policy | Release-blocked | `.github#1605` owns the exact-release Pingora vs patched-`lru` decision and disposition of unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388`; `.github#810` independently owns the public non-fork Dependency Review compare-API HTTP 403 incident. Known-unsound downgrade, blanket advisory waiver, fail-open 403 handling, or substitute-scanner promotion is prohibited | @@ -63,8 +63,8 @@ The EA owner path must consume that released contract and project each approved 7. Draft #17 originally depended on historical #16 `9d5a108b355030e5d13571399dff93332ac019c3` and was 82 commits behind current #16 `49d3fb5d89bf15543c7df8b433809acb8fac88eb`. Ordinary two-parent commit `edceb2c948231aebfd9a1987c1fae6e8e220b84e` preserved historical #17 as first parent while adopting exact current #16 as the second parent and resolution tree; `93cbb28cc3ccde54380e5c1287c2e1b74a1a71e7` reapplied only the still-valid refused-origin traffic contract on the current stack. The contract requires 502, a conservative one-second loopback envelope around configured 200/400 ms connection budgets, readiness, low-cardinality request-error telemetry and independent `frontend` recovery. It is not connected read-timeout, reset, partial-response, streaming or retry/failover evidence. Exact current-head hosted and review evidence must be reacquired. 8. Draft #18 originally depended on historical #17 `5a522b1ce6e58cb879e4dfe33282748a63554473` and was 87 commits behind current #17 before repair. Ordinary two-parent commit `dba4ce7921d2a87a4029043b78ec4a6a6135a729` preserves the historical #18 lineage while adopting exact current #17 `ce2b3032580fa851f9ccd34462801920be947a09` as the resolution tree. The replayed read-stall contract was strengthened: the accepted backend now stays connected and silent until the gateway has already returned, so fixture closure cannot fake `read_ms` enforcement. With `read_ms=100`, the source requires 502 inside a conservative one-second envelope, readiness, request-error telemetry and independent `frontend` recovery. The API contract and primary-source traceability state explicitly that Pingora's read timeout is per read, not a whole-response deadline. Exact current-head hosted/review evidence must be reacquired. 9. Draft #19 was 94 commits behind current #18 and carried an older packaging approach that copied both binaries into one image. Ordinary two-parent succession preserves historical #19 while adopting exact #18 as its resolution tree instead of replaying that superseded packaging design. The current one-binary-per-image pg-erd profile runs as uid/gid 65532 with read-only root, all capabilities dropped, `no-new-privileges`, and only the versioned pg-erd config mounted read-only. Its OCI acceptance now requires `/livez` plus the separately published `/metrics` endpoint to identify Pingora's Prometheus service through `text/plain*`; a bare HTTP 200 is intentionally insufficient, while a pre-traffic metric family remains unnecessary. Exact current-head hosted/review evidence must be reacquired. -10. Draft #20 originally depended on historical #19 `5c80f35fdef12775a59d82ff3e8861f308173e07` and diverged after #19 succession. Ordinary two-parent commit `bbb9070b0d36efe8023bf01c3e6a83eda3e85a0f` preserves historical #20 as first parent while adopting exact current #19 `e0ab23d43d3ef4ae77aaf8475154084d72ff4a95` as second parent and resolution tree, then reapplies the still-valid routed SIGTERM drain contract. Fresh review found a distinct-port fixture race in two sequential bind-and-drop reservations; current repair `df0d7c03a395dda9c4caebd2a70f3eff4ff27db1` holds traffic and metrics reservations simultaneously before releasing them. Generic drain evidence is not transferred. Exact current-head fmt/compile/test/clippy/rustdoc/100% coverage, applicable OCI/supply-chain and fresh review must execute before routed drain is credited. -11. Draft #21 was still based on historical #20 `d81bed16cfa6d8ea8d16064d195a1f8c4c5c7108` after #20 advanced, leaving it 110 commits behind current #20. Ordinary two-parent commit `fe76050e12bda9c27d466c6b8cf38be67210c83e` preserves the historical #21 lineage while adopting exact current #20 `df0d7c03a395dda9c4caebd2a70f3eff4ff27db1` as the resolution tree. The still-valid post-header partial-response traffic contract is replayed on top; its own traffic/metrics ephemeral reservations use the same simultaneous distinct-port discipline rather than reintroducing the #20 fixture race. Historical #21 packaging wording that said both binaries shared one image is deliberately not replayed because current #19/#20 preserve stronger one-binary-per-image packaging. Exact current-head hosted/review evidence must be reacquired. +10. Draft #20 originally depended on historical #19 `5c80f35fdef12775a59d82ff3e8861f308173e07` and diverged after #19 succession. Ordinary two-parent commit `bbb9070b0d36efe8023bf01c3e6a83eda3e85a0f` preserves historical #20 while adopting exact current #19 `e0ab23d43d3ef4ae77aaf8475154084d72ff4a95`. Fresh review first exposed a traffic/metrics ephemeral-port reservation race, repaired by simultaneous reservations at `df0d7c03a395dda9c4caebd2a70f3eff4ff27db1`. CodeRabbit then found that the process-exit deadline was being created only after downstream/backend completion, allowing the documented SIGTERM-relative external budget to be exceeded. Current repair `2b53817b0fa840050adeec177be56ee89358fc72` anchors the absolute exit deadline to `signal_sent_at` immediately before SIGTERM and passes that deadline to `wait_for_exit`; the valid review thread was answered and resolved. Exact current-head hosted and fresh-review evidence must be reacquired. +11. Draft #21 was initially still based on historical #20 `d81bed16cfa6d8ea8d16064d195a1f8c4c5c7108`, 110 commits behind the then-current drain parent. Ordinary two-parent `fe76050e12bda9c27d466c6b8cf38be67210c83e` preserved historical #21 while adopting #20 `df0d7c03...`; the still-valid partial-response test/docs were replayed and the stale two-binaries-per-image wording was deliberately discarded. After #20's SIGTERM-budget repair advanced the parent again, ordinary two-parent commit `d2725f42d56eefded69866e61b997e8ba7b3a15f` preserved that repaired #21 lineage while adopting exact current #20 `2b53817b0fa840050adeec177be56ee89358fc72` and overlaying exactly the same six valid #21 paths. The partial-response fixture uses simultaneous traffic/metrics reservations so it does not reintroduce the #20 race. Exact current-head hosted/review evidence must be reacquired. 12. After the listener/forwarding/runtime-isolation/refusal/read-stall/OCI/drain/partial-response stack is GREEN, extend dedicated pg-erd acceptance with explicit TCP reset, broader streaming/upgraded failure, slow-drip/whole-response lifetime, payload-free observability assertions, and representative routed concurrency/origin-capacity measurements before adopting a production 20 ms p95 objective. A future HTTPS listener must separately prove TLS-derived forwarded scheme instead of reusing the current clear-text `web` assumption. 13. Exact-head supply-chain must build/scan both admitted image profiles and bind their receipts to source. Only after terminal GREEN may the release path publish registry-bound immutable image digests with release SBOM/provenance/reproducibility evidence and rehearse rollback against those exact digests. 14. Satisfy then-live protected-branch review/governance without self-approval, bot-as-human claims, stale evidence transfer, or routine administrator bypass. From 6a03c226240b00dd96cfc11f5d175b253a30a55a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 19:24:04 +0900 Subject: [PATCH 14/43] test: pin partial response failure after downstream commit --- tests/pg_erd_partial_response_traffic.rs | 55 +++++++++++++++++++++--- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/tests/pg_erd_partial_response_traffic.rs b/tests/pg_erd_partial_response_traffic.rs index 3f306d7c..e44a977a 100644 --- a/tests/pg_erd_partial_response_traffic.rs +++ b/tests/pg_erd_partial_response_traffic.rs @@ -9,6 +9,7 @@ use std::io::{ErrorKind, Read, Write}; use std::net::{SocketAddr, TcpListener, TcpStream}; use std::process::{Child, Command, Stdio}; +use std::sync::mpsc; use std::thread; use std::time::{Duration, Instant}; @@ -110,9 +111,11 @@ fn raw_request(address: SocketAddr, request: &[u8]) -> String { response } -fn raw_request_until_terminal( +fn raw_request_until_terminal_after_body_prefix( address: SocketAddr, request: &[u8], + expected_body_prefix: &[u8], + release_origin: mpsc::Sender<()>, ) -> (Vec, DownstreamTermination) { let mut downstream = TcpStream::connect(address).expect("gateway should accept traffic"); downstream @@ -124,11 +127,44 @@ fn raw_request_until_terminal( let mut response = Vec::new(); let mut buffer = [0_u8; 1024]; + let mut origin_released = false; loop { match downstream.read(&mut buffer) { - Ok(0) => return (response, DownstreamTermination::Eof), - Ok(read) => response.extend_from_slice(&buffer[..read]), + Ok(0) => { + assert!( + origin_released, + "downstream terminated before the committed body prefix was observed" + ); + return (response, DownstreamTermination::Eof); + } + Ok(read) => { + response.extend_from_slice(&buffer[..read]); + if !origin_released { + if let Some(header_end) = response + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|position| position + 4) + { + let expected_end = header_end + expected_body_prefix.len(); + if response.len() >= expected_end { + assert_eq!( + &response[header_end..expected_end], + expected_body_prefix, + "downstream must observe the exact committed body prefix before origin termination" + ); + release_origin + .send(()) + .expect("origin should wait for downstream commit evidence"); + origin_released = true; + } + } + } + } Err(error) if error.kind() == ErrorKind::ConnectionReset => { + assert!( + origin_released, + "downstream reset before the committed body prefix was observed" + ); return (response, DownstreamTermination::ConnectionReset); } Err(error) => panic!("partial downstream response should terminate, not stall: {error}"), @@ -159,6 +195,7 @@ fn read_request_headers(stream: &mut TcpStream) -> String { #[test] fn compiled_pg_erd_truncated_response_stays_committed_and_preserves_independent_routing() { + let (release_backend_tx, release_backend_rx) = mpsc::channel(); let backend = TcpListener::bind("127.0.0.1:0").expect("backend fixture should bind"); let backend_address = backend.local_addr().expect("backend address should exist"); let backend_origin = thread::spawn(move || { @@ -168,13 +205,17 @@ fn compiled_pg_erd_truncated_response_stays_committed_and_preserves_independent_ let request = read_request_headers(&mut stream); assert!(request.starts_with("GET /api/partial-response HTTP/1.1\r\n")); - // Once this status/header block is forwarded, a later framing failure cannot be replaced - // with a second HTTP status or silently failed over to another characterized origin. + // Keep the upstream open until the downstream has actually observed this committed prefix. + // Otherwise an immediate FIN can race proxy forwarding and accidentally exercise a + // pre-commit failure phase while still producing the same buffered bytes. stream .write_all( b"HTTP/1.1 200 OK\r\nContent-Length: 20\r\nConnection: close\r\n\r\npartial", ) .expect("partial backend response should be writable"); + release_backend_rx + .recv_timeout(Duration::from_secs(5)) + .expect("downstream should observe the committed prefix before backend close"); }); let frontend = TcpListener::bind("127.0.0.1:0").expect("frontend fixture should bind"); @@ -201,9 +242,11 @@ fn compiled_pg_erd_truncated_response_stays_committed_and_preserves_independent_ ); let _process = start_gateway(&config, gateway_address, metrics_address); - let (partial, termination) = raw_request_until_terminal( + let (partial, termination) = raw_request_until_terminal_after_body_prefix( gateway_address, b"GET /api/partial-response HTTP/1.1\r\nHost: app.example:8080\r\nConnection: close\r\n\r\n", + b"partial", + release_backend_tx, ); assert!( matches!( From db17db4d6e41f9dd1afa9e1fbc2402ad0dc969e8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 19:30:40 +0900 Subject: [PATCH 15/43] docs: bind partial-response fixture to downstream commit --- TEST_STRATEGY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TEST_STRATEGY.md b/TEST_STRATEGY.md index 93e1f1bf..d987097e 100644 --- a/TEST_STRATEGY.md +++ b/TEST_STRATEGY.md @@ -16,7 +16,7 @@ The bounded Admin Config transition has its own executable contract. `tests/pg_e `tests/pg_erd_read_stall_traffic.rs` separates connected upstream inactivity from refusal. The characterized backend accepts `/api/read-stall`, reads the request headers, then remains connected and sends no response bytes until the test explicitly releases it after the gateway has already returned. With `read_ms=100`, the gateway must fail as HTTP 502 inside a conservative one-second outer envelope, preserve `/readyz`, increment the shared request-error counter, and leave an independent `frontend` route usable. Keeping the fixture connection open prevents origin closure from masquerading as the timeout. Because Pingora's `read_timeout` is per successful `read()` rather than a whole-response lifetime, reset, post-commit partial response, slow-drip and whole-response deadline behavior remain separate contracts. -`tests/pg_erd_partial_response_traffic.rs` covers the distinct post-header failure phase. The characterized backend commits HTTP 200 with `Content-Length: 20`, writes only `partial`, then closes normally. Acceptance requires the downstream to retain that committed status/framing and terminate before all 20 bytes arrive rather than receiving an invented second status or silent route failover. `/readyz` must stay HTTP 200, the shared request-error counter must record the framing failure, and an independent `frontend` request must still complete successfully. Traffic and metrics ports are reserved simultaneously before the process starts so ephemeral-port reuse cannot manufacture an invalid listener collision. This orderly-close contract does not transfer evidence to explicit TCP reset, WebSocket/Upgrade, broader streaming failure, or slow-drip/whole-response lifetime. +`tests/pg_erd_partial_response_traffic.rs` covers the distinct post-header failure phase. The characterized backend sends HTTP 200 with `Content-Length: 20` and the body prefix `partial`, then remains open until the downstream reader has observed a complete response header block and that exact body prefix. Only after this acknowledgement does the fixture release the backend to close normally; EOF/reset before acknowledgement fails the test. This prevents scheduler/socket-buffer timing from masquerading as post-commit evidence. Acceptance then requires the downstream to retain the committed status/framing and terminate before all 20 bytes arrive rather than receiving an invented second status or silent route failover. `/readyz` must stay HTTP 200, the shared request-error counter must record the framing failure, and an independent `frontend` request must still complete successfully. Traffic and metrics ports are reserved simultaneously before the process starts so ephemeral-port reuse cannot manufacture an invalid listener collision. This orderly-close contract does not transfer evidence to explicit TCP reset, WebSocket/Upgrade, broader streaming failure, or slow-drip/whole-response lifetime. On Unix, `tests/pg_erd_graceful_shutdown.rs` proves the bounded pg-erd composition root consumes the shared drain policy instead of borrowing generic-binary evidence. The test routes `/api/held` to the characterized backend, holds that response open until the backend has confirmed the in-flight request, then delivers SIGTERM and releases the response during the shared grace period. The downstream must still receive HTTP 200 and the migration process must exit successfully inside the external termination budget. The traffic and metrics reservation sockets are held simultaneously before startup to prevent ephemeral-port reuse from producing a false listener-collision failure. This is source-defined drain acceptance until the unchanged exact head executes it to terminal GREEN. From 1184b78424fe4a6bae3964deed3ef97a3ad369cd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 07:39:23 +0900 Subject: [PATCH 16/43] style(test): apply rustfmt to pg-erd partial-response fixture --- tests/pg_erd_partial_response_traffic.rs | 26 ++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/pg_erd_partial_response_traffic.rs b/tests/pg_erd_partial_response_traffic.rs index e44a977a..db7f9414 100644 --- a/tests/pg_erd_partial_response_traffic.rs +++ b/tests/pg_erd_partial_response_traffic.rs @@ -74,7 +74,10 @@ fn wait_until_listening(address: SocketAddr, process: &mut Child) { if TcpStream::connect_timeout(&address, Duration::from_millis(100)).is_ok() { return; } - assert!(Instant::now() < deadline, "gateway did not start within 10s"); + assert!( + Instant::now() < deadline, + "gateway did not start within 10s" + ); thread::sleep(Duration::from_millis(25)); } } @@ -167,7 +170,9 @@ fn raw_request_until_terminal_after_body_prefix( ); return (response, DownstreamTermination::ConnectionReset); } - Err(error) => panic!("partial downstream response should terminate, not stall: {error}"), + Err(error) => { + panic!("partial downstream response should terminate, not stall: {error}") + } } } } @@ -184,8 +189,13 @@ fn read_request_headers(stream: &mut TcpStream) -> String { let mut bytes = Vec::new(); let mut buffer = [0_u8; 1024]; loop { - let read = stream.read(&mut buffer).expect("origin request should be readable"); - assert!(read > 0, "gateway closed origin request before headers completed"); + let read = stream + .read(&mut buffer) + .expect("origin request should be readable"); + assert!( + read > 0, + "gateway closed origin request before headers completed" + ); bytes.extend_from_slice(&buffer[..read]); if bytes.windows(4).any(|window| window == b"\r\n\r\n") { return String::from_utf8_lossy(&bytes).into_owned(); @@ -209,9 +219,7 @@ fn compiled_pg_erd_truncated_response_stays_committed_and_preserves_independent_ // Otherwise an immediate FIN can race proxy forwarding and accidentally exercise a // pre-commit failure phase while still producing the same buffered bytes. stream - .write_all( - b"HTTP/1.1 200 OK\r\nContent-Length: 20\r\nConnection: close\r\n\r\npartial", - ) + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 20\r\nConnection: close\r\n\r\npartial") .expect("partial backend response should be writable"); release_backend_rx .recv_timeout(Duration::from_secs(5)) @@ -219,7 +227,9 @@ fn compiled_pg_erd_truncated_response_stays_committed_and_preserves_independent_ }); let frontend = TcpListener::bind("127.0.0.1:0").expect("frontend fixture should bind"); - let frontend_address = frontend.local_addr().expect("frontend address should exist"); + let frontend_address = frontend + .local_addr() + .expect("frontend address should exist"); let frontend_origin = thread::spawn(move || { let (mut stream, _) = frontend .accept() From d8b56b661d17156e24263ab59a177c8fda515db6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 08:42:18 +0900 Subject: [PATCH 17/43] test: require exact partial-response error metric sample --- tests/pg_erd_partial_response_traffic.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/pg_erd_partial_response_traffic.rs b/tests/pg_erd_partial_response_traffic.rs index db7f9414..1c10abfb 100644 --- a/tests/pg_erd_partial_response_traffic.rs +++ b/tests/pg_erd_partial_response_traffic.rs @@ -294,8 +294,10 @@ fn compiled_pg_erd_truncated_response_stays_committed_and_preserves_independent_ let metrics = get(metrics_address, "/metrics"); assert!( - metrics.contains("cwl_pingora_gateway_request_errors_total 1"), - "the post-header upstream framing failure must remain visible through low-cardinality error telemetry: {metrics:?}" + metrics + .lines() + .any(|line| line == "cwl_pingora_gateway_request_errors_total 1"), + "the post-header upstream framing failure must expose exactly one request error through low-cardinality telemetry: {metrics:?}" ); let recovered = get(gateway_address, "/after-partial-response"); From 1eabf05af2e42d7bc1b7ca4b7156c07abd156f69 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 10:46:59 +0900 Subject: [PATCH 18/43] docs: explain partial-response test helpers --- tests/pg_erd_partial_response_traffic.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/pg_erd_partial_response_traffic.rs b/tests/pg_erd_partial_response_traffic.rs index 1c10abfb..51e67d71 100644 --- a/tests/pg_erd_partial_response_traffic.rs +++ b/tests/pg_erd_partial_response_traffic.rs @@ -15,9 +15,11 @@ use std::time::{Duration, Instant}; use tempfile::NamedTempFile; +/// Owns the compiled migration child so assertion failures cannot leak a listening test process. struct GatewayProcess(Child); impl Drop for GatewayProcess { + /// Terminates and reaps the child on every teardown path, including a partial-response panic. fn drop(&mut self) { let _ = self.0.kill(); let _ = self.0.wait(); @@ -30,6 +32,7 @@ enum DownstreamTermination { ConnectionReset, } +/// Selects distinct traffic and metrics authorities while both ephemeral reservations remain held. fn reserve_distinct_loopback_addresses() -> (SocketAddr, SocketAddr) { // Hold both ephemeral reservations at once so listener and metrics authority cannot // accidentally collapse to the same port before the migration process binds them. @@ -47,6 +50,7 @@ fn reserve_distinct_loopback_addresses() -> (SocketAddr, SocketAddr) { addresses } +/// Writes the bounded pg-erd fixture used to separate post-commit truncation from read-stall failure. fn write_config( listener: SocketAddr, metrics_listener: SocketAddr, @@ -62,6 +66,7 @@ fn write_config( file } +/// Waits for one gateway listener without treating an early process exit as startup success. fn wait_until_listening(address: SocketAddr, process: &mut Child) { let deadline = Instant::now() + Duration::from_secs(10); loop { @@ -82,6 +87,7 @@ fn wait_until_listening(address: SocketAddr, process: &mut Child) { } } +/// Starts the compiled pg-erd binary and requires both traffic and metrics authorities to bind. fn start_gateway( config: &NamedTempFile, gateway_address: SocketAddr, @@ -99,6 +105,7 @@ fn start_gateway( GatewayProcess(child) } +/// Sends one connection-closing HTTP/1.1 request and captures the complete downstream response. fn raw_request(address: SocketAddr, request: &[u8]) -> String { let mut downstream = TcpStream::connect(address).expect("gateway should accept traffic"); downstream @@ -114,6 +121,7 @@ fn raw_request(address: SocketAddr, request: &[u8]) -> String { response } +/// Releases the origin only after the downstream has observed the committed header and body prefix. fn raw_request_until_terminal_after_body_prefix( address: SocketAddr, request: &[u8], @@ -177,6 +185,7 @@ fn raw_request_until_terminal_after_body_prefix( } } +/// Issues a fixture GET with the characterized downstream authority and explicit connection close. fn get(address: SocketAddr, path: &str) -> String { raw_request( address, @@ -185,6 +194,7 @@ fn get(address: SocketAddr, path: &str) -> String { ) } +/// Reads only through the origin header terminator so the fixture can control the failure phase. fn read_request_headers(stream: &mut TcpStream) -> String { let mut bytes = Vec::new(); let mut buffer = [0_u8; 1024]; @@ -203,6 +213,7 @@ fn read_request_headers(stream: &mut TcpStream) -> String { } } +/// Proves a post-commit origin truncation preserves framing, terminates downstream and keeps recovery usable. #[test] fn compiled_pg_erd_truncated_response_stays_committed_and_preserves_independent_routing() { let (release_backend_tx, release_backend_rx) = mpsc::channel(); From 79fde3f10b5d85d335071e6419afc2e88ee58415 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 12:24:02 +0900 Subject: [PATCH 19/43] docs: retain phase-aware partial-response contract on current parent --- TRD.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TRD.md b/TRD.md index e23ffabb..8040f0f0 100644 --- a/TRD.md +++ b/TRD.md @@ -30,6 +30,8 @@ Non-health requests acquire the process `max_in_flight_requests` budget before u Generic v1 makes one prevalidated upstream peer available per request. The pg-erd migration adapter selects only peers already bound by `MigrationDeliveryPlan`; neither path performs request-controlled service discovery. Domain retries, failover, and idempotency policy are not invented by this runtime. +Failure handling is phase-aware. Before an upstream response header is committed downstream, transport failure may still be represented by the gateway's fail-closed error response under the one-attempt policy. After a valid response header has been committed, a later upstream framing/body failure cannot be rewritten into a second HTTP status or silently failed over: the incomplete downstream response terminates, low-cardinality error telemetry records the failed request, process readiness remains available, and independent routes must remain usable. This is an edge transport invariant, not product retry authority. + ## Health and observability `GET /livez` and `/readyz` return HTTP 200 with an empty, non-cacheable response through the process-local Pingora health boundary. Readiness proves validated configuration plus an active serving path, not product dependency health. In the pg-erd migration profile, consumer `/healthz` remains ordinary routed application traffic and is not confused with process liveness/readiness. From 92b4587f745cde4a4e612d09a66f3ee184ea9835 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 12:27:03 +0900 Subject: [PATCH 20/43] docs: project post-header failure contract into changelog --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aeadc69c..7abb154d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,8 @@ All notable changes are tracked here. No release has been published yet. - Added dedicated compiled pg-erd traffic acceptance for streamed/chunked body overflow and routed in-flight saturation/recovery: the migration process must return 413 above the shared body budget, return 503 in less than one second above the in-flight budget, keep `/readyz` observable, expose the exact single-rejection Prometheus sample, and admit a later routed request after capacity is released. - Added dedicated compiled pg-erd refused-origin recovery acceptance using a Linux TCP socket bound to the characterized backend address without entering LISTEN state. The fixture first proves direct `ECONNREFUSED` while retaining exclusive port ownership, then requires the migration gateway to return 502 within a conservative one-second envelope around the configured 200/400 ms connection budgets, keep `/readyz` 200, expose the exact single-error Prometheus sample, and allow a later independent frontend route to recover. Connected read stall, TCP reset, partial-response/streaming failure, retry and failover behavior remain separate gaps. - Added dedicated compiled pg-erd connected read-stall acceptance with `read_ms=100`: the backend accepts the routed request and remains open without response bytes until the gateway has already failed it, preventing fixture closure from faking timeout behavior. The contract requires 502 inside a conservative one-second envelope, preserved `/readyz`, the exact single-error Prometheus sample, and independent frontend recovery. Pingora `read_timeout` remains a per-read inactivity budget, not a whole-response lifetime; reset, partial-response and slow-drip cases remain open. -- Added dedicated routed pg-erd graceful-drain acceptance: a characterized `/api/held` backend request is held in flight, SIGTERM is sent only after the backend has accepted it, the response is released during the shared grace period, the downstream must still receive HTTP 200, and the migration process must exit successfully inside the external termination budget. Generic drain evidence is not transferred to this composition root; the contract remains source-defined until the unchanged exact head reaches terminal hosted GREEN. +- Added dedicated compiled pg-erd post-header partial-response acceptance: the backend commits HTTP 200 with `Content-Length: 20`, writes only `partial`, and closes only after the downstream has observed that committed header/body prefix. The downstream must retain the committed status/framing and terminate before body completion instead of receiving an invented second status or silent failover; `/readyz`, exact low-cardinality request-error telemetry, and an independent `frontend` route must remain usable. Explicit TCP reset, broader streaming/upgraded failure, and slow-drip/whole-response lifetime remain separate gaps. +- Added dedicated routed pg-erd graceful-drain acceptance: a characterized `/api/held` backend request is held in flight, SIGTERM is sent only after the backend has accepted it, the response is released during the shared grace period, the downstream must still receive HTTP 200, and the migration process must exit successfully inside the external termination budget measured from SIGTERM. Generic drain evidence is not transferred to this composition root. - Added optional per-upstream absolute PEM trust-bundle consumption without taking ownership of certificate issuance/rotation; trust material is loaded fail-closed before listeners open. - Added an executable local-CA TLS test through the compiled gateway that holds CA trust constant and proves SNI/hostname mismatch is rejected. - Added a focused transport-adapter regression proving an upstream without a custom trust bundle leaves Pingora's platform trust roots selected rather than replacing the CA store. @@ -31,11 +32,11 @@ All notable changes are tracked here. No release has been published yet. - Added request-body limits and a distrust-by-default forwarded-header policy. - Added low-cardinality metrics plus credential/cookie-safe access logging through the production path. - Overrode Pingora framework retry/drain defaults with one total upstream attempt, a 5-second SIGTERM grace period, and a 30-second graceful-shutdown timeout. -- Added non-root/read-only-root OCI packaging with an explicit build-time allowlist for the generic and bounded pg-erd process identities. Exact-head OCI acceptance now builds and starts each image under uid/gid 65532, dropped capabilities and `no-new-privileges`; the supply-chain lane builds and vulnerability-scans both candidate images. This is unreleased source-defined acceptance until the current head reaches terminal hosted GREEN. +- Added non-root/read-only-root OCI packaging with an explicit build-time allowlist for the generic and bounded pg-erd process identities. Exact-head OCI acceptance builds and starts each image under uid/gid 65532, dropped capabilities and `no-new-privileges`; the supply-chain lane builds and vulnerability-scans both candidate images. - Extended the dedicated pg-erd OCI acceptance so the least-privilege migration container is not accepted until its process-health `/livez` endpoint answers and the separately published `/metrics` listener identifies the Pingora Prometheus service through its `text/plain` response media type. The check deliberately does not require a metric family before application traffic has emitted one, and it preserves the current one-binary-per-image packaging boundary. - Added a committed dependency lock, fail-closed license/source/advisory policy, exact-source SBOM and image-vulnerability evidence. - Added an exact-head owned-production coverage gate that requires 100% lines and regions without filename/function/branch exclusions; repaired compiler-generated generic startup coverage and structurally impossible literal-header error regions rather than weakening the gate. - Added missing-public-rustdoc enforcement and documentation builds with warnings denied. - Added DDD, product, technical, security, threat, test, operability, configuration, migration-gap, and primary-source traceability documentation. -Release remains blocked on the organization decision for the exact Pingora release versus `RUSTSEC-2026-0253` and the separate time-bounded disposition of unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388` (`ContextualWisdomLab/.github#1605`), restoration of authoritative public non-fork Dependency Review evidence (`ContextualWisdomLab/.github#810`), terminal exact-current-head CI/supply-chain/security/review evidence, representative compiled-binary pg-erd route/header/forwarding/body/backpressure/failure/concurrency/drain and benchmark evidence, an immutable registry digest with provenance and rehearsed rollback, and protected-branch integration. Routed graceful drain now has source acceptance but remains uncredited until terminal exact-head execution. No consumer migration, canary, cutover, or legacy removal is claimed before those release and traffic-contract gates are satisfied. +Release remains blocked on the organization decision for the exact Pingora release versus `RUSTSEC-2026-0253` and the separate time-bounded disposition of unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388` (`ContextualWisdomLab/.github#1605`), restoration of authoritative public non-fork Dependency Review evidence (`ContextualWisdomLab/.github#810`), terminal exact-current-head CI/supply-chain/security/review evidence, representative compiled-binary pg-erd route/header/forwarding/body/backpressure/failure/concurrency/drain and benchmark evidence, an immutable registry digest with provenance and rehearsed rollback, and protected-branch integration. Parent #20 routed graceful-drain acceptance is exact-head hosted GREEN at `d4d4565854cc924a2214de2b67a966d2f253da3e`; the changed #21 post-header partial-response candidate must independently reacquire exact-head execution/review evidence before that child contract is credited. No consumer migration, canary, cutover, or legacy removal is claimed before those release and traffic-contract gates are satisfied. From baef6a04cc1a727d000c97a13b1c374f2aa52086 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 12:27:48 +0900 Subject: [PATCH 21/43] docs: make partial-response traffic strategy current --- TEST_STRATEGY.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/TEST_STRATEGY.md b/TEST_STRATEGY.md index 10e90ff9..be5c7ba5 100644 --- a/TEST_STRATEGY.md +++ b/TEST_STRATEGY.md @@ -16,7 +16,9 @@ The bounded Admin Config transition has its own executable contract. `tests/pg_e `tests/pg_erd_read_stall_traffic.rs` separates connected upstream inactivity from refusal. The characterized backend accepts `/api/read-stall`, reads the request headers, then remains connected and sends no response bytes until the test explicitly releases it after the gateway has already returned. With `read_ms=100`, the gateway must fail as HTTP 502 inside a conservative one-second outer envelope, preserve `/readyz`, expose the exact Prometheus sample `cwl_pingora_gateway_request_errors_total 1`, and leave an independent `frontend` route usable. Keeping the fixture connection open prevents origin closure from masquerading as the timeout. Because Pingora's `read_timeout` is per successful `read()` rather than a whole-response lifetime, reset, post-commit partial response, slow-drip and whole-response deadline behavior remain separate contracts. -On Unix, `tests/pg_erd_graceful_shutdown.rs` proves the bounded pg-erd composition root consumes the shared drain policy instead of borrowing generic-binary evidence. The test routes `/api/held` to the characterized backend, holds that response open until the backend has confirmed the in-flight request, then delivers SIGTERM and releases the response during the shared grace period. The downstream must still receive HTTP 200 and the migration process must exit successfully inside the external termination budget measured from SIGTERM. This is source-defined drain acceptance until the unchanged exact head executes it to terminal GREEN. +`tests/pg_erd_partial_response_traffic.rs` covers the distinct post-header failure phase. The characterized backend sends HTTP 200 with `Content-Length: 20` and the body prefix `partial`, then remains open until the downstream reader has observed a complete response header block and that exact body prefix. Only after this acknowledgement does the fixture release the backend to close normally; EOF/reset before acknowledgement fails the test. This prevents scheduler/socket-buffer timing from masquerading as post-commit evidence. Acceptance requires the downstream to retain the committed status/framing and terminate before all 20 bytes arrive rather than receiving an invented second status or silent route failover. `/readyz` must stay HTTP 200, the shared request-error counter must expose exactly `cwl_pingora_gateway_request_errors_total 1`, and an independent `frontend` request must still complete successfully. Traffic and metrics ports are reserved simultaneously before process startup so ephemeral-port reuse cannot manufacture an invalid listener collision. This orderly-close contract does not transfer evidence to explicit TCP reset, WebSocket/Upgrade, broader streaming failure, or slow-drip/whole-response lifetime. + +On Unix, `tests/pg_erd_graceful_shutdown.rs` proves the bounded pg-erd composition root consumes the shared drain policy instead of borrowing generic-binary evidence. The test routes `/api/held` to the characterized backend, holds that response open until the backend has confirmed the in-flight request, then creates one absolute termination deadline, delivers SIGTERM, and releases the response during the shared grace period. The downstream must still receive HTTP 200 and the migration process must exit successfully before that SIGTERM-relative external termination deadline. Its traffic and metrics reservation sockets are held simultaneously before startup so the fixture cannot false-fail through ephemeral-port reuse. Parent #20 exact `d4d4565854cc924a2214de2b67a966d2f253da3e` has terminal CI `34181336779` and Supply Chain `34181336796` GREEN; descendants must revalidate rather than transfer that receipt. The `oci-runtime` job separately validates artifact composition instead of inferring it from compiled-process tests. The Dockerfile admits only `cwl-pingora-gateway` and `cwl-pingora-pg-erd-migration` as build-time process identities, normalizes the selected executable to one fixed distroless runtime path, and CI builds both image profiles. Each exact candidate must declare uid/gid `65532`, start under a read-only root filesystem with all capabilities dropped and `no-new-privileges`, and consume only a read-only configuration mount. The generic profile must expose local `/livez`. The pg-erd profile must expose local `/livez` on the traffic listener and its separately published `/metrics` listener must identify the Pingora Prometheus service by a `text/plain` response media type before the container is accepted. `tests/pg_erd_oci_metrics_workflow_contract.rs` prevents a bare HTTP 200 from false-greening a mistakenly bound proxy service while deliberately avoiding a metric-family requirement before routed application traffic has emitted one. `examples/pg-erd-migration.yaml` is deliberately an origin-independent OCI smoke fixture; this gate proves the dedicated binary is actually packaged and both process/observability listeners start under the required container isolation but does not claim routed pg-erd parity, origin health, or performance. The supply-chain job additionally builds and vulnerability-scans both image profiles and binds both local image IDs plus per-image scan outputs to the exact source SHA. These workflow contracts count only after terminal success on the unchanged exact head. @@ -24,4 +26,4 @@ The `oci-runtime` job separately validates artifact composition instead of infer Every behavioral migration should begin with characterization against the old owned edge behavior, then add equivalent production-path evidence for Pingora. Static-serving consumers must cover route precedence, SPA fallback, MIME, ETag/cache, Range/HEAD/304/416, redirects/security headers and compression as applicable. Proxy consumers must cover Host/SNI/TLS, forwarding trust, WebSocket/upgrade, limits, timeout/retry behavior, streaming/uploads, saturation/backpressure, errors, health/readiness and drain. Characterized response policies must additionally be exercised through the compiled proxy path before any parity or canary claim. -Release-quality gaps remain: no property/fuzz tests yet; no downstream TLS listener contract; no HTTP/2 or HTTP/3 parity evidence; no tracing evidence; no immutable published registry digest/provenance and rehearsed rollback; and no benchmark against replaced Nginx/Traefik traffic. For the pg-erd candidate specifically, the dedicated source now covers streamed body overflow, in-flight saturation/recovery, refused-origin recovery, connected silent-origin read timeout, routed graceful drain, and OCI process/Prometheus-listener identity, but TCP reset, post-commit partial-response/streaming failure, slow-drip/whole-response lifetime, representative routed origin-capacity load, terminal current-head traffic/OCI/supply-chain execution, shadow/canary and rollback still lack evidence. OCI non-root/read-only-root source acceptance covers both admitted process images, while owned-production 100% line/region coverage, public rustdoc, SBOM/image vulnerability, local-CA upstream TLS, explicit backpressure/recovery, and the minimal k6 loopback path remain exact-head gates. None can be transferred to a changed or consumer-specific head. Production performance claims remain forbidden until representative deployment measurements exist. +Release-quality gaps remain: no property/fuzz tests yet; no downstream TLS listener contract; no HTTP/2 or HTTP/3 parity evidence; no tracing evidence; no immutable published registry digest/provenance and rehearsed rollback; and no benchmark against replaced Nginx/Traefik traffic. For the pg-erd candidate specifically, source now covers streamed body overflow, in-flight saturation/recovery, refused-origin recovery, connected silent-origin read timeout, orderly post-header truncation, routed graceful drain, and OCI process/Prometheus-listener identity. Explicit TCP reset, broader streaming/upgraded failure, slow-drip/whole-response lifetime, representative routed origin-capacity load, shadow/canary and rollback remain open. Parent #20 has exact hosted GREEN, but the changed #21 head must independently pass formatting/compile/test/Clippy/rustdoc/documentation/100%-coverage/load/OCI/Supply Chain before its partial-response contract is credited. OCI non-root/read-only-root source acceptance covers both admitted process images, while public rustdoc, SBOM/image vulnerability, local-CA upstream TLS, explicit backpressure/recovery, and the minimal k6 loopback path remain exact-head gates. None can be transferred to a changed or consumer-specific head. Production performance claims remain forbidden until representative deployment measurements exist. From 92c9b0f461587290f2d08b3b8bdb2d8e0ed9570d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 12:28:55 +0900 Subject: [PATCH 22/43] docs: trace post-commit HTTP failure semantics --- docs/doctoring/TRACEABILITY.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/doctoring/TRACEABILITY.md b/docs/doctoring/TRACEABILITY.md index 68a9cca9..c82fb172 100644 --- a/docs/doctoring/TRACEABILITY.md +++ b/docs/doctoring/TRACEABILITY.md @@ -10,6 +10,8 @@ This file links material technical/security claims to primary standards or upstr | Graceful SIGTERM uses `grace_period_seconds` and `graceful_shutdown_timeout_seconds`, with framework fallbacks when unset | `pingora-core/src/server/mod.rs` and `pingora-core/src/server/configuration/mod.rs` at the pinned commit; CWL v1 sets 5 s grace and 10 s per-runtime graceful timeout explicitly inside a 30 s external termination budget | | Standard upstream request policy supports hop-by-hop/connection-nominated stripping and normalized WebSocket-only HTTP/1 upgrade forwarding | Cloudflare Pingora `HttpUpstreamRequestPolicy` / peer implementation at pinned commit `09696b51bc59315353d96686355861604d0bb48c` | | Pingora `read_timeout` is a per-individual-read inactivity budget and resets after each successful upstream `read()`; it is not a total-response lifetime bound | Cloudflare Pingora `docs/user_guide/peer.md` and `pingora-proxy/src/proxy_h1.rs` at pinned commit `09696b51bc59315353d96686355861604d0bb48c`; the pg-erd read-stall acceptance therefore keeps a connected origin silent without closing its socket and deliberately does not claim slow-drip/whole-response bounding | +| A proxy failure after the upstream response header has already been sent downstream cannot be replaced with a new error response or failover | Cloudflare Pingora `docs/user_guide/failover.md` and `pingora-proxy/src/proxy_h1.rs` at pinned commit `09696b51bc59315353d96686355861604d0bb48c`; the pg-erd partial-response fixture therefore observes the committed status/body prefix before causing origin termination and requires the existing downstream response to terminate rather than become a second response | +| Pingora HTTP/1 body framing treats a body that ends before its declared `Content-Length` as a premature body-end failure, and upstream read failures propagate through the HTTP/1 client/proxy path | `pingora-core/src/protocols/http/v1/body.rs`, `pingora-core/src/protocols/http/v1/client.rs`, and `pingora-proxy/src/proxy_h1.rs` at pinned commit `09696b51bc59315353d96686355861604d0bb48c`; RFC 9112 message-framing requirements | | Pingora's Prometheus HTTP application sets `Content-Type` from `prometheus::TextEncoder::format_type()`; the pinned `prometheus` 0.14 line defines that format as `text/plain; version=0.0.4` | Cloudflare `pingora-prometheus/src/lib.rs` at pinned Pingora commit `09696b51bc59315353d96686355861604d0bb48c` and TiKV `rust-prometheus` v0.14.0 commit `e07efb4f372f1245bf7410b71e822c69877bcb32`, `src/encoder/text.rs`; #19 therefore strips only optional semicolon parameters and requires exact base media type `text/plain` rather than a prefix wildcard | | Pingora OpenSSL peers support a per-peer CA store; when configured it replaces the verification store for that peer while certificate and hostname verification remain separately enabled | `pingora-core/src/upstreams/peer.rs`, `pingora-core/src/connectors/tls/boringssl_openssl/mod.rs`, and `pingora-core/src/protocols/tls/boringssl_openssl/mod.rs` at pinned commit `09696b51bc59315353d96686355861604d0bb48c` | | IPv4-mapped IPv6 addresses represent IPv4 nodes in IPv6 form; Rust's `Ipv6Addr::to_ipv4_mapped` performs the bounded mapped-only canonicalization used by the socket-authority invariant. Linux IPv6 sockets can expose IPv4 peers as mapped IPv6 addresses, so mapped/native authority must not be compared only as unrelated textual address families | RFC 4291 §2.5.5.2; Rust `std::net::Ipv6Addr` documentation; Linux `ipv6(7)` | @@ -39,8 +41,14 @@ Cloudflare. (n.d.). *Pingora upstream peer options* [Source code, commit 09696b5 Cloudflare. (n.d.). *Peer: how to connect to upstream* [Documentation, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/docs/user_guide/peer.md +Cloudflare. (n.d.). *Handling failures and failover* [Documentation, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/docs/user_guide/failover.md + Cloudflare. (n.d.). *Pingora HTTP/1 proxy implementation* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-proxy/src/proxy_h1.rs +Cloudflare. (n.d.). *Pingora HTTP/1 client session* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/protocols/http/v1/client.rs + +Cloudflare. (n.d.). *Pingora HTTP/1 body framing* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/protocols/http/v1/body.rs + Cloudflare. (n.d.). *Pingora Prometheus HTTP application* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-prometheus/src/lib.rs Cloudflare. (n.d.). *Pingora OpenSSL upstream TLS connector* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/connectors/tls/boringssl_openssl/mod.rs From 7b371cb341d42563eb112beab7dd421b7aa51b74 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 12:33:22 +0900 Subject: [PATCH 23/43] docs: make migration gap baseline current for partial-response restack --- docs/product-technical-gap-baseline.md | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 3ad8e25a..69ee430d 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -52,27 +52,33 @@ The documentation-complete #18 head `9749d01ae0e9aae027d7fce1a2c15e6a8358acd9` i Current #19 exact `86a6eb1b8fd5777b578cdbce49f40d52e916cc9b` has independently closed hosted execution: CI `34180261244` completed success for formatting, compile/test, strict Clippy, warning-denied public rustdoc, complete owned-production coverage, resolved dependency-lock verification, generic load, and dual-profile OCI runtime; Supply Chain `34180261252` completed success on the same exact SHA. A fresh owner technical sweep found no actionable source/documentation defect and no review thread. This is technical evidence only and not an independent human `APPROVED` review. -`#20` now ordinary/non-force adopts exact current #19 while preserving the routed SIGTERM drain delta. Two-parent commit `f2c47a1f347de0355fc5dc07785b3def0e9806ae` retains historical #20 `9a5307cc792f05af1534587637a359054a752382` as first parent and exact #19 `86a6eb1b8fd5777b578cdbce49f40d52e916cc9b` as second parent, using the current #19 tree and reapplying only `tests/pg_erd_graceful_shutdown.rs`. Follow-up documentation commits project the preserved drain contract without replaying stale #19 workflow/docs blobs. The test sends SIGTERM only after the characterized backend has accepted `/api/held`, releases the response during the shared grace period, requires downstream HTTP 200, and keeps one absolute external termination deadline measured from signal time. Exact-head hosted/review evidence must be reacquired on the resulting #20 head before drain is credited. +`#20` ordinary/non-force adopted exact #19 while preserving the routed SIGTERM drain delta. Two-parent commit `f2c47a1f347de0355fc5dc07785b3def0e9806ae` retains historical #20 `9a5307cc792f05af1534587637a359054a752382` as first parent and exact #19 `86a6eb1b8fd5777b578cdbce49f40d52e916cc9b` as second parent, using the current #19 tree and reapplying only `tests/pg_erd_graceful_shutdown.rs`. Follow-up documentation commits project the preserved drain contract without replaying stale #19 workflow/docs blobs. The test sends SIGTERM only after the characterized backend has accepted `/api/held`, fixes one absolute external termination deadline at signal time, releases the response during the shared grace period, requires downstream HTTP 200, and requires successful process exit before that same deadline. + +Exact #20 head `d4d4565854cc924a2214de2b67a966d2f253da3e` is terminal hosted GREEN. CI `34181336779` completed success for `test`, `load-contract`, and dual-profile `oci-runtime`; its test lane passed exact checkout, Rust 1.98.0 formatting, compile/test, strict Clippy, warning-denied public rustdoc, complete owned-production line/region coverage enforcement, resolved dependency-lock verification and evidence upload. Supply Chain `34181336796` completed success through committed dependency audit, both candidate-image builds, SPDX SBOM, both image scans, exact-source binding and evidence upload. Current exact-range owner technical review reports no actionable finding and confirms the historical SIGTERM-relative deadline defect remains repaired. This is technical evidence only, not self-approval or independent human `APPROVED` review. + +`#21` owns the next distinct post-header partial-response phase. Historical exact `1eabf05af2e42d7bc1b7ca4b7156c07abd156f69` retained three valid repairs: the backend does not close until the downstream has observed a complete HTTP 200 header plus exact `partial` body prefix; the request-error oracle matches the exact Prometheus line rather than a substring; and every named fixture/test helper carries purpose/constraint rustdoc. Ordinary two-parent commit `bff40ec74448a6e63bac8c408962aad7f7309d4e` preserves that historical #21 head as first parent and exact current #20 `d4d4565854cc924a2214de2b67a966d2f253da3e` as second parent. Its resolution tree starts from current #20 and reapplies only `tests/pg_erd_partial_response_traffic.rs`, so stale parent source/workflow/docs blobs are not overlaid. Follow-up commits `79fde3f10b5d85d335071e6419afc2e88ee58415`, `92b4587f745cde4a4e612d09a66f3ee184ea9835`, `baef6a04cc1a727d000c97a13b1c374f2aa52086`, and `92c9b0f461587290f2d08b3b8bdb2d8e0ed9570d` semantically project the phase-aware TRD, CHANGELOG, test strategy and primary-source traceability onto the current parent tree. + +The #21 acceptance sends `Content-Length: 20` but only the seven-byte prefix `partial`, waits until that committed prefix has been observed downstream, and only then releases the origin to close. It requires the downstream to retain the committed 200/framing and terminate before body completion rather than receive an invented second HTTP status or failover; `/readyz` must remain 200, telemetry must contain exactly `cwl_pingora_gateway_request_errors_total 1`, and an independent `frontend` route must recover. Explicit TCP reset, WebSocket/Upgrade, broader streaming failure, and slow-drip/whole-response lifetime remain separate gaps. Exact-hosted and exact-range review evidence belong only to the final unchanged #21 head and must be reacquired after this documentation projection. ## Capability state and buyer-visible gaps | Area | Current state | Remaining acceptance | | --- | --- | --- | -| Admin Config / network authority | Implemented; #12/#14 exact hosted GREEN; #18 fixture structurally reserves distinct listeners | Revalidate on every descendant restack | +| Admin Config / network authority | Implemented; #12/#14 exact hosted GREEN; #18/#20 fixtures structurally reserve distinct listeners | Revalidate on every descendant restack | | Generic forwarding trust | #15 exact source/review/CI/Supply Chain GREEN and inherited through the current parent stack | Preserve the exact invariant through descendants; no client-IP/trusted-proxy claim until separately characterized | | Edge routing / HTTP policy | Characterized and compiled for pg-erd | Routed production-process parity must remain GREEN through descendant restacks | -| Runtime isolation | Declared/streamed body and in-flight limits implemented; #16 exact real-listener acceptance is hosted GREEN; #20 carries routed SIGTERM drain source acceptance | Reacquire exact #20 hosted drain evidence; origin-capacity semantics remain separate | -| Failure recovery | #17 refused-origin and #18 connected-silent-origin exact hosted/technical GREEN | Preserve through #20+; reset, post-commit truncation and slow-drip/streaming lifetime remain unproven | +| Runtime isolation | Declared/streamed body and in-flight limits implemented; #16 exact real-listener acceptance GREEN; #20 routed SIGTERM drain exact hosted/technical GREEN | Preserve through descendants; origin-capacity semantics remain separate | +| Failure recovery | #17 refused-origin, #18 connected-silent-origin, and #20 drain exact hosted/technical GREEN; #21 carries orderly post-header truncation source acceptance | Reacquire exact #21 hosted/review evidence; explicit reset, broader streaming/upgraded failure, and slow-drip/whole-response lifetime remain unproven | | Upstream TLS | Generic local-CA/SNI verification exists; pg-erd fail-closed trust activation exists | Successful pg-erd TLS listener/origin path and representative TLS performance remain unproven | | Protocols | HTTP/1.1 path exists | Downstream TLS/H2, H2→H1 Cookie behavior, WebSocket/Extended CONNECT and explicit H3/QUIC disposition require separate supplier-capable contracts | -| OCI / supply chain | #19 exact dual-profile candidate build, least privilege, Prometheus-listener identity, audit, SPDX SBOM and image-scan evidence GREEN | Revalidate on exact #20; immutable registry digest, signing/attestation/provenance, release-bound SBOM, reproducibility receipt and rollback rehearsal | -| Performance | Current parent controlled generic loopback remains below the `<20 ms` gate; supplier measured-origin #62 p95 `0.71593235 ms` | Do not treat controlled loopback as production SLO proof; routed pg-erd concurrency/TLS/failure measurements remain required | -| Observability | Low-cardinality counters/logging and separate metrics listener; #16 exact single-rejection and #17/#18 exact single-error telemetry GREEN; #19 exact metrics-listener media-type identity GREEN | Broader payload-free runtime assertions and tracing remain gaps | -| Documentation / review | #19 exact owner technical sweep is clean; #20 touched test helpers carry purpose/constraint rustdoc | Fresh exact-range #20 review after restack; no bot/static review is independent human approval | +| OCI / supply chain | #20 exact dual-profile candidate build, least privilege, Prometheus-listener identity, audit, SPDX SBOM and image-scan evidence GREEN | Revalidate on changed #21; immutable registry digest, signing/attestation/provenance, release-bound SBOM, reproducibility receipt and rollback rehearsal | +| Performance | Parent #20 controlled generic loopback remains under the `<20 ms` gate; supplier measured-origin #62 p95 `0.71593235 ms` | Do not treat controlled loopback as production SLO proof; routed pg-erd concurrency/TLS/failure measurements remain required | +| Observability | Low-cardinality counters/logging and separate metrics listener; #16 exact single-rejection, #17/#18 exact single-error, and #19 metrics-listener identity GREEN; #21 requires exact single-error after post-header truncation | Reacquire #21 exact telemetry evidence; broader payload-free runtime assertions and tracing remain gaps | +| Documentation / review | #20 exact owner technical sweep is clean; #21 source helpers are rustdoc-complete and current docs now describe phase-aware failure | Fresh exact-range #21 review after the final unchanged head; no bot/static review is independent human approval | | Release / migration | No protected release or consumer cutover credit | Immutable release → parity → shadow/canary → rollback rehearsal → cutover → verified Nginx/OpenResty/legacy removal | ## Execution order -The current dependency order is `#54 derivative RED + #62 exact semantics/load GREEN → maintainer-integrated immutable supplier repair → gateway supplier bump and committed lock regeneration → #54 GREEN + preserved #62 GREEN → #56 independent APPROVED/governance → foundation/protected integration → #12 → #14 → #15 → #16 → #17 → #18 exact hosted/technical GREEN → #19 exact hosted/technical GREEN → #20 ordinary non-force succession + exact hosted/review closure → #21 and later descendants parent-first ordinary non-force succession → remaining routed TLS/failure/protocol acceptance → immutable gateway release/SBOM/provenance/reproducibility/rollback → shadow/canary → cutover → verified legacy removal`. +The current dependency order is `#54 derivative RED + #62 exact semantics/load GREEN → maintainer-integrated immutable supplier repair → gateway supplier bump and committed lock regeneration → #54 GREEN + preserved #62 GREEN → #56 independent APPROVED/governance → foundation/protected integration → #12 → #14 → #15 → #16 → #17 → #18 exact hosted/technical GREEN → #19 exact hosted/technical GREEN → #20 exact hosted/technical GREEN → #21 ordinary non-force succession + exact hosted/review closure → #22 and later descendants parent-first ordinary non-force succession → remaining routed TLS/failure/protocol acceptance → immutable gateway release/SBOM/provenance/reproducibility/rollback → shadow/canary → cutover → verified legacy removal`. No Draft state, predecessor receipt, bot review, local image ID, mutable supplier PR, queue state or controlled loopback measurement is treated as protected merge, release, canary, cutover, rollback or legacy-removal evidence. From f32295c3a0d0525d9f0e3f73b41795eac6c1395d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 12:51:00 +0900 Subject: [PATCH 24/43] test: require exact committed content length --- tests/pg_erd_partial_response_traffic.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/pg_erd_partial_response_traffic.rs b/tests/pg_erd_partial_response_traffic.rs index 51e67d71..4fbad5fe 100644 --- a/tests/pg_erd_partial_response_traffic.rs +++ b/tests/pg_erd_partial_response_traffic.rs @@ -286,8 +286,18 @@ fn compiled_pg_erd_truncated_response_stays_committed_and_preserves_independent_ headers.starts_with("http/1.1 200"), "a post-header upstream failure cannot be rewritten as a new status: {headers:?}" ); - assert!( - headers.contains("content-length: 20"), + let content_length_headers: Vec<_> = headers + .lines() + .filter(|line| line.starts_with("content-length:")) + .collect(); + assert_eq!( + content_length_headers.len(), + 1, + "the committed response must retain exactly one Content-Length field: {headers:?}" + ); + assert_eq!( + content_length_headers[0], + "content-length: 20", "the committed response must retain its declared framing for this fixture: {headers:?}" ); let body = &partial[header_end..]; From 8dd0b0efeaf07be473c7c1a593c0f567c2143269 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 12:52:34 +0900 Subject: [PATCH 25/43] docs: record partial-response framing review repair --- docs/product-technical-gap-baseline.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 69ee430d..708a8648 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -58,7 +58,9 @@ Exact #20 head `d4d4565854cc924a2214de2b67a966d2f253da3e` is terminal hosted GRE `#21` owns the next distinct post-header partial-response phase. Historical exact `1eabf05af2e42d7bc1b7ca4b7156c07abd156f69` retained three valid repairs: the backend does not close until the downstream has observed a complete HTTP 200 header plus exact `partial` body prefix; the request-error oracle matches the exact Prometheus line rather than a substring; and every named fixture/test helper carries purpose/constraint rustdoc. Ordinary two-parent commit `bff40ec74448a6e63bac8c408962aad7f7309d4e` preserves that historical #21 head as first parent and exact current #20 `d4d4565854cc924a2214de2b67a966d2f253da3e` as second parent. Its resolution tree starts from current #20 and reapplies only `tests/pg_erd_partial_response_traffic.rs`, so stale parent source/workflow/docs blobs are not overlaid. Follow-up commits `79fde3f10b5d85d335071e6419afc2e88ee58415`, `92b4587f745cde4a4e612d09a66f3ee184ea9835`, `baef6a04cc1a727d000c97a13b1c374f2aa52086`, and `92c9b0f461587290f2d08b3b8bdb2d8e0ed9570d` semantically project the phase-aware TRD, CHANGELOG, test strategy and primary-source traceability onto the current parent tree. -The #21 acceptance sends `Content-Length: 20` but only the seven-byte prefix `partial`, waits until that committed prefix has been observed downstream, and only then releases the origin to close. It requires the downstream to retain the committed 200/framing and terminate before body completion rather than receive an invented second HTTP status or failover; `/readyz` must remain 200, telemetry must contain exactly `cwl_pingora_gateway_request_errors_total 1`, and an independent `frontend` route must recover. Explicit TCP reset, WebSocket/Upgrade, broader streaming failure, and slow-drip/whole-response lifetime remain separate gaps. Exact-hosted and exact-range review evidence belong only to the final unchanged #21 head and must be reacquired after this documentation projection. +The #21 acceptance sends `Content-Length: 20` but only the seven-byte prefix `partial`, waits until that committed prefix has been observed downstream, and only then releases the origin to close. It requires the downstream to retain the committed 200/framing and terminate before body completion rather than receive an invented second HTTP status or failover; `/readyz` must remain 200, telemetry must contain exactly `cwl_pingora_gateway_request_errors_total 1`, and an independent `frontend` route must recover. Explicit TCP reset, WebSocket/Upgrade, broader streaming failure, and slow-drip/whole-response lifetime remain separate gaps. + +Fresh exact-range CodeRabbit review of `d4d456...7b371cb` found one valid test-oracle defect: the framing assertion used substring matching for `content-length: 20`, so a different header name containing that substring or duplicate Content-Length fields could satisfy the check without proving the intended single framing field. Test-only repair `f32295c3a0d0525d9f0e3f73b41795eac6c1395d` now enumerates actual `content-length:` header lines, requires cardinality exactly one, and requires that sole field to equal `content-length: 20`. Production routing, failure semantics, timeout, telemetry and product authority are unchanged. Exact hosted and exact-range review evidence must be reacquired on the final unchanged #21 head after this repair and baseline projection. ## Capability state and buyer-visible gaps @@ -68,13 +70,13 @@ The #21 acceptance sends `Content-Length: 20` but only the seven-byte prefix `pa | Generic forwarding trust | #15 exact source/review/CI/Supply Chain GREEN and inherited through the current parent stack | Preserve the exact invariant through descendants; no client-IP/trusted-proxy claim until separately characterized | | Edge routing / HTTP policy | Characterized and compiled for pg-erd | Routed production-process parity must remain GREEN through descendant restacks | | Runtime isolation | Declared/streamed body and in-flight limits implemented; #16 exact real-listener acceptance GREEN; #20 routed SIGTERM drain exact hosted/technical GREEN | Preserve through descendants; origin-capacity semantics remain separate | -| Failure recovery | #17 refused-origin, #18 connected-silent-origin, and #20 drain exact hosted/technical GREEN; #21 carries orderly post-header truncation source acceptance | Reacquire exact #21 hosted/review evidence; explicit reset, broader streaming/upgraded failure, and slow-drip/whole-response lifetime remain unproven | +| Failure recovery | #17 refused-origin, #18 connected-silent-origin, and #20 drain exact hosted/technical GREEN; #21 carries orderly post-header truncation plus exact single-field framing acceptance | Reacquire exact #21 hosted/review evidence; explicit reset, broader streaming/upgraded failure, and slow-drip/whole-response lifetime remain unproven | | Upstream TLS | Generic local-CA/SNI verification exists; pg-erd fail-closed trust activation exists | Successful pg-erd TLS listener/origin path and representative TLS performance remain unproven | | Protocols | HTTP/1.1 path exists | Downstream TLS/H2, H2→H1 Cookie behavior, WebSocket/Extended CONNECT and explicit H3/QUIC disposition require separate supplier-capable contracts | | OCI / supply chain | #20 exact dual-profile candidate build, least privilege, Prometheus-listener identity, audit, SPDX SBOM and image-scan evidence GREEN | Revalidate on changed #21; immutable registry digest, signing/attestation/provenance, release-bound SBOM, reproducibility receipt and rollback rehearsal | | Performance | Parent #20 controlled generic loopback remains under the `<20 ms` gate; supplier measured-origin #62 p95 `0.71593235 ms` | Do not treat controlled loopback as production SLO proof; routed pg-erd concurrency/TLS/failure measurements remain required | | Observability | Low-cardinality counters/logging and separate metrics listener; #16 exact single-rejection, #17/#18 exact single-error, and #19 metrics-listener identity GREEN; #21 requires exact single-error after post-header truncation | Reacquire #21 exact telemetry evidence; broader payload-free runtime assertions and tracing remain gaps | -| Documentation / review | #20 exact owner technical sweep is clean; #21 source helpers are rustdoc-complete and current docs now describe phase-aware failure | Fresh exact-range #21 review after the final unchanged head; no bot/static review is independent human approval | +| Documentation / review | #20 exact owner technical sweep is clean; #21 source helpers are rustdoc-complete and its framing-oracle review finding is repaired | Fresh exact-range #21 review after the final unchanged head; no bot/static review is independent human approval | | Release / migration | No protected release or consumer cutover credit | Immutable release → parity → shadow/canary → rollback rehearsal → cutover → verified Nginx/OpenResty/legacy removal | ## Execution order From 469e995ee279a4bb174c9ea6834bdd3d885ea27f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 12:55:40 +0900 Subject: [PATCH 26/43] docs: tighten partial-response framing changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7abb154d..6ba0edbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ All notable changes are tracked here. No release has been published yet. - Added dedicated compiled pg-erd traffic acceptance for streamed/chunked body overflow and routed in-flight saturation/recovery: the migration process must return 413 above the shared body budget, return 503 in less than one second above the in-flight budget, keep `/readyz` observable, expose the exact single-rejection Prometheus sample, and admit a later routed request after capacity is released. - Added dedicated compiled pg-erd refused-origin recovery acceptance using a Linux TCP socket bound to the characterized backend address without entering LISTEN state. The fixture first proves direct `ECONNREFUSED` while retaining exclusive port ownership, then requires the migration gateway to return 502 within a conservative one-second envelope around the configured 200/400 ms connection budgets, keep `/readyz` 200, expose the exact single-error Prometheus sample, and allow a later independent frontend route to recover. Connected read stall, TCP reset, partial-response/streaming failure, retry and failover behavior remain separate gaps. - Added dedicated compiled pg-erd connected read-stall acceptance with `read_ms=100`: the backend accepts the routed request and remains open without response bytes until the gateway has already failed it, preventing fixture closure from faking timeout behavior. The contract requires 502 inside a conservative one-second envelope, preserved `/readyz`, the exact single-error Prometheus sample, and independent frontend recovery. Pingora `read_timeout` remains a per-read inactivity budget, not a whole-response lifetime; reset, partial-response and slow-drip cases remain open. -- Added dedicated compiled pg-erd post-header partial-response acceptance: the backend commits HTTP 200 with `Content-Length: 20`, writes only `partial`, and closes only after the downstream has observed that committed header/body prefix. The downstream must retain the committed status/framing and terminate before body completion instead of receiving an invented second status or silent failover; `/readyz`, exact low-cardinality request-error telemetry, and an independent `frontend` route must remain usable. Explicit TCP reset, broader streaming/upgraded failure, and slow-drip/whole-response lifetime remain separate gaps. +- Added dedicated compiled pg-erd post-header partial-response acceptance: the backend commits HTTP 200 with exactly one `Content-Length: 20` field, writes only `partial`, and closes only after the downstream has observed that committed header/body prefix. The downstream must retain the committed status and the single exact framing field, then terminate before body completion instead of receiving an invented second status or silent failover; `/readyz`, exact low-cardinality request-error telemetry, and an independent `frontend` route must remain usable. `X-Content-Length` and duplicate/conflicting `Content-Length` fields cannot satisfy the framing oracle. Explicit TCP reset, broader streaming/upgraded failure, and slow-drip/whole-response lifetime remain separate gaps. - Added dedicated routed pg-erd graceful-drain acceptance: a characterized `/api/held` backend request is held in flight, SIGTERM is sent only after the backend has accepted it, the response is released during the shared grace period, the downstream must still receive HTTP 200, and the migration process must exit successfully inside the external termination budget measured from SIGTERM. Generic drain evidence is not transferred to this composition root. - Added optional per-upstream absolute PEM trust-bundle consumption without taking ownership of certificate issuance/rotation; trust material is loaded fail-closed before listeners open. - Added an executable local-CA TLS test through the compiled gateway that holds CA trust constant and proves SNI/hostname mismatch is rejected. From bbf127f003c9b31c9e94a76a051bc1e80bda68ba Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 12:56:09 +0900 Subject: [PATCH 27/43] docs: lock exact content-length oracle --- TEST_STRATEGY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TEST_STRATEGY.md b/TEST_STRATEGY.md index be5c7ba5..63b40353 100644 --- a/TEST_STRATEGY.md +++ b/TEST_STRATEGY.md @@ -16,7 +16,7 @@ The bounded Admin Config transition has its own executable contract. `tests/pg_e `tests/pg_erd_read_stall_traffic.rs` separates connected upstream inactivity from refusal. The characterized backend accepts `/api/read-stall`, reads the request headers, then remains connected and sends no response bytes until the test explicitly releases it after the gateway has already returned. With `read_ms=100`, the gateway must fail as HTTP 502 inside a conservative one-second outer envelope, preserve `/readyz`, expose the exact Prometheus sample `cwl_pingora_gateway_request_errors_total 1`, and leave an independent `frontend` route usable. Keeping the fixture connection open prevents origin closure from masquerading as the timeout. Because Pingora's `read_timeout` is per successful `read()` rather than a whole-response lifetime, reset, post-commit partial response, slow-drip and whole-response deadline behavior remain separate contracts. -`tests/pg_erd_partial_response_traffic.rs` covers the distinct post-header failure phase. The characterized backend sends HTTP 200 with `Content-Length: 20` and the body prefix `partial`, then remains open until the downstream reader has observed a complete response header block and that exact body prefix. Only after this acknowledgement does the fixture release the backend to close normally; EOF/reset before acknowledgement fails the test. This prevents scheduler/socket-buffer timing from masquerading as post-commit evidence. Acceptance requires the downstream to retain the committed status/framing and terminate before all 20 bytes arrive rather than receiving an invented second status or silent route failover. `/readyz` must stay HTTP 200, the shared request-error counter must expose exactly `cwl_pingora_gateway_request_errors_total 1`, and an independent `frontend` request must still complete successfully. Traffic and metrics ports are reserved simultaneously before process startup so ephemeral-port reuse cannot manufacture an invalid listener collision. This orderly-close contract does not transfer evidence to explicit TCP reset, WebSocket/Upgrade, broader streaming failure, or slow-drip/whole-response lifetime. +`tests/pg_erd_partial_response_traffic.rs` covers the distinct post-header failure phase. The characterized backend sends HTTP 200 with `Content-Length: 20` and the body prefix `partial`, then remains open until the downstream reader has observed a complete response header block and that exact body prefix. Only after this acknowledgement does the fixture release the backend to close normally; EOF/reset before acknowledgement fails the test. This prevents scheduler/socket-buffer timing from masquerading as post-commit evidence. Acceptance requires the downstream to retain the committed status/framing and terminate before all 20 bytes arrive rather than receiving an invented second status or silent route failover. The framing oracle lowercases the received header block, parses individual header lines, admits only actual `content-length:` fields, requires exactly one such field, and requires that sole field to equal `content-length: 20`; `X-Content-Length` and duplicate/conflicting Content-Length fields therefore cannot false-pass. `/readyz` must stay HTTP 200, the shared request-error counter must expose exactly `cwl_pingora_gateway_request_errors_total 1`, and an independent `frontend` request must still complete successfully. Traffic and metrics ports are reserved simultaneously before process startup so ephemeral-port reuse cannot manufacture an invalid listener collision. This orderly-close contract does not transfer evidence to explicit TCP reset, WebSocket/Upgrade, broader streaming failure, or slow-drip/whole-response lifetime. On Unix, `tests/pg_erd_graceful_shutdown.rs` proves the bounded pg-erd composition root consumes the shared drain policy instead of borrowing generic-binary evidence. The test routes `/api/held` to the characterized backend, holds that response open until the backend has confirmed the in-flight request, then creates one absolute termination deadline, delivers SIGTERM, and releases the response during the shared grace period. The downstream must still receive HTTP 200 and the migration process must exit successfully before that SIGTERM-relative external termination deadline. Its traffic and metrics reservation sockets are held simultaneously before startup so the fixture cannot false-fail through ephemeral-port reuse. Parent #20 exact `d4d4565854cc924a2214de2b67a966d2f253da3e` has terminal CI `34181336779` and Supply Chain `34181336796` GREEN; descendants must revalidate rather than transfer that receipt. From e286b912a1092362c41ac3a0c8e03a2aac4e4854 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 12:58:09 +0900 Subject: [PATCH 28/43] test: parse content-length field semantics --- tests/pg_erd_partial_response_traffic.rs | 45 +++++++++++++++++------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/tests/pg_erd_partial_response_traffic.rs b/tests/pg_erd_partial_response_traffic.rs index 4fbad5fe..74bc53a4 100644 --- a/tests/pg_erd_partial_response_traffic.rs +++ b/tests/pg_erd_partial_response_traffic.rs @@ -213,6 +213,32 @@ fn read_request_headers(stream: &mut TcpStream) -> String { } } +/// Extracts Content-Length field values by case-insensitive field identity and trimmed field value. +fn content_length_values(headers: &str) -> Vec<&str> { + headers + .lines() + .filter_map(|line| line.trim_end_matches('\r').split_once(':')) + .filter(|(name, _)| name.eq_ignore_ascii_case("content-length")) + .map(|(_, value)| value.trim()) + .collect() +} + +/// Locks the framing oracle against lookalike names and duplicate/conflicting field values. +#[test] +fn content_length_parser_preserves_field_identity_and_cardinality_evidence() { + assert!(content_length_values("HTTP/1.1 200 OK\r\nX-Content-Length: 20\r\n\r\n").is_empty()); + assert_eq!( + content_length_values("HTTP/1.1 200 OK\r\ncOnTeNt-LeNgTh:\t20\r\n\r\n"), + vec!["20"] + ); + assert_eq!( + content_length_values( + "HTTP/1.1 200 OK\r\nContent-Length: 20\r\ncontent-length: 21\r\n\r\n" + ), + vec!["20", "21"] + ); +} + /// Proves a post-commit origin truncation preserves framing, terminates downstream and keeps recovery usable. #[test] fn compiled_pg_erd_truncated_response_stays_committed_and_preserves_independent_routing() { @@ -281,24 +307,17 @@ fn compiled_pg_erd_truncated_response_stays_committed_and_preserves_independent_ .position(|window| window == b"\r\n\r\n") .map(|position| position + 4) .expect("committed partial response must contain a complete header block"); - let headers = String::from_utf8_lossy(&partial[..header_end]).to_ascii_lowercase(); + let raw_headers = String::from_utf8_lossy(&partial[..header_end]); + let headers = raw_headers.to_ascii_lowercase(); assert!( headers.starts_with("http/1.1 200"), "a post-header upstream failure cannot be rewritten as a new status: {headers:?}" ); - let content_length_headers: Vec<_> = headers - .lines() - .filter(|line| line.starts_with("content-length:")) - .collect(); - assert_eq!( - content_length_headers.len(), - 1, - "the committed response must retain exactly one Content-Length field: {headers:?}" - ); + let content_lengths = content_length_values(raw_headers.as_ref()); assert_eq!( - content_length_headers[0], - "content-length: 20", - "the committed response must retain its declared framing for this fixture: {headers:?}" + content_lengths, + vec!["20"], + "the committed response must retain exactly one Content-Length field whose value is 20: {raw_headers:?}" ); let body = &partial[header_end..]; assert_eq!(body, b"partial"); From d15099271f5632c805fe10539bf7fffcfeec0eea Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 13:00:43 +0900 Subject: [PATCH 29/43] docs: match content-length parser semantics --- TEST_STRATEGY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TEST_STRATEGY.md b/TEST_STRATEGY.md index 63b40353..9e36a803 100644 --- a/TEST_STRATEGY.md +++ b/TEST_STRATEGY.md @@ -16,7 +16,7 @@ The bounded Admin Config transition has its own executable contract. `tests/pg_e `tests/pg_erd_read_stall_traffic.rs` separates connected upstream inactivity from refusal. The characterized backend accepts `/api/read-stall`, reads the request headers, then remains connected and sends no response bytes until the test explicitly releases it after the gateway has already returned. With `read_ms=100`, the gateway must fail as HTTP 502 inside a conservative one-second outer envelope, preserve `/readyz`, expose the exact Prometheus sample `cwl_pingora_gateway_request_errors_total 1`, and leave an independent `frontend` route usable. Keeping the fixture connection open prevents origin closure from masquerading as the timeout. Because Pingora's `read_timeout` is per successful `read()` rather than a whole-response lifetime, reset, post-commit partial response, slow-drip and whole-response deadline behavior remain separate contracts. -`tests/pg_erd_partial_response_traffic.rs` covers the distinct post-header failure phase. The characterized backend sends HTTP 200 with `Content-Length: 20` and the body prefix `partial`, then remains open until the downstream reader has observed a complete response header block and that exact body prefix. Only after this acknowledgement does the fixture release the backend to close normally; EOF/reset before acknowledgement fails the test. This prevents scheduler/socket-buffer timing from masquerading as post-commit evidence. Acceptance requires the downstream to retain the committed status/framing and terminate before all 20 bytes arrive rather than receiving an invented second status or silent route failover. The framing oracle lowercases the received header block, parses individual header lines, admits only actual `content-length:` fields, requires exactly one such field, and requires that sole field to equal `content-length: 20`; `X-Content-Length` and duplicate/conflicting Content-Length fields therefore cannot false-pass. `/readyz` must stay HTTP 200, the shared request-error counter must expose exactly `cwl_pingora_gateway_request_errors_total 1`, and an independent `frontend` request must still complete successfully. Traffic and metrics ports are reserved simultaneously before process startup so ephemeral-port reuse cannot manufacture an invalid listener collision. This orderly-close contract does not transfer evidence to explicit TCP reset, WebSocket/Upgrade, broader streaming failure, or slow-drip/whole-response lifetime. +`tests/pg_erd_partial_response_traffic.rs` covers the distinct post-header failure phase. The characterized backend sends HTTP 200 with `Content-Length: 20` and the body prefix `partial`, then remains open until the downstream reader has observed a complete response header block and that exact body prefix. Only after this acknowledgement does the fixture release the backend to close normally; EOF/reset before acknowledgement fails the test. This prevents scheduler/socket-buffer timing from masquerading as post-commit evidence. Acceptance requires the downstream to retain the committed status/framing and terminate before all 20 bytes arrive rather than receiving an invented second status or silent route failover. The framing oracle parses individual header lines from the original header block, matches `Content-Length` field names case-insensitively, trims field-value optional whitespace, requires exactly one value equal to `20`, and includes a focused regression proving `X-Content-Length` plus duplicate/conflicting Content-Length fields cannot false-pass. `/readyz` must stay HTTP 200, the shared request-error counter must expose exactly `cwl_pingora_gateway_request_errors_total 1`, and an independent `frontend` request must still complete successfully. Traffic and metrics ports are reserved simultaneously before process startup so ephemeral-port reuse cannot manufacture an invalid listener collision. This orderly-close contract does not transfer evidence to explicit TCP reset, WebSocket/Upgrade, broader streaming failure, or slow-drip/whole-response lifetime. On Unix, `tests/pg_erd_graceful_shutdown.rs` proves the bounded pg-erd composition root consumes the shared drain policy instead of borrowing generic-binary evidence. The test routes `/api/held` to the characterized backend, holds that response open until the backend has confirmed the in-flight request, then creates one absolute termination deadline, delivers SIGTERM, and releases the response during the shared grace period. The downstream must still receive HTTP 200 and the migration process must exit successfully before that SIGTERM-relative external termination deadline. Its traffic and metrics reservation sockets are held simultaneously before startup so the fixture cannot false-fail through ephemeral-port reuse. Parent #20 exact `d4d4565854cc924a2214de2b67a966d2f253da3e` has terminal CI `34181336779` and Supply Chain `34181336796` GREEN; descendants must revalidate rather than transfer that receipt. From 51f1242663ccbf164efc50ca2ac74c4d0a1c7126 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 13:01:55 +0900 Subject: [PATCH 30/43] docs: project semantic framing repair --- docs/product-technical-gap-baseline.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 708a8648..2dbbfeff 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -60,7 +60,7 @@ Exact #20 head `d4d4565854cc924a2214de2b67a966d2f253da3e` is terminal hosted GRE The #21 acceptance sends `Content-Length: 20` but only the seven-byte prefix `partial`, waits until that committed prefix has been observed downstream, and only then releases the origin to close. It requires the downstream to retain the committed 200/framing and terminate before body completion rather than receive an invented second HTTP status or failover; `/readyz` must remain 200, telemetry must contain exactly `cwl_pingora_gateway_request_errors_total 1`, and an independent `frontend` route must recover. Explicit TCP reset, WebSocket/Upgrade, broader streaming failure, and slow-drip/whole-response lifetime remain separate gaps. -Fresh exact-range CodeRabbit review of `d4d456...7b371cb` found one valid test-oracle defect: the framing assertion used substring matching for `content-length: 20`, so a different header name containing that substring or duplicate Content-Length fields could satisfy the check without proving the intended single framing field. Test-only repair `f32295c3a0d0525d9f0e3f73b41795eac6c1395d` now enumerates actual `content-length:` header lines, requires cardinality exactly one, and requires that sole field to equal `content-length: 20`. Production routing, failure semantics, timeout, telemetry and product authority are unchanged. Exact hosted and exact-range review evidence must be reacquired on the final unchanged #21 head after this repair and baseline projection. +Fresh exact-range CodeRabbit review of `d4d456...7b371cb` found one valid test-oracle defect: the framing assertion used substring matching for `content-length: 20`, so a different header name containing that substring or duplicate Content-Length fields could satisfy the check without proving the intended single framing field. Initial test-only repair `f32295c3a0d0525d9f0e3f73b41795eac6c1395d` removed the substring oracle. Follow-up `e286b912a1092362c41ac3a0c8e03a2aac4e4854` completed the semantic repair by parsing original header lines, matching `Content-Length` field identity case-insensitively, trimming field-value optional whitespace, requiring the integration response to yield exactly `vec!["20"]`, and adding focused regression cases for `X-Content-Length`, mixed-case field names and duplicate/conflicting values. Production routing, failure semantics, timeout, telemetry and product authority are unchanged. The CodeRabbit thread was answered and resolved after the source repair; exact hosted and exact-range review evidence still must be reacquired on the final unchanged #21 head. ## Capability state and buyer-visible gaps @@ -70,13 +70,13 @@ Fresh exact-range CodeRabbit review of `d4d456...7b371cb` found one valid test-o | Generic forwarding trust | #15 exact source/review/CI/Supply Chain GREEN and inherited through the current parent stack | Preserve the exact invariant through descendants; no client-IP/trusted-proxy claim until separately characterized | | Edge routing / HTTP policy | Characterized and compiled for pg-erd | Routed production-process parity must remain GREEN through descendant restacks | | Runtime isolation | Declared/streamed body and in-flight limits implemented; #16 exact real-listener acceptance GREEN; #20 routed SIGTERM drain exact hosted/technical GREEN | Preserve through descendants; origin-capacity semantics remain separate | -| Failure recovery | #17 refused-origin, #18 connected-silent-origin, and #20 drain exact hosted/technical GREEN; #21 carries orderly post-header truncation plus exact single-field framing acceptance | Reacquire exact #21 hosted/review evidence; explicit reset, broader streaming/upgraded failure, and slow-drip/whole-response lifetime remain unproven | +| Failure recovery | #17 refused-origin, #18 connected-silent-origin, and #20 drain exact hosted/technical GREEN; #21 carries orderly post-header truncation plus semantic single-field framing acceptance | Reacquire exact #21 hosted/review evidence; explicit reset, broader streaming/upgraded failure, and slow-drip/whole-response lifetime remain unproven | | Upstream TLS | Generic local-CA/SNI verification exists; pg-erd fail-closed trust activation exists | Successful pg-erd TLS listener/origin path and representative TLS performance remain unproven | | Protocols | HTTP/1.1 path exists | Downstream TLS/H2, H2→H1 Cookie behavior, WebSocket/Extended CONNECT and explicit H3/QUIC disposition require separate supplier-capable contracts | | OCI / supply chain | #20 exact dual-profile candidate build, least privilege, Prometheus-listener identity, audit, SPDX SBOM and image-scan evidence GREEN | Revalidate on changed #21; immutable registry digest, signing/attestation/provenance, release-bound SBOM, reproducibility receipt and rollback rehearsal | | Performance | Parent #20 controlled generic loopback remains under the `<20 ms` gate; supplier measured-origin #62 p95 `0.71593235 ms` | Do not treat controlled loopback as production SLO proof; routed pg-erd concurrency/TLS/failure measurements remain required | | Observability | Low-cardinality counters/logging and separate metrics listener; #16 exact single-rejection, #17/#18 exact single-error, and #19 metrics-listener identity GREEN; #21 requires exact single-error after post-header truncation | Reacquire #21 exact telemetry evidence; broader payload-free runtime assertions and tracing remain gaps | -| Documentation / review | #20 exact owner technical sweep is clean; #21 source helpers are rustdoc-complete and its framing-oracle review finding is repaired | Fresh exact-range #21 review after the final unchanged head; no bot/static review is independent human approval | +| Documentation / review | #20 exact owner technical sweep is clean; #21 source helpers are rustdoc-complete and its framing-oracle review finding is repaired with dedicated false-positive regression coverage | Fresh exact-range #21 review after the final unchanged head; no bot/static review is independent human approval | | Release / migration | No protected release or consumer cutover credit | Immutable release → parity → shadow/canary → rollback rehearsal → cutover → verified Nginx/OpenResty/legacy removal | ## Execution order From 4d3cf712b89a0b607db1e5f80db2d60ed29f6e1c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 12 Sep 2026 23:00:41 +0900 Subject: [PATCH 31/43] docs: bind partial-response child to repaired parent evidence --- CHANGELOG.md | 2 +- TEST_STRATEGY.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ba0edbb..d5f43392 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,4 +39,4 @@ All notable changes are tracked here. No release has been published yet. - Added missing-public-rustdoc enforcement and documentation builds with warnings denied. - Added DDD, product, technical, security, threat, test, operability, configuration, migration-gap, and primary-source traceability documentation. -Release remains blocked on the organization decision for the exact Pingora release versus `RUSTSEC-2026-0253` and the separate time-bounded disposition of unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388` (`ContextualWisdomLab/.github#1605`), restoration of authoritative public non-fork Dependency Review evidence (`ContextualWisdomLab/.github#810`), terminal exact-current-head CI/supply-chain/security/review evidence, representative compiled-binary pg-erd route/header/forwarding/body/backpressure/failure/concurrency/drain and benchmark evidence, an immutable registry digest with provenance and rehearsed rollback, and protected-branch integration. Parent #20 routed graceful-drain acceptance is exact-head hosted GREEN at `d4d4565854cc924a2214de2b67a966d2f253da3e`; the changed #21 post-header partial-response candidate must independently reacquire exact-head execution/review evidence before that child contract is credited. No consumer migration, canary, cutover, or legacy removal is claimed before those release and traffic-contract gates are satisfied. +Release remains blocked on the organization decision for the exact Pingora release versus `RUSTSEC-2026-0253` and the separate time-bounded disposition of unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388` (`ContextualWisdomLab/.github#1605`), restoration of authoritative public non-fork Dependency Review evidence (`ContextualWisdomLab/.github#810`), terminal exact-current-head CI/supply-chain/security/review evidence, representative compiled-binary pg-erd route/header/forwarding/body/backpressure/failure/concurrency/drain and benchmark evidence, an immutable registry digest with provenance and rehearsed rollback, and protected-branch integration. Parent #20 has moved by ordinary repair to `ab70a6c75da1677374597d2302a18a59fe4c3850` and must reacquire its own exact-head evidence; the changed #21 partial-response candidate must independently reacquire exact-head execution/review evidence as well. No predecessor GREEN transfers across either movement. No consumer migration, canary, cutover, or legacy removal is claimed before those release and traffic-contract gates are satisfied. diff --git a/TEST_STRATEGY.md b/TEST_STRATEGY.md index 9e36a803..0fc33a91 100644 --- a/TEST_STRATEGY.md +++ b/TEST_STRATEGY.md @@ -18,7 +18,7 @@ The bounded Admin Config transition has its own executable contract. `tests/pg_e `tests/pg_erd_partial_response_traffic.rs` covers the distinct post-header failure phase. The characterized backend sends HTTP 200 with `Content-Length: 20` and the body prefix `partial`, then remains open until the downstream reader has observed a complete response header block and that exact body prefix. Only after this acknowledgement does the fixture release the backend to close normally; EOF/reset before acknowledgement fails the test. This prevents scheduler/socket-buffer timing from masquerading as post-commit evidence. Acceptance requires the downstream to retain the committed status/framing and terminate before all 20 bytes arrive rather than receiving an invented second status or silent route failover. The framing oracle parses individual header lines from the original header block, matches `Content-Length` field names case-insensitively, trims field-value optional whitespace, requires exactly one value equal to `20`, and includes a focused regression proving `X-Content-Length` plus duplicate/conflicting Content-Length fields cannot false-pass. `/readyz` must stay HTTP 200, the shared request-error counter must expose exactly `cwl_pingora_gateway_request_errors_total 1`, and an independent `frontend` request must still complete successfully. Traffic and metrics ports are reserved simultaneously before process startup so ephemeral-port reuse cannot manufacture an invalid listener collision. This orderly-close contract does not transfer evidence to explicit TCP reset, WebSocket/Upgrade, broader streaming failure, or slow-drip/whole-response lifetime. -On Unix, `tests/pg_erd_graceful_shutdown.rs` proves the bounded pg-erd composition root consumes the shared drain policy instead of borrowing generic-binary evidence. The test routes `/api/held` to the characterized backend, holds that response open until the backend has confirmed the in-flight request, then creates one absolute termination deadline, delivers SIGTERM, and releases the response during the shared grace period. The downstream must still receive HTTP 200 and the migration process must exit successfully before that SIGTERM-relative external termination deadline. Its traffic and metrics reservation sockets are held simultaneously before startup so the fixture cannot false-fail through ephemeral-port reuse. Parent #20 exact `d4d4565854cc924a2214de2b67a966d2f253da3e` has terminal CI `34181336779` and Supply Chain `34181336796` GREEN; descendants must revalidate rather than transfer that receipt. +On Unix, `tests/pg_erd_graceful_shutdown.rs` proves the bounded pg-erd composition root consumes the shared drain policy instead of borrowing generic-binary evidence. The test routes `/api/held` to the characterized backend, holds that response open until the backend has confirmed the in-flight request, then creates one absolute termination deadline, delivers SIGTERM, and releases the response during the shared grace period. The downstream must still receive HTTP 200 and the migration process must exit successfully before that SIGTERM-relative external termination deadline. Its traffic and metrics reservation sockets are held simultaneously before startup so the fixture cannot false-fail through ephemeral-port reuse. Current parent #20 exact `ab70a6c75da1677374597d2302a18a59fe4c3850` is an ordinary ancestry/single-writer repair and must independently reacquire hosted evidence; historical `d4d4565...` GREEN is predecessor evidence only and does not transfer. The `oci-runtime` job separately validates artifact composition instead of inferring it from compiled-process tests. The Dockerfile admits only `cwl-pingora-gateway` and `cwl-pingora-pg-erd-migration` as build-time process identities, normalizes the selected executable to one fixed distroless runtime path, and CI builds both image profiles. Each exact candidate must declare uid/gid `65532`, start under a read-only root filesystem with all capabilities dropped and `no-new-privileges`, and consume only a read-only configuration mount. The generic profile must expose local `/livez`. The pg-erd profile must expose local `/livez` on the traffic listener and its separately published `/metrics` listener must identify the Pingora Prometheus service by a `text/plain` response media type before the container is accepted. `tests/pg_erd_oci_metrics_workflow_contract.rs` prevents a bare HTTP 200 from false-greening a mistakenly bound proxy service while deliberately avoiding a metric-family requirement before routed application traffic has emitted one. `examples/pg-erd-migration.yaml` is deliberately an origin-independent OCI smoke fixture; this gate proves the dedicated binary is actually packaged and both process/observability listeners start under the required container isolation but does not claim routed pg-erd parity, origin health, or performance. The supply-chain job additionally builds and vulnerability-scans both image profiles and binds both local image IDs plus per-image scan outputs to the exact source SHA. These workflow contracts count only after terminal success on the unchanged exact head. @@ -26,4 +26,4 @@ The `oci-runtime` job separately validates artifact composition instead of infer Every behavioral migration should begin with characterization against the old owned edge behavior, then add equivalent production-path evidence for Pingora. Static-serving consumers must cover route precedence, SPA fallback, MIME, ETag/cache, Range/HEAD/304/416, redirects/security headers and compression as applicable. Proxy consumers must cover Host/SNI/TLS, forwarding trust, WebSocket/upgrade, limits, timeout/retry behavior, streaming/uploads, saturation/backpressure, errors, health/readiness and drain. Characterized response policies must additionally be exercised through the compiled proxy path before any parity or canary claim. -Release-quality gaps remain: no property/fuzz tests yet; no downstream TLS listener contract; no HTTP/2 or HTTP/3 parity evidence; no tracing evidence; no immutable published registry digest/provenance and rehearsed rollback; and no benchmark against replaced Nginx/Traefik traffic. For the pg-erd candidate specifically, source now covers streamed body overflow, in-flight saturation/recovery, refused-origin recovery, connected silent-origin read timeout, orderly post-header truncation, routed graceful drain, and OCI process/Prometheus-listener identity. Explicit TCP reset, broader streaming/upgraded failure, slow-drip/whole-response lifetime, representative routed origin-capacity load, shadow/canary and rollback remain open. Parent #20 has exact hosted GREEN, but the changed #21 head must independently pass formatting/compile/test/Clippy/rustdoc/documentation/100%-coverage/load/OCI/Supply Chain before its partial-response contract is credited. OCI non-root/read-only-root source acceptance covers both admitted process images, while public rustdoc, SBOM/image vulnerability, local-CA upstream TLS, explicit backpressure/recovery, and the minimal k6 loopback path remain exact-head gates. None can be transferred to a changed or consumer-specific head. Production performance claims remain forbidden until representative deployment measurements exist. +Release-quality gaps remain: no property/fuzz tests yet; no downstream TLS listener contract; no HTTP/2 or HTTP/3 parity evidence; no tracing evidence; no immutable published registry digest/provenance and rehearsed rollback; and no benchmark against replaced Nginx/Traefik traffic. For the pg-erd candidate specifically, source now covers streamed body overflow, in-flight saturation/recovery, refused-origin recovery, connected silent-origin read timeout, orderly post-header truncation, routed graceful drain, and OCI process/Prometheus-listener identity. Explicit TCP reset, broader streaming/upgraded failure, slow-drip/whole-response lifetime, representative routed origin-capacity load, shadow/canary and rollback remain open. Current parent #20 and this changed #21 head must each independently pass their exact-head formatting/compile/test/Clippy/rustdoc/100%-coverage/load/OCI/Supply Chain gates before their respective contracts are credited. OCI non-root/read-only-root source acceptance covers both admitted process images, while public rustdoc, SBOM/image vulnerability, local-CA upstream TLS, explicit backpressure/recovery, and the minimal k6 loopback path remain exact-head gates. None can be transferred to a changed or consumer-specific head. Production performance claims remain forbidden until representative deployment measurements exist. From f8436a14bc0a4c33cfd1645b2cee6855d7498454 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 09:38:19 +0900 Subject: [PATCH 32/43] test(partial): harden listener handoff and readiness --- tests/pg_erd_partial_response_traffic.rs | 82 +++++++++++++++++++----- 1 file changed, 65 insertions(+), 17 deletions(-) diff --git a/tests/pg_erd_partial_response_traffic.rs b/tests/pg_erd_partial_response_traffic.rs index 74bc53a4..2fb9c42d 100644 --- a/tests/pg_erd_partial_response_traffic.rs +++ b/tests/pg_erd_partial_response_traffic.rs @@ -15,6 +15,8 @@ use std::time::{Duration, Instant}; use tempfile::NamedTempFile; +const MAX_RESPONSE_HEADER_BYTES: usize = 64 * 1024; + /// Owns the compiled migration child so assertion failures cannot leak a listening test process. struct GatewayProcess(Child); @@ -32,22 +34,20 @@ enum DownstreamTermination { ConnectionReset, } -/// Selects distinct traffic and metrics authorities while both ephemeral reservations remain held. -fn reserve_distinct_loopback_addresses() -> (SocketAddr, SocketAddr) { - // Hold both ephemeral reservations at once so listener and metrics authority cannot - // accidentally collapse to the same port before the migration process binds them. +/// Holds distinct traffic and metrics reservations until the compiled child is ready to bind them. +fn reserve_distinct_loopback_listeners() -> (TcpListener, TcpListener) { let traffic = TcpListener::bind("127.0.0.1:0").expect("traffic port should be reservable"); let metrics = TcpListener::bind("127.0.0.1:0").expect("metrics port should be reservable"); - let addresses = ( + assert_ne!( traffic .local_addr() .expect("traffic reservation should expose an address"), metrics .local_addr() .expect("metrics reservation should expose an address"), + "traffic and metrics reservations must remain distinct" ); - assert_ne!(addresses.0, addresses.1); - addresses + (traffic, metrics) } /// Writes the bounded pg-erd fixture used to separate post-commit truncation from read-stall failure. @@ -66,28 +66,66 @@ fn write_config( file } -/// Waits for one gateway listener without treating an early process exit as startup success. -fn wait_until_listening(address: SocketAddr, process: &mut Child) { +/// Waits for a bounded complete HTTP 200 response instead of treating bare TCP accept as readiness. +fn wait_until_http_ok(address: SocketAddr, path: &str, process: &mut Child) { let deadline = Instant::now() + Duration::from_secs(10); + let request = format!("GET {path} HTTP/1.1\r\nHost: gateway.local\r\nConnection: close\r\n\r\n"); loop { if let Some(status) = process .try_wait() .expect("gateway process state should be readable") { - panic!("gateway exited before accepting traffic: {status}"); + panic!("gateway exited before {path} became ready: {status}"); } - if TcpStream::connect_timeout(&address, Duration::from_millis(100)).is_ok() { - return; + + if let Ok(mut stream) = TcpStream::connect_timeout(&address, Duration::from_millis(100)) { + stream + .set_read_timeout(Some(Duration::from_millis(250))) + .expect("readiness read timeout should be configurable"); + stream + .set_write_timeout(Some(Duration::from_millis(250))) + .expect("readiness write timeout should be configurable"); + if stream.write_all(request.as_bytes()).is_ok() { + let mut response = Vec::new(); + let mut buffer = [0_u8; 1024]; + loop { + match stream.read(&mut buffer) { + Ok(0) => break, + Ok(read) => { + response.extend_from_slice(&buffer[..read]); + if response.len() > MAX_RESPONSE_HEADER_BYTES { + break; + } + if response.windows(4).any(|window| window == b"\r\n\r\n") { + if response.starts_with(b"HTTP/1.1 200 ") { + return; + } + break; + } + } + Err(error) + if matches!( + error.kind(), + ErrorKind::WouldBlock | ErrorKind::TimedOut + ) => + { + break; + } + Err(_) => break, + } + } + } } + assert!( Instant::now() < deadline, - "gateway did not start within 10s" + "gateway did not expose HTTP 200 on {path} within 10s" ); thread::sleep(Duration::from_millis(25)); } } -/// Starts the compiled pg-erd binary and requires both traffic and metrics authorities to bind. +/// Starts the compiled pg-erd binary and requires application-level traffic and metrics readiness. fn start_gateway( config: &NamedTempFile, gateway_address: SocketAddr, @@ -100,8 +138,8 @@ fn start_gateway( .stderr(Stdio::null()) .spawn() .expect("compiled pg-erd migration binary should start"); - wait_until_listening(gateway_address, &mut child); - wait_until_listening(metrics_address, &mut child); + wait_until_http_ok(gateway_address, "/readyz", &mut child); + wait_until_http_ok(metrics_address, "/metrics", &mut child); GatewayProcess(child) } @@ -280,13 +318,23 @@ fn compiled_pg_erd_truncated_response_stays_committed_and_preserves_independent_ .expect("frontend recovery response should be writable"); }); - let (gateway_address, metrics_address) = reserve_distinct_loopback_addresses(); + let (gateway_reservation, metrics_reservation) = reserve_distinct_loopback_listeners(); + let gateway_address = gateway_reservation + .local_addr() + .expect("traffic reservation should expose an address"); + let metrics_address = metrics_reservation + .local_addr() + .expect("metrics reservation should expose an address"); let config = write_config( gateway_address, metrics_address, backend_address, frontend_address, ); + // Release the exact reservations only at the compiled child-bind handoff; retaining them through + // config construction prevents another fixture from reclaiming either selected authority early. + drop(gateway_reservation); + drop(metrics_reservation); let _process = start_gateway(&config, gateway_address, metrics_address); let (partial, termination) = raw_request_until_terminal_after_body_prefix( From 64b8bc9da41be176d77348fbec606f28358b725a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 09:50:25 +0900 Subject: [PATCH 33/43] test(partial): bound request reads and parse status exactly --- tests/pg_erd_partial_response_traffic.rs | 52 +++++++++++++++++++----- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/tests/pg_erd_partial_response_traffic.rs b/tests/pg_erd_partial_response_traffic.rs index 2fb9c42d..5acc6e06 100644 --- a/tests/pg_erd_partial_response_traffic.rs +++ b/tests/pg_erd_partial_response_traffic.rs @@ -234,17 +234,24 @@ fn get(address: SocketAddr, path: &str) -> String { /// Reads only through the origin header terminator so the fixture can control the failure phase. fn read_request_headers(stream: &mut TcpStream) -> String { + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .expect("origin request timeout should be configurable"); let mut bytes = Vec::new(); let mut buffer = [0_u8; 1024]; loop { let read = stream .read(&mut buffer) - .expect("origin request should be readable"); + .expect("origin request should complete within the fixture timeout"); assert!( read > 0, "gateway closed origin request before headers completed" ); bytes.extend_from_slice(&buffer[..read]); + assert!( + bytes.len() <= MAX_RESPONSE_HEADER_BYTES, + "origin request headers exceeded the bounded fixture limit" + ); if bytes.windows(4).any(|window| window == b"\r\n\r\n") { return String::from_utf8_lossy(&bytes).into_owned(); } @@ -261,6 +268,20 @@ fn content_length_values(headers: &str) -> Vec<&str> { .collect() } +/// Parses only an exact HTTP/1.1 three-digit response status line for the wire oracle. +fn http_1_1_status_code(response: &str) -> Option { + let status_line = response.lines().next()?.trim_end_matches('\r'); + let mut fields = status_line.splitn(3, ' '); + if fields.next()? != "HTTP/1.1" { + return None; + } + let code = fields.next()?; + if code.len() != 3 || !code.bytes().all(|byte| byte.is_ascii_digit()) || fields.next().is_none() { + return None; + } + code.parse().ok() +} + /// Locks the framing oracle against lookalike names and duplicate/conflicting field values. #[test] fn content_length_parser_preserves_field_identity_and_cardinality_evidence() { @@ -277,6 +298,15 @@ fn content_length_parser_preserves_field_identity_and_cardinality_evidence() { ); } +/// Locks status evidence to exact HTTP/1.1 protocol and a three-digit code. +#[test] +fn status_parser_rejects_prefix_and_protocol_lookalikes() { + assert_eq!(http_1_1_status_code("HTTP/1.1 200 OK\r\n\r\n"), Some(200)); + assert_eq!(http_1_1_status_code("HTTP/1.1 2000 Bad\r\n\r\n"), None); + assert_eq!(http_1_1_status_code("http/1.1 200 OK\r\n\r\n"), None); + assert_eq!(http_1_1_status_code("HTTP/2 200 OK\r\n\r\n"), None); +} + /// Proves a post-commit origin truncation preserves framing, terminates downstream and keeps recovery usable. #[test] fn compiled_pg_erd_truncated_response_stays_committed_and_preserves_independent_routing() { @@ -356,10 +386,10 @@ fn compiled_pg_erd_truncated_response_stays_committed_and_preserves_independent_ .map(|position| position + 4) .expect("committed partial response must contain a complete header block"); let raw_headers = String::from_utf8_lossy(&partial[..header_end]); - let headers = raw_headers.to_ascii_lowercase(); - assert!( - headers.starts_with("http/1.1 200"), - "a post-header upstream failure cannot be rewritten as a new status: {headers:?}" + assert_eq!( + http_1_1_status_code(raw_headers.as_ref()), + Some(200), + "a post-header upstream failure cannot be rewritten as a new status: {raw_headers:?}" ); let content_lengths = content_length_values(raw_headers.as_ref()); assert_eq!( @@ -375,8 +405,9 @@ fn compiled_pg_erd_truncated_response_stays_committed_and_preserves_independent_ ); let readiness = get(gateway_address, "/readyz"); - assert!( - readiness.starts_with("HTTP/1.1 200"), + assert_eq!( + http_1_1_status_code(&readiness), + Some(200), "one truncated upstream response must not poison process readiness: {readiness:?}" ); @@ -389,8 +420,9 @@ fn compiled_pg_erd_truncated_response_stays_committed_and_preserves_independent_ ); let recovered = get(gateway_address, "/after-partial-response"); - assert!( - recovered.starts_with("HTTP/1.1 200"), + assert_eq!( + http_1_1_status_code(&recovered), + Some(200), "an independent characterized route must remain usable after a truncated response: {recovered:?}" ); assert!(recovered.ends_with("\r\n\r\nrecovered")); @@ -401,4 +433,4 @@ fn compiled_pg_erd_truncated_response_stays_committed_and_preserves_independent_ backend_origin .join() .expect("partial backend fixture should complete"); -} +} \ No newline at end of file From 59b49396f5a468b96d0d6efaa0bccf4b98560ce1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 09:50:59 +0900 Subject: [PATCH 34/43] docs(partial): reconcile changelog on current parent --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 812b7edd..18264ccf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ All notable changes are tracked here. No release has been published yet. - Added dedicated compiled pg-erd refused-origin recovery acceptance using a Linux TCP socket bound to the characterized backend address without entering LISTEN state. The fixture first proves direct `ECONNREFUSED` while retaining exclusive port ownership, then requires the migration gateway to return 502 within a conservative one-second envelope around the configured 200/400 ms connection budgets, keep `/readyz` 200, expose the exact single-error Prometheus sample, and allow a later independent frontend route to recover. Connected read stall, TCP reset, partial-response/streaming failure, retry and failover behavior remain separate gaps. - Added dedicated compiled pg-erd connected read-stall acceptance with `read_ms=100`: the backend accepts the routed request and remains open without response bytes until the gateway has already failed it, preventing fixture closure from faking timeout behavior. The contract requires 502 inside a conservative one-second envelope, preserved `/readyz`, the exact single-error Prometheus sample, and independent frontend recovery. Pingora `read_timeout` remains a per-read inactivity budget, not a whole-response lifetime; reset, partial-response and slow-drip cases remain open. - Added dedicated routed pg-erd graceful-drain acceptance: a characterized `/api/held` backend request is held in flight, SIGTERM is sent only after the backend has accepted it, the response is released during the shared grace period, the downstream must still receive HTTP 200, and the migration process must exit successfully inside the absolute external termination budget. The fixture retains traffic and metrics socket reservations through config construction, releases them only at child-bind handoff, and admits the drain case only after bounded `/readyz` HTTP/1.1 200 readiness; generic drain evidence is not transferred to this composition root. +- Added dedicated compiled pg-erd post-header truncation acceptance: the backend commits exact HTTP/1.1 200 framing with one `Content-Length: 20` field and the seven-byte `partial` prefix, remains open until downstream commit evidence is observed, then closes. The fixture requires the committed response to terminate incomplete without a second status or failover, records exactly one request error, preserves `/readyz`, and proves an independent frontend route still works. Status parsing now rejects protocol/prefix lookalikes, origin request-header reads are time/size bounded, and content-length parsing rejects lookalike or duplicate/conflicting fields. - Added optional per-upstream absolute PEM trust-bundle consumption without taking ownership of certificate issuance/rotation; trust material is loaded fail-closed before listeners open. - Added an executable local-CA TLS test through the compiled gateway that holds CA trust constant and proves SNI/hostname mismatch is rejected. - Added a focused transport-adapter regression proving an upstream without a custom trust bundle leaves Pingora's platform trust roots selected rather than replacing the CA store. From b4590a0a81f54768833198462de7ddc2eca35598 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 09:51:25 +0900 Subject: [PATCH 35/43] docs(partial): reconcile test strategy on current parent --- TEST_STRATEGY.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/TEST_STRATEGY.md b/TEST_STRATEGY.md index faab7075..cc9d6235 100644 --- a/TEST_STRATEGY.md +++ b/TEST_STRATEGY.md @@ -18,10 +18,12 @@ The bounded Admin Config transition has its own executable contract. `tests/pg_e On Unix, `tests/pg_erd_graceful_shutdown.rs` proves the bounded pg-erd composition root consumes the shared drain policy instead of borrowing generic-binary evidence. The fixture retains both ephemeral gateway listener reservations through config construction and releases them only at child-bind handoff, then requires a bounded complete `/readyz` HTTP/1.1 200 before test traffic. It routes `/api/held` to the characterized backend, holds that response until the backend confirms the request is in flight, sends SIGTERM, releases the response during the shared grace period, requires downstream HTTP 200 completion, and requires the migration process to exit before the absolute termination deadline anchored at signal delivery. This contract counts only when the unchanged exact head executes it to terminal GREEN. +`tests/pg_erd_partial_response_traffic.rs` covers the distinct post-header failure phase. The backend sends exact HTTP/1.1 200 with one `Content-Length: 20` field and the body prefix `partial`, then remains open until the downstream reader has observed the complete response header block and exact prefix. Only after that acknowledgement does the fixture allow origin close, preventing an immediate FIN from accidentally exercising a pre-commit failure phase. Acceptance requires the committed response to terminate before all 20 bytes arrive, forbids a rewritten second status or silent failover, preserves `/readyz`, records exactly `cwl_pingora_gateway_request_errors_total 1`, and proves an independent `frontend` route still completes. The status oracle accepts only exact HTTP/1.1 plus a three-digit code, the framing oracle rejects `X-Content-Length` and duplicate/conflicting Content-Length fields, origin request-header reads are bounded by five seconds and 64 KiB, and traffic/metrics reservations remain held until child-bind handoff. This contract is source-defined until the unchanged exact head reaches terminal hosted GREEN. + The `oci-runtime` job validates artifact composition separately from compiled-process tests. The Dockerfile admits only `cwl-pingora-gateway` and `cwl-pingora-pg-erd-migration` as build-time process identities, normalizes the selected executable to one fixed distroless runtime path, and CI builds both image profiles. Each exact candidate must declare uid/gid `65532`, start under a read-only root filesystem with all capabilities dropped and `no-new-privileges`, and consume only a read-only configuration mount. The generic profile must expose local `/livez`. The pg-erd profile must expose local `/livez` on the traffic listener and its separately published `/metrics` listener must identify the Pingora Prometheus service by exact base media type `text/plain` after stripping only optional semicolon parameters. `tests/pg_erd_oci_metrics_workflow_contract.rs` prevents a bare HTTP 200 or prefix wildcard from false-greening a mistakenly bound proxy service while deliberately avoiding a metric-family requirement before routed application traffic has emitted one. `examples/pg-erd-migration.yaml` is an origin-independent OCI smoke fixture, not routed parity evidence. The supply-chain job additionally builds and vulnerability-scans both profiles and binds both local image IDs plus per-image scan outputs to the exact source SHA. These workflow contracts count only after terminal success on the unchanged exact head. `tests/load/gateway_smoke.js` is a separate concurrent traffic contract executed with checksum-pinned k6 2.2.0 against the release-mode generic gateway binary and `tests/load/upstream_fixture.py`, a deterministic local HTTP/1.1 upstream. The contract sends 400 requests across four virtual users, requires every response to preserve the expected status/body, requires zero failed HTTP requests, and gates loopback `http_req_duration` p95 below 20 ms. `tests/load_origin_readiness_workflow_contract.rs` requires the measured origin to be ready before gateway startup. `tests/load_evidence_workflow_contract.rs` requires a successful k6 path to produce a nonempty summary while ensuring a pre-k6 failure cannot be replaced by a secondary missing-artifact failure. This is a regression bound for the minimal local generic path, not evidence that the pg-erd multi-route path or an Internet, TLS, multi-hop, consumer production path satisfies a 20 ms p95 SLO. A dedicated pg-erd routed load contract remains required before adopting the 20 ms objective for that migration. Every behavioral migration should begin with characterization against the old owned edge behavior, then add equivalent production-path evidence for Pingora. Static-serving consumers must cover route precedence, SPA fallback, MIME, ETag/cache, Range/HEAD/304/416, redirects/security headers, and compression as applicable. Proxy consumers must cover Host/SNI/TLS, forwarding trust, WebSocket/upgrade, limits, timeout/retry behavior, streaming/uploads, saturation/backpressure, errors, health/readiness, and drain. Characterized response policies must additionally be exercised through the compiled proxy path before any parity or canary claim. -Release-quality gaps remain: no property/fuzz tests yet; no downstream TLS listener contract; no HTTP/2 or HTTP/3 parity evidence; no tracing evidence; no immutable published registry digest/provenance and rehearsed rollback; and no benchmark against replaced Nginx/Traefik traffic. For the pg-erd candidate specifically, dedicated source now covers streamed body overflow, in-flight saturation/recovery, refused-origin recovery, connected silent-origin read timeout, routed graceful drain, and OCI process/Prometheus-listener identity, but TCP reset, post-commit partial-response/streaming failure, slow-drip/whole-response lifetime, representative routed origin-capacity load, terminal current-head traffic/OCI/supply-chain execution, shadow/canary, and rollback still lack evidence. OCI non-root/read-only-root source acceptance covers both admitted process images, while owned-production 100% line/region coverage, public rustdoc, SBOM/image vulnerability, local-CA upstream TLS, explicit backpressure/recovery, and the minimal k6 loopback path remain exact-head gates. None can be transferred to a changed or consumer-specific head. Production performance claims remain forbidden until representative deployment measurements exist. \ No newline at end of file +Release-quality gaps remain: no property/fuzz tests yet; no downstream TLS listener contract; no HTTP/2 or HTTP/3 parity evidence; no tracing evidence; no immutable published registry digest/provenance and rehearsed rollback; and no benchmark against replaced Nginx/Traefik traffic. For the pg-erd candidate specifically, dedicated source now covers streamed body overflow, in-flight saturation/recovery, refused-origin recovery, connected silent-origin read timeout, routed graceful drain, orderly post-header truncation, and OCI process/Prometheus-listener identity, but TCP reset, broader streaming/upgraded failure, slow-drip/whole-response lifetime, representative routed origin-capacity load, terminal current-head traffic/OCI/supply-chain execution, shadow/canary, and rollback still lack evidence. OCI non-root/read-only-root source acceptance covers both admitted process images, while owned-production 100% line/region coverage, public rustdoc, SBOM/image vulnerability, local-CA upstream TLS, explicit backpressure/recovery, and the minimal k6 loopback path remain exact-head gates. None can be transferred to a changed or consumer-specific head. Production performance claims remain forbidden until representative deployment measurements exist. \ No newline at end of file From c3ecf2a5c27fa0655461e3fa53958466037ee3e3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 09:53:45 +0900 Subject: [PATCH 36/43] style(partial): apply rustfmt-compatible status parser layout --- tests/pg_erd_partial_response_traffic.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/pg_erd_partial_response_traffic.rs b/tests/pg_erd_partial_response_traffic.rs index 5acc6e06..15b833e8 100644 --- a/tests/pg_erd_partial_response_traffic.rs +++ b/tests/pg_erd_partial_response_traffic.rs @@ -276,7 +276,10 @@ fn http_1_1_status_code(response: &str) -> Option { return None; } let code = fields.next()?; - if code.len() != 3 || !code.bytes().all(|byte| byte.is_ascii_digit()) || fields.next().is_none() { + if code.len() != 3 + || !code.bytes().all(|byte| byte.is_ascii_digit()) + || fields.next().is_none() + { return None; } code.parse().ok() From 149ab17b25cf8e3ee9700e6c517479c6f29e88e5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 09:55:51 +0900 Subject: [PATCH 37/43] style(partial): apply exact Rust 1.98 rustfmt diff --- tests/pg_erd_partial_response_traffic.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/pg_erd_partial_response_traffic.rs b/tests/pg_erd_partial_response_traffic.rs index 15b833e8..cf34b6dc 100644 --- a/tests/pg_erd_partial_response_traffic.rs +++ b/tests/pg_erd_partial_response_traffic.rs @@ -69,7 +69,8 @@ fn write_config( /// Waits for a bounded complete HTTP 200 response instead of treating bare TCP accept as readiness. fn wait_until_http_ok(address: SocketAddr, path: &str, process: &mut Child) { let deadline = Instant::now() + Duration::from_secs(10); - let request = format!("GET {path} HTTP/1.1\r\nHost: gateway.local\r\nConnection: close\r\n\r\n"); + let request = + format!("GET {path} HTTP/1.1\r\nHost: gateway.local\r\nConnection: close\r\n\r\n"); loop { if let Some(status) = process .try_wait() @@ -276,9 +277,7 @@ fn http_1_1_status_code(response: &str) -> Option { return None; } let code = fields.next()?; - if code.len() != 3 - || !code.bytes().all(|byte| byte.is_ascii_digit()) - || fields.next().is_none() + if code.len() != 3 || !code.bytes().all(|byte| byte.is_ascii_digit()) || fields.next().is_none() { return None; } @@ -436,4 +435,4 @@ fn compiled_pg_erd_truncated_response_stays_committed_and_preserves_independent_ backend_origin .join() .expect("partial backend fixture should complete"); -} \ No newline at end of file +} From 1486d70873834383d983124e0097a50947316efd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 09:57:57 +0900 Subject: [PATCH 38/43] docs(partial): preserve current-parent technical authority --- TRD.md | 49 ++++++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/TRD.md b/TRD.md index 8040f0f0..eec49a6c 100644 --- a/TRD.md +++ b/TRD.md @@ -2,52 +2,55 @@ ## Runtime -Rust edition 2021 with manifest MSRV `1.98.0` on this branch. Cloudflare Pingora and `pingora-prometheus` are pinned to exact public upstream revision `09696b51bc59315353d96686355861604d0bb48c`; mutable branch or contributor-PR dependencies are not release authority. +This branch uses Rust edition 2021 with `rust-version = "1.98.0"`. The Pingora crates are pinned to Cloudflare Pingora `0.8.0` at exact upstream Git revision `09696b51bc59315353d96686355861604d0bb48c`; mutable branch, tag, or contributor-PR resolution is not release authority. -The generic production composition root is `src/bin/cwl-pingora-gateway.rs`. It parses one explicit `--config`, validates the transport-neutral contract before granting network authority, constructs `GatewayProxy`, exposes the dedicated metrics listener, adds the downstream TCP listener to Pingora `http_proxy_service`, and delegates serving and shutdown to Pingora's `Server` lifecycle. +There are two composition roots with deliberately different contracts: -The characterized `pg-erd-cloud` migration uses the separate `src/bin/cwl-pingora-pg-erd-migration.rs` composition root and `PgErdMigrationConfig`. Keeping a separate binary prevents the generic v1 contract from being widened into a product-routing configuration language. Product authentication/authorization, business routing, certificate issuance/ACME, Wardnet/EgressWeave policy, and Keyverse identity remain outside this process boundary. +- `src/bin/cwl-pingora-gateway.rs` activates generic version-1 `GatewayConfig` and the one-upstream `GatewayProxy`. +- `src/bin/cwl-pingora-pg-erd-migration.rs` activates only the bounded `PgErdMigrationConfig` profile and `MigrationGatewayProxy` for the characterized pg-erd edge surface. -## Contract +Both parse an explicit `--config` path before creating listeners and delegate process lifecycle to Pingora's server lifecycle. The separate binaries prevent the generic v1 configuration language from being widened implicitly by one consumer migration. -Generic configuration version 1 is strict YAML with `deny_unknown_fields`. Required top-level fields are `version`, `listener`, `metrics_listener`, `max_request_body_bytes`, `max_in_flight_requests`, `upstream_keepalive_pool_size`, and `upstreams`. Version 1 accepts exactly one upstream; it does not provide a generic product route table or request-controlled destination. +## Generic edge contract -Traffic and metrics listeners must use non-zero, non-overlapping effective socket authority. Validation rejects same-family wildcard/concrete overlap, IPv6-wildcard/IPv4 dual-stack ambiguity, native IPv4 versus IPv4-mapped IPv6 aliases, and native/mapped or mapped-to-mapped IPv4 wildcard aliases while preserving distinct concrete non-aliased addresses. Request-body, in-flight, and keepalive-pool budgets must be positive. +Generic configuration version 1 is strict YAML with unknown fields denied. It admits one explicit non-zero listener, a distinct non-zero metrics authority, exactly one non-zero upstream socket, a positive request-body budget, positive process in-flight and upstream keepalive budgets, and explicit positive upstream I/O budgets. Listener/metrics validation rejects effective socket-authority overlap: equal sockets, same-family wildcard aliases, native/IPv4-mapped IPv4 aliases, and IPv6-wildcard/IPv4 same-port ambiguity. Distinct concrete non-aliased addresses remain independent. HTTPS upstreams require SNI and certificate/hostname verification; cleartext upstreams must not carry SNI or trust-bundle data. No request may select arbitrary upstream authority dynamically. -Each upstream has a stable non-empty name, a non-zero concrete socket address, `tls`, optional `sni`, optional absolute `trust_bundle_file`, and explicit positive `connection_ms`, `total_connection_ms`, `read_ms`, `write_ms`, and `idle_ms` budgets. TLS upstreams require non-empty SNI. Pingora `HttpPeer` enables certificate and hostname verification. When `trust_bundle_file` is configured, the loaded PEM certificates become that peer's CA store rather than being silently merged with platform roots. Clear-text upstreams may define neither SNI nor a trust bundle. The gateway does not issue, renew, or rotate certificates. +Requests with a parseable `Content-Length` above the configured limit fail with 413 before upstream selection. Streamed body bytes are counted against the same bound. Saturated process admission fails with 503 and the lease is released when the request completes or aborts. -`PgErdMigrationConfig` is a bounded Admin Config contract, not a second generic router. It admits operator-supplied listener/metrics sockets, non-zero runtime budgets, and concrete transport/TLS data only for the already characterized `backend` and `frontend` identities. Route selection and response-security policy remain compiled migration contracts. Missing, extra, renamed, zero-port, or overlapping transport authority fails before listener activation. +## Bounded pg-erd Admin Config -`PgErdMigrationConfig` also derives public Serde `Deserialize`; callers therefore are not forced through `PgErdMigrationConfig::from_yaml`. The public `build_proxy()` activation boundary revalidates the complete deterministic Admin Config contract before delivery peers or runtime limits are materialized. Only after that revalidation may `RuntimeIsolationLimits::from_validated` reuse the proven-positive budgets, so direct deserialization cannot bypass version, listener-authority, runtime, keepalive, or transport-authority invariants. +`PgErdMigrationConfig` is not a generic route language. Operators may provide only the traffic listener, metrics listener, positive body/in-flight/keepalive budgets, and concrete transport/TLS values for the already characterized `backend` and `frontend` identities. Route precedence, response-security fields and admitted upstream names remain compiled migration semantics. Product authentication/authorization, business routing, domain response semantics, Keyverse identity and Wardnet/EgressWeave verdicts remain outside this bounded context. -## Request policy +Configuration validation fails before listener activation on unsupported versions, zero ports or runtime budgets, overlapping traffic/metrics socket authority, zero keepalive capacity, missing/duplicate/extra/renamed upstream authority, or invalid upstream TLS/transport data. The migration profile consumes the same shared effective socket-authority invariant as generic v1 while preserving its characterized zero-transport-authority error surface. -Pingora's standard upstream request policy supplies the pinned supplier's hop-by-hop and `Connection`-nomination sanitation. The generic gateway additionally removes client-provided `Forwarded`, `X-Forwarded-For`, `X-Forwarded-Host`, `X-Forwarded-Port`, `X-Forwarded-Proto`, `X-Forwarded-Server`, and `X-Real-IP`, then emits only gateway-owned `Forwarded: proto=http` for the v1 clear-text downstream listener. Generic v1 deliberately makes no client-IP identity or downstream proxy-provenance claim. +`PgErdMigrationConfig` derives public Serde `Deserialize`, so callers are not forced to enter through `PgErdMigrationConfig::from_yaml`. The public activation boundary therefore revalidates the complete deterministic configuration in `build_proxy()` before delivery peers or runtime limits are materialized. Only after that check may `RuntimeIsolationLimits::from_validated` reuse the proven positive budgets. Direct deserialization cannot bypass version, listener-authority, runtime, keepalive or transport-authority invariants. -The pg-erd migration adapter also discards request-controlled forwarding identity before rebuilding only the characterized compatibility fields from accepted transport/request authority. The current captured Traefik entry point is clear-text, so its forwarded scheme is explicitly `http`; HTTPS requires a separate TLS-derived contract rather than inference. +Custom upstream trust-bundle bytes are not preloaded during YAML parsing. Peer/trust materialization happens once during `build_proxy()` before the composition root creates listeners, reducing validate-then-reload drift for operator-supplied trust material. -Non-health requests acquire the process `max_in_flight_requests` budget before upstream selection and fail closed with HTTP 503 at capacity. Requests with a parseable `Content-Length` above `max_request_body_bytes` fail with HTTP 413 before upstream selection; streamed body bytes are counted and fail with 413 if the same bound is exceeded. Pingora's parser retains its own finite protocol limits, but an operator-controlled smaller HTTP/1 header byte/count budget remains a separate edge-policy gap. +## Request and forwarding policy -Generic v1 makes one prevalidated upstream peer available per request. The pg-erd migration adapter selects only peers already bound by `MigrationDeliveryPlan`; neither path performs request-controlled service discovery. Domain retries, failover, and idempotency policy are not invented by this runtime. +Pingora's standard upstream-request policy handles hop-by-hop and connection-nominated headers. The generic gateway additionally removes request-controlled `Forwarded`, `X-Forwarded-For`, `X-Forwarded-Host`, `X-Forwarded-Port`, `X-Forwarded-Proto`, `X-Forwarded-Server`, and `X-Real-IP`, then emits only gateway-owned `Forwarded: proto=http` for its current cleartext downstream contract. Generic v1 deliberately makes no client-IP identity or downstream proxy-provenance claim. + +The pg-erd migration callback uses the separate Ingress Forwarding Policy. Request-controlled `Forwarded`, `X-Forwarded-*`, `X-Real-IP` and legacy `X-Forwarded-Server` values are discarded. `X-Forwarded-For` and `X-Real-IP` are rebuilt from the accepted client socket, `X-Forwarded-Host` preserves original Host authority, and `X-Forwarded-Port` comes from an explicit Host port or the admitted scheme default rather than the process listener bind. The currently characterized legacy entry point is cleartext `web`, so downstream scheme is explicitly `http`; HTTPS forwarding semantics require a separate downstream-TLS contract. Failure handling is phase-aware. Before an upstream response header is committed downstream, transport failure may still be represented by the gateway's fail-closed error response under the one-attempt policy. After a valid response header has been committed, a later upstream framing/body failure cannot be rewritten into a second HTTP status or silently failed over: the incomplete downstream response terminates, low-cardinality error telemetry records the failed request, process readiness remains available, and independent routes must remain usable. This is an edge transport invariant, not product retry authority. -## Health and observability +## Health, observability and graceful lifecycle -`GET /livez` and `/readyz` return HTTP 200 with an empty, non-cacheable response through the process-local Pingora health boundary. Readiness proves validated configuration plus an active serving path, not product dependency health. In the pg-erd migration profile, consumer `/healthz` remains ordinary routed application traffic and is not confused with process liveness/readiness. +`/livez` and `/readyz` are gateway process endpoints served locally through the production Pingora path and do not become consumer routes. Pg-erd `/healthz` remains characterized product traffic to `backend`. Shared observability is low-cardinality and payload-free: request path/query, headers, cookies, credentials, customer payloads and product identifiers are outside the shared telemetry contract. -The shared process exposes bounded Prometheus counters for request completion, request errors, observed request-body bytes, and backpressure rejection. The canonical gateway log vocabulary records only low-cardinality transport completion facts; authorization headers, cookies, credentials, customer payloads, and unbounded product route labels are outside this shared observability contract. +Both composition roots use the shared Pingora server policy and bounded graceful shutdown. Process tests terminate successful children through the graceful path so LLVM coverage profiles can flush; emergency cleanup remains a test-harness fallback rather than the normal lifecycle. -## Packaging +## Packaging and release boundary -The Docker builder is digest-pinned `rust:1.98.0-bookworm`; the final image is digest-pinned `gcr.io/distroless/base-nossl-debian13:nonroot`. The pinned Pingora OpenSSL path is vendored, so the final image does not carry Debian `libssl`. The Dockerfile exposes only one build-time selector, `CWL_GATEWAY_BIN`, and fail-closes unless its value is exactly `cwl-pingora-gateway` or `cwl-pingora-pg-erd-migration`. The selected executable is normalized to one fixed runtime path before the distroless stage, so a final image contains one admitted process identity rather than both binaries or a runtime shell selector. +The Docker builder is digest-pinned Rust 1.98.0 Bookworm and the final image is digest-pinned distroless Debian 13 `base-nossl` non-root. `CWL_GATEWAY_BIN` is a build-time-only fail-closed allowlist of exactly `cwl-pingora-gateway` and `cwl-pingora-pg-erd-migration`; the selected executable is normalized to one fixed runtime path, so the final image contains one admitted process identity and no runtime shell selector. -Both image profiles run as uid/gid `65532`, have no intentional application writes, and are required to remain compatible with a read-only root filesystem, all capabilities dropped, and `no-new-privileges`. The OCI gate builds the default generic image and an explicit pg-erd image, then independently starts each exact candidate under those restrictions with a read-only configuration mount and requires `/livez` to become reachable. `examples/pg-erd-migration.yaml` is bounded smoke configuration for this process-level OCI proof; it does not substitute for routed origin/load/failure acceptance. Until the exact current head reaches terminal success, the new workflow is source-defined acceptance rather than hosted GREEN evidence. +Exact-head OCI acceptance builds both profiles and starts each as uid/gid `65532` under read-only-root, all-capabilities-dropped and `no-new-privileges` restrictions with a read-only configuration mount. The supply-chain lane builds and vulnerability-scans both candidate images, binds both local image IDs and per-image scan outputs to the exact source SHA, and keeps failure diagnostics distinct from promotion-shaped success evidence. These are unreleased candidate receipts only. -The candidate supply-chain lane builds and vulnerability-scans both admitted images and binds both local image IDs plus per-image scan outputs to the exact source SHA. Its dependency SBOM describes the shared committed Rust dependency graph. A protected release still requires registry-bound immutable image digests, release-bound SBOM/provenance/reproducibility evidence, and rollback rehearsal; no Draft PR head or local image ID is a deployable release identity. +A protected release remains blocked until exact-head CI, strict Clippy, warning-denied rustdoc, 100% owned production line/region coverage, load/runtime and supply-chain evidence are terminal GREEN; protected review/governance is satisfied without bypass; supplier/advisory policy is clean; and an immutable image digest, release-bound SBOM/provenance/reproducibility and rollback evidence exist. Source capability, predecessor GREEN or a mutable image/tag does not establish release, parity, shadow, canary, cutover or legacy-removal state. ## Protocol and migration limits -Generic v1 is a clear-text downstream HTTP proxy with one explicit upstream per process. Downstream TLS termination, HTTP/2 admission, H2→H1 Cookie normalization, HTTP/3/QUIC, WebSocket/Extended CONNECT, dynamic reload, Kubernetes Gateway API, and consumer-specific multi-route behavior are versioned increments with separate realistic RED→GREEN evidence. +Generic v1 remains a cleartext downstream HTTP proxy with one explicit upstream per process. Downstream TLS termination, HTTP/2 admission, H2-to-H1 Cookie normalization, HTTP/3/QUIC, WebSocket/Extended CONNECT, dynamic reload, Kubernetes Gateway API, and consumer-specific multi-route behavior are versioned increments with separate realistic RED-to-GREEN evidence. -The concrete pg-erd migration stack is a bounded consumer-characterization adapter and does not widen generic v1. Source presence is not parity. Promotion still requires unchanged exact-head formatting, compile/test, strict Clippy, rustdoc, owned-production coverage, routed traffic/load/failure evidence, terminal dedicated OCI/supply-chain execution, immutable release identity, consumer deployment pin, shadow/canary, rollback rehearsal, protected cutover, and verified legacy removal. +The pg-erd migration stack is a bounded consumer-characterization adapter and does not widen generic v1. Promotion still requires unchanged exact-head formatting, compile/test, strict Clippy, rustdoc, owned-production coverage, routed traffic/load/failure evidence, immutable release identity, consumer deployment pin, shadow/canary, rollback rehearsal, protected cutover, and verified legacy removal. From 490030bf3f092fe6ea9ab33bab4482a322c2d4b3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 09:58:37 +0900 Subject: [PATCH 39/43] docs(partial): preserve current traceability and add failure-phase sources --- docs/doctoring/TRACEABILITY.md | 59 +++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/docs/doctoring/TRACEABILITY.md b/docs/doctoring/TRACEABILITY.md index c82fb172..300a57ed 100644 --- a/docs/doctoring/TRACEABILITY.md +++ b/docs/doctoring/TRACEABILITY.md @@ -1,41 +1,44 @@ # Primary-Source and APA-7 Traceability -This file links material technical/security claims to primary standards or upstream sources. Revalidate version/advisory claims immediately before release. +This file links material technical/security claims to primary standards or upstream sources. Revalidate version, branch, release, and advisory claims immediately before promotion or release. Repository observations in this revision were taken on 2026-09-13 KST (2026-09-12 UTC); the explicit timezone prevents the local-date evidence from being mistaken for a future UTC observation. | Claim | Source | | --- | --- | -| Pingora server/proxy composition and graceful server lifecycle | Cloudflare Pingora source at pinned commit `09696b51bc59315353d96686355861604d0bb48c`, the protected upstream `main` head observed on 2026-09-01 | -| Pingora downstream sessions expose accepted client/server socket addresses; Pingora socket addresses expose IP socket values through `as_inet()` | `pingora-core/src/protocols/http/server.rs`, `pingora-proxy/tests/utils/server_utils.rs`, and `pingora-core/src/protocols/l4/socket.rs` at pinned commit `09696b51bc59315353d96686355861604d0bb48c`; these are the transport observations used by the pg-erd forwarding adapter | -| Pingora's server default `max_retries` is 16, while the proxy loop copies that field and loops while its attempt counter is below the value | `pingora-core/src/server/configuration/mod.rs` and `pingora-proxy/src/lib.rs` at pinned commit `09696b51bc59315353d96686355861604d0bb48c`; CWL v1 therefore sets the field to `1` for one total attempt | -| Graceful SIGTERM uses `grace_period_seconds` and `graceful_shutdown_timeout_seconds`, with framework fallbacks when unset | `pingora-core/src/server/mod.rs` and `pingora-core/src/server/configuration/mod.rs` at the pinned commit; CWL v1 sets 5 s grace and 10 s per-runtime graceful timeout explicitly inside a 30 s external termination budget | -| Standard upstream request policy supports hop-by-hop/connection-nominated stripping and normalized WebSocket-only HTTP/1 upgrade forwarding | Cloudflare Pingora `HttpUpstreamRequestPolicy` / peer implementation at pinned commit `09696b51bc59315353d96686355861604d0bb48c` | -| Pingora `read_timeout` is a per-individual-read inactivity budget and resets after each successful upstream `read()`; it is not a total-response lifetime bound | Cloudflare Pingora `docs/user_guide/peer.md` and `pingora-proxy/src/proxy_h1.rs` at pinned commit `09696b51bc59315353d96686355861604d0bb48c`; the pg-erd read-stall acceptance therefore keeps a connected origin silent without closing its socket and deliberately does not claim slow-drip/whole-response bounding | -| A proxy failure after the upstream response header has already been sent downstream cannot be replaced with a new error response or failover | Cloudflare Pingora `docs/user_guide/failover.md` and `pingora-proxy/src/proxy_h1.rs` at pinned commit `09696b51bc59315353d96686355861604d0bb48c`; the pg-erd partial-response fixture therefore observes the committed status/body prefix before causing origin termination and requires the existing downstream response to terminate rather than become a second response | -| Pingora HTTP/1 body framing treats a body that ends before its declared `Content-Length` as a premature body-end failure, and upstream read failures propagate through the HTTP/1 client/proxy path | `pingora-core/src/protocols/http/v1/body.rs`, `pingora-core/src/protocols/http/v1/client.rs`, and `pingora-proxy/src/proxy_h1.rs` at pinned commit `09696b51bc59315353d96686355861604d0bb48c`; RFC 9112 message-framing requirements | -| Pingora's Prometheus HTTP application sets `Content-Type` from `prometheus::TextEncoder::format_type()`; the pinned `prometheus` 0.14 line defines that format as `text/plain; version=0.0.4` | Cloudflare `pingora-prometheus/src/lib.rs` at pinned Pingora commit `09696b51bc59315353d96686355861604d0bb48c` and TiKV `rust-prometheus` v0.14.0 commit `e07efb4f372f1245bf7410b71e822c69877bcb32`, `src/encoder/text.rs`; #19 therefore strips only optional semicolon parameters and requires exact base media type `text/plain` rather than a prefix wildcard | -| Pingora OpenSSL peers support a per-peer CA store; when configured it replaces the verification store for that peer while certificate and hostname verification remain separately enabled | `pingora-core/src/upstreams/peer.rs`, `pingora-core/src/connectors/tls/boringssl_openssl/mod.rs`, and `pingora-core/src/protocols/tls/boringssl_openssl/mod.rs` at pinned commit `09696b51bc59315353d96686355861604d0bb48c` | -| IPv4-mapped IPv6 addresses represent IPv4 nodes in IPv6 form; Rust's `Ipv6Addr::to_ipv4_mapped` performs the bounded mapped-only canonicalization used by the socket-authority invariant. Linux IPv6 sockets can expose IPv4 peers as mapped IPv6 addresses, so mapped/native authority must not be compared only as unrelated textual address families | RFC 4291 §2.5.5.2; Rust `std::net::Ipv6Addr` documentation; Linux `ipv6(7)` | -| Traefik normally adds `X-Forwarded-For`, `X-Real-Ip`, `X-Forwarded-Host`, `X-Forwarded-Port`, `X-Forwarded-Proto`, and `X-Forwarded-Server` when proxying HTTP | Traefik official Getting Started FAQ, current documentation revalidated 2026-09-02 | -| Incoming Traefik `X-Forwarded-*` identity is trusted only when an EntryPoint explicitly configures trusted IPs or insecure trust; insecure mode is not recommended for production | Traefik official EntryPoints documentation, current documentation revalidated 2026-09-02 | +| The current gateway stack is compiled against exact Pingora source `09696b51bc59315353d96686355861604d0bb48c`; that SHA is a candidate source pin, not current upstream protected-branch authority | `Cargo.toml`/`Cargo.lock` in the current gateway stack plus Cloudflare Pingora source at commit `09696b51bc59315353d96686355861604d0bb48c` | +| Cloudflare Pingora protected `main` observed on 2026-09-13 KST (2026-09-12 UTC) is `4487f7b2ab50f159e4a2cf4f6a6b813f61bb6e19`; the workspace at that exact still declares `derivative = "2.2.0"` | Cloudflare Pingora GitHub branch API and root `Cargo.toml` at `4487f7b2ab50f159e4a2cf4f6a6b813f61bb6e19` | +| Pingora 0.9.0 is the latest published release observed on 2026-09-13 KST (2026-09-12 UTC); GitHub published it on 2026-09-09 and its release notes label the release 2026-09-04. The release is not immutable and has no attached assets, so recency alone is not CWL release authority | Cloudflare Pingora GitHub Release `0.9.0` | +| Pingora server/proxy composition and graceful server lifecycle used by the current candidate | Cloudflare Pingora source at pinned commit `09696b51bc59315353d96686355861604d0bb48c` | +| Pingora downstream sessions expose the accepted client socket address. The pg-erd forwarding adapter combines that client IP with the original request Host authority; it deliberately does not use the process listener socket as external `X-Forwarded-Port` authority because container/Service/NAT publishing can rewrite that port | `pingora-core/src/protocols/http/server.rs`, `pingora-core/src/protocols/l4/socket.rs`, and `pingora-proxy/src/lib.rs` at candidate commit `09696b51bc59315353d96686355861604d0bb48c`; RFC 9110 authority semantics | +| Pingora's default `fail_to_proxy` preserves explicit `HTTPStatus`, maps upstream failures to 502, non-dead downstream failures to 400, internal/unset failures to 500, suppresses a response for dead downstream I/O, and defaults downstream reuse to false. The pg-erd adapter mirrors that status/reuse mapping while applying its characterized response-header policy to writable local error responses | `pingora-proxy/src/proxy_trait.rs` and `pingora-core/src/protocols/http/server.rs` at candidate commit `09696b51bc59315353d96686355861604d0bb48c` | +| Pingora's server default `max_retries` is 16, while the proxy loop copies that field and loops while its attempt counter is below the value | `pingora-core/src/server/configuration/mod.rs` and `pingora-proxy/src/lib.rs` at candidate commit `09696b51bc59315353d96686355861604d0bb48c`; CWL v1 sets the field to `1` for one total attempt | +| Graceful SIGTERM uses `grace_period_seconds` and `graceful_shutdown_timeout_seconds`, with framework fallbacks when unset | `pingora-core/src/server/mod.rs` and `pingora-core/src/server/configuration/mod.rs` at the candidate commit; CWL v1 sets 5 s grace and 10 s per-runtime graceful timeout explicitly inside a 30 s external termination budget | +| Standard upstream request policy supports hop-by-hop/connection-nominated stripping and normalized WebSocket-only HTTP/1 upgrade forwarding | Cloudflare Pingora `HttpUpstreamRequestPolicy` / peer implementation at candidate commit `09696b51bc59315353d96686355861604d0bb48c` | +| Pingora `read_timeout` is a per-individual-read inactivity budget and resets after each successful upstream `read()`; it is not a total-response lifetime bound | Cloudflare Pingora `docs/user_guide/peer.md` and `pingora-proxy/src/proxy_h1.rs` at candidate commit `09696b51bc59315353d96686355861604d0bb48c`; the pg-erd read-stall acceptance therefore keeps a connected origin silent without closing its socket and deliberately does not claim slow-drip/whole-response bounding | +| A proxy failure after the upstream response header has already been committed downstream cannot be represented as a new second status or silently treated as failover | Cloudflare Pingora `docs/user_guide/failover.md` and `pingora-proxy/src/proxy_h1.rs` at candidate commit `09696b51bc59315353d96686355861604d0bb48c`; the pg-erd partial-response fixture observes the committed status/body prefix before origin termination and requires the existing downstream response to terminate incomplete | +| Pingora HTTP/1 body framing treats a body ending before its declared `Content-Length` as premature body termination, and upstream read failures propagate through the HTTP/1 client/proxy path | `pingora-core/src/protocols/http/v1/body.rs`, `pingora-core/src/protocols/http/v1/client.rs`, and `pingora-proxy/src/proxy_h1.rs` at candidate commit `09696b51bc59315353d96686355861604d0bb48c`; RFC 9112 message-framing requirements | +| Pingora's Prometheus HTTP application sets `Content-Type` from `prometheus::TextEncoder::format_type()`; the pinned `prometheus` 0.14 line defines that format as `text/plain; version=0.0.4` | Cloudflare `pingora-prometheus/src/lib.rs` at candidate commit `09696b51bc59315353d96686355861604d0bb48c` and TiKV `rust-prometheus` v0.14.0 commit `e07efb4f372f1245bf7410b71e822c69877bcb32`, `src/encoder/text.rs`; the OCI metrics acceptance strips only optional semicolon parameters and requires exact base media type `text/plain` rather than a prefix wildcard | +| Pingora OpenSSL peers support a per-peer CA store; when configured it replaces the verification store for that peer while certificate and hostname verification remain separately enabled | `pingora-core/src/upstreams/peer.rs`, `pingora-core/src/connectors/tls/boringssl_openssl/mod.rs`, and `pingora-core/src/protocols/tls/boringssl_openssl/mod.rs` at candidate commit `09696b51bc59315353d96686355861604d0bb48c` | +| IPv4-mapped IPv6 addresses represent IPv4 nodes in IPv6 form; Rust `Ipv6Addr::to_ipv4_mapped` provides the mapped-only canonicalization used by the socket-authority invariant. Linux IPv6 sockets can expose IPv4 peers as mapped IPv6 addresses, so mapped/native authority cannot be treated as unrelated textual families | RFC 4291 §2.5.5.2; Rust `std::net::Ipv6Addr` documentation; Linux `ipv6(7)` | +| Traefik normally adds `X-Forwarded-For`, `X-Real-Ip`, `X-Forwarded-Host`, `X-Forwarded-Port`, `X-Forwarded-Proto`, and `X-Forwarded-Server` when proxying HTTP | Traefik official Getting Started FAQ, revalidated 2026-09-13 KST (2026-09-12 UTC) | +| Incoming Traefik `X-Forwarded-*` identity is trusted only when an EntryPoint explicitly configures trusted IPs or insecure trust; insecure mode is not recommended for production | Traefik official EntryPoints documentation, revalidated 2026-09-13 KST (2026-09-12 UTC) | | `pg-erd-cloud` can use `X-Forwarded-For` for rate-limit/observability client identity only under an explicit trust switch and tells operators to enable it only behind a sanitizing ingress | `ContextualWisdomLab/pg-erd-cloud@8dc746920c12988f082e914879d95e13c9693535`: `.env.example`, `backend/app/rate_limit.py`, `backend/app/observability.py`, `docs/api-security-checklist.md` | | Forwarded-header grammar and trust semantics | RFC 7239 | -| HTTP semantics | RFC 9110 | +| HTTP semantics and field-content requirements | RFC 9110 | | HTTP/1.1 message framing/hop-by-hop requirements | RFC 9112 | | HTTP/2 framing and connection semantics | RFC 9113 | -| HTTP/3 semantics over QUIC | RFC 9114; HTTP/3 is not claimed implemented by this v1 candidate until executable listener/interoperability evidence exists | +| HTTP/3 semantics over QUIC | RFC 9114; HTTP/3 is not claimed implemented by this candidate until executable listener/interoperability evidence exists | | Current TLS 1.3 protocol semantics and application identity-verification responsibility | RFC 9846, published July 2026, which obsoletes RFC 8446 and points applications to RFC 9525 for identity verification | | New protocols using TLS must require TLS 1.3 | RFC 9852, BCP 195, July 2026; this gateway is not claiming a new application protocol and still requires explicit migration-time protocol compatibility evidence | | March 2026 Pingora request-smuggling/cache-key advisories are patched in 0.8.0 | GitHub Security Advisories GHSA-xq2h-p299-vjwv, GHSA-hj7x-879w-vrp7, GHSA-f93w-pcj3-rggc | -| Pingora 0.8.1 is the latest release observed on 2026-09-01 and bounds default HTTP/2 server limits | Cloudflare Pingora GitHub Releases, 0.8.1, 2026-06-04 | -| The pinned upstream head is seven commits after the prior security-resolution pin `6463ad6407a1d3fe256f1951dd0ecb054477e3f6`; the relevant retry/grace configuration remains unchanged at the new head | GitHub compare `6463ad6...09696b5` plus the exact `ServerConf` source at `09696b5` | -| Rust 1.98.1 is the current stable point release observed on 2026-09-07 and fixes a Rust 1.98.0 vtable-generation miscompilation that could place a null pointer where a trait-object function pointer was required | Rust Release Team, Rust 1.98.1 announcement, 2026-09-03; this external authority does not by itself promote the repository's separately gated compiler prerequisite | -| OCI runtime-spec 1.3.0 is the latest released runtime specification observed on 2026-09-07 | Open Container Initiative runtime-spec v1.3.0 release notice, 2025-11-04; runtime hardening claims still require executable container evidence | -| OCI image-spec 1.1.1 is the latest released image specification observed on 2026-09-07 | Open Container Initiative image-spec v1.1.1 release notice, 2025-04-02; image-format conformance does not prove non-root, read-only-root, capability or `no-new-privileges` runtime behavior, which remains an executable acceptance concern | -| `lru` versions before 0.18.2 are affected by RUSTSEC-2026-0253 | RustSec advisory RUSTSEC-2026-0253; the upstream pin includes the first-fixed `lru` dependency change, but release must use a committed audited lock | +| `derivative 2.2.0` is unmaintained under RUSTSEC-2024-0388 and remains present in current upstream protected `main`; supplier promotion therefore remains fail-closed until a maintainer-integrated, release-qualified disposition exists | RustSec RUSTSEC-2024-0388; Cloudflare Pingora issue #889; Cloudflare `Cargo.toml@4487f7b2ab50f159e4a2cf4f6a6b813f61bb6e19` | +| The current gateway CI stack still runs Rust 1.98.0, while Rust 1.98.1 is the latest stable release observed on 2026-09-13 KST (2026-09-12 UTC) and fixes a vtable-generation miscompilation in 1.98.0 that could emit undefined behavior | Rust Release Team, Rust 1.98.1 announcement, 2026-09-03; compiler promotion is owned by gateway PR #56 and is not silently folded into this migration callback slice | +| OCI runtime-spec 1.3.0 is the latest released runtime specification observed in the current gateway documentation | Open Container Initiative runtime-spec v1.3.0 release notice, 2025-11-04; runtime hardening claims still require executable container evidence | +| OCI image-spec 1.1.1 is the latest released image specification observed in the current gateway documentation | Open Container Initiative image-spec v1.1.1 release notice, 2025-04-02; image-format conformance does not prove non-root/read-only-root/capability/no-new-privileges runtime behavior | +| `lru` versions before 0.18.2 are affected by RUSTSEC-2026-0253 | RustSec advisory RUSTSEC-2026-0253; supplier promotion must use a committed audited lock rather than infer safety from a moving upstream branch | ## References -Cloudflare. (2026, June 4). *Pingora 0.8.1*. GitHub. https://github.com/cloudflare/pingora/releases/tag/0.8.1 +Cloudflare. (2026, September 9). *Pingora 0.9.0*. GitHub. https://github.com/cloudflare/pingora/releases/tag/0.9.0 Cloudflare. (n.d.). *Pingora upstream peer options* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/upstreams/peer.rs @@ -49,20 +52,22 @@ Cloudflare. (n.d.). *Pingora HTTP/1 client session* [Source code, commit 09696b5 Cloudflare. (n.d.). *Pingora HTTP/1 body framing* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/protocols/http/v1/body.rs -Cloudflare. (n.d.). *Pingora Prometheus HTTP application* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-prometheus/src/lib.rs - Cloudflare. (n.d.). *Pingora OpenSSL upstream TLS connector* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/connectors/tls/boringssl_openssl/mod.rs Cloudflare. (n.d.). *Pingora downstream HTTP session* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/protocols/http/server.rs Cloudflare. (n.d.). *Pingora L4 socket address* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/protocols/l4/socket.rs +Cloudflare. (n.d.). *Pingora proxy failure policy* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-proxy/src/proxy_trait.rs + Cloudflare. (n.d.). *Pingora server configuration* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/server/configuration/mod.rs Cloudflare. (n.d.). *Pingora server lifecycle* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/server/mod.rs Cloudflare. (n.d.). *Pingora proxy implementation* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-proxy/src/lib.rs +Cloudflare. (n.d.). *Pingora Prometheus HTTP application* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-prometheus/src/lib.rs + Cloudflare. (2026). *HTTP request smuggling via premature upgrade* (GHSA-xq2h-p299-vjwv). GitHub Security Advisories. https://github.com/cloudflare/pingora/security/advisories/GHSA-xq2h-p299-vjwv Cloudflare. (2026). *HTTP request smuggling via HTTP/1.0 and Transfer-Encoding misparsing* (GHSA-hj7x-879w-vrp7). GitHub Security Advisories. https://github.com/cloudflare/pingora/security/advisories/GHSA-hj7x-879w-vrp7 @@ -99,6 +104,8 @@ Open Container Initiative. (2025, April 2). *OCI image-spec v1.1.1 release notic Open Container Initiative. (2025, November 4). *OCI runtime-spec v1.3.0 release notice*. https://opencontainers.org/release-notices/v1-3-0-runtime-spec/ -Rust Release Team. (2026, September 3). *Announcing Rust 1.98.1*. Rust Blog. https://blog.rust-lang.org/2026/09/03/Rust-1.98.1/ +Rust Release Team. (2026, September 3). *Announcing Rust 1.98.1*. Rust Blog. https://blog.rust-lang.org/releases/1.98.1/ + +Rust Secure Code Working Group. (2024). *RUSTSEC-2024-0388: derivative is unmaintained*. RustSec Advisory Database. https://rustsec.org/advisories/RUSTSEC-2024-0388.html Rust Secure Code Working Group. (2026, August 11). *RUSTSEC-2026-0253: lru—memory safety issue under panic*. RustSec Advisory Database. https://rustsec.org/advisories/RUSTSEC-2026-0253.html From 62eadc936ca7092f28dfa5f1a80b417fd1aa1841 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 10:01:37 +0900 Subject: [PATCH 40/43] docs(partial): return traceability to dedicated writer --- docs/doctoring/TRACEABILITY.md | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/docs/doctoring/TRACEABILITY.md b/docs/doctoring/TRACEABILITY.md index 300a57ed..1eaab2fa 100644 --- a/docs/doctoring/TRACEABILITY.md +++ b/docs/doctoring/TRACEABILITY.md @@ -14,8 +14,6 @@ This file links material technical/security claims to primary standards or upstr | Graceful SIGTERM uses `grace_period_seconds` and `graceful_shutdown_timeout_seconds`, with framework fallbacks when unset | `pingora-core/src/server/mod.rs` and `pingora-core/src/server/configuration/mod.rs` at the candidate commit; CWL v1 sets 5 s grace and 10 s per-runtime graceful timeout explicitly inside a 30 s external termination budget | | Standard upstream request policy supports hop-by-hop/connection-nominated stripping and normalized WebSocket-only HTTP/1 upgrade forwarding | Cloudflare Pingora `HttpUpstreamRequestPolicy` / peer implementation at candidate commit `09696b51bc59315353d96686355861604d0bb48c` | | Pingora `read_timeout` is a per-individual-read inactivity budget and resets after each successful upstream `read()`; it is not a total-response lifetime bound | Cloudflare Pingora `docs/user_guide/peer.md` and `pingora-proxy/src/proxy_h1.rs` at candidate commit `09696b51bc59315353d96686355861604d0bb48c`; the pg-erd read-stall acceptance therefore keeps a connected origin silent without closing its socket and deliberately does not claim slow-drip/whole-response bounding | -| A proxy failure after the upstream response header has already been committed downstream cannot be represented as a new second status or silently treated as failover | Cloudflare Pingora `docs/user_guide/failover.md` and `pingora-proxy/src/proxy_h1.rs` at candidate commit `09696b51bc59315353d96686355861604d0bb48c`; the pg-erd partial-response fixture observes the committed status/body prefix before origin termination and requires the existing downstream response to terminate incomplete | -| Pingora HTTP/1 body framing treats a body ending before its declared `Content-Length` as premature body termination, and upstream read failures propagate through the HTTP/1 client/proxy path | `pingora-core/src/protocols/http/v1/body.rs`, `pingora-core/src/protocols/http/v1/client.rs`, and `pingora-proxy/src/proxy_h1.rs` at candidate commit `09696b51bc59315353d96686355861604d0bb48c`; RFC 9112 message-framing requirements | | Pingora's Prometheus HTTP application sets `Content-Type` from `prometheus::TextEncoder::format_type()`; the pinned `prometheus` 0.14 line defines that format as `text/plain; version=0.0.4` | Cloudflare `pingora-prometheus/src/lib.rs` at candidate commit `09696b51bc59315353d96686355861604d0bb48c` and TiKV `rust-prometheus` v0.14.0 commit `e07efb4f372f1245bf7410b71e822c69877bcb32`, `src/encoder/text.rs`; the OCI metrics acceptance strips only optional semicolon parameters and requires exact base media type `text/plain` rather than a prefix wildcard | | Pingora OpenSSL peers support a per-peer CA store; when configured it replaces the verification store for that peer while certificate and hostname verification remain separately enabled | `pingora-core/src/upstreams/peer.rs`, `pingora-core/src/connectors/tls/boringssl_openssl/mod.rs`, and `pingora-core/src/protocols/tls/boringssl_openssl/mod.rs` at candidate commit `09696b51bc59315353d96686355861604d0bb48c` | | IPv4-mapped IPv6 addresses represent IPv4 nodes in IPv6 form; Rust `Ipv6Addr::to_ipv4_mapped` provides the mapped-only canonicalization used by the socket-authority invariant. Linux IPv6 sockets can expose IPv4 peers as mapped IPv6 addresses, so mapped/native authority cannot be treated as unrelated textual families | RFC 4291 §2.5.5.2; Rust `std::net::Ipv6Addr` documentation; Linux `ipv6(7)` | @@ -42,16 +40,6 @@ Cloudflare. (2026, September 9). *Pingora 0.9.0*. GitHub. https://github.com/clo Cloudflare. (n.d.). *Pingora upstream peer options* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/upstreams/peer.rs -Cloudflare. (n.d.). *Peer: how to connect to upstream* [Documentation, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/docs/user_guide/peer.md - -Cloudflare. (n.d.). *Handling failures and failover* [Documentation, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/docs/user_guide/failover.md - -Cloudflare. (n.d.). *Pingora HTTP/1 proxy implementation* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-proxy/src/proxy_h1.rs - -Cloudflare. (n.d.). *Pingora HTTP/1 client session* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/protocols/http/v1/client.rs - -Cloudflare. (n.d.). *Pingora HTTP/1 body framing* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/protocols/http/v1/body.rs - Cloudflare. (n.d.). *Pingora OpenSSL upstream TLS connector* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/connectors/tls/boringssl_openssl/mod.rs Cloudflare. (n.d.). *Pingora downstream HTTP session* [Source code, commit 09696b51bc59315353d96686355861604d0bb48c]. GitHub. https://github.com/cloudflare/pingora/blob/09696b51bc59315353d96686355861604d0bb48c/pingora-core/src/protocols/http/server.rs @@ -108,4 +96,4 @@ Rust Release Team. (2026, September 3). *Announcing Rust 1.98.1*. Rust Blog. htt Rust Secure Code Working Group. (2024). *RUSTSEC-2024-0388: derivative is unmaintained*. RustSec Advisory Database. https://rustsec.org/advisories/RUSTSEC-2024-0388.html -Rust Secure Code Working Group. (2026, August 11). *RUSTSEC-2026-0253: lru—memory safety issue under panic*. RustSec Advisory Database. https://rustsec.org/advisories/RUSTSEC-2026-0253.html +Rust Secure Code Working Group. (2026, August 11). *RUSTSEC-2026-0253: lru—memory safety issue under panic*. RustSec Advisory Database. https://rustsec.org/advisories/RUSTSEC-2026-0253.html \ No newline at end of file From e13a3775b03549ca029f3a080e2c53a6a95c51f9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 13:22:25 +0900 Subject: [PATCH 41/43] test(partial): restore post-commit truncation contract --- CHANGELOG.md | 9 +- TEST_STRATEGY.md | 8 +- TRD.md | 2 + tests/pg_erd_partial_response_traffic.rs | 438 +++++++++++++++++++++++ 4 files changed, 450 insertions(+), 7 deletions(-) create mode 100644 tests/pg_erd_partial_response_traffic.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 14fc28ea..18264ccf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,8 +22,9 @@ All notable changes are tracked here. No release has been published yet. - Added process-local fail-fast backpressure: non-health requests above the in-flight budget receive HTTP 503, health remains observable, rejection telemetry increments, and capacity is released after request completion or failure. - Added dedicated compiled pg-erd traffic acceptance for streamed/chunked body overflow and routed in-flight saturation/recovery: the migration process must return 413 above the shared body budget, return 503 in less than one second above the in-flight budget, keep `/readyz` observable, expose the exact single-rejection Prometheus sample, and admit a later routed request after capacity is released. - Added dedicated compiled pg-erd refused-origin recovery acceptance using a Linux TCP socket bound to the characterized backend address without entering LISTEN state. The fixture first proves direct `ECONNREFUSED` while retaining exclusive port ownership, then requires the migration gateway to return 502 within a conservative one-second envelope around the configured 200/400 ms connection budgets, keep `/readyz` 200, expose the exact single-error Prometheus sample, and allow a later independent frontend route to recover. Connected read stall, TCP reset, partial-response/streaming failure, retry and failover behavior remain separate gaps. -- Added dedicated compiled pg-erd connected read-stall acceptance with `read_ms=100`: the backend accepts the routed request, completes bounded request-header receipt, records that causal point, and remains open without response bytes until the gateway has already failed it. The contract requires 502 no earlier than a conservative 50 ms lower bound from completed origin headers and still inside a one-second outer envelope, preserves `/readyz`, requires the exact single-error Prometheus sample, and proves independent frontend recovery. Pingora `read_timeout` remains a per-read inactivity budget, not a whole-response lifetime; reset, post-commit partial response, slow-drip and whole-response-deadline cases remain open. -- Added dedicated routed pg-erd graceful-drain acceptance: a characterized `/api/held` backend request is held in flight, backend request-header receipt must complete within five seconds and 64 KiB before the request is considered admitted, SIGTERM is sent only after that causal point, the response is released during the shared grace period, the downstream must still receive HTTP 200, and the migration process must exit successfully inside the single absolute external termination budget anchored at signal delivery. The fixture retains traffic and metrics socket reservations through config construction, releases them only at child-bind handoff, and admits the drain case only after bounded `/readyz` HTTP/1.1 200 readiness; generic drain evidence is not transferred to this composition root. +- Added dedicated compiled pg-erd connected read-stall acceptance with `read_ms=100`: the backend accepts the routed request and remains open without response bytes until the gateway has already failed it, preventing fixture closure from faking timeout behavior. The contract requires 502 inside a conservative one-second envelope, preserved `/readyz`, the exact single-error Prometheus sample, and independent frontend recovery. Pingora `read_timeout` remains a per-read inactivity budget, not a whole-response lifetime; reset, partial-response and slow-drip cases remain open. +- Added dedicated routed pg-erd graceful-drain acceptance: a characterized `/api/held` backend request is held in flight, SIGTERM is sent only after the backend has accepted it, the response is released during the shared grace period, the downstream must still receive HTTP 200, and the migration process must exit successfully inside the absolute external termination budget. The fixture retains traffic and metrics socket reservations through config construction, releases them only at child-bind handoff, and admits the drain case only after bounded `/readyz` HTTP/1.1 200 readiness; generic drain evidence is not transferred to this composition root. +- Added dedicated compiled pg-erd post-header truncation acceptance: the backend commits exact HTTP/1.1 200 framing with one `Content-Length: 20` field and the seven-byte `partial` prefix, remains open until downstream commit evidence is observed, then closes. The fixture requires the committed response to terminate incomplete without a second status or failover, records exactly one request error, preserves `/readyz`, and proves an independent frontend route still works. Status parsing now rejects protocol/prefix lookalikes, origin request-header reads are time/size bounded, and content-length parsing rejects lookalike or duplicate/conflicting fields. - Added optional per-upstream absolute PEM trust-bundle consumption without taking ownership of certificate issuance/rotation; trust material is loaded fail-closed before listeners open. - Added an executable local-CA TLS test through the compiled gateway that holds CA trust constant and proves SNI/hostname mismatch is rejected. - Added a focused transport-adapter regression proving an upstream without a custom trust bundle leaves Pingora's platform trust roots selected rather than replacing the CA store. @@ -33,11 +34,11 @@ All notable changes are tracked here. No release has been published yet. - Added low-cardinality metrics plus credential/cookie-safe access logging through the production path. - Overrode Pingora framework retry/drain defaults with one total upstream attempt, a 5-second SIGTERM grace period, and a 30-second graceful-shutdown timeout. - Added non-root/read-only-root OCI packaging with a fail-closed build-time allowlist for the generic and bounded pg-erd process identities; exact-head OCI acceptance builds and starts both profiles under uid/gid 65532, dropped capabilities and `no-new-privileges`, while the supply-chain lane builds and vulnerability-scans both candidate images. -- Extended pg-erd OCI acceptance so promotion also requires the separately published `/metrics` listener to identify the Prometheus service by exact base media type `text/plain`; a bare HTTP 200 or prefix-wildcard media-type match is insufficient. +- Extended the pg-erd OCI acceptance so the least-privilege migration container is not accepted until process `/livez` responds and the separately published `/metrics` listener identifies the Pingora Prometheus service through exact base media type `text/plain` after stripping only optional semicolon parameters. A bare HTTP 200 or prefix-wildcard media-type match is insufficient. - Added a committed dependency lock, fail-closed license/source/advisory policy, exact-source SBOM and image-vulnerability evidence. - Added an exact-head owned-production coverage gate that requires 100% lines and regions without filename/function/branch exclusions; repaired compiler-generated generic startup coverage and structurally impossible literal-header error regions rather than weakening the gate. - Added missing-public-rustdoc enforcement and documentation builds with warnings denied. - Added a load-workflow contract that proves the measured loopback origin is ready before gateway startup so fixture races cannot be counted as gateway latency or availability behavior, and separately preserves the primary failure when k6 never produces a summary. - Added DDD, product, technical, security, threat, test, operability, configuration, migration-gap, and primary-source traceability documentation. -Release remains blocked on a maintainer-integrated and release-qualified disposition of unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388`, the exact Pingora supplier/protocol gates tracked by the foundation stack, the Rust compiler promotion owned by #56, terminal exact-current CI/supply-chain/security/review evidence, central required-workflow convergence and independent approval, representative pg-erd routed concurrency/origin-capacity/network-failure/drain and benchmark evidence, an immutable package/image identity with SBOM/provenance/reproducibility and rehearsed rollback, and protected-branch integration. No consumer migration, shadow/canary, cutover, or legacy removal is claimed before those release and traffic-contract gates are satisfied. +Release remains blocked on a maintainer-integrated and release-qualified disposition of unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388`, the exact Pingora supplier/protocol gates tracked by the foundation stack, the Rust compiler promotion owned by #56, terminal exact-current CI/supply-chain/security/review evidence, central required-workflow convergence and independent approval, representative pg-erd routed concurrency/origin-capacity/network-failure/drain and benchmark evidence, an immutable package/image identity with SBOM/provenance/reproducibility and rehearsed rollback, and protected-branch integration. No consumer migration, shadow/canary, cutover, or legacy removal is claimed before those release and traffic-contract gates are satisfied. \ No newline at end of file diff --git a/TEST_STRATEGY.md b/TEST_STRATEGY.md index 704c4340..cc9d6235 100644 --- a/TEST_STRATEGY.md +++ b/TEST_STRATEGY.md @@ -14,9 +14,11 @@ The bounded Admin Config transition has its own executable contract. `tests/pg_e `tests/pg_erd_upstream_failure_traffic.rs` adds the next distinct failure phase through the dedicated compiled process. On Linux it binds the characterized backend TCP address without calling `listen(2)`, so the test retains exclusive port ownership while connection attempts receive `ECONNREFUSED`; a direct `TcpStream::connect_timeout` precondition must observe `ConnectionRefused` before gateway traffic begins. If another process has stolen the selected port, fixture setup fails instead of allowing false-GREEN evidence. The migration gateway must then return HTTP 502 within a conservative one-second outer envelope around the configured 200 ms connection / 400 ms total-connection budgets, keep `/readyz` at HTTP 200, expose the exact Prometheus sample `cwl_pingora_gateway_request_errors_total 1`, and route an independent fallback request successfully to `frontend`. Exact-line matching prevents values such as `10` or `11` from false-passing the single-error contract. This contract does not claim connected read-stall, TCP reset, post-commit truncation, slow-drip/whole-response lifetime, retry, or failover behavior. -`tests/pg_erd_read_stall_traffic.rs` separates connected upstream inactivity from refusal. The characterized backend accepts `/api/read-stall`, reads request headers under a five-second/64 KiB fixture bound, records that causal point, then remains connected and sends no response bytes until the test explicitly releases it after the gateway has already returned. Traffic and metrics loopback sockets remain reserved through config construction and are released only at the child-bind handoff. With `read_ms=100`, the gateway must fail as HTTP 502 no earlier than a conservative 50 ms lower bound measured from completed origin request-header receipt and still inside the one-second outer envelope, preserve `/readyz`, expose the exact Prometheus sample `cwl_pingora_gateway_request_errors_total 1`, and leave an independent `frontend` route usable. Keeping the fixture connection open prevents origin closure from masquerading as the timeout, while the lower bound prevents an unrelated immediate 502 from masquerading as the configured read-inactivity path. Because Pingora's `read_timeout` is per successful `read()` rather than a whole-response lifetime, reset, post-commit partial response, slow-drip and whole-response deadline behavior remain separate contracts. +`tests/pg_erd_read_stall_traffic.rs` separates connected upstream inactivity from refusal. The characterized backend accepts `/api/read-stall`, reads the request headers, then remains connected and sends no response bytes until the test explicitly releases it after the gateway has already returned. With `read_ms=100`, the gateway must fail as HTTP 502 inside a conservative one-second outer envelope, preserve `/readyz`, expose the exact Prometheus sample `cwl_pingora_gateway_request_errors_total 1`, and leave an independent `frontend` route usable. Keeping the fixture connection open prevents origin closure from masquerading as the timeout. Because Pingora's `read_timeout` is per successful `read()` rather than a whole-response lifetime, reset, post-commit partial response, slow-drip and whole-response deadline behavior remain separate contracts. -On Unix, `tests/pg_erd_graceful_shutdown.rs` proves the bounded pg-erd composition root consumes the shared drain policy instead of borrowing generic-binary evidence. Traffic and metrics listener reservations survive config construction and are released only at child-bind handoff; readiness requires a bounded complete `/readyz` HTTP/1.1 200. The characterized backend accepts `/api/held`, and request-header receipt itself is bounded to five seconds and 64 KiB before the fixture declares the request in flight. Only then does the test send SIGTERM, release the held response during the shared grace period, require downstream HTTP 200 completion, and require successful migration-process exit before the single absolute external termination deadline anchored at signal delivery. This contract counts only when the unchanged exact head executes it to terminal GREEN. +On Unix, `tests/pg_erd_graceful_shutdown.rs` proves the bounded pg-erd composition root consumes the shared drain policy instead of borrowing generic-binary evidence. The fixture retains both ephemeral gateway listener reservations through config construction and releases them only at child-bind handoff, then requires a bounded complete `/readyz` HTTP/1.1 200 before test traffic. It routes `/api/held` to the characterized backend, holds that response until the backend confirms the request is in flight, sends SIGTERM, releases the response during the shared grace period, requires downstream HTTP 200 completion, and requires the migration process to exit before the absolute termination deadline anchored at signal delivery. This contract counts only when the unchanged exact head executes it to terminal GREEN. + +`tests/pg_erd_partial_response_traffic.rs` covers the distinct post-header failure phase. The backend sends exact HTTP/1.1 200 with one `Content-Length: 20` field and the body prefix `partial`, then remains open until the downstream reader has observed the complete response header block and exact prefix. Only after that acknowledgement does the fixture allow origin close, preventing an immediate FIN from accidentally exercising a pre-commit failure phase. Acceptance requires the committed response to terminate before all 20 bytes arrive, forbids a rewritten second status or silent failover, preserves `/readyz`, records exactly `cwl_pingora_gateway_request_errors_total 1`, and proves an independent `frontend` route still completes. The status oracle accepts only exact HTTP/1.1 plus a three-digit code, the framing oracle rejects `X-Content-Length` and duplicate/conflicting Content-Length fields, origin request-header reads are bounded by five seconds and 64 KiB, and traffic/metrics reservations remain held until child-bind handoff. This contract is source-defined until the unchanged exact head reaches terminal hosted GREEN. The `oci-runtime` job validates artifact composition separately from compiled-process tests. The Dockerfile admits only `cwl-pingora-gateway` and `cwl-pingora-pg-erd-migration` as build-time process identities, normalizes the selected executable to one fixed distroless runtime path, and CI builds both image profiles. Each exact candidate must declare uid/gid `65532`, start under a read-only root filesystem with all capabilities dropped and `no-new-privileges`, and consume only a read-only configuration mount. The generic profile must expose local `/livez`. The pg-erd profile must expose local `/livez` on the traffic listener and its separately published `/metrics` listener must identify the Pingora Prometheus service by exact base media type `text/plain` after stripping only optional semicolon parameters. `tests/pg_erd_oci_metrics_workflow_contract.rs` prevents a bare HTTP 200 or prefix wildcard from false-greening a mistakenly bound proxy service while deliberately avoiding a metric-family requirement before routed application traffic has emitted one. `examples/pg-erd-migration.yaml` is an origin-independent OCI smoke fixture, not routed parity evidence. The supply-chain job additionally builds and vulnerability-scans both profiles and binds both local image IDs plus per-image scan outputs to the exact source SHA. These workflow contracts count only after terminal success on the unchanged exact head. @@ -24,4 +26,4 @@ The `oci-runtime` job validates artifact composition separately from compiled-pr Every behavioral migration should begin with characterization against the old owned edge behavior, then add equivalent production-path evidence for Pingora. Static-serving consumers must cover route precedence, SPA fallback, MIME, ETag/cache, Range/HEAD/304/416, redirects/security headers, and compression as applicable. Proxy consumers must cover Host/SNI/TLS, forwarding trust, WebSocket/upgrade, limits, timeout/retry behavior, streaming/uploads, saturation/backpressure, errors, health/readiness, and drain. Characterized response policies must additionally be exercised through the compiled proxy path before any parity or canary claim. -Release-quality gaps remain: no property/fuzz tests yet; no downstream TLS listener contract; no HTTP/2 or HTTP/3 parity evidence; no tracing evidence; no immutable published registry digest/provenance and rehearsed rollback; and no benchmark against replaced Nginx/Traefik traffic. For the pg-erd candidate specifically, dedicated source now covers streamed body overflow, in-flight saturation/recovery, refused-origin recovery, connected silent-origin read timeout, routed graceful drain, and OCI process/Prometheus-listener identity, but TCP reset, post-commit partial-response/streaming failure, slow-drip/whole-response lifetime, representative routed origin-capacity load, terminal current-head traffic/OCI/supply-chain execution, shadow/canary, and rollback still lack evidence. OCI non-root/read-only-root source acceptance covers both admitted process images, while owned-production 100% line/region coverage, public rustdoc, SBOM/image vulnerability, local-CA upstream TLS, explicit backpressure/recovery, and the minimal k6 loopback path remain exact-head gates. None can be transferred to a changed or consumer-specific head. Production performance claims remain forbidden until representative deployment measurements exist. +Release-quality gaps remain: no property/fuzz tests yet; no downstream TLS listener contract; no HTTP/2 or HTTP/3 parity evidence; no tracing evidence; no immutable published registry digest/provenance and rehearsed rollback; and no benchmark against replaced Nginx/Traefik traffic. For the pg-erd candidate specifically, dedicated source now covers streamed body overflow, in-flight saturation/recovery, refused-origin recovery, connected silent-origin read timeout, routed graceful drain, orderly post-header truncation, and OCI process/Prometheus-listener identity, but TCP reset, broader streaming/upgraded failure, slow-drip/whole-response lifetime, representative routed origin-capacity load, terminal current-head traffic/OCI/supply-chain execution, shadow/canary, and rollback still lack evidence. OCI non-root/read-only-root source acceptance covers both admitted process images, while owned-production 100% line/region coverage, public rustdoc, SBOM/image vulnerability, local-CA upstream TLS, explicit backpressure/recovery, and the minimal k6 loopback path remain exact-head gates. None can be transferred to a changed or consumer-specific head. Production performance claims remain forbidden until representative deployment measurements exist. \ No newline at end of file diff --git a/TRD.md b/TRD.md index 6ac7c064..eec49a6c 100644 --- a/TRD.md +++ b/TRD.md @@ -33,6 +33,8 @@ Pingora's standard upstream-request policy handles hop-by-hop and connection-nom The pg-erd migration callback uses the separate Ingress Forwarding Policy. Request-controlled `Forwarded`, `X-Forwarded-*`, `X-Real-IP` and legacy `X-Forwarded-Server` values are discarded. `X-Forwarded-For` and `X-Real-IP` are rebuilt from the accepted client socket, `X-Forwarded-Host` preserves original Host authority, and `X-Forwarded-Port` comes from an explicit Host port or the admitted scheme default rather than the process listener bind. The currently characterized legacy entry point is cleartext `web`, so downstream scheme is explicitly `http`; HTTPS forwarding semantics require a separate downstream-TLS contract. +Failure handling is phase-aware. Before an upstream response header is committed downstream, transport failure may still be represented by the gateway's fail-closed error response under the one-attempt policy. After a valid response header has been committed, a later upstream framing/body failure cannot be rewritten into a second HTTP status or silently failed over: the incomplete downstream response terminates, low-cardinality error telemetry records the failed request, process readiness remains available, and independent routes must remain usable. This is an edge transport invariant, not product retry authority. + ## Health, observability and graceful lifecycle `/livez` and `/readyz` are gateway process endpoints served locally through the production Pingora path and do not become consumer routes. Pg-erd `/healthz` remains characterized product traffic to `backend`. Shared observability is low-cardinality and payload-free: request path/query, headers, cookies, credentials, customer payloads and product identifiers are outside the shared telemetry contract. diff --git a/tests/pg_erd_partial_response_traffic.rs b/tests/pg_erd_partial_response_traffic.rs new file mode 100644 index 00000000..cf34b6dc --- /dev/null +++ b/tests/pg_erd_partial_response_traffic.rs @@ -0,0 +1,438 @@ +//! Real-listener partial upstream response acceptance for the dedicated pg-erd migration binary. +//! +//! This contract distinguishes a response that fails after its status/header block has already +//! been received from failures that occur before downstream response commitment. It proves that a +//! truncated characterized origin response is not rewritten into an invented retry/failover, +//! leaves process health observable, records the transport failure, and does not poison an +//! independent characterized route. + +use std::io::{ErrorKind, Read, Write}; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::process::{Child, Command, Stdio}; +use std::sync::mpsc; +use std::thread; +use std::time::{Duration, Instant}; + +use tempfile::NamedTempFile; + +const MAX_RESPONSE_HEADER_BYTES: usize = 64 * 1024; + +/// Owns the compiled migration child so assertion failures cannot leak a listening test process. +struct GatewayProcess(Child); + +impl Drop for GatewayProcess { + /// Terminates and reaps the child on every teardown path, including a partial-response panic. + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DownstreamTermination { + Eof, + ConnectionReset, +} + +/// Holds distinct traffic and metrics reservations until the compiled child is ready to bind them. +fn reserve_distinct_loopback_listeners() -> (TcpListener, TcpListener) { + let traffic = TcpListener::bind("127.0.0.1:0").expect("traffic port should be reservable"); + let metrics = TcpListener::bind("127.0.0.1:0").expect("metrics port should be reservable"); + assert_ne!( + traffic + .local_addr() + .expect("traffic reservation should expose an address"), + metrics + .local_addr() + .expect("metrics reservation should expose an address"), + "traffic and metrics reservations must remain distinct" + ); + (traffic, metrics) +} + +/// Writes the bounded pg-erd fixture used to separate post-commit truncation from read-stall failure. +fn write_config( + listener: SocketAddr, + metrics_listener: SocketAddr, + backend: SocketAddr, + frontend: SocketAddr, +) -> NamedTempFile { + let mut file = NamedTempFile::new().expect("temporary config should be writable"); + writeln!( + file, + "version: 1\nlistener: {listener}\nmetrics_listener: {metrics_listener}\nmax_request_body_bytes: 8\nmax_in_flight_requests: 8\nupstream_keepalive_pool_size: 4\nupstreams:\n - name: backend\n address: {backend}\n tls: false\n timeouts:\n connection_ms: 200\n total_connection_ms: 400\n read_ms: 500\n write_ms: 1000\n idle_ms: 5000\n - name: frontend\n address: {frontend}\n tls: false\n timeouts:\n connection_ms: 200\n total_connection_ms: 400\n read_ms: 1000\n write_ms: 1000\n idle_ms: 5000" + ) + .expect("migration config should be written"); + file +} + +/// Waits for a bounded complete HTTP 200 response instead of treating bare TCP accept as readiness. +fn wait_until_http_ok(address: SocketAddr, path: &str, process: &mut Child) { + let deadline = Instant::now() + Duration::from_secs(10); + let request = + format!("GET {path} HTTP/1.1\r\nHost: gateway.local\r\nConnection: close\r\n\r\n"); + loop { + if let Some(status) = process + .try_wait() + .expect("gateway process state should be readable") + { + panic!("gateway exited before {path} became ready: {status}"); + } + + if let Ok(mut stream) = TcpStream::connect_timeout(&address, Duration::from_millis(100)) { + stream + .set_read_timeout(Some(Duration::from_millis(250))) + .expect("readiness read timeout should be configurable"); + stream + .set_write_timeout(Some(Duration::from_millis(250))) + .expect("readiness write timeout should be configurable"); + if stream.write_all(request.as_bytes()).is_ok() { + let mut response = Vec::new(); + let mut buffer = [0_u8; 1024]; + loop { + match stream.read(&mut buffer) { + Ok(0) => break, + Ok(read) => { + response.extend_from_slice(&buffer[..read]); + if response.len() > MAX_RESPONSE_HEADER_BYTES { + break; + } + if response.windows(4).any(|window| window == b"\r\n\r\n") { + if response.starts_with(b"HTTP/1.1 200 ") { + return; + } + break; + } + } + Err(error) + if matches!( + error.kind(), + ErrorKind::WouldBlock | ErrorKind::TimedOut + ) => + { + break; + } + Err(_) => break, + } + } + } + } + + assert!( + Instant::now() < deadline, + "gateway did not expose HTTP 200 on {path} within 10s" + ); + thread::sleep(Duration::from_millis(25)); + } +} + +/// Starts the compiled pg-erd binary and requires application-level traffic and metrics readiness. +fn start_gateway( + config: &NamedTempFile, + gateway_address: SocketAddr, + metrics_address: SocketAddr, +) -> GatewayProcess { + let mut child = Command::new(env!("CARGO_BIN_EXE_cwl-pingora-pg-erd-migration")) + .args(["--config", config.path().to_str().expect("UTF-8 temp path")]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("compiled pg-erd migration binary should start"); + wait_until_http_ok(gateway_address, "/readyz", &mut child); + wait_until_http_ok(metrics_address, "/metrics", &mut child); + GatewayProcess(child) +} + +/// Sends one connection-closing HTTP/1.1 request and captures the complete downstream response. +fn raw_request(address: SocketAddr, request: &[u8]) -> String { + let mut downstream = TcpStream::connect(address).expect("gateway should accept traffic"); + downstream + .set_read_timeout(Some(Duration::from_secs(5))) + .expect("downstream timeout should be configurable"); + downstream + .write_all(request) + .expect("downstream request should be writable"); + let mut response = String::new(); + downstream + .read_to_string(&mut response) + .expect("gateway response should be readable"); + response +} + +/// Releases the origin only after the downstream has observed the committed header and body prefix. +fn raw_request_until_terminal_after_body_prefix( + address: SocketAddr, + request: &[u8], + expected_body_prefix: &[u8], + release_origin: mpsc::Sender<()>, +) -> (Vec, DownstreamTermination) { + let mut downstream = TcpStream::connect(address).expect("gateway should accept traffic"); + downstream + .set_read_timeout(Some(Duration::from_secs(5))) + .expect("downstream timeout should be configurable"); + downstream + .write_all(request) + .expect("downstream request should be writable"); + + let mut response = Vec::new(); + let mut buffer = [0_u8; 1024]; + let mut origin_released = false; + loop { + match downstream.read(&mut buffer) { + Ok(0) => { + assert!( + origin_released, + "downstream terminated before the committed body prefix was observed" + ); + return (response, DownstreamTermination::Eof); + } + Ok(read) => { + response.extend_from_slice(&buffer[..read]); + if !origin_released { + if let Some(header_end) = response + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|position| position + 4) + { + let expected_end = header_end + expected_body_prefix.len(); + if response.len() >= expected_end { + assert_eq!( + &response[header_end..expected_end], + expected_body_prefix, + "downstream must observe the exact committed body prefix before origin termination" + ); + release_origin + .send(()) + .expect("origin should wait for downstream commit evidence"); + origin_released = true; + } + } + } + } + Err(error) if error.kind() == ErrorKind::ConnectionReset => { + assert!( + origin_released, + "downstream reset before the committed body prefix was observed" + ); + return (response, DownstreamTermination::ConnectionReset); + } + Err(error) => { + panic!("partial downstream response should terminate, not stall: {error}") + } + } + } +} + +/// Issues a fixture GET with the characterized downstream authority and explicit connection close. +fn get(address: SocketAddr, path: &str) -> String { + raw_request( + address, + format!("GET {path} HTTP/1.1\r\nHost: app.example:8080\r\nConnection: close\r\n\r\n") + .as_bytes(), + ) +} + +/// Reads only through the origin header terminator so the fixture can control the failure phase. +fn read_request_headers(stream: &mut TcpStream) -> String { + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .expect("origin request timeout should be configurable"); + let mut bytes = Vec::new(); + let mut buffer = [0_u8; 1024]; + loop { + let read = stream + .read(&mut buffer) + .expect("origin request should complete within the fixture timeout"); + assert!( + read > 0, + "gateway closed origin request before headers completed" + ); + bytes.extend_from_slice(&buffer[..read]); + assert!( + bytes.len() <= MAX_RESPONSE_HEADER_BYTES, + "origin request headers exceeded the bounded fixture limit" + ); + if bytes.windows(4).any(|window| window == b"\r\n\r\n") { + return String::from_utf8_lossy(&bytes).into_owned(); + } + } +} + +/// Extracts Content-Length field values by case-insensitive field identity and trimmed field value. +fn content_length_values(headers: &str) -> Vec<&str> { + headers + .lines() + .filter_map(|line| line.trim_end_matches('\r').split_once(':')) + .filter(|(name, _)| name.eq_ignore_ascii_case("content-length")) + .map(|(_, value)| value.trim()) + .collect() +} + +/// Parses only an exact HTTP/1.1 three-digit response status line for the wire oracle. +fn http_1_1_status_code(response: &str) -> Option { + let status_line = response.lines().next()?.trim_end_matches('\r'); + let mut fields = status_line.splitn(3, ' '); + if fields.next()? != "HTTP/1.1" { + return None; + } + let code = fields.next()?; + if code.len() != 3 || !code.bytes().all(|byte| byte.is_ascii_digit()) || fields.next().is_none() + { + return None; + } + code.parse().ok() +} + +/// Locks the framing oracle against lookalike names and duplicate/conflicting field values. +#[test] +fn content_length_parser_preserves_field_identity_and_cardinality_evidence() { + assert!(content_length_values("HTTP/1.1 200 OK\r\nX-Content-Length: 20\r\n\r\n").is_empty()); + assert_eq!( + content_length_values("HTTP/1.1 200 OK\r\ncOnTeNt-LeNgTh:\t20\r\n\r\n"), + vec!["20"] + ); + assert_eq!( + content_length_values( + "HTTP/1.1 200 OK\r\nContent-Length: 20\r\ncontent-length: 21\r\n\r\n" + ), + vec!["20", "21"] + ); +} + +/// Locks status evidence to exact HTTP/1.1 protocol and a three-digit code. +#[test] +fn status_parser_rejects_prefix_and_protocol_lookalikes() { + assert_eq!(http_1_1_status_code("HTTP/1.1 200 OK\r\n\r\n"), Some(200)); + assert_eq!(http_1_1_status_code("HTTP/1.1 2000 Bad\r\n\r\n"), None); + assert_eq!(http_1_1_status_code("http/1.1 200 OK\r\n\r\n"), None); + assert_eq!(http_1_1_status_code("HTTP/2 200 OK\r\n\r\n"), None); +} + +/// Proves a post-commit origin truncation preserves framing, terminates downstream and keeps recovery usable. +#[test] +fn compiled_pg_erd_truncated_response_stays_committed_and_preserves_independent_routing() { + let (release_backend_tx, release_backend_rx) = mpsc::channel(); + let backend = TcpListener::bind("127.0.0.1:0").expect("backend fixture should bind"); + let backend_address = backend.local_addr().expect("backend address should exist"); + let backend_origin = thread::spawn(move || { + let (mut stream, _) = backend + .accept() + .expect("routed request should reach the characterized backend authority"); + let request = read_request_headers(&mut stream); + assert!(request.starts_with("GET /api/partial-response HTTP/1.1\r\n")); + + // Keep the upstream open until the downstream has actually observed this committed prefix. + // Otherwise an immediate FIN can race proxy forwarding and accidentally exercise a + // pre-commit failure phase while still producing the same buffered bytes. + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 20\r\nConnection: close\r\n\r\npartial") + .expect("partial backend response should be writable"); + release_backend_rx + .recv_timeout(Duration::from_secs(5)) + .expect("downstream should observe the committed prefix before backend close"); + }); + + let frontend = TcpListener::bind("127.0.0.1:0").expect("frontend fixture should bind"); + let frontend_address = frontend + .local_addr() + .expect("frontend address should exist"); + let frontend_origin = thread::spawn(move || { + let (mut stream, _) = frontend + .accept() + .expect("fallback request should reach the independent frontend authority"); + let request = read_request_headers(&mut stream); + assert!(request.starts_with("GET /after-partial-response HTTP/1.1\r\n")); + stream + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 9\r\nConnection: close\r\n\r\nrecovered", + ) + .expect("frontend recovery response should be writable"); + }); + + let (gateway_reservation, metrics_reservation) = reserve_distinct_loopback_listeners(); + let gateway_address = gateway_reservation + .local_addr() + .expect("traffic reservation should expose an address"); + let metrics_address = metrics_reservation + .local_addr() + .expect("metrics reservation should expose an address"); + let config = write_config( + gateway_address, + metrics_address, + backend_address, + frontend_address, + ); + // Release the exact reservations only at the compiled child-bind handoff; retaining them through + // config construction prevents another fixture from reclaiming either selected authority early. + drop(gateway_reservation); + drop(metrics_reservation); + let _process = start_gateway(&config, gateway_address, metrics_address); + + let (partial, termination) = raw_request_until_terminal_after_body_prefix( + gateway_address, + b"GET /api/partial-response HTTP/1.1\r\nHost: app.example:8080\r\nConnection: close\r\n\r\n", + b"partial", + release_backend_tx, + ); + assert!( + matches!( + termination, + DownstreamTermination::Eof | DownstreamTermination::ConnectionReset + ), + "a committed truncated response must terminate the downstream connection" + ); + let header_end = partial + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|position| position + 4) + .expect("committed partial response must contain a complete header block"); + let raw_headers = String::from_utf8_lossy(&partial[..header_end]); + assert_eq!( + http_1_1_status_code(raw_headers.as_ref()), + Some(200), + "a post-header upstream failure cannot be rewritten as a new status: {raw_headers:?}" + ); + let content_lengths = content_length_values(raw_headers.as_ref()); + assert_eq!( + content_lengths, + vec!["20"], + "the committed response must retain exactly one Content-Length field whose value is 20: {raw_headers:?}" + ); + let body = &partial[header_end..]; + assert_eq!(body, b"partial"); + assert!( + body.len() < 20, + "fixture must terminate before its declared response body completes" + ); + + let readiness = get(gateway_address, "/readyz"); + assert_eq!( + http_1_1_status_code(&readiness), + Some(200), + "one truncated upstream response must not poison process readiness: {readiness:?}" + ); + + let metrics = get(metrics_address, "/metrics"); + assert!( + metrics + .lines() + .any(|line| line == "cwl_pingora_gateway_request_errors_total 1"), + "the post-header upstream framing failure must expose exactly one request error through low-cardinality telemetry: {metrics:?}" + ); + + let recovered = get(gateway_address, "/after-partial-response"); + assert_eq!( + http_1_1_status_code(&recovered), + Some(200), + "an independent characterized route must remain usable after a truncated response: {recovered:?}" + ); + assert!(recovered.ends_with("\r\n\r\nrecovered")); + + frontend_origin + .join() + .expect("frontend recovery fixture should complete"); + backend_origin + .join() + .expect("partial backend fixture should complete"); +} From 9b1e7b347122620b566bcaccdca03fa43c45fe19 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 13:23:33 +0900 Subject: [PATCH 42/43] docs(partial): preserve current parent failure contracts --- CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18264ccf..9fd5fe07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,9 +22,9 @@ All notable changes are tracked here. No release has been published yet. - Added process-local fail-fast backpressure: non-health requests above the in-flight budget receive HTTP 503, health remains observable, rejection telemetry increments, and capacity is released after request completion or failure. - Added dedicated compiled pg-erd traffic acceptance for streamed/chunked body overflow and routed in-flight saturation/recovery: the migration process must return 413 above the shared body budget, return 503 in less than one second above the in-flight budget, keep `/readyz` observable, expose the exact single-rejection Prometheus sample, and admit a later routed request after capacity is released. - Added dedicated compiled pg-erd refused-origin recovery acceptance using a Linux TCP socket bound to the characterized backend address without entering LISTEN state. The fixture first proves direct `ECONNREFUSED` while retaining exclusive port ownership, then requires the migration gateway to return 502 within a conservative one-second envelope around the configured 200/400 ms connection budgets, keep `/readyz` 200, expose the exact single-error Prometheus sample, and allow a later independent frontend route to recover. Connected read stall, TCP reset, partial-response/streaming failure, retry and failover behavior remain separate gaps. -- Added dedicated compiled pg-erd connected read-stall acceptance with `read_ms=100`: the backend accepts the routed request and remains open without response bytes until the gateway has already failed it, preventing fixture closure from faking timeout behavior. The contract requires 502 inside a conservative one-second envelope, preserved `/readyz`, the exact single-error Prometheus sample, and independent frontend recovery. Pingora `read_timeout` remains a per-read inactivity budget, not a whole-response lifetime; reset, partial-response and slow-drip cases remain open. -- Added dedicated routed pg-erd graceful-drain acceptance: a characterized `/api/held` backend request is held in flight, SIGTERM is sent only after the backend has accepted it, the response is released during the shared grace period, the downstream must still receive HTTP 200, and the migration process must exit successfully inside the absolute external termination budget. The fixture retains traffic and metrics socket reservations through config construction, releases them only at child-bind handoff, and admits the drain case only after bounded `/readyz` HTTP/1.1 200 readiness; generic drain evidence is not transferred to this composition root. -- Added dedicated compiled pg-erd post-header truncation acceptance: the backend commits exact HTTP/1.1 200 framing with one `Content-Length: 20` field and the seven-byte `partial` prefix, remains open until downstream commit evidence is observed, then closes. The fixture requires the committed response to terminate incomplete without a second status or failover, records exactly one request error, preserves `/readyz`, and proves an independent frontend route still works. Status parsing now rejects protocol/prefix lookalikes, origin request-header reads are time/size bounded, and content-length parsing rejects lookalike or duplicate/conflicting fields. +- Added dedicated compiled pg-erd connected read-stall acceptance with `read_ms=100`: the backend accepts the routed request, completes bounded request-header receipt, records that causal point, and remains open without response bytes until the gateway has already failed it. The contract requires 502 no earlier than a conservative 50 ms lower bound from completed origin headers and still inside a one-second outer envelope, preserves `/readyz`, requires the exact single-error Prometheus sample, and proves independent frontend recovery. Pingora `read_timeout` remains a per-read inactivity budget, not a whole-response lifetime; reset, post-commit partial response, slow-drip and whole-response-deadline cases remain open. +- Added dedicated routed pg-erd graceful-drain acceptance: a characterized `/api/held` backend request is held in flight, backend request-header receipt must complete within five seconds and 64 KiB before the request is considered admitted, SIGTERM is sent only after that causal point, the response is released during the shared grace period, the downstream must still receive HTTP 200, and the migration process must exit successfully inside the single absolute external termination budget anchored at signal delivery. The fixture retains traffic and metrics socket reservations through config construction, releases them only at child-bind handoff, and admits the drain case only after bounded `/readyz` HTTP/1.1 200 readiness; generic drain evidence is not transferred to this composition root. +- Added dedicated compiled pg-erd post-header truncation acceptance: the backend commits exact HTTP/1.1 200 framing with one `Content-Length: 20` field and the seven-byte `partial` prefix, remains open until downstream commit evidence is observed, then closes. The fixture requires the committed response to terminate incomplete without a second status or failover, records exactly one request error, preserves `/readyz`, and proves an independent frontend route still works. Status parsing rejects protocol/prefix lookalikes, origin request-header reads are time/size bounded, and content-length parsing rejects lookalike or duplicate/conflicting fields. - Added optional per-upstream absolute PEM trust-bundle consumption without taking ownership of certificate issuance/rotation; trust material is loaded fail-closed before listeners open. - Added an executable local-CA TLS test through the compiled gateway that holds CA trust constant and proves SNI/hostname mismatch is rejected. - Added a focused transport-adapter regression proving an upstream without a custom trust bundle leaves Pingora's platform trust roots selected rather than replacing the CA store. @@ -34,11 +34,11 @@ All notable changes are tracked here. No release has been published yet. - Added low-cardinality metrics plus credential/cookie-safe access logging through the production path. - Overrode Pingora framework retry/drain defaults with one total upstream attempt, a 5-second SIGTERM grace period, and a 30-second graceful-shutdown timeout. - Added non-root/read-only-root OCI packaging with a fail-closed build-time allowlist for the generic and bounded pg-erd process identities; exact-head OCI acceptance builds and starts both profiles under uid/gid 65532, dropped capabilities and `no-new-privileges`, while the supply-chain lane builds and vulnerability-scans both candidate images. -- Extended the pg-erd OCI acceptance so the least-privilege migration container is not accepted until process `/livez` responds and the separately published `/metrics` listener identifies the Pingora Prometheus service through exact base media type `text/plain` after stripping only optional semicolon parameters. A bare HTTP 200 or prefix-wildcard media-type match is insufficient. +- Extended pg-erd OCI acceptance so promotion also requires the separately published `/metrics` listener to identify the Prometheus service by exact base media type `text/plain`; a bare HTTP 200 or prefix-wildcard media-type match is insufficient. - Added a committed dependency lock, fail-closed license/source/advisory policy, exact-source SBOM and image-vulnerability evidence. - Added an exact-head owned-production coverage gate that requires 100% lines and regions without filename/function/branch exclusions; repaired compiler-generated generic startup coverage and structurally impossible literal-header error regions rather than weakening the gate. - Added missing-public-rustdoc enforcement and documentation builds with warnings denied. - Added a load-workflow contract that proves the measured loopback origin is ready before gateway startup so fixture races cannot be counted as gateway latency or availability behavior, and separately preserves the primary failure when k6 never produces a summary. - Added DDD, product, technical, security, threat, test, operability, configuration, migration-gap, and primary-source traceability documentation. -Release remains blocked on a maintainer-integrated and release-qualified disposition of unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388`, the exact Pingora supplier/protocol gates tracked by the foundation stack, the Rust compiler promotion owned by #56, terminal exact-current CI/supply-chain/security/review evidence, central required-workflow convergence and independent approval, representative pg-erd routed concurrency/origin-capacity/network-failure/drain and benchmark evidence, an immutable package/image identity with SBOM/provenance/reproducibility and rehearsed rollback, and protected-branch integration. No consumer migration, shadow/canary, cutover, or legacy removal is claimed before those release and traffic-contract gates are satisfied. \ No newline at end of file +Release remains blocked on a maintainer-integrated and release-qualified disposition of unmaintained `derivative 2.2.0` / `RUSTSEC-2024-0388`, the exact Pingora supplier/protocol gates tracked by the foundation stack, the Rust compiler promotion owned by #56, terminal exact-current CI/supply-chain/security/review evidence, central required-workflow convergence and independent approval, representative pg-erd routed concurrency/origin-capacity/network-failure/drain and benchmark evidence, an immutable package/image identity with SBOM/provenance/reproducibility and rehearsed rollback, and protected-branch integration. No consumer migration, shadow/canary, cutover, or legacy removal is claimed before those release and traffic-contract gates are satisfied. From 4e7907b63345ea62362c6b8e826c35839679ed24 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 13:24:01 +0900 Subject: [PATCH 43/43] docs(partial): preserve current parent test contracts --- TEST_STRATEGY.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/TEST_STRATEGY.md b/TEST_STRATEGY.md index cc9d6235..b9ea8000 100644 --- a/TEST_STRATEGY.md +++ b/TEST_STRATEGY.md @@ -14,9 +14,9 @@ The bounded Admin Config transition has its own executable contract. `tests/pg_e `tests/pg_erd_upstream_failure_traffic.rs` adds the next distinct failure phase through the dedicated compiled process. On Linux it binds the characterized backend TCP address without calling `listen(2)`, so the test retains exclusive port ownership while connection attempts receive `ECONNREFUSED`; a direct `TcpStream::connect_timeout` precondition must observe `ConnectionRefused` before gateway traffic begins. If another process has stolen the selected port, fixture setup fails instead of allowing false-GREEN evidence. The migration gateway must then return HTTP 502 within a conservative one-second outer envelope around the configured 200 ms connection / 400 ms total-connection budgets, keep `/readyz` at HTTP 200, expose the exact Prometheus sample `cwl_pingora_gateway_request_errors_total 1`, and route an independent fallback request successfully to `frontend`. Exact-line matching prevents values such as `10` or `11` from false-passing the single-error contract. This contract does not claim connected read-stall, TCP reset, post-commit truncation, slow-drip/whole-response lifetime, retry, or failover behavior. -`tests/pg_erd_read_stall_traffic.rs` separates connected upstream inactivity from refusal. The characterized backend accepts `/api/read-stall`, reads the request headers, then remains connected and sends no response bytes until the test explicitly releases it after the gateway has already returned. With `read_ms=100`, the gateway must fail as HTTP 502 inside a conservative one-second outer envelope, preserve `/readyz`, expose the exact Prometheus sample `cwl_pingora_gateway_request_errors_total 1`, and leave an independent `frontend` route usable. Keeping the fixture connection open prevents origin closure from masquerading as the timeout. Because Pingora's `read_timeout` is per successful `read()` rather than a whole-response lifetime, reset, post-commit partial response, slow-drip and whole-response deadline behavior remain separate contracts. +`tests/pg_erd_read_stall_traffic.rs` separates connected upstream inactivity from refusal. The characterized backend accepts `/api/read-stall`, reads request headers under a five-second/64 KiB fixture bound, records that causal point, then remains connected and sends no response bytes until the test explicitly releases it after the gateway has already returned. Traffic and metrics loopback sockets remain reserved through config construction and are released only at the child-bind handoff. With `read_ms=100`, the gateway must fail as HTTP 502 no earlier than a conservative 50 ms lower bound measured from completed origin request-header receipt and still inside the one-second outer envelope, preserve `/readyz`, expose the exact Prometheus sample `cwl_pingora_gateway_request_errors_total 1`, and leave an independent `frontend` route usable. Keeping the fixture connection open prevents origin closure from masquerading as the timeout, while the lower bound prevents an unrelated immediate 502 from masquerading as the configured read-inactivity path. Because Pingora's `read_timeout` is per successful `read()` rather than a whole-response lifetime, reset, post-commit partial response, slow-drip and whole-response deadline behavior remain separate contracts. -On Unix, `tests/pg_erd_graceful_shutdown.rs` proves the bounded pg-erd composition root consumes the shared drain policy instead of borrowing generic-binary evidence. The fixture retains both ephemeral gateway listener reservations through config construction and releases them only at child-bind handoff, then requires a bounded complete `/readyz` HTTP/1.1 200 before test traffic. It routes `/api/held` to the characterized backend, holds that response until the backend confirms the request is in flight, sends SIGTERM, releases the response during the shared grace period, requires downstream HTTP 200 completion, and requires the migration process to exit before the absolute termination deadline anchored at signal delivery. This contract counts only when the unchanged exact head executes it to terminal GREEN. +On Unix, `tests/pg_erd_graceful_shutdown.rs` proves the bounded pg-erd composition root consumes the shared drain policy instead of borrowing generic-binary evidence. Traffic and metrics listener reservations survive config construction and are released only at child-bind handoff; readiness requires a bounded complete `/readyz` HTTP/1.1 200. The characterized backend accepts `/api/held`, and request-header receipt itself is bounded to five seconds and 64 KiB before the fixture declares the request in flight. Only then does the test send SIGTERM, release the held response during the shared grace period, require downstream HTTP 200 completion, and require successful migration-process exit before the single absolute external termination deadline anchored at signal delivery. This contract counts only when the unchanged exact head executes it to terminal GREEN. `tests/pg_erd_partial_response_traffic.rs` covers the distinct post-header failure phase. The backend sends exact HTTP/1.1 200 with one `Content-Length: 20` field and the body prefix `partial`, then remains open until the downstream reader has observed the complete response header block and exact prefix. Only after that acknowledgement does the fixture allow origin close, preventing an immediate FIN from accidentally exercising a pre-commit failure phase. Acceptance requires the committed response to terminate before all 20 bytes arrive, forbids a rewritten second status or silent failover, preserves `/readyz`, records exactly `cwl_pingora_gateway_request_errors_total 1`, and proves an independent `frontend` route still completes. The status oracle accepts only exact HTTP/1.1 plus a three-digit code, the framing oracle rejects `X-Content-Length` and duplicate/conflicting Content-Length fields, origin request-header reads are bounded by five seconds and 64 KiB, and traffic/metrics reservations remain held until child-bind handoff. This contract is source-defined until the unchanged exact head reaches terminal hosted GREEN. @@ -26,4 +26,4 @@ The `oci-runtime` job validates artifact composition separately from compiled-pr Every behavioral migration should begin with characterization against the old owned edge behavior, then add equivalent production-path evidence for Pingora. Static-serving consumers must cover route precedence, SPA fallback, MIME, ETag/cache, Range/HEAD/304/416, redirects/security headers, and compression as applicable. Proxy consumers must cover Host/SNI/TLS, forwarding trust, WebSocket/upgrade, limits, timeout/retry behavior, streaming/uploads, saturation/backpressure, errors, health/readiness, and drain. Characterized response policies must additionally be exercised through the compiled proxy path before any parity or canary claim. -Release-quality gaps remain: no property/fuzz tests yet; no downstream TLS listener contract; no HTTP/2 or HTTP/3 parity evidence; no tracing evidence; no immutable published registry digest/provenance and rehearsed rollback; and no benchmark against replaced Nginx/Traefik traffic. For the pg-erd candidate specifically, dedicated source now covers streamed body overflow, in-flight saturation/recovery, refused-origin recovery, connected silent-origin read timeout, routed graceful drain, orderly post-header truncation, and OCI process/Prometheus-listener identity, but TCP reset, broader streaming/upgraded failure, slow-drip/whole-response lifetime, representative routed origin-capacity load, terminal current-head traffic/OCI/supply-chain execution, shadow/canary, and rollback still lack evidence. OCI non-root/read-only-root source acceptance covers both admitted process images, while owned-production 100% line/region coverage, public rustdoc, SBOM/image vulnerability, local-CA upstream TLS, explicit backpressure/recovery, and the minimal k6 loopback path remain exact-head gates. None can be transferred to a changed or consumer-specific head. Production performance claims remain forbidden until representative deployment measurements exist. \ No newline at end of file +Release-quality gaps remain: no property/fuzz tests yet; no downstream TLS listener contract; no HTTP/2 or HTTP/3 parity evidence; no tracing evidence; no immutable published registry digest/provenance and rehearsed rollback; and no benchmark against replaced Nginx/Traefik traffic. For the pg-erd candidate specifically, dedicated source now covers streamed body overflow, in-flight saturation/recovery, refused-origin recovery, connected silent-origin read timeout, routed graceful drain, orderly post-header truncation, and OCI process/Prometheus-listener identity, but TCP reset, broader streaming/upgraded failure, slow-drip/whole-response lifetime, representative routed origin-capacity load, terminal current-head traffic/OCI/supply-chain execution, shadow/canary, and rollback still lack evidence. OCI non-root/read-only-root source acceptance covers both admitted process images, while owned-production 100% line/region coverage, public rustdoc, SBOM/image vulnerability, local-CA upstream TLS, explicit backpressure/recovery, and the minimal k6 loopback path remain exact-head gates. None can be transferred to a changed or consumer-specific head. Production performance claims remain forbidden until representative deployment measurements exist.