From 3701f266817515949e74de5a09bfb3f38974810b Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 18:53:23 -0700 Subject: [PATCH 01/10] Lower Headers, Request and Response members that undici-types declares as properties @types/node's fetch declarations (undici-types) spell the Headers methods and the body mixin as function-typed properties (readonly append: (...) => void), where scriptc's own declarations use methods. The receiver call lowering treated such a standard-library property as a stored callback: Headers.forEach was refused at build time and append/get/set/... compiled to a runtime 'is not a function'. Standard-library members now take the shared receiver dispatch either way. Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/lowering/lower-calls.ts | 12 +++++++ .../node-types/fetch-handle-members.ts.json | 6 ++++ .../node-types/fetch-handle-members.ts | 32 +++++++++++++++++++ tests/harness/project-config.test.ts | 21 ++++++++++++ 4 files changed, 71 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/node-types/fetch-handle-members.ts.json create mode 100644 tests/fixtures/node-types/fetch-handle-members.ts diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 5a48387825..464499ba96 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -3665,8 +3665,20 @@ function lowerDynReceiverMethodCall( ) ) return true; + // A standard-library member spelled as a function-typed property + // (undici-types declares Headers.append/forEach/... and the body mixin + // as `readonly append: (...) => void`; scriptc's own declarations use + // methods) is still the handle's native method: it takes the shared + // receiver dispatch below either way (which also calls a plain object's + // own member), so the lowering does not depend on which declaration + // file supplies the type. + const libMethod = + prop !== undefined && + prop.declarations.length > 0 && + prop.declarations.every((decl) => lowerer.isStdlibFile(decl.getSourceFile())); return ( prop !== undefined && + !libMethod && prop.declarations.some( (decl) => ts.isPropertyDeclaration(decl) || ts.isPropertySignature(decl), ) diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/node-types/fetch-handle-members.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/node-types/fetch-handle-members.ts.json new file mode 100644 index 0000000000..80464768ac --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/node-types/fetch-handle-members.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/node-types/fetch-handle-members.ts" + ], + "diags": [] +} diff --git a/tests/fixtures/node-types/fetch-handle-members.ts b/tests/fixtures/node-types/fetch-handle-members.ts new file mode 100644 index 0000000000..07f8f689ef --- /dev/null +++ b/tests/fixtures/node-types/fetch-handle-members.ts @@ -0,0 +1,32 @@ +// undici-types (@types/node's fetch declarations) spells the Headers methods +// and the Request/Response body mixin as function-typed properties; they +// must lower to the same native handle methods as the shipped declarations. +async function main(): Promise { + const h = new Headers([["b", "2"], ["a", "1"]]); + h.append("set-cookie", "x=1"); + h.append("set-cookie", "y=2"); + h.set("c", "3"); + h.delete("b"); + const seen: string[] = []; + h.forEach((value, name) => seen.push(`${name}=${value}`)); + console.log(seen.join(","), h.get("a"), h.has("b"), h.getSetCookie().join("|")); + const r1 = new Response("body-text", { status: 202, statusText: "Acc", headers: h }); + console.log(r1.status, r1.statusText, r1.ok, r1.headers.get("c"), r1.bodyUsed); + console.log(await r1.text(), r1.bodyUsed); + const r2 = new Response(JSON.stringify({ k: 1 })); + console.log(JSON.stringify(await r2.json())); + const r3 = new Response(new Uint8Array([1, 2, 3])); + console.log((await r3.arrayBuffer()).byteLength); + const r4 = new Response("abc"); + console.log((await r4.bytes()).length); + const r5 = Response.json({ z: true }, { status: 200 }); + console.log(r5.headers.get("content-type"), await r5.text()); + const req = new Request("http://x.test/a?b=1", { method: "POST", body: "payload", headers: { q: "1" } }); + console.log(req.method, req.url, req.headers.get("q"), await req.text()); + const req2 = new Request("http://x.test/j", { method: "PUT", body: "{\"n\":2}" }); + console.log(JSON.stringify(await req2.json())); + const ac = new AbortController(); + ac.abort(); + console.log(ac.signal.aborted); +} +main(); diff --git a/tests/harness/project-config.test.ts b/tests/harness/project-config.test.ts index 802fbd3e19..d98f12da37 100644 --- a/tests/harness/project-config.test.ts +++ b/tests/harness/project-config.test.ts @@ -52,6 +52,27 @@ test("node-types: filesystem capacity supports typed number and BigInt results u expect(comparableStderr(native.stderr)).toBe(node.stderr); }); +test("node-types: Headers, Request and Response members typed by undici-types run natively", async () => { + const outDir = outDirFor("fetch-handle-members"); + const entry = join(nodeTypesDir, "fetch-handle-members.ts"); + const result = await compile(entry, { + outPath: join( + outDir, + process.platform === "win32" ? "fetch-handle-members.exe" : "fetch-handle-members", + ), + outDir, + sanitize, + }); + expect(result.ok, !result.ok ? JSON.stringify(result.diagnostics, null, 2) : "").toBe(true); + if (!result.ok) return; + const [native, node] = await Promise.all([ + execFileAsync(result.binaryPath), + execFileAsync(process.execPath, ["--disable-warning=MODULE_TYPELESS_PACKAGE_JSON", entry]), + ]); + expect(native.stdout).toBe(node.stdout); + expect(comparableStderr(native.stderr)).toBe(node.stderr); +}); + test("node-types: http header arrays and server.address() follow @types/node's types", async () => { const outDir = outDirFor("http-headers-address"); const entry = join(nodeTypesDir, "http-headers-address.ts"); From 6e67b1da09ea5638b46f1b7b784649a9e3971883 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 18:53:23 -0700 Subject: [PATCH 02/10] Run the poll phase once per event-loop iteration The net unit drained its poller every time the loop came back to the I/O station, including the restarts that drain microtasks. I/O that became ready during the iteration's own callbacks (a loopback connect started by one of them) was delivered before the check phase, so setImmediate callbacks queued earlier ran after it; Node delivers it in the next iteration's poll phase. The poller now drains once per iteration (a full batch keeps draining), reopening after the check phase. Back-to-back undici requests now reuse connections like Node, so the fixtures drop their setImmediate turn between requests. Co-Authored-By: Claude Opus 5.5 --- .../6251-net-connect-poll-phase-order.ts.json | 6 ++ packages/runtime/src/scr_async.c | 20 ++++++ packages/runtime/src/scr_net.c | 22 +++++- .../6251-net-connect-poll-phase-order.ts | 70 +++++++++++++++++++ tests/fixtures/npm/cases/undici5/main.ts | 1 - tests/fixtures/npm/cases/undici7/main.ts | 1 - 6 files changed, 115 insertions(+), 5 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6251-net-connect-poll-phase-order.ts.json create mode 100644 tests/corpus/6251-net-connect-poll-phase-order.ts diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6251-net-connect-poll-phase-order.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6251-net-connect-poll-phase-order.ts.json new file mode 100644 index 0000000000..0d0190a322 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6251-net-connect-poll-phase-order.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6251-net-connect-poll-phase-order.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_async.c b/packages/runtime/src/scr_async.c index 88ad0f7bde..de48d6b616 100644 --- a/packages/runtime/src/scr_async.c +++ b/packages/runtime/src/scr_async.c @@ -4149,6 +4149,19 @@ static int scr_workers_pollfd(void) { * macrotask). */ bool scr_loop_has_ready(void) { return scr_ready_len > 0 || scr_nt_head != NULL; } +/* The poll phase runs once per loop iteration, like libuv's: the net unit + * drains the poller at most once between check phases. I/O that becomes + * ready while this iteration's callbacks run (a loopback connect started + * by one of them) waits for the next iteration, so immediates queued in + * the meantime fire first — Node's poll → check order. */ +static SCR_TL bool scr_loop_io_open = true; +bool scr_loop_io_take(void) { + bool open = scr_loop_io_open; + scr_loop_io_open = false; + return open; +} +void scr_loop_io_reopen(void) { scr_loop_io_open = true; } + /* The process.nextTick queue to exhaustion, ticks queued meanwhile * included. false: a tick threw (uncaught, pending). */ static bool scr_loop_run_ticks(void) { @@ -4618,6 +4631,7 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { * every macrotask; pending ticks or a woken fiber restart the turn * first (the timer loop's break above), so the phase only runs on a * drained checkpoint. */ + bool check_ran = false; if (scr_ready_len == 0 && scr_nt_head == NULL && scr_nunhandled == 0 && scr_pending_immediates > 0) { size_t end = scr_nimmediates; while (scr_immediates_head < end) { @@ -4648,7 +4662,13 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { scr_immediates_head = 0; scr_nimmediates = 0; } + check_ran = true; } + /* The next poll phase opens once this iteration's check phase ran (or + * had nothing to run). A check phase deferred for pending microtasks + * keeps it closed, so the next iteration reaches the check phase + * before polling again. */ + if (check_ran || scr_pending_immediates == 0) scr_loop_io_open = true; } return rejection_failed; } diff --git a/packages/runtime/src/scr_net.c b/packages/runtime/src/scr_net.c index f13906454c..9684aebe97 100644 --- a/packages/runtime/src/scr_net.c +++ b/packages/runtime/src/scr_net.c @@ -4908,17 +4908,28 @@ static int scr_net_pollfd(void) { return scr_net_poller != NULL ? scrp_poller_fd(scr_net_poller) : -1; } +bool scr_loop_io_take(void); +void scr_loop_io_reopen(void); + static void scr_net_dispatch(void) { if (!scr_net_servers && !scr_net_socks && !scr_net_proto_pending()) { return; } + /* One poller drain per loop iteration (scr_loop_io_take); a full batch + * keeps draining, like libuv's poll phase. Deferred emits (the sweep) + * still run on every call. */ + bool may_drain = scr_loop_io_take(); for (;;) { scr_net_sweep(); if (scr_exc_pending()) return; if (scr_net_poller == NULL) return; ScrPollerEvent evs[64]; - int n = scrp_drain(scr_net_poller, evs, 64); + int n = 0; + if (may_drain) { + n = scrp_drain(scr_net_poller, evs, 64); + may_drain = n == 64; + } for (int i = 0; i < n; i++) { void *udata = evs[i].udata; if (!udata) continue; @@ -4992,8 +5003,13 @@ static void scr_net_dispatch(void) { } if (scr_exc_pending()) return; } - if (!scr_net_flags_pending()) return; - if (scr_loop_has_ready()) return; /* microtasks interleave first */ + if (!scr_net_flags_pending() && !may_drain) return; + if (scr_loop_has_ready()) { + /* microtasks interleave first; a full batch's rest drains on the + * next call, still within this poll phase */ + if (may_drain) scr_loop_io_reopen(); + return; + } } } diff --git a/tests/corpus/6251-net-connect-poll-phase-order.ts b/tests/corpus/6251-net-connect-poll-phase-order.ts new file mode 100644 index 0000000000..cfe09292ae --- /dev/null +++ b/tests/corpus/6251-net-connect-poll-phase-order.ts @@ -0,0 +1,70 @@ +// A connection started by a poll-phase callback completes in the NEXT loop +// iteration's poll phase: the callback's nextTick and promise jobs run +// first, then the check phase's setImmediate, then the new socket's +// 'connect' (libuv/Node order; loopback connects complete at once, so an +// event loop that polls again before the check phase reorders them). The +// second half is the same through an awaited promise, the shape of an HTTP +// client that dials a fresh connection after the previous response. +import * as net from "node:net"; + +const log: string[] = []; +const server = net.createServer((socket) => { + socket.on("data", (chunk: Buffer) => socket.write(chunk)); +}); + +function dial(port: number, label: string): Promise { + return new Promise((resolve) => { + const socket = net.connect({ port, host: "127.0.0.1" }); + socket.on("connect", () => { + log.push(`connect ${label}`); + resolve(socket); + }); + }); +} + +function roundTrip(socket: net.Socket, label: string): Promise { + return new Promise((resolve) => { + socket.once("data", () => { + log.push(`data ${label}`); + setImmediate(() => log.push(`immediate after data ${label}`)); + process.nextTick(() => log.push(`tick after data ${label}`)); + Promise.resolve().then(() => log.push(`promise after data ${label}`)); + resolve(); + }); + socket.write("ping"); + }); +} + +server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = address !== null && typeof address === "object" ? address.port : 0; + const first = net.connect({ port, host: "127.0.0.1" }); + first.on("connect", () => { + log.push("connect first"); + first.write("ping"); + }); + first.once("data", () => { + log.push("data first"); + setImmediate(() => log.push("immediate from data")); + process.nextTick(() => log.push("tick from data")); + Promise.resolve().then(() => log.push("promise from data")); + const second = net.connect({ port, host: "127.0.0.1" }); + second.on("connect", async () => { + log.push("connect second"); + second.destroy(); + first.destroy(); + // Awaited: each response's continuation dials the next connection. + const a = await dial(port, "a"); + await roundTrip(a, "a"); + log.push("continuation after a"); + const b = await dial(port, "b"); + await roundTrip(b, "b"); + a.destroy(); + b.destroy(); + setImmediate(() => { + server.close(); + console.log(log.join("\n")); + }); + }); + }); +}); diff --git a/tests/fixtures/npm/cases/undici5/main.ts b/tests/fixtures/npm/cases/undici5/main.ts index a65601a099..b9c2eafc9e 100644 --- a/tests/fixtures/npm/cases/undici5/main.ts +++ b/tests/fixtures/npm/cases/undici5/main.ts @@ -195,7 +195,6 @@ server.listen(0, "127.0.0.1", async () => { for (let i = 0; i < 4; i++) { const r = await request(`${origin}/r${i}`, { dispatcher: agent }); await r.body.text(); - await turn(); } console.log("connections opened:", agent.connections); console.log("stats:", agent.statsFor(origin)); diff --git a/tests/fixtures/npm/cases/undici7/main.ts b/tests/fixtures/npm/cases/undici7/main.ts index 24b608e612..054175e4d6 100644 --- a/tests/fixtures/npm/cases/undici7/main.ts +++ b/tests/fixtures/npm/cases/undici7/main.ts @@ -201,7 +201,6 @@ server.listen(0, "127.0.0.1", async () => { for (let i = 0; i < 4; i++) { const r = await request(`${origin}/r${i}`, { dispatcher: agent }); await r.body.text(); - await turn(); } console.log("connections opened:", agent.connections); console.log("stats:", agent.statsFor(origin)); From 4e0e7fc1afbb114163ad1bd55c598f10db9d5a00 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 19:55:18 -0700 Subject: [PATCH 03/10] Box URL into checked-dynamic unions beside fetch handles fetch's input under @types/node is string | URL | Request. A union with a checked-dynamic arm collapses to the checked-dynamic representation only when every sibling survives the collapse; URL did not qualify, so the type (and anything typed as typeof globalThis.fetch, such as a custom fetch cast through unknown) did not compile. URL's handle boxes by reference, keeps its accessors and setters, answers instanceof URL and checks back out to the typed value, so URL arms now join the collapse. A fetch() input typed string | URL also converts through the checked-dynamic input path instead of being narrowed to a string, which threw on the URL arm. Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/lowering/lower-stdlib.ts | 22 +++++++-- packages/compiler/src/frontend/type-mapper.ts | 27 +++++++---- .../6252-fetch-input-url-unions.ts.json | 6 +++ .../node-types/fetch-input-unions.ts.json | 6 +++ tests/corpus/6252-fetch-input-url-unions.ts | 48 +++++++++++++++++++ .../fixtures/node-types/fetch-input-unions.ts | 48 +++++++++++++++++++ tests/harness/project-config.test.ts | 21 ++++++++ 7 files changed, 165 insertions(+), 13 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6252-fetch-input-url-unions.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/node-types/fetch-input-unions.ts.json create mode 100644 tests/corpus/6252-fetch-input-url-unions.ts create mode 100644 tests/fixtures/node-types/fetch-input-unions.ts diff --git a/packages/compiler/src/frontend/lowering/lower-stdlib.ts b/packages/compiler/src/frontend/lowering/lower-stdlib.ts index 2f0e1c66f9..68a5145c3a 100644 --- a/packages/compiler/src/frontend/lowering/lower-stdlib.ts +++ b/packages/compiler/src/frontend/lowering/lower-stdlib.ts @@ -619,6 +619,20 @@ export function lowerFetchCall(lowerer: Lowerer, call: ts.CallExpression): IrExp : lowerer.lowerExpr(argument); stmts.push({ kind: "exprStmt", expr: discarded, loc: discarded.loc }); } + // A union input holding a URL arm (`string | URL`) converts through the + // checked-dynamic input path, which reads the URL's href at the call + // like the bare-URL arm below; narrowing it to a string would throw on + // the URL arm. + const unionWithUrl = + input.type.kind === "union" && + (lowerer.unions.get(input.type.unionId)?.arms.some((arm) => arm.kind === "url") ?? false) && + lowerer.dynConvertible(input.type); + const dynInput: IrExpr | null = + input.type.kind === "dyn" + ? inputRef + : unionWithUrl + ? { kind: "dynFrom", value: inputRef, type: DYN, loc: locOf(inputNode) } + : null; const url: IrExpr = input.type.kind === "url" ? { @@ -628,13 +642,13 @@ export function lowerFetchCall(lowerer: Lowerer, call: ts.CallExpression): IrExp type: STRING, loc: locOf(inputNode), } - : input.type.kind === "dyn" - ? inputRef + : dynInput !== null + ? dynInput : lowerer.coerceInto(inputNode, inputRef, STRING); const answer: IrExpr = { kind: "libCall", - fn: input.type.kind === "dyn" ? "fetch.input" : "fetch.start", - args: [input.type.kind === "dyn" ? inputRef : url, initRef], + fn: dynInput !== null ? "fetch.input" : "fetch.start", + args: [url, initRef], type: { kind: "promise", inner: DYN }, loc, }; diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 90b5d1e9f8..24ff477f53 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -1360,8 +1360,9 @@ export function mapAsyncGeneratorAnnotation(type: ts.Type, ctx: TypeMapperCtx): if ( !symbol || !["AsyncIterator", "AsyncIterableIterator", "AsyncIterable"].includes(symbol.name) || - !symbol.declarations - .some((d) => ts.isInterfaceDeclaration(d) && ctx.isStdlibFile(d.getSourceFile())) + !symbol.declarations.some( + (d) => ts.isInterfaceDeclaration(d) && ctx.isStdlibFile(d.getSourceFile()), + ) ) return null; const args = checker.getTypeArguments(type as ts.TypeReference); @@ -2513,12 +2514,11 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { if ( psym && WEBCRYPTO_DYN_TYPES.has(psym.name) && - psym.declarations - .some( - (d) => - (ts.isClassDeclaration(d) || ts.isInterfaceDeclaration(d)) && - ctx.isStdlibFile(d.getSourceFile()), - ) + psym.declarations.some( + (d) => + (ts.isClassDeclaration(d) || ts.isInterfaceDeclaration(d)) && + ctx.isStdlibFile(d.getSourceFile()), + ) ) { return DYN; } @@ -4184,7 +4184,9 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { // semantics would DEGRADE under the collapse — class instances, // Maps/Sets, functions (closure identity, `x === String` // narrowing), promises, regexes, generators, handles — keep their - // existing union homes and fences below. A pending recursive + // existing union homes and fences below. URL is the + // exception: its handle boxes by reference and keeps every member, + // so it collapses too (dynSubsumableUnionArm). A pending recursive // placeholder cannot collapse (back-references already hold the // union id); the degenerate recursive spelling keeps its fence. if ( @@ -6285,6 +6287,13 @@ function dynSubsumableUnionArm(arm: IrType, ctx: TypeMapperCtx): boolean { case "undefinedT": case "nullT": return true; + // URL boxes BY REFERENCE (its checked-dynamic handle keeps identity, + // reads and writes its members through the native accessors, answers + // `instanceof URL`, and checks back out to the typed value), so it + // rides the collapse beside fetch's Request/Response handles: + // `string | URL | Request`, fetch's own input type under @types/node. + case "url": + return true; case "record": case "array": return canConvertToDyn( diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6252-fetch-input-url-unions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6252-fetch-input-url-unions.ts.json new file mode 100644 index 0000000000..13010068a6 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6252-fetch-input-url-unions.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6252-fetch-input-url-unions.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/node-types/fetch-input-unions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/node-types/fetch-input-unions.ts.json new file mode 100644 index 0000000000..15aba2051c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/node-types/fetch-input-unions.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/node-types/fetch-input-unions.ts" + ], + "diags": [] +} diff --git a/tests/corpus/6252-fetch-input-url-unions.ts b/tests/corpus/6252-fetch-input-url-unions.ts new file mode 100644 index 0000000000..3394fe5b46 --- /dev/null +++ b/tests/corpus/6252-fetch-input-url-unions.ts @@ -0,0 +1,48 @@ +// `typeof globalThis.fetch` under @types/node takes `string | URL | Request`: +// unions holding URL handles beside fetch handles compile (URL boxes by +// reference into the checked-dynamic representation), and a custom fetch +// cast to that type through `unknown` keeps working, including an extra +// init member written through a record view (the dispatcher pattern). +import * as http from "node:http"; +type FetchImpl = typeof globalThis.fetch; +interface DispatchingInit { + method?: string; + headers?: Record; + dispatcher?: { name: string }; +} +function customFetch(input: string | URL, init?: DispatchingInit): Promise { + console.log("custom fetch", typeof input === "string" ? "string" : "url", init?.dispatcher?.name ?? "none"); + return fetch(input, { method: init?.method, headers: init?.headers }); +} +const server = http.createServer((req, res) => res.end(`ok ${req.method} ${req.url}`)); +server.listen(0, "127.0.0.1", async () => { + const address = server.address(); + const port = address !== null && typeof address === "object" ? address.port : 0; + const impl: FetchImpl = customFetch as unknown as FetchImpl; + const init: RequestInit = { method: "POST" }; + (init as Record).dispatcher = { name: "pinned" }; + const res = await impl(`http://127.0.0.1:${port}/a`, init); + console.log(res.status, await res.text()); + const viaUrl = await impl(new URL(`http://127.0.0.1:${port}/b`)); + console.log(viaUrl.status, await viaUrl.text()); + const direct = await fetch(new URL(`http://127.0.0.1:${port}/c`)); + console.log(direct.status, await direct.text()); + server.close(); +}); + +function describeInput(input: string | Request | URL): string { + if (typeof input === "string") return `string ${input}`; + if (input instanceof URL) return `url ${input.href} ${input.hostname}`; + return `request ${input.method} ${input.url}`; +} +console.log(describeInput("http://a.test/x")); +const u = new URL("http://b.test:8080/p?q=1"); +console.log(describeInput(u)); +console.log(describeInput(new Request("http://c.test/r", { method: "PUT" }))); +const items: (string | URL | Request)[] = [u, "s"]; +console.log(items.length, items[0] === u, String(items[0])); +let slot: string | URL | Request = u; +if (slot instanceof URL) { slot.pathname = "/changed"; } +console.log(u.href); +const params: string | Request | URL = new URL("http://d.test/?b=2"); +console.log(typeof params === "string" ? params : params instanceof URL ? params.searchParams.get("b") : "req"); diff --git a/tests/fixtures/node-types/fetch-input-unions.ts b/tests/fixtures/node-types/fetch-input-unions.ts new file mode 100644 index 0000000000..3394fe5b46 --- /dev/null +++ b/tests/fixtures/node-types/fetch-input-unions.ts @@ -0,0 +1,48 @@ +// `typeof globalThis.fetch` under @types/node takes `string | URL | Request`: +// unions holding URL handles beside fetch handles compile (URL boxes by +// reference into the checked-dynamic representation), and a custom fetch +// cast to that type through `unknown` keeps working, including an extra +// init member written through a record view (the dispatcher pattern). +import * as http from "node:http"; +type FetchImpl = typeof globalThis.fetch; +interface DispatchingInit { + method?: string; + headers?: Record; + dispatcher?: { name: string }; +} +function customFetch(input: string | URL, init?: DispatchingInit): Promise { + console.log("custom fetch", typeof input === "string" ? "string" : "url", init?.dispatcher?.name ?? "none"); + return fetch(input, { method: init?.method, headers: init?.headers }); +} +const server = http.createServer((req, res) => res.end(`ok ${req.method} ${req.url}`)); +server.listen(0, "127.0.0.1", async () => { + const address = server.address(); + const port = address !== null && typeof address === "object" ? address.port : 0; + const impl: FetchImpl = customFetch as unknown as FetchImpl; + const init: RequestInit = { method: "POST" }; + (init as Record).dispatcher = { name: "pinned" }; + const res = await impl(`http://127.0.0.1:${port}/a`, init); + console.log(res.status, await res.text()); + const viaUrl = await impl(new URL(`http://127.0.0.1:${port}/b`)); + console.log(viaUrl.status, await viaUrl.text()); + const direct = await fetch(new URL(`http://127.0.0.1:${port}/c`)); + console.log(direct.status, await direct.text()); + server.close(); +}); + +function describeInput(input: string | Request | URL): string { + if (typeof input === "string") return `string ${input}`; + if (input instanceof URL) return `url ${input.href} ${input.hostname}`; + return `request ${input.method} ${input.url}`; +} +console.log(describeInput("http://a.test/x")); +const u = new URL("http://b.test:8080/p?q=1"); +console.log(describeInput(u)); +console.log(describeInput(new Request("http://c.test/r", { method: "PUT" }))); +const items: (string | URL | Request)[] = [u, "s"]; +console.log(items.length, items[0] === u, String(items[0])); +let slot: string | URL | Request = u; +if (slot instanceof URL) { slot.pathname = "/changed"; } +console.log(u.href); +const params: string | Request | URL = new URL("http://d.test/?b=2"); +console.log(typeof params === "string" ? params : params instanceof URL ? params.searchParams.get("b") : "req"); diff --git a/tests/harness/project-config.test.ts b/tests/harness/project-config.test.ts index d98f12da37..226d3872a9 100644 --- a/tests/harness/project-config.test.ts +++ b/tests/harness/project-config.test.ts @@ -73,6 +73,27 @@ test("node-types: Headers, Request and Response members typed by undici-types ru expect(comparableStderr(native.stderr)).toBe(node.stderr); }); +test("node-types: fetch's string | URL | Request input and a custom fetch cast to typeof fetch", async () => { + const outDir = outDirFor("fetch-input-unions"); + const entry = join(nodeTypesDir, "fetch-input-unions.ts"); + const result = await compile(entry, { + outPath: join( + outDir, + process.platform === "win32" ? "fetch-input-unions.exe" : "fetch-input-unions", + ), + outDir, + sanitize, + }); + expect(result.ok, !result.ok ? JSON.stringify(result.diagnostics, null, 2) : "").toBe(true); + if (!result.ok) return; + const [native, node] = await Promise.all([ + execFileAsync(result.binaryPath), + execFileAsync(process.execPath, ["--disable-warning=MODULE_TYPELESS_PACKAGE_JSON", entry]), + ]); + expect(native.stdout).toBe(node.stdout); + expect(comparableStderr(native.stderr)).toBe(node.stderr); +}); + test("node-types: http header arrays and server.address() follow @types/node's types", async () => { const outDir = outDirFor("http-headers-address"); const entry = join(nodeTypesDir, "http-headers-address.ts"); From 87895c340e4de2284af6f5c5fc0c38cf10194c6a Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 19 Oct 2026 22:08:08 -0500 Subject: [PATCH 04/10] Test the undici shims against a peer that sends malformed responses The undici5 and undici7 npm cases now also run peers.ts against Node with the vendored undici: header values with CR or LF, chunk-size lines that are valid, malformed, overflowing or 1 MiB long, and fetch bodies that are endless (read once, then cancelled), gzipped, truncated, empty, corrupt, cut off, aborted after the headers, redirected while endless, or carry six content codings. The undici5 case leaves out the Host values, which undici 5.12 writes into the request head, and the two fetch cases where undici 5.12 differs (six codings, and an endless redirect, which fails with a TypeError in 5.12). package-shims.test.ts compiles two programs in copies of the cases: one reads 64 MiB chunk-size lines and 64 MiB plain and gzipped fetch bodies from a separate Node peer and checks that RSS stays under 48 MiB (plain lane), the other checks that the 5.x profile rejects a Host value with CR or LF with undici 5.29.0's error. --- packages/compiler/shims/undici/core/client.ts | 217 +++++++++++- packages/compiler/shims/undici/core/fetch.ts | 327 +++++++++++++++--- .../compiler/shims/undici/core/request.ts | 8 + .../fixtures/npm/cases/undici5/main.ts.json | 1 + .../fixtures/npm/cases/undici7/main.ts.json | 1 + tests/fixtures/npm/cases/undici5/main.ts | 2 + tests/fixtures/npm/cases/undici5/peers.ts | 183 ++++++++++ tests/fixtures/npm/cases/undici7/main.ts | 2 + tests/fixtures/npm/cases/undici7/peers.ts | 197 +++++++++++ .../fixtures/package-shims/undici-bounded.ts | 29 ++ .../package-shims/undici-hostile-peer.mjs | 49 +++ tests/fixtures/package-shims/undici5-host.ts | 26 ++ tests/harness/package-shims.test.ts | 70 +++- 13 files changed, 1041 insertions(+), 71 deletions(-) create mode 100644 tests/fixtures/npm/cases/undici5/peers.ts create mode 100644 tests/fixtures/npm/cases/undici7/peers.ts create mode 100644 tests/fixtures/package-shims/undici-bounded.ts create mode 100644 tests/fixtures/package-shims/undici-hostile-peer.mjs create mode 100644 tests/fixtures/package-shims/undici5-host.ts diff --git a/packages/compiler/shims/undici/core/client.ts b/packages/compiler/shims/undici/core/client.ts index 24ebe54d02..2d0f5c361b 100644 --- a/packages/compiler/shims/undici/core/client.ts +++ b/packages/compiler/shims/undici/core/client.ts @@ -782,6 +782,50 @@ function errorCode(err: Error): string { const CRLF = Buffer.from("\r\n"); +// Chunk-size line states (llhttp's chunk_size* and chunk_extension* nodes). +const SIZE_FIRST = 0; +const SIZE_DIGITS = 1; +const SIZE_AFTER = 2; +const SIZE_EXT = 3; +const SIZE_EXT_NAME = 4; +const SIZE_EXT_VALUE = 5; +const SIZE_EXT_QUOTED = 6; +const SIZE_EXT_PAIR = 7; +const SIZE_EXT_QUOTED_DONE = 8; +const SIZE_CR = 9; + +function hexDigit(c: number): number { + if (c >= 0x30 && c <= 0x39) return c - 0x30; + if (c >= 0x41 && c <= 0x46) return c - 0x37; + if (c >= 0x61 && c <= 0x66) return c - 0x57; + return -1; +} + +/** tchar: ALPHA, DIGIT and !#$%&'*+-.^_`|~. */ +const TOKEN_BYTES = tokenBytes(); + +function tokenBytes(): Uint8Array { + const table = new Uint8Array(256); + const marks = "!#$%&'*+-.^_`|~"; + for (let i = 0; i < marks.length; i++) table[marks.charCodeAt(i)] = 1; + for (let c = 0x30; c <= 0x39; c++) table[c] = 1; + for (let c = 0x41; c <= 0x5a; c++) { + table[c] = 1; + table[c + 0x20] = 1; + } + return table; +} + +/** qdtext: HTAB, SP and 0x21-0xff but '"' and '\\'. */ +function isQuotedByte(c: number): boolean { + return c === 0x09 || c === 0x20 || (c >= 0x21 && c !== 0x22 && c !== 0x5c); +} + +/** The byte after '\\' in a quoted-pair: HTAB, SP, VCHAR or obs-text. */ +function isQuotedPairByte(c: number): boolean { + return c === 0x09 || c === 0x20 || (c >= 0x21 && c !== 0x7f); +} + /** An incremental HTTP/1.1 response parser (the subset of llhttp undici * relies on): status line, raw header pairs, Content-Length, chunked and * read-until-close bodies, trailers, and pause/resume back-pressure. */ @@ -805,6 +849,9 @@ class ResponseParser { bytesRead: number; trailers: Buffer[]; skipBody: boolean; + sizeState: number; + sizeDigits: number; + chunkSize: number; constructor(client: Client, socket: Socket) { this.client = client; @@ -826,6 +873,9 @@ class ResponseParser { this.bytesRead = 0; this.trailers = []; this.skipBody = false; + this.sizeState = SIZE_FIRST; + this.sizeDigits = 0; + this.chunkSize = 0; } reset(): void { @@ -868,11 +918,154 @@ class ResponseParser { } } - fail(message: string, code: string): void { - this.client.__onParseError(this.socket, new HTTPParserError(message, code, "")); + /** A parse error with llhttp's reason: undici 7 puts it inside its + * protocol message and, as its error table maps names to numbers only, + * sets no code; undici 5 uses the reason as the message and the code. */ + fail(reason: string, code: string): void { + const err = + this.client.__profile.major >= 7 + ? new HTTPParserError(`Response does not match the HTTP/1.1 protocol (${reason})`) + : new HTTPParserError(reason, code, ""); + this.client.__onParseError(this.socket, err); this.state = 99; } + /** Reads the chunk-size line one byte at a time as llhttp does, so a line + * that never ends holds no memory: hex digits (past 16 significant ones + * the size overflows), then extensions, checked and dropped, then CRLF. + * The 7.x profile follows llhttp 9 (undici 7), the 5.x profile llhttp 8 + * in its lenient mode (undici 5). Returns true when the line is complete + * and chunkSize holds its size; false when it needs more bytes or failed. */ + chunkSizeLine(): boolean { + const buf = this.buf; + if (this.sizeState === SIZE_FIRST) { + // The common line, hex digits then CRLF in this buffer, in one pass; + // any other line takes the byte loop below from its start. + let size = 0; + let digits = 0; + let j = 0; + for (; j < buf.length && j < 32; j++) { + const d = hexDigit(buf[j]!); + if (d === -1) break; + if (digits > 0 || d !== 0) { + digits++; + size = size * 16 + d; + } + } + if (j > 0 && digits <= 16 && j + 1 < buf.length && buf[j] === 0x0d && buf[j + 1] === 0x0a) { + this.chunkSize = size; + this.buf = buf.subarray(j + 2); + return true; + } + } + const v7 = this.client.__profile.major >= 7; + let i = 0; + while (i < buf.length) { + const c = buf[i]!; + const state = this.sizeState; + if (state === SIZE_FIRST || state === SIZE_DIGITS) { + const d = hexDigit(c); + if (d === -1) { + if (state === SIZE_FIRST) { + this.fail("Invalid character in chunk size", "INVALID_CHUNK_SIZE"); + return false; + } + this.sizeState = SIZE_AFTER; + continue; + } + if (state === SIZE_FIRST) { + this.chunkSize = 0; + this.sizeDigits = 0; + this.sizeState = SIZE_DIGITS; + } + if (this.sizeDigits > 0 || d !== 0) { + if (this.sizeDigits === 16) { + this.fail("Chunk size overflow", "INVALID_CHUNK_SIZE"); + return false; + } + this.sizeDigits++; + this.chunkSize = this.chunkSize * 16 + d; + } + } else if (state === SIZE_AFTER) { + if (c === 0x0d) this.sizeState = SIZE_CR; + else if (c === 0x3b) this.sizeState = SIZE_EXT; + else if (c === 0x0a && v7) { + this.fail("Missing expected CR after chunk size", "CR_EXPECTED"); + return false; + } else { + this.fail("Invalid character in chunk size", "INVALID_CHUNK_SIZE"); + return false; + } + } else if (state === SIZE_EXT) { + if (c === 0x20 || c === 0x0d) { + this.fail("Invalid character in chunk extensions", "STRICT"); + return false; + } + this.sizeState = SIZE_EXT_NAME; + continue; + } else if (state === SIZE_EXT_NAME) { + if (c === 0x3d) this.sizeState = SIZE_EXT_VALUE; + else if (c === 0x3b) this.sizeState = SIZE_EXT; + else if (c === 0x0d) this.sizeState = SIZE_CR; + else if (c === 0x0a && v7) { + this.fail("Missing expected CR after chunk extension name", "CR_EXPECTED"); + return false; + } else if (TOKEN_BYTES[c] !== 1) { + this.fail("Invalid character in chunk extensions name", "STRICT"); + return false; + } + } else if (state === SIZE_EXT_VALUE) { + if (c === 0x22) this.sizeState = SIZE_EXT_QUOTED; + // llhttp 8 goes on with the chunk-size node after a value's ';'. + else if (c === 0x3b) this.sizeState = v7 ? SIZE_EXT : SIZE_AFTER; + else if (c === 0x0d) this.sizeState = SIZE_CR; + else if (c === 0x0a && v7) { + this.fail("Missing expected CR after chunk extension value", "CR_EXPECTED"); + return false; + } else if (TOKEN_BYTES[c] !== 1) { + this.fail("Invalid character in chunk extensions value", "STRICT"); + return false; + } + } else if (state === SIZE_EXT_QUOTED) { + if (c === 0x22) this.sizeState = SIZE_EXT_QUOTED_DONE; + else if (c === 0x5c && v7) this.sizeState = SIZE_EXT_PAIR; + else if (!isQuotedByte(c)) { + this.fail("Invalid character in chunk extensions quoted value", "STRICT"); + return false; + } + } else if (state === SIZE_EXT_PAIR) { + if (!isQuotedPairByte(c)) { + this.fail("Invalid quoted-pair in chunk extensions quoted value", "STRICT"); + return false; + } + this.sizeState = SIZE_EXT_QUOTED; + } else if (state === SIZE_EXT_QUOTED_DONE) { + if (c === 0x3b) this.sizeState = SIZE_EXT; + else if (c === 0x0d) this.sizeState = SIZE_CR; + else if (c === 0x0a && v7) { + this.fail("Missing expected CR after chunk extension value", "CR_EXPECTED"); + return false; + } else { + this.fail("Invalid character in chunk extensions quote value", "STRICT"); + return false; + } + } else { + // SIZE_CR: llhttp 9 needs the LF; llhttp 8's lenient mode skips + // whatever byte follows the CR. + if (c !== 0x0a && v7) { + this.fail("Expected LF after chunk size", "STRICT"); + return false; + } + this.buf = buf.subarray(i + 1); + this.sizeState = SIZE_FIRST; + return true; + } + i++; + } + this.buf = Buffer.alloc(0); + return false; + } + line(): Buffer | null { const idx = this.buf.indexOf(CRLF); if (idx === -1) return null; @@ -897,7 +1090,12 @@ class ResponseParser { const text = line.toString("latin1"); const m = /^HTTP\/1\.([01]) (\d{3})(?: (.*))?$/.exec(text); if (m === null) { - this.fail("Response does not match the HTTP/1.1 protocol", "INVALID_CONSTANT"); + const v7 = this.client.__profile.major >= 7; + if (!text.startsWith("HTTP/")) { + this.fail(v7 ? "Expected HTTP/, RTSP/ or ICE/" : "Expected HTTP/", "INVALID_CONSTANT"); + } else { + this.fail(v7 ? "Invalid status code" : "Invalid response status", "INVALID_STATUS"); + } return; } this.versionMinor = Number(m[1]); @@ -1007,18 +1205,11 @@ class ResponseParser { } } else if (this.state === 3) { // Chunk size line. - const line = this.line(); - if (line === null) return; - const sizeText = line.toString("latin1").split(";")[0]!.trim(); - if (!/^[0-9a-fA-F]+$/.test(sizeText)) { - this.fail("Invalid character in chunk size", "INVALID_CHUNK_SIZE"); - return; - } - const size = parseInt(sizeText, 16); - if (size === 0) { + if (!this.chunkSizeLine()) return; + if (this.chunkSize === 0) { this.state = 5; } else { - this.remaining = size; + this.remaining = this.chunkSize; this.state = 4; } } else if (this.state === 4) { diff --git a/packages/compiler/shims/undici/core/fetch.ts b/packages/compiler/shims/undici/core/fetch.ts index 9b287a7bd8..54a0ce0d52 100644 --- a/packages/compiler/shims/undici/core/fetch.ts +++ b/packages/compiler/shims/undici/core/fetch.ts @@ -2,14 +2,16 @@ // program-installed global dispatcher) it is the runtime's own fetch, which // implements the same WHATWG algorithm. With one, the request goes through // that dispatcher (its connector, lookup and pool), following undici's -// fetch for request headers, redirects and errors; the response body is -// buffered into a native Response. +// fetch for request headers, redirects and errors. The promise resolves +// when the response headers arrive, and the Response body streams from the +// connection with undici's back-pressure (see dispatchOnce). // // Known gaps: a Response built here reports `url` as "" and `redirected` as // false (the runtime's Response has no settable URL yet), and https requests // offer `gzip, deflate` rather than undici's `br, gzip, deflate` (the runtime // has no Brotli decoder). -import { gunzipSync, inflateSync } from "node:zlib"; +import type { Transform } from "node:stream"; +import * as zlib from "node:zlib"; import type { Dispatcher } from "./dispatcher.ts"; import { getGlobalDispatcher, hasCustomGlobalDispatcher } from "./api.ts"; import type { DispatchHandlers, Profile } from "./types.ts"; @@ -115,7 +117,9 @@ interface RawResponse { status: number; statusText: string; headers: string[]; - body: Buffer; + /** null when the response has no body to expose (null-body status, HEAD, + * or a redirect fetch follows); its bytes are then read and dropped. */ + body: ReadableStream | null; } const redirectStatuses = [301, 302, 303, 307, 308]; @@ -125,7 +129,7 @@ export function undiciFetch( input: string | URL, init: UndiciRequestInit | undefined, createGlobal: () => Dispatcher, - _profile: Profile, + profile: Profile, ): Promise { const explicit = init !== undefined ? init.dispatcher : undefined; if (explicit === undefined && !hasCustomGlobalDispatcher()) { @@ -139,7 +143,7 @@ export function undiciFetch( }); } const dispatcher = explicit ?? getGlobalDispatcher(createGlobal); - return fetchThrough(dispatcher, input, init ?? {}); + return fetchThrough(dispatcher, input, init ?? {}, profile); } function normalizeMethod(method: string): string { @@ -153,6 +157,7 @@ async function fetchThrough( dispatcher: Dispatcher, input: string | URL, init: UndiciRequestInit, + profile: Profile, ): Promise { let url = typeof input === "string" ? new URL(input) : new URL(input.href); let method = normalizeMethod(init.method ?? "GET"); @@ -197,7 +202,7 @@ async function fetchThrough( let raw: RawResponse; try { - raw = await dispatchOnce(dispatcher, url, method, pairs, body, signal); + raw = await dispatchOnce(dispatcher, url, method, pairs, body, signal, redirectMode, profile); } catch (err) { if (signal !== null && signal.aborted) throw abortReason(signal); throw new TypeError("fetch failed", { cause: err }); @@ -243,15 +248,11 @@ async function fetchThrough( continue; } - const nullBody = nullBodyStatuses.includes(raw.status) || method === "HEAD"; - return new Response( - nullBody ? null : decode(raw.body, responseHeaders.get("content-encoding")), - { - status: raw.status, - statusText: raw.statusText, - headers: responseHeaders, - }, - ); + return new Response(raw.body, { + status: raw.status, + statusText: raw.statusText, + headers: responseHeaders, + }); } } @@ -324,20 +325,28 @@ function headerListOf( return list; } -function decode(body: Buffer, encoding: string | null): Buffer { - if (encoding === null || body.length === 0) return body; - let out = body; - const codings = encoding - .toLowerCase() - .split(",") - .map((c) => c.trim()) - .reverse(); +/** undici 7's limit on a response's content codings, applied in both + * profiles: undici 5 has none, and builds one decoder per listed coding. */ +const maxContentEncodings = 5; + +/** Whether every coding has a decoder; otherwise the body passes through + * as sent. */ +function decodable(codings: string[]): boolean { + if (codings.length === 0) return false; for (const coding of codings) { - if (coding === "gzip" || coding === "x-gzip") out = gunzipSync(out); - else if (coding === "deflate") out = inflateSync(out); - else return body; + if (coding !== "gzip" && coding !== "x-gzip" && coding !== "deflate") return false; } - return out; + return true; +} + +/** A decoder for one coding. undici 7 decodes with Z_SYNC_FLUSH, which + * accepts a truncated stream; undici 5 uses zlib's defaults. */ +function decoderFor(coding: string, profile: Profile): Transform { + const options = + profile.major >= 7 + ? { flush: zlib.constants.Z_SYNC_FLUSH, finishFlush: zlib.constants.Z_SYNC_FLUSH } + : {}; + return coding === "deflate" ? zlib.createInflate(options) : zlib.createGunzip(options); } function abortReason(signal: AbortSignal): Error { @@ -347,6 +356,16 @@ function abortReason(signal: AbortSignal): Error { : new DOMException("This operation was aborted", "AbortError"); } +/** Unread bytes held beside the response stream before the connection + * pauses: Node's Readable highWaterMark, which undici's fetch body uses. */ +const bodyHighWaterMark = 64 * 1024; + +/** Dispatches one request and resolves when its response headers arrive. + * The body streams as undici's fetch reads one: up to bodyHighWaterMark + * unread bytes wait beside the stream, past that the connection (or the + * last decoder) pauses until the reader pulls again; cancelling the stream + * aborts the request, and an error after the headers errors the stream + * with TypeError('terminated'). */ function dispatchOnce( dispatcher: Dispatcher, url: URL, @@ -354,50 +373,248 @@ function dispatchOnce( headers: string[], body: Buffer | null, signal: AbortSignal | null, + redirectMode: "follow" | "manual" | "error", + profile: Profile, ): Promise { return new Promise((resolve, reject) => { - let status = 0; - let statusText = ""; - let rawHeaders: string[] = []; - const chunks: Buffer[] = []; - let settled = false; + let resolved = false; + let done = false; + let dump = false; let abortFn: ((err?: Error) => void) | null = null; - const onAbort = (): void => { + + // The response body: its stream controller, the unread bytes beside it, + // and how to restart the paused producer. + let controller: ReadableStreamDefaultController | null = null; + let hasBody = false; + const queue: Buffer[] = []; + let queued = 0; + let ended = false; + let closed = false; + let onDrain: (() => void) | null = null; + // The decoders, first-applied last; their listeners hold this scope, so + // releaseDecoders drops them once the body ends, fails or is cancelled. + let decoding = false; + let decoders: Transform[] = []; + let first: Transform | null = null; + let encodedBytes = 0; + let resumeConnection: (() => void) | null = null; + + const abortRequest = (err: Error): void => { const abort = abortFn; - if (abort !== null && signal !== null) abort(abortReason(signal)); + if (abort !== null && !done) abort(err); }; - if (signal !== null) signal.addEventListener("abort", onAbort); - const finish = (err: Error | null): void => { - if (settled) return; - settled = true; + const onAbort = (): void => { + if (signal === null) return; + const reason = abortReason(signal); + // After the headers undici 7 errors the body with the signal's + // reason; undici 5 aborts its fetch controller without one, so the + // body gets the controller's default AbortError. + errorBody( + profile.major >= 7 ? reason : new DOMException("The operation was aborted.", "AbortError"), + ); + abortRequest(reason); + }; + const stopListening = (): void => { if (signal !== null) signal.removeEventListener("abort", onAbort); - if (err !== null) reject(err); - else resolve({ status, statusText, headers: rawHeaders, body: Buffer.concat(chunks) }); }; + if (signal !== null) signal.addEventListener("abort", onAbort); + + const releaseDecoders = (): void => { + const chain = decoders; + decoders = []; + first = null; + for (const decoder of chain) { + decoder.removeAllListeners(); + decoder.destroy(); + } + }; + const closeBody = (): void => { + closed = true; + controller!.close(); + stopListening(); + }; + const errorBody = (err: unknown): void => { + if (!hasBody || closed) return; + closed = true; + queue.length = 0; + queued = 0; + controller!.error(err); + stopListening(); + releaseDecoders(); + }; + const endBody = (): void => { + if (closed) return; + ended = true; + if (queue.length === 0) closeBody(); + }; + /** Takes a chunk; false when the producer should pause until onDrain. */ + const pushBody = (chunk: Buffer): boolean => { + if (closed) return true; + if (queue.length === 0 && (controller!.desiredSize ?? 0) > 0) { + controller!.enqueue(new Uint8Array(chunk)); + return true; + } + queue.push(chunk); + queued += chunk.length; + return queued < bodyHighWaterMark; + }; + const pullBody = (): void => { + if (closed) return; + const next = queue.shift(); + if (next !== undefined) { + queued -= next.length; + controller!.enqueue(new Uint8Array(next)); + } + if (queue.length === 0 && ended) { + closeBody(); + return; + } + const drain = onDrain; + if (drain !== null && queued < bodyHighWaterMark) { + onDrain = null; + drain(); + } + }; + const cancelBody = (reason: unknown): void => { + if (closed) return; + closed = true; + queue.length = 0; + queued = 0; + stopListening(); + releaseDecoders(); + abortRequest( + reason instanceof Error + ? reason + : new DOMException("The operation was aborted.", "AbortError"), + ); + }; + const failBody = (err: Error): void => { + errorBody(new TypeError("terminated", { cause: err })); + abortRequest(err); + }; + const handler: DispatchHandlers = { onConnect: (abort: (err?: Error) => void): void => { abortFn = abort; if (signal !== null && signal.aborted) abort(abortReason(signal)); }, - onHeaders: ( - statusCode: number, - raw: Buffer[], - _resume: () => void, - text: string, - ): boolean => { + onHeaders: (statusCode: number, raw: Buffer[], resume: () => void, text: string): boolean => { if (statusCode < 200) return true; - status = statusCode; - statusText = text; - rawHeaders = []; - for (const b of raw) rawHeaders.push(b.toString("latin1")); + resolved = true; + resumeConnection = resume; + const rawHeaders: string[] = []; + let location = false; + let encoding: string | null = null; + for (let i = 0; i + 1 < raw.length; i += 2) { + const name = raw[i]!.toString("latin1"); + const value = raw[i + 1]!.toString("latin1"); + rawHeaders.push(name, value); + const lower = name.toLowerCase(); + if (lower === "location") location = true; + else if (lower === "content-encoding") + encoding = encoding === null ? value : `${encoding}, ${value}`; + } + const follows = + redirectStatuses.includes(statusCode) && location && redirectMode !== "manual"; + if (follows || nullBodyStatuses.includes(statusCode) || method === "HEAD") { + dump = true; + stopListening(); + resolve({ status: statusCode, statusText: text, headers: rawHeaders, body: null }); + return true; + } + const codings = + encoding === null + ? [] + : encoding + .toLowerCase() + .split(",") + .map((c) => c.trim()); + if (codings.length > maxContentEncodings) { + dump = true; + stopListening(); + reject( + new Error( + `too many content-encodings in response: ${codings.length}, maximum allowed is ${maxContentEncodings}`, + ), + ); + abortRequest(new DOMException("The operation was aborted.", "AbortError")); + return true; + } + hasBody = true; + const stream = new ReadableStream({ + start: (c: ReadableStreamDefaultController): void => { + controller = c; + }, + pull: (): void => pullBody(), + cancel: (reason: unknown): void => cancelBody(reason), + }); + if (decodable(codings)) { + // Decoders run last-applied coding first, piped into each other; + // the last one feeds the body. + decoding = true; + let last: Transform | null = null; + for (let i = codings.length - 1; i >= 0; i--) { + const decoder = decoderFor(codings[i]!, profile); + decoder.on("error", (err: Error) => failBody(err)); + if (last === null) first = decoder; + else last.pipe(decoder); + decoders.push(decoder); + last = decoder; + } + const output = last!; + output.on("data", (chunk: Buffer) => { + if (!pushBody(chunk)) { + output.pause(); + onDrain = (): void => { + output.resume(); + }; + } + }); + output.on("end", () => { + releaseDecoders(); + endBody(); + }); + first!.on("drain", () => resume()); + } + resolve({ status: statusCode, statusText: text, headers: rawHeaders, body: stream }); return true; }, onData: (chunk: Buffer): boolean => { - chunks.push(chunk); - return true; + if (dump || !hasBody || closed) return true; + if (decoding) { + const decoder = first; + if (decoder === null) return true; + encodedBytes += chunk.length; + return decoder.write(chunk); + } + if (pushBody(chunk)) return true; + onDrain = resumeConnection; + return false; + }, + onComplete: (_trailers: string[] | null): void => { + done = true; + if (!hasBody) { + stopListening(); + return; + } + // zlib rejects an empty stream; undici ends such a body unread. + const decoder = first; + if (decoder !== null && encodedBytes > 0) { + decoder.end(); + } else { + releaseDecoders(); + endBody(); + } + }, + onError: (err: Error): void => { + done = true; + if (!resolved) { + stopListening(); + reject(err); + return; + } + errorBody(new TypeError("terminated", { cause: err })); }, - onComplete: (_trailers: string[] | null): void => finish(null), - onError: (err: Error): void => finish(err), }; dispatcher.dispatch( { diff --git a/packages/compiler/shims/undici/core/request.ts b/packages/compiler/shims/undici/core/request.ts index a876e42f08..6e0f680221 100644 --- a/packages/compiler/shims/undici/core/request.ts +++ b/packages/compiler/shims/undici/core/request.ts @@ -227,8 +227,16 @@ export class Request { function processHeader(request: Request, key: string, val: string, profile: Profile): void { const lower = key.toLowerCase(); + // undici 7 checks the name and every value before the special cases; + // undici 5 checks the first Host value before storing it. Either way a + // CR or LF in a value never reaches the request head. + if (profile.major >= 7) { + if (!tokenRegExp.test(lower)) throw new InvalidArgumentError("invalid header key"); + if (headerCharRegex.test(val)) throw new InvalidArgumentError(`invalid ${key} header`); + } if (lower === "host") { if (request.host === null) { + if (headerCharRegex.test(val)) throw new InvalidArgumentError(`invalid ${key} header`); request.host = val; return; } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/undici5/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/undici5/main.ts.json index 80e48f7f14..dec84be6a0 100644 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/undici5/main.ts.json +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/undici5/main.ts.json @@ -15,6 +15,7 @@ "/packages/compiler/shims/undici/core/fetch.ts", "/packages/compiler/shims/undici/core/errors-v5.ts", "/packages/compiler/shims/undici/v5.ts", + "/tests/fixtures/npm/cases/undici5/peers.ts", "/tests/fixtures/npm/cases/undici5/main.ts" ], "diags": [] diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/undici7/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/undici7/main.ts.json index d4b65f4aea..75f724bde9 100644 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/undici7/main.ts.json +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/undici7/main.ts.json @@ -15,6 +15,7 @@ "/packages/compiler/shims/undici/core/fetch.ts", "/packages/compiler/shims/undici/core/errors-v7.ts", "/packages/compiler/shims/undici/v7.ts", + "/tests/fixtures/npm/cases/undici7/peers.ts", "/tests/fixtures/npm/cases/undici7/main.ts" ], "diags": [] diff --git a/tests/fixtures/npm/cases/undici5/main.ts b/tests/fixtures/npm/cases/undici5/main.ts index a65601a099..37f653495f 100644 --- a/tests/fixtures/npm/cases/undici5/main.ts +++ b/tests/fixtures/npm/cases/undici5/main.ts @@ -18,6 +18,7 @@ import { setGlobalDispatcher, } from "undici"; import type { Dispatcher } from "undici"; +import { checkPeers } from "./peers.ts"; const server = http.createServer((req, res) => { const url = req.url ?? "/"; @@ -294,5 +295,6 @@ server.listen(0, "127.0.0.1", async () => { await agent.close(); server.close(); + await checkPeers(); console.log("done"); }); diff --git a/tests/fixtures/npm/cases/undici5/peers.ts b/tests/fixtures/npm/cases/undici5/peers.ts new file mode 100644 index 0000000000..300fa6b359 --- /dev/null +++ b/tests/fixtures/npm/cases/undici5/peers.ts @@ -0,0 +1,183 @@ +// Requests against a raw TCP peer that sends malformed or endless +// responses: header values with CR or LF never reach the wire, a +// chunk-size line is read as llhttp reads it (so an endless one holds no +// memory), and fetch() resolves at the headers with a body that streams, +// cancels and fails as undici's does. +import * as net from "node:net"; +import type { Socket } from "node:net"; +import { gzipSync } from "node:zlib"; +import { Agent, errors, fetch, request } from "undici"; + +function describe(err: unknown): string { + // HTTPParserError extends Error, not UndiciError. + if (err instanceof errors.HTTPParserError) return `${err.name} ${err.code ?? ""} ${err.message}`; + if (err instanceof errors.UndiciError) return `${err.name} ${err.code} ${err.message}`; + if (err instanceof DOMException) return `${err.name} ${err.message}`; + if (err instanceof Error) { + const code = (err as NodeJS.ErrnoException).code ?? ""; + const cause = err.cause !== undefined ? ` cause: ${describe(err.cause)}` : ""; + return `${err.name} ${code} ${err.message}${cause}`; + } + return String(err); +} + +const MiB = 1024 * 1024; +const text = "hello ".repeat(2000); +const packed = gzipSync(text); + +// Chunked bodies whose chunk-size lines the client reads. Each is a +// complete response, or fails at the size line. +const chunkCases: [string, string][] = [ + ["plain", "5\r\nhello\r\n0\r\n\r\n"], + ["extensions", '5;a=b;c="d e";f\r\nhello\r\n0\r\n\r\n'], + ["quoted pair", '5;a="b\\"c"\r\nhello\r\n0\r\n\r\n'], + ["value then extension", "5;a=b;c=d\r\nhello\r\n0\r\n\r\n"], + ["space in name", "5;a b\r\nhello\r\n0\r\n\r\n"], + ["empty extension", "5;\r\nhello\r\n0\r\n\r\n"], + ["bad value", "5;a=b c\r\nhello\r\n0\r\n\r\n"], + ["bad quoted value", '5;a="b\x01"\r\nhello\r\n0\r\n\r\n'], + ["after quoted value", '5;a="b"c\r\nhello\r\n0\r\n\r\n'], + ["LF after name", "5;a\nhello\r\n0\r\n\r\n"], + ["LF after value", "5;a=b\nhello\r\n0\r\n\r\n"], + ["space after size", "5 \r\nhello\r\n0\r\n\r\n"], + ["space before size", " 5\r\nhello\r\n0\r\n\r\n"], + ["empty size", "\r\nhello\r\n0\r\n\r\n"], + ["not hex", "g\r\nhello\r\n0\r\n\r\n"], + ["LF only", "5\nhello\r\n0\r\n\r\n"], + ["CR then other byte", "5\rXhello\r\n0\r\n\r\n"], + ["16 hex digits", `${"0".repeat(11)}00005\r\nhello\r\n0\r\n\r\n`], + ["17 significant digits", "10000000000000000\r\nhello\r\n0\r\n\r\n"], + ["1 MiB of leading zeros", `${"0".repeat(MiB)}5\r\nhello\r\n0\r\n\r\n`], + ["1 MiB extension", `5;a=${"b".repeat(MiB)}\r\nhello\r\n0\r\n\r\n`], + ["1 MiB of digits", `1${"0".repeat(MiB)}\r\nhello\r\n0\r\n\r\n`], + ["1 MiB of spaces", `5${" ".repeat(MiB)}\r\nhello\r\n0\r\n\r\n`], +]; + +let chunkBody = ""; +let closes = 0; + +const peer = net.createServer((s: Socket) => { + s.on("error", () => {}); + s.on("close", () => { + closes++; + }); + let head = ""; + s.on("data", (d: Buffer) => { + head += d.toString("latin1"); + if (!head.includes("\r\n\r\n")) return; + const lines = head.split("\r\n"); + const path = lines[0]!.split(" ")[1]!; + head = ""; + const send = (status: string, fields: string, body: string): void => { + s.write(`HTTP/1.1 ${status}\r\n${fields}\r\n`); + s.write(body); + }; + if (path === "/echo") { + const received = lines.slice(1).filter((l) => l !== ""); + send("200 OK", `Content-Length: ${JSON.stringify(received).length}\r\n`, JSON.stringify(received)); + } else if (path === "/chunked") { + send("200 OK", "Transfer-Encoding: chunked\r\n", chunkBody); + } else if (path === "/endless") { + s.write("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n"); + const timer = setInterval(() => s.write("4\r\ndata\r\n"), 5); + s.on("close", () => clearInterval(timer)); + } else if (path === "/gzip") { + s.write(`HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: ${packed.length}\r\n\r\n`); + for (let i = 0; i < packed.length; i += 50) s.write(packed.subarray(i, i + 50)); + } else if (path === "/gzip-truncated") { + const part = packed.subarray(0, packed.length - 12); + s.write(`HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: ${part.length}\r\n\r\n`); + s.write(part); + } else if (path === "/gzip-empty") { + send("200 OK", "Content-Encoding: gzip\r\nContent-Length: 0\r\n", ""); + } else if (path === "/gzip-bad") { + send("200 OK", "Content-Encoding: gzip\r\nContent-Length: 8\r\n", "not gzip"); + } else if (path === "/cut") { + s.write("HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n0123456789"); + setTimeout(() => s.destroy(), 20); + } else if (path === "/no-content") { + send("204 No Content", "", ""); + } else { + send("200 OK", "Content-Length: 5\r\n", "plain"); + } + }); +}); + +export async function checkPeers(): Promise { + await new Promise((resolve) => peer.listen(0, "127.0.0.1", () => resolve())); + const origin = `http://127.0.0.1:${(peer.address() as net.AddressInfo).port}`; + const dispatcher = new Agent(); + + console.log("== peers: header values"); + // undici 5.12 writes a Host value with CR or LF into the request head; + // the built-in implementation rejects it as undici 5.29 does + // (package-shims.test.ts covers that). + for (const [name, value] of [["x-test", "a\r\nX-Injected: 1"]]) { + try { + const res = await request(`${origin}/echo`, { dispatcher, headers: { [name!]: value! } }); + console.log(JSON.stringify(value), "sent:", await res.body.text()); + } catch (err) { + console.log(JSON.stringify(value), describe(err)); + } + } + + console.log("== peers: chunk-size lines"); + for (const [label, body] of chunkCases) { + chunkBody = body; + try { + const res = await request(`${origin}/chunked`, { dispatcher }); + console.log(label, res.statusCode, JSON.stringify(await res.body.text())); + } catch (err) { + console.log(label, describe(err)); + } + } + + console.log("== peers: fetch bodies"); + const tryFetch = async ( + label: string, + path: string, + read: (res: Response) => Promise, + method = "GET", + signal: AbortSignal | null = null, + ): Promise => { + try { + const res = await fetch(`${origin}${path}`, { dispatcher, method, signal }); + console.log(label, res.status, res.body === null ? "null body" : "body"); + await read(res); + } catch (err) { + console.log(label, describe(err)); + } + }; + await tryFetch("endless", "/endless", async (res) => { + const reader = res.body!.getReader(); + const first = await reader.read(); + console.log("first chunk:", first.done, new TextDecoder().decode(first.value!).slice(0, 4)); + const before = closes; + await reader.cancel(); + await new Promise((resolve) => setTimeout(resolve, 100)); + console.log("cancel closed the connection:", closes > before); + }); + await tryFetch("gzip", "/gzip", async (res) => console.log("text matches:", (await res.text()) === text)); + await tryFetch("gzip-truncated", "/gzip-truncated", async (res) => + console.log("length", (await res.text()).length), + ); + await tryFetch("gzip-empty", "/gzip-empty", async (res) => console.log(JSON.stringify(await res.text()))); + await tryFetch("gzip-bad", "/gzip-bad", async (res) => console.log(JSON.stringify(await res.text()))); + await tryFetch("cut", "/cut", async (res) => console.log(JSON.stringify(await res.text()))); + const controller = new AbortController(); + await tryFetch( + "abort after headers", + "/endless", + async (res) => { + controller.abort(); + console.log(await res.text()); + }, + "GET", + controller.signal, + ); + await tryFetch("no-content", "/no-content", async (res) => console.log(JSON.stringify(await res.text()))); + await tryFetch("head", "/plain", async (res) => console.log(JSON.stringify(await res.text())), "HEAD"); + + await dispatcher.close(); + peer.close(); +} diff --git a/tests/fixtures/npm/cases/undici7/main.ts b/tests/fixtures/npm/cases/undici7/main.ts index 24b608e612..8ded547fbb 100644 --- a/tests/fixtures/npm/cases/undici7/main.ts +++ b/tests/fixtures/npm/cases/undici7/main.ts @@ -20,6 +20,7 @@ import { setGlobalDispatcher, } from "undici"; import type { Dispatcher } from "undici"; +import { checkPeers } from "./peers.ts"; const server = http.createServer((req, res) => { const url = req.url ?? "/"; @@ -357,5 +358,6 @@ server.listen(0, "127.0.0.1", async () => { await agent.close(); server.close(); + await checkPeers(); console.log("done"); }); diff --git a/tests/fixtures/npm/cases/undici7/peers.ts b/tests/fixtures/npm/cases/undici7/peers.ts new file mode 100644 index 0000000000..3c2c54fbc9 --- /dev/null +++ b/tests/fixtures/npm/cases/undici7/peers.ts @@ -0,0 +1,197 @@ +// Requests against a raw TCP peer that sends malformed or endless +// responses: header values with CR or LF never reach the wire, a +// chunk-size line is read as llhttp reads it (so an endless one holds no +// memory), and fetch() resolves at the headers with a body that streams, +// cancels and fails as undici's does. +import * as net from "node:net"; +import type { Socket } from "node:net"; +import { gzipSync } from "node:zlib"; +import { Agent, errors, fetch, request } from "undici"; + +function describe(err: unknown): string { + // HTTPParserError extends Error, not UndiciError. + if (err instanceof errors.HTTPParserError) return `${err.name} ${err.code ?? ""} ${err.message}`; + if (err instanceof errors.UndiciError) return `${err.name} ${err.code} ${err.message}`; + if (err instanceof DOMException) return `${err.name} ${err.message}`; + if (err instanceof Error) { + const code = (err as NodeJS.ErrnoException).code ?? ""; + const cause = err.cause !== undefined ? ` cause: ${describe(err.cause)}` : ""; + return `${err.name} ${code} ${err.message}${cause}`; + } + return String(err); +} + +const MiB = 1024 * 1024; +const text = "hello ".repeat(2000); +const packed = gzipSync(text); + +// Chunked bodies whose chunk-size lines the client reads. Each is a +// complete response, or fails at the size line. +const chunkCases: [string, string][] = [ + ["plain", "5\r\nhello\r\n0\r\n\r\n"], + ["extensions", '5;a=b;c="d e";f\r\nhello\r\n0\r\n\r\n'], + ["quoted pair", '5;a="b\\"c"\r\nhello\r\n0\r\n\r\n'], + ["value then extension", "5;a=b;c=d\r\nhello\r\n0\r\n\r\n"], + ["space in name", "5;a b\r\nhello\r\n0\r\n\r\n"], + ["empty extension", "5;\r\nhello\r\n0\r\n\r\n"], + ["bad value", "5;a=b c\r\nhello\r\n0\r\n\r\n"], + ["bad quoted value", '5;a="b\x01"\r\nhello\r\n0\r\n\r\n'], + ["after quoted value", '5;a="b"c\r\nhello\r\n0\r\n\r\n'], + ["LF after name", "5;a\nhello\r\n0\r\n\r\n"], + ["LF after value", "5;a=b\nhello\r\n0\r\n\r\n"], + ["space after size", "5 \r\nhello\r\n0\r\n\r\n"], + ["space before size", " 5\r\nhello\r\n0\r\n\r\n"], + ["empty size", "\r\nhello\r\n0\r\n\r\n"], + ["not hex", "g\r\nhello\r\n0\r\n\r\n"], + ["LF only", "5\nhello\r\n0\r\n\r\n"], + ["CR then other byte", "5\rXhello\r\n0\r\n\r\n"], + ["16 hex digits", `${"0".repeat(11)}00005\r\nhello\r\n0\r\n\r\n`], + ["17 significant digits", "10000000000000000\r\nhello\r\n0\r\n\r\n"], + ["1 MiB of leading zeros", `${"0".repeat(MiB)}5\r\nhello\r\n0\r\n\r\n`], + ["1 MiB extension", `5;a=${"b".repeat(MiB)}\r\nhello\r\n0\r\n\r\n`], + ["1 MiB of digits", `1${"0".repeat(MiB)}\r\nhello\r\n0\r\n\r\n`], + ["1 MiB of spaces", `5${" ".repeat(MiB)}\r\nhello\r\n0\r\n\r\n`], +]; + +let chunkBody = ""; +let redirectSocket: Socket | null = null; +let closes = 0; + +const peer = net.createServer((s: Socket) => { + s.on("error", () => {}); + s.on("close", () => { + closes++; + }); + let head = ""; + s.on("data", (d: Buffer) => { + head += d.toString("latin1"); + if (!head.includes("\r\n\r\n")) return; + const lines = head.split("\r\n"); + const path = lines[0]!.split(" ")[1]!; + head = ""; + const send = (status: string, fields: string, body: string): void => { + s.write(`HTTP/1.1 ${status}\r\n${fields}\r\n`); + s.write(body); + }; + if (path === "/echo") { + const received = lines.slice(1).filter((l) => l !== ""); + send("200 OK", `Content-Length: ${JSON.stringify(received).length}\r\n`, JSON.stringify(received)); + } else if (path === "/chunked") { + send("200 OK", "Transfer-Encoding: chunked\r\n", chunkBody); + } else if (path === "/endless") { + s.write("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n"); + const timer = setInterval(() => s.write("4\r\ndata\r\n"), 5); + s.on("close", () => clearInterval(timer)); + } else if (path === "/gzip") { + s.write(`HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: ${packed.length}\r\n\r\n`); + for (let i = 0; i < packed.length; i += 50) s.write(packed.subarray(i, i + 50)); + } else if (path === "/gzip-truncated") { + const part = packed.subarray(0, packed.length - 12); + s.write(`HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: ${part.length}\r\n\r\n`); + s.write(part); + } else if (path === "/gzip-empty") { + send("200 OK", "Content-Encoding: gzip\r\nContent-Length: 0\r\n", ""); + } else if (path === "/gzip-bad") { + send("200 OK", "Content-Encoding: gzip\r\nContent-Length: 8\r\n", "not gzip"); + } else if (path === "/cut") { + s.write("HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n0123456789"); + setTimeout(() => s.destroy(), 20); + } else if (path === "/many-codings") { + send("200 OK", "Content-Encoding: gzip, gzip, gzip, gzip, gzip, gzip\r\nContent-Length: 2\r\n", "ok"); + } else if (path === "/redirect-endless") { + redirectSocket = s; + s.write("HTTP/1.1 302 Found\r\nLocation: /plain\r\nTransfer-Encoding: chunked\r\n\r\n"); + const timer = setInterval(() => s.write("4\r\ndata\r\n"), 5); + s.on("close", () => clearInterval(timer)); + } else if (path === "/no-content") { + send("204 No Content", "", ""); + } else { + send("200 OK", "Content-Length: 5\r\n", "plain"); + } + }); +}); + +export async function checkPeers(): Promise { + await new Promise((resolve) => peer.listen(0, "127.0.0.1", () => resolve())); + const origin = `http://127.0.0.1:${(peer.address() as net.AddressInfo).port}`; + const dispatcher = new Agent(); + + console.log("== peers: header values"); + for (const [name, value] of [ + ["host", "a\r\nX-Injected: 1"], + ["host", "a\nb"], + ["x-test", "a\r\nX-Injected: 1"], + ]) { + try { + const res = await request(`${origin}/echo`, { dispatcher, headers: { [name!]: value! } }); + console.log(JSON.stringify(value), "sent:", await res.body.text()); + } catch (err) { + console.log(JSON.stringify(value), describe(err)); + } + } + + console.log("== peers: chunk-size lines"); + for (const [label, body] of chunkCases) { + chunkBody = body; + try { + const res = await request(`${origin}/chunked`, { dispatcher }); + console.log(label, res.statusCode, JSON.stringify(await res.body.text())); + } catch (err) { + console.log(label, describe(err)); + } + } + + console.log("== peers: fetch bodies"); + const tryFetch = async ( + label: string, + path: string, + read: (res: Response) => Promise, + method = "GET", + signal: AbortSignal | null = null, + ): Promise => { + try { + const res = await fetch(`${origin}${path}`, { dispatcher, method, signal }); + console.log(label, res.status, res.body === null ? "null body" : "body"); + await read(res); + } catch (err) { + console.log(label, describe(err)); + } + }; + await tryFetch("endless", "/endless", async (res) => { + const reader = res.body!.getReader(); + const first = await reader.read(); + console.log("first chunk:", first.done, new TextDecoder().decode(first.value!).slice(0, 4)); + const before = closes; + await reader.cancel(); + await new Promise((resolve) => setTimeout(resolve, 100)); + console.log("cancel closed the connection:", closes > before); + }); + await tryFetch("gzip", "/gzip", async (res) => console.log("text matches:", (await res.text()) === text)); + await tryFetch("gzip-truncated", "/gzip-truncated", async (res) => + console.log("length", (await res.text()).length), + ); + await tryFetch("gzip-empty", "/gzip-empty", async (res) => console.log(JSON.stringify(await res.text()))); + await tryFetch("gzip-bad", "/gzip-bad", async (res) => console.log(JSON.stringify(await res.text()))); + await tryFetch("cut", "/cut", async (res) => console.log(JSON.stringify(await res.text()))); + const controller = new AbortController(); + await tryFetch( + "abort after headers", + "/endless", + async (res) => { + controller.abort(); + console.log(await res.text()); + }, + "GET", + controller.signal, + ); + await tryFetch("many-codings", "/many-codings", async (res) => console.log(await res.text())); + await tryFetch("redirect-endless", "/redirect-endless", async (res) => + console.log("followed:", await res.text()), + ); + await tryFetch("no-content", "/no-content", async (res) => console.log(JSON.stringify(await res.text()))); + await tryFetch("head", "/plain", async (res) => console.log(JSON.stringify(await res.text())), "HEAD"); + + if (redirectSocket !== null) (redirectSocket as Socket).destroy(); + await dispatcher.close(); + peer.close(); +} diff --git a/tests/fixtures/package-shims/undici-bounded.ts b/tests/fixtures/package-shims/undici-bounded.ts new file mode 100644 index 0000000000..a531a66493 --- /dev/null +++ b/tests/fixtures/package-shims/undici-bounded.ts @@ -0,0 +1,29 @@ +// Reads every response of undici-hostile-peer.mjs (port in argv[2]): the +// endless chunk-size lines through request(), the large and the gzipped +// bodies through fetch() with a pause after the first chunk. Memory stays +// bounded by the parser's and the body stream's buffers. +import { Agent, fetch, request } from "undici"; + +const MiB = 1024 * 1024; +const origin = `http://127.0.0.1:${process.argv[2]}`; +const dispatcher = new Agent(); +for (const path of ["/extension", "/zeros"]) { + const res = await request(`${origin}${path}`, { dispatcher }); + console.log(path, res.statusCode, await res.body.text()); +} +for (const path of ["/big", "/bomb"]) { + const res = await fetch(`${origin}${path}`, { dispatcher }); + const reader = res.body!.getReader(); + const first = await reader.read(); + let total = first.value!.length; + await new Promise((resolve) => setTimeout(resolve, 500)); + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + total += value!.length; + } + console.log(path, res.status, total / MiB, "MiB"); +} +const rss = process.memoryUsage().rss; +console.log(rss < 48 * MiB ? "rss under 48 MiB" : `rss ${Math.round(rss / MiB)} MiB`); +await dispatcher.close(); diff --git a/tests/fixtures/package-shims/undici-hostile-peer.mjs b/tests/fixtures/package-shims/undici-hostile-peer.mjs new file mode 100644 index 0000000000..ce6c5881c6 --- /dev/null +++ b/tests/fixtures/package-shims/undici-hostile-peer.mjs @@ -0,0 +1,49 @@ +// A peer for package-shims.test.ts: prints its port, then answers +// /extension a chunk whose size line carries 64 MiB of extension, +// /zeros a chunk size with 64 MiB of leading zeros, +// /big a 64 MiB body, as fast as the socket takes it, +// /bomb 64 MiB of zeros, gzipped, +// and exits when its stdin closes. +import * as net from "node:net"; +import * as zlib from "node:zlib"; + +const MiB = 1024 * 1024; +const size = 64 * MiB; +const bomb = zlib.gzipSync(Buffer.alloc(size)); + +function stream(s, head, piece, tail) { + s.write(head); + let sent = 0; + const pump = () => { + while (sent < size) { + sent += piece.length; + if (!s.write(piece)) return void s.once("drain", pump); + } + s.write(tail); + }; + pump(); +} + +const server = net.createServer((s) => { + s.on("error", () => {}); + let head = ""; + s.on("data", (d) => { + head += d.toString("latin1"); + if (!head.includes("\r\n\r\n")) return; + const path = head.split(" ")[1]; + head = ""; + const chunked = "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n"; + if (path === "/extension") { + stream(s, `${chunked}5;a=`, Buffer.alloc(64 * 1024, 98), "\r\nhello\r\n0\r\n\r\n"); + } else if (path === "/zeros") { + stream(s, chunked, Buffer.alloc(64 * 1024, 48), "5\r\nhello\r\n0\r\n\r\n"); + } else if (path === "/big") { + stream(s, `HTTP/1.1 200 OK\r\nContent-Length: ${size}\r\n\r\n`, Buffer.alloc(64 * 1024, 97), ""); + } else { + s.write(`HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: ${bomb.length}\r\n\r\n`); + s.write(bomb); + } + }); +}); +server.listen(0, "127.0.0.1", () => console.log(server.address().port)); +process.stdin.on("end", () => process.exit(0)).resume(); diff --git a/tests/fixtures/package-shims/undici5-host.ts b/tests/fixtures/package-shims/undici5-host.ts new file mode 100644 index 0000000000..92b395ff83 --- /dev/null +++ b/tests/fixtures/package-shims/undici5-host.ts @@ -0,0 +1,26 @@ +// A Host header value with CR or LF, through the undici 5.x profile: it +// throws before the request is written, as undici 5.29 does (5.12, the +// vendored fixture version, writes the value into the request head). +import * as http from "node:http"; +import { Agent, errors, request } from "undici"; + +const server = http.createServer((_req, res) => { + res.statusCode = 204; + res.end(); +}); +server.listen(0, "127.0.0.1", async () => { + const origin = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const dispatcher = new Agent(); + for (const host of ["ok.test", "a\r\nX-Injected: 1", "a\nb"]) { + try { + const res = await request(`${origin}/`, { dispatcher, headers: { host } }); + await res.body.dump(); + console.log(JSON.stringify(host), res.statusCode); + } catch (err) { + if (err instanceof errors.UndiciError) console.log(JSON.stringify(host), err.name, err.code, err.message); + else throw err; + } + } + await dispatcher.close(); + server.close(); +}); diff --git a/tests/harness/package-shims.test.ts b/tests/harness/package-shims.test.ts index 3b31c5e330..0be52d2df2 100644 --- a/tests/harness/package-shims.test.ts +++ b/tests/harness/package-shims.test.ts @@ -6,10 +6,14 @@ * package, and the package's code never runs; * - a shimmed package that is not installed, or whose major version has no * shim, resolves exactly as before (here: refused like any missing or - * uncompiled package). + * uncompiled package); + * - the undici shims keep memory bounded against a peer that sends endless + * chunk-size lines and large or compressed bodies, and the 5.x profile + * rejects a Host value with CR or LF as undici 5.29 does (the vendored + * 5.12 writes it into the request). */ -import { execFile } from "node:child_process"; -import { cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { execFile, spawn } from "node:child_process"; +import { copyFileSync, cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { promisify } from "node:util"; @@ -19,6 +23,7 @@ import { compile } from "@scriptc/compiler"; const execFileAsync = promisify(execFile); const repoRoot = join(import.meta.dirname, "../.."); const casesRoot = join(repoRoot, "tests/fixtures/npm/cases"); +const shimFixtures = join(repoRoot, "tests/fixtures/package-shims"); const sanitize = process.env["SCRIPTC_SAN"] === "1"; const scratch = mkdtempSync(join(tmpdir(), "scriptc-package-shims-")); @@ -49,6 +54,37 @@ function installEncoding(dir: string): void { ); } +/** Compiles `entry` (a file in tests/fixtures/package-shims) as main.ts of + * a copy of the npm case `name`, which carries the package. */ +async function buildInCase(name: string, entry: string): Promise<{ dir: string; binary: string }> { + const dir = copyCase(name, `${name}-${entry.replace(/\..*$/, "")}`); + copyFileSync(join(shimFixtures, entry), join(dir, "main.ts")); + const out = join(dir, ".scriptc"); + const result = await compile(join(dir, "main.ts"), { + outPath: join(out, "program"), + outDir: out, + sanitize, + }); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + return { dir, binary: result.binaryPath }; +} + +/** Runs `binary ` against undici-hostile-peer.mjs. */ +async function runAgainstPeer(binary: string, cwd: string): Promise { + const peer = spawn(process.execPath, [join(shimFixtures, "undici-hostile-peer.mjs")], { + stdio: ["pipe", "pipe", "inherit"], + }); + try { + const port = await new Promise((resolve, reject) => { + peer.stdout.once("data", (d: Buffer) => resolve(d.toString().trim())); + peer.once("exit", (code) => reject(new Error(`peer exited with ${code}`))); + }); + return await run(binary, [port], cwd); + } finally { + peer.stdin.end(); + } +} + describe("package shims", () => { test("an unloaded package is never loaded, even when installed", async () => { const original = join(casesRoot, "unloaded-optional-dependency"); @@ -112,4 +148,32 @@ describe("package shims", () => { expect(result.diagnostics.map((d) => d.message).join("\n")).not.toContain("shims/undici"); } }, 180_000); + + for (const name of ["undici7", "undici5"]) { + test(`${name}: endless chunk-size lines and large bodies stay in bounded memory`, async () => { + const { dir, binary } = await buildInCase(name, "undici-bounded.ts"); + const out = await runAgainstPeer(binary, dir); + expect(out.split("\n").slice(0, 4)).toEqual([ + "/extension 200 hello", + "/zeros 200 hello", + "/big 200 64 MiB", + "/bomb 200 64 MiB", + ]); + // ASan's shadow memory and quarantine keep freed buffers mapped. + if (!sanitize) expect(out.split("\n")[4]).toBe("rss under 48 MiB"); + }, 300_000); + } + + test("undici5: a Host value with CR or LF throws before the request is written", async () => { + const { dir, binary } = await buildInCase("undici5", "undici5-host.ts"); + // undici 5.29.0's output. + expect(await run(binary, [], dir)).toBe( + [ + '"ok.test" 204', + '"a\\r\\nX-Injected: 1" InvalidArgumentError UND_ERR_INVALID_ARG invalid host header', + '"a\\nb" InvalidArgumentError UND_ERR_INVALID_ARG invalid host header', + "", + ].join("\n"), + ); + }, 180_000); }); From 9548ab3663d2c89261c9213d5f8fa23ce2a48e6f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 19 Oct 2026 23:38:15 -0500 Subject: [PATCH 05/10] Give the undici npm cases a 600 s timeout The undici cases compile the built-in undici implementation's module graph; a sanitized build of it on a CI runner outlasted the default 120 s per case. --- .../compiler/test/ts7/resolver-parity.test.ts | 15 +++++- .../compiler/test/undici-shim-types.test.ts | 46 +++++++++---------- tests/harness/node-types-timers.test.ts | 14 ++++-- tests/harness/npm.test.ts | 7 ++- 4 files changed, 50 insertions(+), 32 deletions(-) diff --git a/packages/compiler/test/ts7/resolver-parity.test.ts b/packages/compiler/test/ts7/resolver-parity.test.ts index b1788d4da9..b3190ae9f9 100644 --- a/packages/compiler/test/ts7/resolver-parity.test.ts +++ b/packages/compiler/test/ts7/resolver-parity.test.ts @@ -22,6 +22,7 @@ import { resolveTypeDirective, setProjectPathMappings, } from "../../src/frontend/resolve.js"; +import { packageShimFor } from "../../src/frontend/package-shims.js"; import { isRelativeSpecifier } from "../../src/frontend/workspace-registry.js"; import { options5 } from "./harness.js"; @@ -143,8 +144,18 @@ test("relative and bare specifiers across the whole test tree resolve identicall // for bare specifiers are exactly the package.json-mediated // resolutions inside the project — resolveProjectImport must agree. const oursProj = resolveProjectImport(file, spec); - const theirsProj = - reference && !reference.isExternalLibraryImport ? reference.resolvedFileName : null; + // The deliberate package-shim delta: a package scriptc implements + // itself (package-shims.ts) answers its shim module here, where + // 5.9.3 answers the installed package as an external import; the + // installed package's answer is still compared below. + const shimmed = packageShimFor(file, spec) !== null; + const theirsProj = shimmed + ? reference && reference.isExternalLibraryImport + ? packageShimFor(file, spec) + : null + : reference && !reference.isExternalLibraryImport + ? reference.resolvedFileName + : null; if (oursProj !== theirsProj) { failures.push(`${file} '${spec}' (project): ours=${oursProj} ts5=${theirsProj}`); } diff --git a/packages/compiler/test/undici-shim-types.test.ts b/packages/compiler/test/undici-shim-types.test.ts index c7bc4ccaf3..a0b770add2 100644 --- a/packages/compiler/test/undici-shim-types.test.ts +++ b/packages/compiler/test/undici-shim-types.test.ts @@ -18,9 +18,12 @@ * program the check builds includes them), since real projects compile * them with their own Node types. */ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import { join } from "node:path"; -import ts5 from "typescript5"; import { describe, expect, test } from "vitest"; +import * as ad from "../src/frontend/ts7/adapter.js"; +import { createProgram, getPreEmitDiagnostics } from "../src/frontend/ts7/program-adapter.js"; const repoRoot = join(import.meta.dirname, "../../.."); const shimRoot = join(repoRoot, "packages/compiler/shims/undici"); @@ -282,31 +285,29 @@ type _clientApi = ExpectNever, keyof real.Clie `; } -function check(files: Record): string[] { - const options: ts5.CompilerOptions = { +/** The diagnostics of one program, checked by the TypeScript 7 checker. */ +function check(name: string, text: string): string[] { + const dir = mkdtempSync(join(tmpdir(), "scriptc-undici-shim-types-")); + const file = join(dir, name); + writeFileSync(file, text); + const program = createProgram([file], { strict: true, noEmit: true, skipLibCheck: true, allowImportingTsExtensions: true, - module: ts5.ModuleKind.ESNext, - moduleResolution: ts5.ModuleResolutionKind.Bundler, - target: ts5.ScriptTarget.ES2022, + module: ad.ModuleKind.ESNext as number, + moduleResolution: ad.ModuleResolutionKind.Bundler as number, + target: ad.ScriptTarget.ES2022 as number, lib: ["lib.es2023.d.ts"], types: ["node"], typeRoots: [join(repoRoot, "node_modules/@types")], - }; - const host = ts5.createCompilerHost(options); - const readFile = host.readFile.bind(host); - const fileExists = host.fileExists.bind(host); - host.readFile = (name) => files[name] ?? readFile(name); - host.fileExists = (name) => name in files || fileExists(name); - const program = ts5.createProgram(Object.keys(files), options, host); - return ts5.getPreEmitDiagnostics(program).map((d) => { - const where = d.file - ? `${d.file.fileName}:${d.file.getLineAndCharacterOfPosition(d.start ?? 0).line + 1}` - : ""; - return `${where} ${ts5.flattenDiagnosticMessageText(d.messageText, "\n")}`; }); + try { + return getPreEmitDiagnostics(program).map((d) => `${d.fileName ?? ""}:${d.pos} ${d.text}`); + } finally { + program.dispose(); + rmSync(dir, { recursive: true, force: true }); + } } describe.each([5, 7])("undici %i.x shim types", (major) => { @@ -314,18 +315,15 @@ describe.each([5, 7])("undici %i.x shim types", (major) => { const shim = join(shimRoot, `v${major}.ts`); test("a program written against undici type-checks against the real declarations", () => { - const file = join(repoRoot, `.undici-usage-real-${major}.ts`); - expect(check({ [file]: helpers + usage(major).replace(/UNDICI/g, real) })).toEqual([]); + expect(check("usage-real.ts", helpers + usage(major).replace(/UNDICI/g, real))).toEqual([]); }); test("the same program type-checks against the shim", () => { - const file = join(repoRoot, `.undici-usage-shim-${major}.ts`); - expect(check({ [file]: helpers + usage(major).replace(/UNDICI/g, shim) })).toEqual([]); + expect(check("usage-shim.ts", helpers + usage(major).replace(/UNDICI/g, shim))).toEqual([]); }); test("the shim's exports and records are a compatible subset", () => { - const file = join(repoRoot, `.undici-shape-${major}.ts`); const text = helpers + shape(major).replace(/REAL/g, real).replace(/SHIM/g, shim); - expect(check({ [file]: text })).toEqual([]); + expect(check("shape.ts", text)).toEqual([]); }); }); diff --git a/tests/harness/node-types-timers.test.ts b/tests/harness/node-types-timers.test.ts index 257d644ffc..faaafcca9d 100644 --- a/tests/harness/node-types-timers.test.ts +++ b/tests/harness/node-types-timers.test.ts @@ -53,7 +53,7 @@ const program = `class Holder { } const holder = new Holder(); holder.timer = setTimeout(() => console.log("never"), 50); -const kept = setTimeout(() => console.log("fired", holder.timer === null), 10); +const kept = setTimeout(() => console.log("fired", holder.timer === null), 200); kept.unref().ref(); kept.refresh(); console.log("hasRef", kept.hasRef()); @@ -84,10 +84,14 @@ test("@types/node 20 class-declared timer handles compile and match Node", async }); if (!result.ok) throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); - const [node, native] = await Promise.all([ - execFileAsync(process.execPath, [join(dir, "main.ts")], { cwd: dir, encoding: "utf8" }), - execFileAsync(result.binaryPath, [], { cwd: dir, encoding: "utf8" }), - ]); + // One after the other, and the timer 200 ms out: under load Node's own + // startup can outlast a short timer, which then runs before the + // immediate. + const node = await execFileAsync(process.execPath, [join(dir, "main.ts")], { + cwd: dir, + encoding: "utf8", + }); + const native = await execFileAsync(result.binaryPath, [], { cwd: dir, encoding: "utf8" }); expect(native.stdout).toBe(node.stdout); expect(native.stdout).toBe("hasRef true\nimmediate\nfired true\n"); }, 180_000); diff --git a/tests/harness/npm.test.ts b/tests/harness/npm.test.ts index 54207f2464..f6f31f35c3 100644 --- a/tests/harness/npm.test.ts +++ b/tests/harness/npm.test.ts @@ -113,6 +113,11 @@ const KNOWN: Record = { const cases = shardSelect(npmCases(fixturesRoot), (c) => c.name); +/** Per-case timeouts past the default 120 s: the undici cases compile the + * built-in undici implementation's module graph, which a sanitized build + * on a CI runner takes minutes to compile. */ +const TIMEOUTS: Record = { undici5: 600_000, undici7: 600_000 }; + interface RunResult { stdout: Buffer; stderr: Buffer; @@ -235,7 +240,7 @@ describe(`npm native (${cases.length} programs${sanitize ? ", sanitized" : ""}${ } await expectMatchesNode(c, built.binary); }, - 120_000, + TIMEOUTS[c.name] ?? 120_000, ); } }); From c2ae71a6faf307e5661a4a6bcd73ac17f6ef657a Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 02:07:00 -0500 Subject: [PATCH 06/10] Bound full-batch poller drains to 48 per poll phase A full batch of 64 events polled again without returning to the event loop. When synchronous socket callbacks kept 64 events ready on every drain, timers and setImmediate callbacks never ran. The poll phase now drains at most 48 full batches, the count libuv's uv__io_poll uses, and the budget carries across the reopen that lets microtasks run, so the remaining events wait for the next loop iteration. --- packages/runtime/src/scr_async.c | 13 +++++++++++-- packages/runtime/src/scr_net.c | 8 +++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/packages/runtime/src/scr_async.c b/packages/runtime/src/scr_async.c index de48d6b616..2ab35e85d0 100644 --- a/packages/runtime/src/scr_async.c +++ b/packages/runtime/src/scr_async.c @@ -4153,13 +4153,19 @@ bool scr_loop_has_ready(void) { return scr_ready_len > 0 || scr_nt_head != NULL; * drains the poller at most once between check phases. I/O that becomes * ready while this iteration's callbacks run (a loopback connect started * by one of them) waits for the next iteration, so immediates queued in - * the meantime fire first — Node's poll → check order. */ + * the meantime fire first — Node's poll → check order. A full batch polls + * again without waiting, at most SCR_LOOP_IO_BATCHES times per phase (the + * count libuv's uv__io_poll uses); later events wait for the next + * iteration, so timers and immediates still run while sockets stay busy. */ +enum { SCR_LOOP_IO_BATCHES = 48 }; static SCR_TL bool scr_loop_io_open = true; +static SCR_TL int scr_loop_io_batches = 0; bool scr_loop_io_take(void) { bool open = scr_loop_io_open; scr_loop_io_open = false; return open; } +bool scr_loop_io_more(void) { return ++scr_loop_io_batches < SCR_LOOP_IO_BATCHES; } void scr_loop_io_reopen(void) { scr_loop_io_open = true; } /* The process.nextTick queue to exhaustion, ticks queued meanwhile @@ -4668,7 +4674,10 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { * had nothing to run). A check phase deferred for pending microtasks * keeps it closed, so the next iteration reaches the check phase * before polling again. */ - if (check_ran || scr_pending_immediates == 0) scr_loop_io_open = true; + if (check_ran || scr_pending_immediates == 0) { + scr_loop_io_open = true; + scr_loop_io_batches = 0; + } } return rejection_failed; } diff --git a/packages/runtime/src/scr_net.c b/packages/runtime/src/scr_net.c index 9684aebe97..f12840d91a 100644 --- a/packages/runtime/src/scr_net.c +++ b/packages/runtime/src/scr_net.c @@ -4909,6 +4909,7 @@ static int scr_net_pollfd(void) { } bool scr_loop_io_take(void); +bool scr_loop_io_more(void); void scr_loop_io_reopen(void); static void scr_net_dispatch(void) { @@ -4917,8 +4918,9 @@ static void scr_net_dispatch(void) { return; } /* One poller drain per loop iteration (scr_loop_io_take); a full batch - * keeps draining, like libuv's poll phase. Deferred emits (the sweep) - * still run on every call. */ + * keeps draining within the phase's batch budget (scr_loop_io_more), like + * libuv's poll phase. Deferred emits (the sweep) still run on every + * call. */ bool may_drain = scr_loop_io_take(); for (;;) { scr_net_sweep(); @@ -4928,7 +4930,7 @@ static void scr_net_dispatch(void) { int n = 0; if (may_drain) { n = scrp_drain(scr_net_poller, evs, 64); - may_drain = n == 64; + may_drain = n == 64 && scr_loop_io_more(); } for (int i = 0; i < n; i++) { void *udata = evs[i].udata; From 4470bab3082ed93c8fbbab836d1c6faec493d126 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Tue, 20 Oct 2026 07:58:16 -0700 Subject: [PATCH 07/10] Give the undici shim's Dispatcher an explicit constructor The implicit constructor forwarded its arguments to EventEmitter's options-taking constructor, which compiles to a deferred runtime fence. Dispatcher takes no options, so it calls super() directly. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/shims/undici/core/dispatcher.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/compiler/shims/undici/core/dispatcher.ts b/packages/compiler/shims/undici/core/dispatcher.ts index cd195b5dad..d9d35c33f3 100644 --- a/packages/compiler/shims/undici/core/dispatcher.ts +++ b/packages/compiler/shims/undici/core/dispatcher.ts @@ -38,6 +38,12 @@ export interface ResponseData { const defaultProfile: Profile = { major: 7 }; export class Dispatcher extends EventEmitter { + // An explicit constructor: the implicit one forwards its arguments to + // EventEmitter's options-taking constructor, which stays a runtime fence. + constructor() { + super(); + } + dispatch(_options: DispatchOptions, _handler: DispatchHandlers): boolean { throw new Error("not implemented"); } From 8aca6d653db32d942890fa8cc66b8db513662671 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 10:48:36 -0500 Subject: [PATCH 08/10] Abort the request of a redirect that fetch follows or rejects A redirect fetch followed or rejected was marked to drain: its body was read and dropped until the server ended it, and the abort listener was already removed. A body that never ends kept the connection busy after cancellation and stalled a pool of one connection. Following or rejecting a redirect now aborts its request, so the connection closes. undici 7 leaves that connection open with its body paused, which stalls the same pool; the follow-up request here gets a new connection. --- packages/compiler/shims/undici/core/fetch.ts | 9 +++++++-- .../compiler/test/ts7/resolver-parity.test.ts | 20 ++++++++++--------- .../package-shims/undici-hostile-peer.mjs | 15 ++++++++++++++ .../fixtures/package-shims/undici-redirect.ts | 19 ++++++++++++++++++ tests/harness/package-shims.test.ts | 17 +++++++++++++++- 5 files changed, 68 insertions(+), 12 deletions(-) create mode 100644 tests/fixtures/package-shims/undici-redirect.ts diff --git a/packages/compiler/shims/undici/core/fetch.ts b/packages/compiler/shims/undici/core/fetch.ts index 54a0ce0d52..ebca55d717 100644 --- a/packages/compiler/shims/undici/core/fetch.ts +++ b/packages/compiler/shims/undici/core/fetch.ts @@ -117,8 +117,9 @@ interface RawResponse { status: number; statusText: string; headers: string[]; - /** null when the response has no body to expose (null-body status, HEAD, - * or a redirect fetch follows); its bytes are then read and dropped. */ + /** null when the response has no body to expose: a null-body status or + * HEAD (its bytes, if any, are read and dropped), or a redirect fetch + * follows or rejects (the request is aborted, closing its connection). */ body: ReadableStream | null; } @@ -520,6 +521,10 @@ function dispatchOnce( dump = true; stopListening(); resolve({ status: statusCode, statusText: text, headers: rawHeaders, body: null }); + // A redirect that is followed or rejected never reads its body: + // abort the request so the connection is released instead of + // draining a body that may never end. + if (follows) abortRequest(new DOMException("The operation was aborted.", "AbortError")); return true; } const codings = diff --git a/packages/compiler/test/ts7/resolver-parity.test.ts b/packages/compiler/test/ts7/resolver-parity.test.ts index b3190ae9f9..b5d5875e73 100644 --- a/packages/compiler/test/ts7/resolver-parity.test.ts +++ b/packages/compiler/test/ts7/resolver-parity.test.ts @@ -147,15 +147,17 @@ test("relative and bare specifiers across the whole test tree resolve identicall // The deliberate package-shim delta: a package scriptc implements // itself (package-shims.ts) answers its shim module here, where // 5.9.3 answers the installed package as an external import; the - // installed package's answer is still compared below. - const shimmed = packageShimFor(file, spec) !== null; - const theirsProj = shimmed - ? reference && reference.isExternalLibraryImport - ? packageShimFor(file, spec) - : null - : reference && !reference.isExternalLibraryImport - ? reference.resolvedFileName - : null; + // installed package's answer is still compared below. Resolved + // answers use forward slashes on every platform, as 5.9.3's do. + const shim = packageShimFor(file, spec); + const theirsProj = + shim !== null + ? reference && reference.isExternalLibraryImport + ? shim.replaceAll("\\", "/") + : null + : reference && !reference.isExternalLibraryImport + ? reference.resolvedFileName + : null; if (oursProj !== theirsProj) { failures.push(`${file} '${spec}' (project): ours=${oursProj} ts5=${theirsProj}`); } diff --git a/tests/fixtures/package-shims/undici-hostile-peer.mjs b/tests/fixtures/package-shims/undici-hostile-peer.mjs index ce6c5881c6..b55e3226f3 100644 --- a/tests/fixtures/package-shims/undici-hostile-peer.mjs +++ b/tests/fixtures/package-shims/undici-hostile-peer.mjs @@ -3,6 +3,9 @@ // /zeros a chunk size with 64 MiB of leading zeros, // /big a 64 MiB body, as fast as the socket takes it, // /bomb 64 MiB of zeros, gzipped, +// /redirect-endless a 302 to /redirect-closed whose body never ends, +// /redirect-closed "closed" once the last /redirect-endless connection +// closed (waiting up to a second), else "open", // and exits when its stdin closes. import * as net from "node:net"; import * as zlib from "node:zlib"; @@ -24,6 +27,8 @@ function stream(s, head, piece, tail) { pump(); } +let redirectClosed = Promise.resolve(); + const server = net.createServer((s) => { s.on("error", () => {}); let head = ""; @@ -39,6 +44,16 @@ const server = net.createServer((s) => { stream(s, chunked, Buffer.alloc(64 * 1024, 48), "5\r\nhello\r\n0\r\n\r\n"); } else if (path === "/big") { stream(s, `HTTP/1.1 200 OK\r\nContent-Length: ${size}\r\n\r\n`, Buffer.alloc(64 * 1024, 97), ""); + } else if (path === "/redirect-endless") { + redirectClosed = new Promise((resolve) => s.on("close", resolve)); + s.write("HTTP/1.1 302 Found\r\nLocation: /redirect-closed\r\nTransfer-Encoding: chunked\r\n\r\n"); + const timer = setInterval(() => s.write("4\r\ndata\r\n"), 5); + s.on("close", () => clearInterval(timer)); + } else if (path === "/redirect-closed") { + const timeout = new Promise((resolve) => setTimeout(() => resolve("open"), 1000)); + void Promise.race([redirectClosed.then(() => "closed"), timeout]).then((state) => { + s.write(`HTTP/1.1 200 OK\r\nContent-Length: ${state.length}\r\n\r\n${state}`); + }); } else { s.write(`HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: ${bomb.length}\r\n\r\n`); s.write(bomb); diff --git a/tests/fixtures/package-shims/undici-redirect.ts b/tests/fixtures/package-shims/undici-redirect.ts new file mode 100644 index 0000000000..de9700fcfc --- /dev/null +++ b/tests/fixtures/package-shims/undici-redirect.ts @@ -0,0 +1,19 @@ +// Follows and rejects redirects whose bodies never end +// (undici-hostile-peer.mjs, port in argv[2]) through a pool of one +// connection. The discarded redirect's request is aborted, so its +// connection closes, the follow-up request gets a connection, and the peer +// reports the redirect's connection closed. +import { Agent, fetch } from "undici"; + +const origin = `http://127.0.0.1:${process.argv[2]}`; +const dispatcher = new Agent({ connections: 1 }); +const followed = await fetch(`${origin}/redirect-endless`, { dispatcher }); +console.log("follow", followed.status, await followed.text()); +try { + await fetch(`${origin}/redirect-endless`, { dispatcher, redirect: "error" }); +} catch (err) { + console.log("error mode", (err as Error).name, (err as Error).message); +} +const after = await fetch(`${origin}/redirect-closed`, { dispatcher }); +console.log("after error mode", await after.text()); +await dispatcher.close(); diff --git a/tests/harness/package-shims.test.ts b/tests/harness/package-shims.test.ts index 0be52d2df2..447052b81f 100644 --- a/tests/harness/package-shims.test.ts +++ b/tests/harness/package-shims.test.ts @@ -10,7 +10,10 @@ * - the undici shims keep memory bounded against a peer that sends endless * chunk-size lines and large or compressed bodies, and the 5.x profile * rejects a Host value with CR or LF as undici 5.29 does (the vendored - * 5.12 writes it into the request). + * 5.12 writes it into the request); + * - fetch aborts a redirect it follows or rejects, so a redirect body that + * never ends does not keep its connection busy (undici 7 leaves that + * connection open with its body paused). */ import { execFile, spawn } from "node:child_process"; import { copyFileSync, cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; @@ -164,6 +167,18 @@ describe("package shims", () => { }, 300_000); } + test("undici7: fetch closes the connection of a redirect it follows or rejects", async () => { + const { dir, binary } = await buildInCase("undici7", "undici-redirect.ts"); + expect(await runAgainstPeer(binary, dir)).toBe( + [ + "follow 200 closed", + "error mode TypeError fetch failed", + "after error mode closed", + "", + ].join("\n"), + ); + }, 180_000); + test("undici5: a Host value with CR or LF throws before the request is written", async () => { const { dir, binary } = await buildInCase("undici5", "undici5-host.ts"); // undici 5.29.0's output. From 33354cc4817fcad1726937085babbee55ab80722 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 10:50:19 -0500 Subject: [PATCH 09/10] Abort the request of a redirect that fetch follows or rejects A redirect fetch followed or rejected was marked to drain: its body was read and dropped until the server ended it, and the abort listener was already removed. A body that never ends kept the connection busy after cancellation and stalled a pool of one connection. Following or rejecting a redirect now aborts its request, so the connection closes. undici 7 leaves that connection open with its body paused, which stalls the same pool; the follow-up request here gets a new connection. --- packages/compiler/shims/undici/core/fetch.ts | 9 +++++++-- .../package-shims/undici-hostile-peer.mjs | 15 +++++++++++++++ .../fixtures/package-shims/undici-redirect.ts | 19 +++++++++++++++++++ tests/harness/package-shims.test.ts | 17 ++++++++++++++++- 4 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/package-shims/undici-redirect.ts diff --git a/packages/compiler/shims/undici/core/fetch.ts b/packages/compiler/shims/undici/core/fetch.ts index 54a0ce0d52..ebca55d717 100644 --- a/packages/compiler/shims/undici/core/fetch.ts +++ b/packages/compiler/shims/undici/core/fetch.ts @@ -117,8 +117,9 @@ interface RawResponse { status: number; statusText: string; headers: string[]; - /** null when the response has no body to expose (null-body status, HEAD, - * or a redirect fetch follows); its bytes are then read and dropped. */ + /** null when the response has no body to expose: a null-body status or + * HEAD (its bytes, if any, are read and dropped), or a redirect fetch + * follows or rejects (the request is aborted, closing its connection). */ body: ReadableStream | null; } @@ -520,6 +521,10 @@ function dispatchOnce( dump = true; stopListening(); resolve({ status: statusCode, statusText: text, headers: rawHeaders, body: null }); + // A redirect that is followed or rejected never reads its body: + // abort the request so the connection is released instead of + // draining a body that may never end. + if (follows) abortRequest(new DOMException("The operation was aborted.", "AbortError")); return true; } const codings = diff --git a/tests/fixtures/package-shims/undici-hostile-peer.mjs b/tests/fixtures/package-shims/undici-hostile-peer.mjs index ce6c5881c6..b55e3226f3 100644 --- a/tests/fixtures/package-shims/undici-hostile-peer.mjs +++ b/tests/fixtures/package-shims/undici-hostile-peer.mjs @@ -3,6 +3,9 @@ // /zeros a chunk size with 64 MiB of leading zeros, // /big a 64 MiB body, as fast as the socket takes it, // /bomb 64 MiB of zeros, gzipped, +// /redirect-endless a 302 to /redirect-closed whose body never ends, +// /redirect-closed "closed" once the last /redirect-endless connection +// closed (waiting up to a second), else "open", // and exits when its stdin closes. import * as net from "node:net"; import * as zlib from "node:zlib"; @@ -24,6 +27,8 @@ function stream(s, head, piece, tail) { pump(); } +let redirectClosed = Promise.resolve(); + const server = net.createServer((s) => { s.on("error", () => {}); let head = ""; @@ -39,6 +44,16 @@ const server = net.createServer((s) => { stream(s, chunked, Buffer.alloc(64 * 1024, 48), "5\r\nhello\r\n0\r\n\r\n"); } else if (path === "/big") { stream(s, `HTTP/1.1 200 OK\r\nContent-Length: ${size}\r\n\r\n`, Buffer.alloc(64 * 1024, 97), ""); + } else if (path === "/redirect-endless") { + redirectClosed = new Promise((resolve) => s.on("close", resolve)); + s.write("HTTP/1.1 302 Found\r\nLocation: /redirect-closed\r\nTransfer-Encoding: chunked\r\n\r\n"); + const timer = setInterval(() => s.write("4\r\ndata\r\n"), 5); + s.on("close", () => clearInterval(timer)); + } else if (path === "/redirect-closed") { + const timeout = new Promise((resolve) => setTimeout(() => resolve("open"), 1000)); + void Promise.race([redirectClosed.then(() => "closed"), timeout]).then((state) => { + s.write(`HTTP/1.1 200 OK\r\nContent-Length: ${state.length}\r\n\r\n${state}`); + }); } else { s.write(`HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: ${bomb.length}\r\n\r\n`); s.write(bomb); diff --git a/tests/fixtures/package-shims/undici-redirect.ts b/tests/fixtures/package-shims/undici-redirect.ts new file mode 100644 index 0000000000..de9700fcfc --- /dev/null +++ b/tests/fixtures/package-shims/undici-redirect.ts @@ -0,0 +1,19 @@ +// Follows and rejects redirects whose bodies never end +// (undici-hostile-peer.mjs, port in argv[2]) through a pool of one +// connection. The discarded redirect's request is aborted, so its +// connection closes, the follow-up request gets a connection, and the peer +// reports the redirect's connection closed. +import { Agent, fetch } from "undici"; + +const origin = `http://127.0.0.1:${process.argv[2]}`; +const dispatcher = new Agent({ connections: 1 }); +const followed = await fetch(`${origin}/redirect-endless`, { dispatcher }); +console.log("follow", followed.status, await followed.text()); +try { + await fetch(`${origin}/redirect-endless`, { dispatcher, redirect: "error" }); +} catch (err) { + console.log("error mode", (err as Error).name, (err as Error).message); +} +const after = await fetch(`${origin}/redirect-closed`, { dispatcher }); +console.log("after error mode", await after.text()); +await dispatcher.close(); diff --git a/tests/harness/package-shims.test.ts b/tests/harness/package-shims.test.ts index 0be52d2df2..447052b81f 100644 --- a/tests/harness/package-shims.test.ts +++ b/tests/harness/package-shims.test.ts @@ -10,7 +10,10 @@ * - the undici shims keep memory bounded against a peer that sends endless * chunk-size lines and large or compressed bodies, and the 5.x profile * rejects a Host value with CR or LF as undici 5.29 does (the vendored - * 5.12 writes it into the request). + * 5.12 writes it into the request); + * - fetch aborts a redirect it follows or rejects, so a redirect body that + * never ends does not keep its connection busy (undici 7 leaves that + * connection open with its body paused). */ import { execFile, spawn } from "node:child_process"; import { copyFileSync, cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; @@ -164,6 +167,18 @@ describe("package shims", () => { }, 300_000); } + test("undici7: fetch closes the connection of a redirect it follows or rejects", async () => { + const { dir, binary } = await buildInCase("undici7", "undici-redirect.ts"); + expect(await runAgainstPeer(binary, dir)).toBe( + [ + "follow 200 closed", + "error mode TypeError fetch failed", + "after error mode closed", + "", + ].join("\n"), + ); + }, 180_000); + test("undici5: a Host value with CR or LF throws before the request is written", async () => { const { dir, binary } = await buildInCase("undici5", "undici5-host.ts"); // undici 5.29.0's output. From 9eae4af46c02ece2c7ea630d74d83aafbc56ac73 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 20:08:11 -0500 Subject: [PATCH 10/10] Drop the undici shim type test from the TypeScript 5 import allowlist The test checks the shims with the TypeScript 7 checker and no longer imports typescript5, so test-ts7 reported its allowlist entry as stale. --- scripts/test-ts7.mjs | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/test-ts7.mjs b/scripts/test-ts7.mjs index 5ee1f50fca..e022c1ff5f 100644 --- a/scripts/test-ts7.mjs +++ b/scripts/test-ts7.mjs @@ -30,7 +30,6 @@ const ALLOWED_TYPESCRIPT5_IMPORTS = new Set([ "packages/compiler/src/library/semantic-source.ts", "packages/compiler/test/ts7/harness.ts", "packages/compiler/test/ts7/resolver-parity.test.ts", - "packages/compiler/test/undici-shim-types.test.ts", ]); function sourceFilesUnder(dir) {