Carry lifetime regions across await and reuse freed region chunks

benchmarks/regions/README.md

@@ -7,6 +7,7 @@ Programs that build generations of `ProgramLifetime` objects and free each gener
 | `generations.ts` | Per-file syntax trees and binder symbols in one region per file, checker state (types, signatures, node and symbol links in paged tables, literal and union caches) in one region per generation. `edit` re-parses one file per generation, `all` re-parses every file. |
 | `compiler.ts` | A tokenizer, recursive-descent parser, binder and checker over generated sources, rebuilt from scratch in one region per build, like a watch mode that does not reuse syntax trees. |
 | `emit.ts` | A large live graph of ordinary objects with parent links, standing in for a type checker, and a per-file emit that builds its output in a region released right after. |
+| `requests.ts` | Batches of concurrent async request handlers, each in its own region across its awaits: tokenize a JSON body, parse it into a syntax tree that points at its tokens, validate it into issues and an index, render a response, release the region. |
 
 ```console
 $ node benchmarks/regions/run.mjs --baseline=../scriptc-main --runs=7

benchmarks/regions/emit.ts

@@ -54,7 +54,7 @@ let level: Type[] = [new Type(id++, undefined)];
 while (id < typeCount) {
   const next: Type[] = [];
   for (const t of level) for (let k = 0; k < 4 && id < typeCount; k++) next.push(new Type(id++, t));
-  roots.push(...next);
+  for (const t of next) roots.push(t);
   level = next;
 }
 

benchmarks/regions/requests.ts

@@ -0,0 +1,221 @@
+// An async request handler: each request tokenizes a JSON body, awaits (as
+// if reading more of it), parses the tokens into a syntax tree whose nodes
+// point back at their tokens, awaits again (as if loading reference data),
+// validates the tree into issues held in an index, and renders a response.
+// Requests run in batches of concurrent handlers, so the handlers of one
+// batch interleave at every await.
+//
+//   requests <requests> <concurrency> <items-per-body> [plain]
+//
+// Each request runs in its own LifetimeRegion, released when its response
+// is rendered; `plain` never constructs a region.
+import { readFileSync } from "node:fs";
+import type { ProgramLifetime } from "@scriptc/types";
+import { LifetimeRegion } from "@scriptc/experimental";
+
+const T_PUNCT = 0;
+const T_STR = 1;
+const T_NUM = 2;
+const T_WORD = 3;
+
+class Token implements ProgramLifetime {
+  readonly kind: number;
+  readonly text: string;
+  readonly pos: number;
+  constructor(kind: number, text: string, pos: number) {
+    this.kind = kind;
+    this.text = text;
+    this.pos = pos;
+  }
+}
+
+const N_OBJECT = 0;
+const N_ARRAY = 1;
+const N_STRING = 2;
+const N_NUMBER = 3;
+const N_LITERAL = 4;
+
+class JsonNode implements ProgramLifetime {
+  readonly kind: number;
+  readonly token: Token;
+  parent: JsonNode | undefined = undefined;
+  readonly keys: string[] = [];
+  readonly children: JsonNode[] = [];
+  constructor(kind: number, token: Token) {
+    this.kind = kind;
+    this.token = token;
+  }
+}
+
+class Issue implements ProgramLifetime {
+  readonly path: string;
+  readonly node: JsonNode;
+  readonly message: string;
+  constructor(path: string, node: JsonNode, message: string) {
+    this.path = path;
+    this.node = node;
+    this.message = message;
+  }
+}
+
+// ---- request bodies (deterministic) ----
+
+function body(id: number, items: number): string {
+  let seed = 7 + id * 7919;
+  const rand = (n: number): number => {
+    seed = (seed * 1103515245 + 12345) % 2147483648;
+    return seed % n;
+  };
+  const parts: string[] = [];
+  for (let i = 0; i < items; i++) {
+    const qty = rand(12) - 1;
+    const sku = rand(9) === 0 ? "" : `sku-${rand(5000)}`;
+    const tags = Array.from({ length: rand(4) }, (_, k) => `"t${k}${rand(10)}"`).join(",");
+    parts.push(`{"sku":"${sku}","qty":${qty},"price":${rand(10000) / 100},"gift":${rand(2) === 0},"tags":[${tags}]}`);
+  }
+  return `{"order":${id},"customer":"c${id % 97}","items":[${parts.join(",")}]}`;
+}
+
+// ---- handler ----
+
+function tokenize(text: string): Token[] {
+  const tokens: Token[] = [];
+  let i = 0;
+  while (i < text.length) {
+    const c = text.charCodeAt(i);
+    const start = i;
+    if (c === 34) {
+      i++;
+      while (text.charCodeAt(i) !== 34) i++;
+      i++;
+      tokens.push(new Token(T_STR, text.slice(start + 1, i - 1), start));
+    } else if (c === 45 || (c >= 48 && c <= 57)) {
+      i++;
+      while (i < text.length && (text.charCodeAt(i) === 46 || (text.charCodeAt(i) >= 48 && text.charCodeAt(i) <= 57))) i++;
+      tokens.push(new Token(T_NUM, text.slice(start, i), start));
+    } else if (c >= 97 && c <= 122) {
+      while (i < text.length && text.charCodeAt(i) >= 97 && text.charCodeAt(i) <= 122) i++;
+      tokens.push(new Token(T_WORD, text.slice(start, i), start));
+    } else {
+      i++;
+      tokens.push(new Token(T_PUNCT, text[start]!, start));
+    }
+  }
+  return tokens;
+}
+
+class Parser {
+  private at = 0;
+  private readonly tokens: Token[];
+  constructor(tokens: Token[]) {
+    this.tokens = tokens;
+  }
+  value(): JsonNode {
+    const token = this.tokens[this.at++]!;
+    if (token.kind === T_STR) return new JsonNode(N_STRING, token);
+    if (token.kind === T_NUM) return new JsonNode(N_NUMBER, token);
+    if (token.kind === T_WORD) return new JsonNode(N_LITERAL, token);
+    const node = new JsonNode(token.text === "{" ? N_OBJECT : N_ARRAY, token);
+    const close = token.text === "{" ? "}" : "]";
+    while (this.tokens[this.at]!.text !== close) {
+      if (node.kind === N_OBJECT) {
+        node.keys.push(this.tokens[this.at]!.text);
+        this.at += 2; // the key and the colon
+      }
+      const child = this.value();
+      child.parent = node;
+      node.children.push(child);
+      if (this.tokens[this.at]!.text === ",") this.at++;
+    }
+    this.at++;
+    return node;
+  }
+}
+
+function member(node: JsonNode, key: string): JsonNode | undefined {
+  const i = node.keys.indexOf(key);
+  return i < 0 ? undefined : node.children[i];
+}
+
+function validate(root: JsonNode, issues: Issue[], index: Map<string, JsonNode>): number {
+  let total = 0;
+  const items = member(root, "items")!;
+  for (let i = 0; i < items.children.length; i++) {
+    const item = items.children[i]!;
+    const path = `items[${i}]`;
+    const sku = member(item, "sku")!;
+    if (sku.token.text === "") issues.push(new Issue(`${path}.sku`, sku, "empty sku"));
+    else if (index.has(sku.token.text)) issues.push(new Issue(`${path}.sku`, sku, `duplicate of ${index.get(sku.token.text)!.token.pos}`));
+    else index.set(sku.token.text, item);
+    const qty = Number(member(item, "qty")!.token.text);
+    if (qty <= 0) issues.push(new Issue(`${path}.qty`, member(item, "qty")!, "quantity must be positive"));
+    else total += qty * Number(member(item, "price")!.token.text);
+    const tags = member(item, "tags")!;
+    for (const tag of tags.children) if (tag.token.text.endsWith("0")) issues.push(new Issue(`${path}.tags`, tag, "reserved tag"));
+  }
+  return total;
+}
+
+const hop = (): Promise<void> => new Promise((resolve) => setImmediate(resolve));
+
+async function handle(text: string): Promise<string> {
+  const tokens = tokenize(text);
+  await hop();
+  const root = new Parser(tokens).value();
+  await hop();
+  const issues: Issue[] = [];
+  const index = new Map<string, JsonNode>();
+  const total = validate(root, issues, index);
+  await hop();
+  const first = issues[0];
+  return `${member(root, "order")!.token.text} ${tokens.length} ${index.size} ${issues.length} ${total.toFixed(2)}` +
+    (first === undefined ? "" : ` ${first.path}@${first.node.token.pos}`);
+}
+
+// ---- driver ----
+
+function rssMb(key: string): number {
+  const m = new RegExp(`${key}:\\s*(\\d+) kB`).exec(readFileSync("/proc/self/status", "utf8"));
+  return m === null ? 0 : Number(m[1]) / 1024;
+}
+
+const args = process.argv.slice(2);
+const requests = Number(args[0] ?? "4000");
+const concurrency = Number(args[1] ?? "16");
+const items = Number(args[2] ?? "100");
+const useRegions = args[3] !== "plain";
+
+async function serve(id: number): Promise<string> {
+  const text = body(id, items);
+  if (!useRegions) return await handle(text);
+  const region = new LifetimeRegion();
+  const response = await region.run(() => handle(text));
+  region.release();
+  return response;
+}
+
+async function main(): Promise<void> {
+  let hash = 0;
+  let lines = 0;
+  let sample = "";
+  let rssFirst = 0;
+  const t0 = performance.now();
+  for (let next = 0; next < requests; next += concurrency) {
+    const batch: Promise<string>[] = [];
+    for (let k = next; k < next + concurrency && k < requests; k++) batch.push(serve(k));
+    for (const response of await Promise.all(batch)) {
+      for (let i = 0; i < response.length; i++) hash = (hash * 31 + response.charCodeAt(i)) % 1000000007;
+      lines++;
+      if (lines === 1) sample = response;
+    }
+    if (next === 0) rssFirst = rssMb("VmRSS");
+  }
+  const ms = performance.now() - t0;
+  console.log(`${lines} responses, hash ${hash}, first ${sample}`);
+  console.log(
+    `total ${ms.toFixed(1)} ms per-request ${((ms * 1000) / requests).toFixed(1)} us` +
+      ` rss-first ${rssFirst.toFixed(1)} MB rss-last ${rssMb("VmRSS").toFixed(1)} MB peak ${rssMb("VmHWM").toFixed(1)} MB`,
+  );
+}
+
+main();

benchmarks/regions/run.mjs

@@ -39,6 +39,8 @@ const CONFIGS = [
   { name: "compiler-one-shot", program: "compiler.ts", args: ["400", "60", "1"] },
   // Per-file emit regions next to a large live heap of ordinary objects.
   { name: "emit-files", program: "emit.ts", args: ["400000", "200", "2000"] },
+  // Concurrent async request handlers, one region per request across its awaits.
+  { name: "requests-async", program: "requests.ts", args: ["4000", "16", "100"] },
 ];
 
 const work = join(tmpdir(), `scriptc-region-bench-${process.pid}`);
@@ -93,7 +95,7 @@ for (const config of CONFIGS) {
       const lines = r.stdout.trim().split("\n");
       outputs.set(name, lines[0].replace(/ gen0 .*/, ""));
       const metrics = {};
-      for (const [, key, value] of lines.at(-1).matchAll(/([a-z0-9-]+) ([0-9.]+) (?:ms|MB)/g))
+      for (const [, key, value] of lines.at(-1).matchAll(/([a-z0-9-]+) ([0-9.]+) (?:ms|us|MB)/g))
         metrics[key] = Number(value);
       samples[name].push(metrics);
     }

docs/content/docs/experimental.mdx

@@ -73,9 +73,9 @@ scriptc region heap passes: 0
   <dt><code>new LifetimeRegion()</code></dt>
   <dd>Creates a region. Creating a region also retries the released regions that could not be freed yet.</dd>
   <dt><code>region.run(fn)</code></dt>
-  <dd>Calls <code>fn</code> with the region as the current region and returns its result. <code>ProgramLifetime</code> instances created during the call belong to the region. Calls nest: a <code>run</code> inside the callback makes another region current until it returns, and the previous region is current again however the callback finishes, including when it throws.</dd>
+  <dd>Calls <code>fn</code> with the region as the current region and returns its result. <code>ProgramLifetime</code> instances created during the call belong to the region, and so do those that async functions called during it create after an <code>await</code> (see <a href="#async-work">Async work</a>). Calls nest: a <code>run</code> inside the callback makes another region current until it returns, and the previous region is current again however the callback finishes, including when it throws.</dd>
   <dt><code>region.release()</code></dt>
-  <dd>Requests that the region be freed. scriptc frees it as soon as nothing outside the region refers to any of its instances, which may be at once, at a later release or region creation, or never. A region is never freed while a <code>run</code> call on it is active. After <code>release()</code>, <code>run</code> still calls its callback, and the instances it creates belong to the default region.</dd>
+  <dd>Requests that the region be freed. scriptc frees it as soon as nothing outside the region refers to any of its instances, which may be at once, at a later release or region creation, or never. A region is never freed while a <code>run</code> call on it is active, or while async work started inside it has not finished. After <code>release()</code>, <code>run</code> still calls its callback, and the instances it creates belong to the default region.</dd>
 </dl>
 
 `ProgramLifetime` instances created outside every `run` call belong to the default region, which is never freed. Ordinary objects (instances of classes without the contract, arrays, maps, closures) are reference counted as always; a region frees the ones that only its instances refer to.
@@ -92,7 +92,62 @@ A released region stays allocated, and its instances stay usable, while anything
 - an instance of the default region, which is never freed;
 - a promise, generator or `Error` that holds an instance, until it is no longer referenced.
 
-Work that resumes after an `await` inside the callback runs outside the region, so the instances it creates belong to the default region and keep the instances they refer to allocated.
+### Async work
+
+An async function called inside `run` keeps the region for its whole run: the instances it creates after each `await` belong to the region too, as do those of the async functions it calls. A server can give every request its own region, even while requests interleave at their awaits:
+
+```ts:requests.ts
+import type { ProgramLifetime } from "@scriptc/types";
+import { LifetimeRegion } from "@scriptc/experimental";
+
+class Part implements ProgramLifetime {
+  readonly text: string;
+  previous: Part | undefined = undefined;
+  constructor(text: string) {
+    this.text = text;
+  }
+}
+
+const read = (): Promise<void> => new Promise((resolve) => setImmediate(resolve));
+
+async function handle(id: number): Promise<string> {
+  let last: Part | undefined;
+  for (let i = 0; i < 1000; i++) {
+    if (i % 100 === 0) await read(); // the next chunk of the request body
+    const part = new Part(`${id}.${i}`);
+    part.previous = last;
+    last = part;
+  }
+  return `request ${id}: last part ${last!.text}`;
+}
+
+async function serve(id: number): Promise<string> {
+  const region = new LifetimeRegion();
+  try {
+    return await region.run(() => handle(id));
+  } finally {
+    region.release();
+  }
+}
+
+Promise.all([1, 2, 3].map(serve)).then((responses) => {
+  for (const response of responses) console.log(response);
+});
+```
+
+Each request's 1000 parts are freed when the request releases its region:
+
+```console
+$ scriptc build requests.ts -o requests >/dev/null
+$ SCRIPTC_REGION_STATS=1 ./requests >/dev/null
+scriptc region 1: freed 1000 objects, 0 owned objects, 1 chunks (2.0 MiB); cycles 0.0 ms, walk 0.0 ms, free 0.0 ms
+scriptc region 2: freed 1000 objects, 0 owned objects, 1 chunks (2.0 MiB); cycles 0.0 ms, walk 0.0 ms, free 0.0 ms
+scriptc region 3: freed 1000 objects, 0 owned objects, 1 chunks (2.0 MiB); cycles 0.0 ms, walk 0.0 ms, free 0.0 ms
+scriptc regions: 3 created, 3 released, 3 freed, 0 pinned, 0 kept
+scriptc region heap passes: 0
+```
+
+A region released while async work started inside it is still pending stays allocated until that work has finished, and the work keeps creating its instances in the region. Callbacks that the event loop calls (timers, `queueMicrotask`, event listeners) and the bodies of async generators run outside the region; synchronous generators run in the region of the code that resumes them.
 
 ### Threads
 
@@ -102,7 +157,7 @@ Each thread has its own regions and its own default region. A region belongs to
 
 Programs that never construct a `LifetimeRegion` compile exactly as they would without the package. In programs that do, every retain and release of a `ProgramLifetime` instance that belongs to a region updates a count of the region chunk that holds it. Instances of the default region are not counted, and until the program creates its first region, a retain or release only tests whether counting has started.
 
-Freeing a region visits each of its instances and the ordinary objects they own. A region that the check would keep may be kept only by unreachable cycles of ordinary objects that refer into it, so before keeping it scriptc runs the cycle collector over the thread's heap and checks again.
+Freeing a region visits each of its instances and the ordinary objects they own. Each thread keeps up to 8 MiB of the memory of freed regions for its next regions, so a program that frees a region per request or per batch does not map and fault in new pages every time. A region that the check would keep may be kept only by unreachable cycles of ordinary objects that refer into it, so before keeping it scriptc runs the cycle collector over the thread's heap and checks again.
 
 `SCRIPTC_REGION_STATS=1` prints one line per release attempt, freed, `kept` (with the number of references the check could not explain) or `pinned` (published), and at exit a summary per thread and the number of those cycle-collector passes over the thread's heap.
 

docs/test/fixtures/migration-baseline.json

@@ -273,6 +273,10 @@
         "text": "When a region is freed",
         "id": "when-a-region-is-freed"
       },
+      {
+        "text": "Async work",
+        "id": "async-work"
+      },
       {
         "text": "Threads",
         "id": "threads"

packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5023-lifetime-region-await.ts.json

@@ -0,0 +1,6 @@
+{
+ "order": [
+  "<repo>/tests/corpus/5023-lifetime-region-await.ts"
+ ],
+ "diags": []
+}

packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5024-lifetime-region-await-forms.ts.json

@@ -0,0 +1,6 @@
+{
+ "order": [
+  "<repo>/tests/corpus/5024-lifetime-region-await-forms.ts"
+ ],
+ "diags": []
+}

packages/experimental/README.md

@@ -21,7 +21,7 @@ function rebuild(): Program {
 
 - `run(fn)` calls `fn` with the region as the current region and returns its result. `ProgramLifetime` instances created during the call belong to the region; calls nest.
 - Instances created outside every `run` call belong to the default region, which is never freed.
-- Work that resumes after an `await` inside `fn` runs outside the region.
+- Async work started inside `fn` keeps its region: what it creates after an `await` belongs to the region too, and the region is not freed before that work has finished.
 - `release()` requests that the region be freed. scriptc frees it once nothing outside the region refers to any of its instances; until then the region stays allocated. Later `run` calls on a released region allocate in the default region.
 - A region that holds a published object (`@scriptc/threads`) is never freed.
 

packages/experimental/index.d.ts

@@ -7,13 +7,14 @@ export declare class LifetimeRegion {
   constructor();
   /** Calls `fn` with this region as the current region and returns its
    * result. ProgramLifetime instances created during the call, outside
-   * nested `run` calls of other regions, belong to this region. Work that
-   * resumes after an `await` inside `fn` runs outside the region. */
+   * nested `run` calls of other regions, belong to this region, and so do
+   * those that async functions called during it create after an `await`. */
   run<T>(fn: () => T): T;
   /** Requests that the region be freed. Its instances are freed once
-   * nothing outside the region refers to any of them; until then the
-   * region stays allocated. Later `run` calls allocate outside the region.
-   * Never observable except through memory use. */
+   * nothing outside the region refers to any of them and the async work
+   * started inside it has finished; until then the region stays allocated.
+   * Later `run` calls allocate outside the region. Never observable except
+   * through memory use. */
   release(): void;
 }