Compile classic CommonJS packages: late requires, JSON requires, nested builtin destructuring, settled require cycles

docs/content/docs/dependencies.mdx

@@ -69,6 +69,10 @@ $ scriptc build app.ts --npm-static=three -o app >/dev/null
 
 Naming a package skips the declaration requirement above, but the package still has to compile. `--npm-static=auto` uses the default automatic selection. The option accepts a comma-separated list and can be repeated.
 
+## CommonJS packages
+
+CommonJS packages compile with Node.js's module semantics. Each `require()` evaluates its module at the call, once, and later calls return the cached exports. A package can assign `module.exports` and run code before it requires its dependencies. If that code reads a `const` or `let` require binding before its `require` runs, the read throws the same `ReferenceError` as Node.js. A `var` binding that earlier code can reach is reported at build time. A `require()` of a JSON file, such as a package's own `package.json`, returns one object parsed at build time, shared by every module that requires it. A require cycle compiles when each of its modules assigns its exports before requiring anything, as semver's classes do, and uses the other modules only inside functions that its top-level code can't call. Other require cycles are reported at build time. The [limitations](/docs/limitations) page lists the supported `require` forms.
+
 ## Unsupported code inside a package
 
 A JavaScript package can contain operations scriptc can't compile, often in functions your program never calls. These don't fail the build: each one compiles to code that throws a runtime error if it ever runs. Coverage reports list them as deferred sites, so review the list and test the parts of the package your program uses.

docs/content/docs/limitations.mdx

@@ -32,6 +32,8 @@ title: "Limitations"
 
 <include>../limitations/language-features/dynamic-imports.mdx</include>
 
+<include>../limitations/language-features/commonjs-require.mdx</include>
+
 ### Types and class definitions
 
 <include>../limitations/types-and-class-definitions/bigint.mdx</include>

docs/content/limitations/language-features/commonjs-require.mdx

@@ -0,0 +1 @@
+- **CommonJS `require`:** Literal `require()` calls of compiled modules, supported builtins, and JSON documents compile. Reading a top-level `const` or `let` require binding before its `require` runs throws the same `ReferenceError` as Node.js. A `var` binding that code above its `require` can reach is unsupported, and so is a read that compiles to no checkable expression, such as exporting the binding before the `require`. JSON documents are parsed at build time, and every `require()` of a document returns the same object. Destructuring the exports of a module supports plain and renamed properties, plus nested patterns for builtin sub-modules such as `util.types`. Any pattern works on a JSON document. Defaults, rest elements, and nested patterns over other modules are unsupported. A require cycle compiles when each of its modules assigns all of its exports before its first `require` and uses the other modules of the cycle only inside functions that its top-level code can't call. Other require cycles are rejected with `SC1016`.

packages/compiler/src/frontend/lowering/builtins/module-bindings.ts

@@ -34,21 +34,48 @@ export function builtinImportOf(
   // binding (`const { readFileSync } = require("fs")`, renames via
   // `{ readFileSync: rf }`) keys the same tables.
   if (ts.isBindingElement(decl) && decl.name !== undefined && ts.isIdentifier(decl.name)) {
-    const varDecl = decl.parent?.parent;
+    // A NESTED pattern names a member of a sub-module (`const { types: {
+    // isPromise } } = require("util")`): util.types IS the util/types
+    // module object, so each level composes the module name.
+    const path: string[] = [];
+    let pattern: ts.Node | undefined = decl.parent;
+    let varDecl = pattern?.parent;
+    while (
+      pattern !== undefined &&
+      ts.isObjectBindingPattern(pattern) &&
+      varDecl !== undefined &&
+      ts.isBindingElement(varDecl)
+    ) {
+      const key = varDecl.propertyName;
+      if (varDecl.initializer !== undefined || key === undefined || !ts.isIdentifier(key)) {
+        return null;
+      }
+      path.unshift(key.text);
+      pattern = varDecl.parent;
+      varDecl = pattern?.parent;
+    }
     if (
-      ts.isObjectBindingPattern(decl.parent) &&
+      pattern !== undefined &&
+      ts.isObjectBindingPattern(pattern) &&
+      varDecl !== undefined &&
       ts.isVariableDeclaration(varDecl) &&
       varDecl.initializer !== undefined
     ) {
       const spec = requireSpecOf(varDecl.initializer);
-      const module =
+      let module =
         spec !== null
           ? canonicalBuiltinModule(spec)
-          : // The one-hop alias twin: `const crypto = require('crypto');
-            // const { createSign } = crypto;` — test/common's idiom. The
-            // destructure re-binds module members under local names, so the
-            // bindings key the same tables as the direct-require form.
-            builtinNamespaceDestructureModuleOf(lowerer, varDecl);
+          : path.length > 0
+            ? null
+            : // The one-hop alias twin: `const crypto = require('crypto');
+              // const { createSign } = crypto;` — test/common's idiom. The
+              // destructure re-binds module members under local names, so the
+              // bindings key the same tables as the direct-require form.
+              builtinNamespaceDestructureModuleOf(lowerer, varDecl);
+      for (const key of path) {
+        if (module === null) break;
+        module = canonicalBuiltinModule(`${module}/${key}`);
+      }
       if (module === null) return null;
       const member =
         decl.propertyName && ts.isIdentifier(decl.propertyName)

packages/compiler/src/frontend/lowering/json-require.ts

@@ -0,0 +1,140 @@
+/* CommonJS require() of JSON modules. Node parses the document at the
+ * first require and caches the resulting object: every later require, from
+ * any module, answers that same object, so a mutation through one requirer
+ * is visible to all of them. The document is data known at build time, so
+ * the object bakes from the parsed text (comptimeValueToIr, as JSON import
+ * declarations do) into one global per document. The parse has no
+ * observable effect, so each requiring module creates the object in its
+ * %init prelude unless an earlier module already did, and every require
+ * call of the document reads the shared global. A require binding of a
+ * JSON module is ordinary storage holding that object — destructuring
+ * copies its properties at the require, exactly like Node. ES-module JSON
+ * imports keep their own record (Node's ESM loader caches separately). */
+import * as ts from "../ts7/adapter.js";
+import type { Lowerer } from "./lowerer.js";
+import { PoisonError } from "./lowerer.js";
+import type { FileParts } from "./lower-modules.js";
+import { isCjsJsFile, locOf, requireSpecOf, resolveImport } from "../program.js";
+import { createRequireCalleeFileOf } from "./builtins/module-bindings.js";
+import { invalidJsonModuleDiag } from "../../diagnostics/diagnostic.js";
+import { BOOL, type IrExpr, type IrGlobal, type IrType } from "../../ir/ir.js";
+
+/** One required JSON document: its shared global, the run-once flag, and
+ * the parsed value with the type it bakes to. */
+export interface JsonRequireModule {
+  global: IrGlobal;
+  loadedId: string;
+  parsed: unknown;
+  type: IrType;
+}
+
+/** The JSON document a CommonJS `require("…")` call loads, or null. */
+export function jsonRequireTarget(
+  lowerer: Lowerer,
+  call: ts.Expression | undefined,
+): ts.SourceFile | null {
+  if (call === undefined || !ts.isCallExpression(call)) return null;
+  const spec = requireSpecOf(call);
+  if (spec === null || !ts.isIdentifier(call.expression)) return null;
+  const sf = call.getSourceFile();
+  if (!isCjsJsFile(sf, lowerer.program)) return null;
+  if (createRequireCalleeFileOf(lowerer, call.expression) !== null) return null;
+  const target = resolveImport(lowerer.program, sf, spec, "require");
+  return target !== null && target.fileName.endsWith(".json") ? target : null;
+}
+
+/** A declaration whose initializer is a require() of a JSON document. */
+export function isJsonRequireDecl(lowerer: Lowerer, decl: ts.VariableDeclaration): boolean {
+  return jsonRequireTarget(lowerer, decl.initializer) !== null;
+}
+
+/** Registers the shared global of every JSON document a CommonJS module of
+ * the program requires, and the guarded creation in each requiring
+ * module's %init prelude. */
+export function collectJsonRequires(lowerer: Lowerer, parts: FileParts[]): void {
+  for (const fp of parts) {
+    const sf = fp.sf;
+    if (sf.isDeclarationFile || !isCjsJsFile(sf, lowerer.program)) continue;
+    if (!sf.text.includes(".json")) continue;
+    const calls: ts.CallExpression[] = [];
+    ts.walkPreorder(sf, (node) => {
+      if (ts.isCallExpression(node) && requireSpecOf(node) !== null) calls.push(node);
+      return undefined;
+    });
+    const created = new Set<JsonRequireModule>();
+    for (const call of calls) {
+      const jsonSf = jsonRequireTarget(lowerer, call);
+      if (jsonSf === null) continue;
+      try {
+        let mod = lowerer.jsonRequireModules.get(jsonSf.fileName);
+        if (mod === undefined) {
+          const tsType = lowerer.typeOf(call);
+          const mapped = lowerer.mapTypeOf(tsType);
+          if (!mapped || !lowerer.comptimeBakeable(mapped)) lowerer.badType(call, tsType);
+          let parsed: unknown;
+          try {
+            parsed = JSON.parse(jsonSf.text);
+          } catch (e) {
+            lowerer.pushDiag(
+              invalidJsonModuleDiag(
+                jsonSf.fileName,
+                e instanceof Error ? e.message : String(e),
+                locOf(call),
+              ),
+            );
+            throw new PoisonError();
+          }
+          const id = `%g.jsonRequire.${lowerer.jsonRequireModules.size}`;
+          const global: IrGlobal = { id, name: "%json", type: mapped, mutable: true };
+          const loadedId = `${id}%loaded`;
+          lowerer.globalsList.push(global, {
+            id: loadedId,
+            name: "%loaded",
+            type: BOOL,
+            mutable: true,
+          });
+          mod = { global, loadedId, parsed, type: mapped };
+          lowerer.jsonRequireModules.set(jsonSf.fileName, mod);
+        }
+        lowerer.jsonRequireCalls.set(call, mod);
+        if (created.has(mod)) continue;
+        created.add(mod);
+        const loc = locOf(call);
+        const value = lowerer.comptimeValueToIr(mod.parsed, mod.type, "$", call);
+        const actions = lowerer.jsonInitActions.get(sf) ?? [];
+        lowerer.jsonInitActions.set(sf, actions);
+        actions.push({
+          kind: "if",
+          cond: {
+            kind: "unary",
+            op: "!",
+            operand: { kind: "varRef", localId: mod.loadedId, type: BOOL, loc },
+            type: BOOL,
+            loc,
+          },
+          then: [
+            {
+              kind: "assign",
+              localId: mod.loadedId,
+              value: { kind: "boolLit", value: true, type: BOOL, loc },
+              loc,
+            },
+            { kind: "assign", localId: mod.global.id, value, loc },
+          ],
+          else_: null,
+          loc,
+        });
+      } catch (e) {
+        if (!(e instanceof PoisonError)) throw e;
+        // diagnostic already recorded
+      }
+    }
+  }
+}
+
+/** A require() call of a JSON document: the shared object. */
+export function lowerJsonRequireCall(lowerer: Lowerer, call: ts.CallExpression): IrExpr | null {
+  const mod = lowerer.jsonRequireCalls.get(call);
+  if (mod === undefined) return null;
+  return { kind: "varRef", localId: mod.global.id, type: mod.global.type, loc: locOf(call) };
+}

packages/compiler/src/frontend/lowering/lower-calls.ts

@@ -59,6 +59,7 @@ import {
 } from "../../ir/ir.js";
 import { lowerProtocolMethodCall } from "./class-protocol-dispatch.js";
 import { isJsSourceFile, locOf, requireSpecOf } from "../program.js";
+import { lowerJsonRequireCall } from "./json-require.js";
 import { lowerAssertInstanceCall } from "./assert-instances.js";
 import { isGenericCallableMemberType, typeKey } from "../type-mapper.js";
 import { dynFallbackType, jsFuncNameOf, newFnCtx, staticImportNamespaceType } from "./lowerer.js";
@@ -223,6 +224,10 @@ export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr {
     if (object) return object;
   }
 
+  // require() of a JSON document: the shared parsed object.
+  const json = lowerJsonRequireCall(lowerer, expr);
+  if (json) return json;
+
   // A call whose chain ROOTS at an ambient-undefined name (`declare
   // const value: Y | undefined; value?.foo("a")`, `declare function
   // chain...; chain(o).mapValues(f).value()`, a trap binding's read):

packages/compiler/src/frontend/lowering/lower-modules.ts

@@ -35,6 +35,8 @@ import {
   withClassInitFlag,
   withCycleInitFlags,
 } from "./cycle-initialization.js";
+import { markRequireEarlyBindings } from "./require-initialization.js";
+import { collectJsonRequires, isJsonRequireDecl } from "./json-require.js";
 import {
   invalidJsonModuleDiag,
   unsupportedPackageImportDiag,
@@ -409,13 +411,15 @@ export function collectProgram(lowerer: Lowerer, parts: FileParts[]): void {
       for (const fp of parts) lowerer.collectGlobals(fp.sf, fp.topStmts);
       lowerer.collectNpmImports(parts);
       lowerer.collectJsonImports(parts);
+      collectJsonRequires(lowerer, parts);
     } finally {
       lowerer.diagSink = null;
     }
   } else {
     for (const fp of parts) lowerer.collectGlobals(fp.sf, fp.topStmts);
     lowerer.collectNpmImports(parts);
     lowerer.collectJsonImports(parts);
+    collectJsonRequires(lowerer, parts);
   }
   for (const decl of computed) {
     const symbol = declSymbolOf(lowerer, decl);
@@ -493,6 +497,7 @@ export function collectProgram(lowerer: Lowerer, parts: FileParts[]): void {
     lowerer.collecting = wasCollecting;
   }
   markCycleEarlyBindings(lowerer);
+  markRequireEarlyBindings(lowerer);
 }
 
 export function collectNpmImports(lowerer: Lowerer, parts: FileParts[]): void {
@@ -1532,8 +1537,9 @@ export function collectGlobals(
       isRequireStatement(stmt) &&
       !(
         ts.isVariableStatement(stmt) &&
-        stmt.declarationList.declarations.some((decl) =>
-          isNativeFfiRequire(lowerer, decl.initializer),
+        stmt.declarationList.declarations.some(
+          (decl) =>
+            isNativeFfiRequire(lowerer, decl.initializer) || isJsonRequireDecl(lowerer, decl),
         )
       )
     ) {