[POC] Compile augmented npm packages from scriptc-pm source overlays (--sources)

.gitignore

@@ -13,6 +13,8 @@ node_modules/
 !tests/fixtures/npm/cases/*/node_modules/
 !tests/library-mode/*/node_modules/
 !tests/fixtures/provenance/node_modules/
+!tests/fixtures/source-overlays/project/node_modules/
+!tests/fixtures/source-overlays/project/node_modules/.pnpm/*/node_modules/
 !tests/fixtures/fetch/node_modules/
 !tests/fixtures/gateway-e2e/node_modules/
 !tests/fixtures/node-types/node_modules/
@@ -32,6 +34,7 @@ output/
 !tests/fixtures/npm/cases/workspace-copied/members/*/dist/
 !tests/fixtures/npm/cases/workspace-copied/node_modules/*/dist/
 !tests/fixtures/provenance/node_modules/*/dist/
+!tests/fixtures/source-overlays/project/node_modules/.pnpm/**/dist/
 .scriptc/
 *.log
 .DS_Store

packages/cli/src/bootstrap.ts

@@ -66,6 +66,7 @@ async function tryFastPath(): Promise<number | null> {
     values["emit-ir"] ||
     values.lib ||
     values["provenance-sources"] ||
+    process.argv.some((arg) => arg === "--sources" || arg.startsWith("--sources=")) ||
     (values["external-types"] ?? []).length > 0
   )
     return null;
@@ -191,6 +192,7 @@ async function coverageFastPath(
     values.optimization !== undefined ||
     values["windows-subsystem"] !== undefined ||
     values["provenance-sources"] ||
+    process.argv.some((arg) => arg === "--sources" || arg.startsWith("--sources=")) ||
     values["emit-ir"] ||
     (values["npm-static"] ?? []).length > 0 ||
     (values["external-types"] ?? []).length > 0

packages/compiler/src/cli/command.ts

@@ -13,10 +13,16 @@ import { buildEnvelope } from "../diagnostics/envelope.js";
 import { renderWarnings } from "../diagnostics/render.js";
 import { isNativeOptimization } from "../backend/optimization.js";
 import { setProvenanceSources } from "../frontend/provenance-registry.js";
+import {
+  findSourceManifest,
+  installSourceOverlays,
+  loadSourceManifest,
+  sourceOverlayFallbacks,
+} from "../frontend/source-overlays.js";
 import type { CliHost, NativeCacheWarmProfile } from "./host.js";
 import { resolveOutputOptions } from "./output-options.js";
 import { selectOutputPaths } from "./paths.js";
-import { CLI_OPTIONS, USAGE } from "./usage.js";
+import { CLI_OPTIONS, normalizeCliArgs, USAGE } from "./usage.js";
 
 /* The exit discipline: NEVER process.exit() after writing output. stdout/
  * stderr to a PIPE are async streams — process.exit() drops whatever libuv
@@ -46,7 +52,12 @@ function parseCli(
   typeof parseArgs<{ options: typeof CLI_OPTIONS; allowPositionals: true; allowNegative: true }>
 > {
   try {
-    return parseArgs({ args, options: CLI_OPTIONS, allowPositionals: true, allowNegative: true });
+    return parseArgs({
+      args: normalizeCliArgs(args),
+      options: CLI_OPTIONS,
+      allowPositionals: true,
+      allowNegative: true,
+    });
   } catch (err) {
     // parseArgs appends a paragraph about `--` and positionals to the
     // unknown-option message; the first sentence is the part that names
@@ -57,6 +68,34 @@ function parseCli(
   }
 }
 
+/** --sources: reads the manifest (the given path, or the nearest
+ * `.scriptc-pm/manifest.json` above `anchor`) and installs its overlays
+ * for the compile. Without the flag, any previous set is cleared. */
+function installSourcesFlag(value: string | undefined, anchor: string): void {
+  installSourceOverlays(null);
+  if (value === undefined) return;
+  const manifestPath = value !== "" ? resolve(value) : findSourceManifest(anchor);
+  if (manifestPath === null) {
+    fail(
+      `scriptc: --sources found no .scriptc-pm/manifest.json above ${dirname(anchor)} (run scriptc-pm sync, or pass --sources=<manifest>)`,
+    );
+  }
+  try {
+    installSourceOverlays(loadSourceManifest(manifestPath));
+  } catch (e) {
+    fail(`scriptc: ${e instanceof Error ? e.message : String(e)}`);
+  }
+}
+
+/** One stderr line per overlay instance the build fell back on. */
+function reportSourceFallbacks(): void {
+  for (const f of sourceOverlayFallbacks()) {
+    process.stderr.write(
+      `scriptc: sources: ${f.name}@${f.version} compiles from published JS (source overlay dropped: ${f.reason})\n`,
+    );
+  }
+}
+
 async function main(args: string[], host: CliHost): Promise<number> {
   const { values, positionals } = parseCli(args);
   const externalTypeArgs = values["external-types"] ?? [];
@@ -72,6 +111,9 @@ async function main(args: string[], host: CliHost): Promise<number> {
   }
 
   const [command, inputArg] = positionals;
+  if (values.sources !== undefined && values["provenance-sources"]) {
+    fail(`--sources and --provenance-sources are alternatives\n\n${USAGE}`);
+  }
   if (command === "cache") {
     if (inputArg !== "warm")
       fail(`unknown cache command "${inputArg ?? ""}" (supported: warm)\n\n${USAGE}`);
@@ -88,6 +130,7 @@ async function main(args: string[], host: CliHost): Promise<number> {
       values["detached-exit"] ||
       (values["npm-static"] ?? []).length > 0 ||
       values["provenance-sources"] ||
+      values.sources !== undefined ||
       externalTypeArgs.length > 0 ||
       values.out !== undefined ||
       values["emit-ir"] ||
@@ -159,6 +202,7 @@ async function main(args: string[], host: CliHost): Promise<number> {
       );
     }
     const profilePath = resolve(profileArg);
+    installSourcesFlag(values.sources, profilePath);
     const libOutDir = values.out
       ? dirname(resolve(values.out))
       : join(dirname(profilePath), ".scriptc");
@@ -169,6 +213,7 @@ async function main(args: string[], host: CliHost): Promise<number> {
       emitIr: values["emit-ir"],
       sanitize: values.sanitize,
     });
+    reportSourceFallbacks();
     if (!result.ok) {
       const color = process.stderr.isTTY ?? false;
       process.stderr.write(
@@ -322,6 +367,7 @@ async function main(args: string[], host: CliHost): Promise<number> {
   // source "paths" at creation): attestations and source trees fetch (or
   // ride the content-addressed cache / the offline manifest), the registry
   // installs, and every fallback prints as a note — never a failure.
+  installSourcesFlag(values.sources, input);
   const provenance = values["provenance-sources"]
     ? await host.resolveProvenanceSources(input)
     : null;
@@ -387,6 +433,7 @@ async function main(args: string[], host: CliHost): Promise<number> {
       ...(ffiProfilePath !== undefined ? { ffiProfilePath } : {}),
       ...(printNativeLinkInfo ? { nativeLinkInfo: true } : {}),
     });
+    if (command === "build") reportSourceFallbacks();
     if (!result.ok) {
       if (printDiagnostics) {
         const envelope = buildEnvelope({

packages/compiler/src/cli/usage.ts

@@ -59,6 +59,13 @@ Options:
                      direct import: own .d.ts, unminified JS, no build-transform
                      markers). Packages that fail preflight are refused with a
                      diagnostic. Eligible dependencies compile automatically.
+      --sources[=<manifest>]
+                     EXPERIMENTAL: compile augmented npm package instances
+                     from the TypeScript source overlays a scriptc-pm
+                     manifest describes (default: the nearest
+                     .scriptc-pm/manifest.json above the entry). An overlay
+                     that does not compile falls back to the package's
+                     published JS
       --provenance-sources
                      EXPERIMENTAL: compile npm dependencies from their
                      provenance-attested SOURCE (fetched at the attested
@@ -73,6 +80,22 @@ Options:
   -v, --version      print the version
 `;
 
+/** `--sources` takes an optional value (`--sources` or `--sources=<path>`),
+ * which parseArgs cannot express: the bare flag becomes an empty value,
+ * meaning "find the manifest". A separate `--sources <path>` argument is
+ * not accepted, so a following positional is never swallowed. */
+export function normalizeCliArgs(args: readonly string[]): string[] {
+  const out: string[] = [];
+  for (const arg of args) {
+    if (arg === "--") {
+      out.push(...args.slice(out.length));
+      break;
+    }
+    out.push(arg === "--sources" ? "--sources=" : arg);
+  }
+  return out;
+}
+
 export const CLI_OPTIONS = {
   out: { type: "string", short: "o" },
   emit: { type: "string" },
@@ -89,6 +112,7 @@ export const CLI_OPTIONS = {
   ffi: { type: "string" },
   "npm-static": { type: "string", multiple: true },
   "provenance-sources": { type: "boolean", default: false },
+  sources: { type: "string" },
   "external-types": { type: "string", multiple: true },
   lib: { type: "boolean", default: false },
   profile: { type: "string" },

packages/compiler/src/coverage/report.ts

@@ -20,7 +20,7 @@ import {
 } from "../diagnostics/envelope.js";
 import { renderDiagnostics } from "../diagnostics/render.js";
 import type { LowerStats } from "../frontend/lowering/lowerer.js";
-import type { ProvenanceSources } from "../frontend/provenance-registry.js";
+import type { SourceOverlayReport } from "../frontend/source-overlays.js";
 
 export interface CoverageInput {
   file: string;
@@ -41,11 +41,12 @@ export interface CoverageInput {
   /** Each dependency's native compilation outcome, with the first refusal
    * reason for a refused package. */
   npmStatic?: NpmStaticStatus[];
-  /** --provenance-sources: the resolved source-mapped packages and the
-   * fallback notes — the report renders per-package attribution from
-   * statsByFile (statements inside each package's source dir). */
-  provenance?: ProvenanceSources;
-  /** --provenance-sources: per-file statement attribution from the emit
+  /** --sources / --provenance-sources: the packages compiled from source
+   * overlays, the ones that fell back, and the producer's notes — the
+   * report renders per-package attribution from statsByFile (statements
+   * inside each package's overlay dir). */
+  sourceOverlays?: SourceOverlayReport;
+  /** --sources / --provenance-sources: per-file statement attribution from the emit
    * pass (LowerResult.statsByFile). */
   statsByFile?: Map<string, { total: number; failed: number }>;
   /** --provenance-sources: diagnostics of elided pure-annotated dead
@@ -55,11 +56,15 @@ export interface CoverageInput {
   preflightFailed: boolean;
 }
 
-/** One dependency's native compilation outcome. */
+/** One dependency's native compilation outcome. "source" compiles the
+ * package instance (`name@version`) from its --sources overlay;
+ * "source-fallback" dropped the overlay and the package compiles from its
+ * published JS (its own row says how that went). */
 export interface NpmStaticStatus {
   package: string;
-  status: "static" | "refused";
-  /** The first refusal reason for a refused package. */
+  status: "static" | "refused" | "source" | "source-fallback";
+  /** The first refusal reason for a refused package; the drop reason for a
+   * source fallback. */
   detail?: string;
 }
 
@@ -194,29 +199,40 @@ export function renderCoverage(input: CoverageInput, opts: CoverageRenderOptions
     out.push(`  ${c(DIM, "npm package compilation:")}`);
     const widestP = Math.max(...npmStatic.map((s) => s.package.length));
     for (const s of npmStatic) {
+      const detail = s.detail !== undefined ? ` ${c(DIM, `(${s.detail})`)}` : "";
       const status =
         s.status === "static"
           ? c(GREEN, "static")
-          : c(RED, "refused") + (s.detail !== undefined ? ` ${c(DIM, `(${s.detail})`)}` : "");
+          : s.status === "source"
+            ? c(GREEN, "static from source")
+            : s.status === "source-fallback"
+              ? c(YELLOW, "published JS") + detail
+              : c(RED, "refused") + detail;
       out.push(`    ${s.package.padEnd(widestP)}  ${status}`);
     }
     out.push("");
   }
 
-  // --provenance-sources: per-package attribution — the answer to "did
-  // the DEPENDENCY's statements compile static?", computed by summing
-  // statsByFile entries under each package's source dir. Elided
-  // pure-annotated dead consts and fallback notes render here too, so
-  // the provenance story is complete in one place.
-  if (input.provenance !== undefined) {
-    const prov = input.provenance;
-    out.push(
-      `  ${c(BOLD, "provenance sources")}   ${c(DIM, "(attested source compiled as program modules)")}`,
-    );
-    for (const pkg of prov.packages) {
+  // --sources / --provenance-sources: per-package attribution — the
+  // answer to "did the DEPENDENCY's statements compile static?", computed
+  // by summing statsByFile entries under each overlay dir. Elided
+  // pure-annotated dead consts, fallbacks, and producer notes render here
+  // too, so the source story is complete in one place.
+  if (input.sourceOverlays !== undefined) {
+    const overlays = input.sourceOverlays;
+    out.push(`  ${c(BOLD, "source overlays")}   ${c(DIM, `(${overlays.origin})`)}`);
+    for (const pkg of overlays.packages) {
+      const origin = pkg.origin !== null ? `  ${c(DIM, pkg.origin)}` : "";
+      out.push(`    ${pkg.name}@${pkg.version}${origin}`);
+      if (pkg.fallback !== null) {
+        out.push(
+          `      ${c(YELLOW, "published JS")} ${c(DIM, `(source overlay dropped: ${pkg.fallback})`)}`,
+        );
+        continue;
+      }
       let pTotal = 0;
       let pFailed = 0;
-      const dir = pkg.dir.endsWith("/") ? pkg.dir : `${pkg.dir}/`;
+      const dir = `${pkg.dir}/`;
       for (const [file, s] of input.statsByFile ??
         new Map<string, { total: number; failed: number }>()) {
         if (!file.startsWith(dir)) continue;
@@ -229,21 +245,14 @@ export function renderCoverage(input: CoverageInput, opts: CoverageRenderOptions
       const pOk = pTotal - pFailed;
       const pPct = pTotal === 0 ? 100 : Math.floor((pOk / pTotal) * 100);
       const pColor = pPct === 100 ? GREEN : pPct >= 75 ? YELLOW : RED;
-      const shortRepo = pkg.repo
-        .replace(/^git\+/, "")
-        .replace(/^https:\/\//, "")
-        .replace(/@refs\/.*$/, "");
-      out.push(
-        `    ${pkg.name}@${pkg.version}  ${c(DIM, `${shortRepo} @ ${pkg.commit.slice(0, 12)}`)}`,
-      );
       out.push(
         `      ${pTotal} statements: ${pOk} static ${c(pColor, c(BOLD, `(${pPct}%)`))}, ${pFailed} fenced` +
           (elided > 0
             ? ` ${c(DIM, `(${elided} fence${elided === 1 ? "" : "s"} elided as @__PURE__ dead consts — uses would fence per site)`)}`
             : ""),
       );
     }
-    for (const note of prov.notes) out.push(`    ${c(YELLOW, "note")}  ${note}`);
+    for (const note of overlays.notes) out.push(`    ${c(YELLOW, "note")}  ${note}`);
     out.push("");
   }
 

packages/compiler/src/coverage/verdict-cache.ts

@@ -19,6 +19,7 @@ import {
   frontendInputsStillMatch,
   type FrontendInputSnapshot,
 } from "../frontend/input-tracker.js";
+import { sourceOverlayCacheIdentity } from "../frontend/source-overlays.js";
 
 const SCHEMA = "scriptc.coverage-verdict.v2";
 
@@ -97,6 +98,10 @@ export function verdictKey(
       return null;
     }
   }
+  // --sources: the overlay set's identity joins the key; a set that cannot
+  // be cached (the network-fetching pilot) never replays.
+  const overlays = sourceOverlayCacheIdentity();
+  if (overlays === null) return null;
   const npmStatic =
     opts.npmStatic === undefined
       ? ""
@@ -120,13 +125,14 @@ export function verdictKey(
       opts.ffiProfilePath ?? "",
       ffi,
       external.join("\n"),
+      ...(overlays !== undefined ? [`sources:${overlays}`] : []),
     ].join("\0"),
   );
 }
 
 function cacheable(coverage: AnalyzeResult["coverage"]): CachedCoverage | null {
   if (
-    coverage.provenance !== undefined ||
+    coverage.sourceOverlays !== undefined ||
     coverage.statsByFile !== undefined ||
     coverage.provenanceElided !== undefined
   )

packages/compiler/src/executable/executable-cache.ts

@@ -121,6 +121,9 @@ export interface EarlyExecutableCacheOptions {
   nodeVersion: string;
   implementation: string;
   implementationDependencies: CompilerImplementationDependency[];
+  /** --sources: the overlay set's identity (manifest lock digest and
+   * bytes). Omitted without --sources, preserving existing keys. */
+  sources?: string;
 }
 
 export type EarlyExecutableRouteOptions = Omit<
@@ -232,6 +235,7 @@ function cacheKey(options: EarlyExecutableCacheOptions): string {
     options.nodeVersion,
     options.implementation,
     ...ffiParts,
+    ...(options.sources !== undefined ? [`sources:${options.sources}`] : []),
   ]);
 }
 
@@ -280,6 +284,7 @@ function routeKey(options: Omit<EarlyExecutableRouteOptions, "nativeEnvironment"
   } else {
     hash.update(resolve(options.ffiProfile.path)).update("\0").update(options.ffiProfile.bytes);
   }
+  if (options.sources !== undefined) hash.update(`\0sources:${options.sources}`);
   return hash.digest("hex");
 }