diff --git a/.env.example b/.env.example index b8c5e542b..d630c2ab1 100644 --- a/.env.example +++ b/.env.example @@ -29,15 +29,7 @@ ANTHROPIC_API_KEY= # DEMO_SEED= -# Dev orchestration overrides (bin/dev.ts, bin/seed.ts) -# -# bin/dev.ts publishes the seed tool packages as the seed admin against -# the seed tenant. Set these to point it at a non-seed hub without -# editing the script. Unset values fall back to the seed defaults shown. -# HUB_ADMIN_EMAIL=alice@example.com -# HUB_ADMIN_PASSWORD=password123 -# HUB_TENANT_SLUG=acme -# HUB_TENANT_NAME=Acme Corp +# Seed overrides (bin/seed.ts) # # Base URL bin/seed.ts targets when seeding via the hub API. # HUB_URL=http://localhost:3000 diff --git a/DEV.md b/DEV.md index 99e385957..fa58fc29a 100644 --- a/DEV.md +++ b/DEV.md @@ -55,36 +55,32 @@ code and have working defaults. All are optional; set them only to override the default behavior. They are commented out in the example files next to the matching service config. -| Variable | Read by | Default | Purpose | -| -------------------------------------- | -------------------------------------------- | ----------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `HUB_ADMIN_EMAIL` | `bin/dev.ts`, `bin/publish-tool-packages.ts` | `alice@example.com` | Admin identity `bin/dev.ts` uses to publish the seed tool packages. | -| `HUB_ADMIN_PASSWORD` | `bin/dev.ts`, `bin/publish-tool-packages.ts` | `password123` | Password for that admin identity. | -| `HUB_TENANT_SLUG` | `bin/dev.ts` | `acme` | Tenant the seed tool packages are published into. | -| `HUB_TENANT_NAME` | `bin/dev.ts` | `Acme Corp` | Display name for that tenant. | -| `HUB_URL` | `bin/seed` | `http://localhost:3000` | Base URL `bin/seed` targets when seeding via the hub API. | -| `HUB_API_ORIGIN` | hub (`apps/hub`) | `BETTER_AUTH_BASE_URL` | The http or https origin a sidecar reaches the hub REST API at. A workflow run's hub token is pinned to it, so set it when sidecars reach the hub at an address other than its public URL (a remote or containerized sidecar cannot reach `localhost`). The value must be an origin: the hub refuses to start on any other value, including one with a path. When the value comes from `BETTER_AUTH_BASE_URL` and carries more than an origin (a path, credentials, a query or a fragment), the hub uses its origin and logs a warning. | -| `HUB_MAX_TARBALL_BYTES` | hub (`apps/hub`) | 10 MiB | Per-tarball cap for tool packages uploaded to the package registry. | -| `DB_STATEMENT_TIMEOUT_MS` | hub (`apps/hub`), db scripts (`bin/`) | 60000 (60s) | Positive integer milliseconds, up to 2147483647. Cancels SQL statements exceeding this duration, including lock waits. | -| `PG_SCHEMA` | hub (`apps/hub`) | unset | Pins the hub to a postgres schema. Integration-test-only; leave unset normally. | -| `WORKFLOW_DEFAULT_MAX_LIFETIME` | hub (`apps/hub`) | `7d` | Maximum deployment lifetime when no tenant or installed workflow sets one. See `docs/workflow-lifecycle-policy.md`. | -| `WORKFLOW_DEFAULT_MAX_DISCONNECTED` | hub (`apps/hub`) | `15m` | How long a deployment's sidecar may stay disconnected before the Hub fails it, when no tenant or installed workflow sets it. | -| `WORKFLOW_DEFAULT_RETENTION_COMPLETED` | hub (`apps/hub`) | `30m` | How long a completed deployment keeps its capacity when no tenant or installed workflow sets it. | -| `WORKFLOW_DEFAULT_RETENTION_FAILED` | hub (`apps/hub`) | `24h` | How long a failed deployment keeps its capacity for inspection when no tenant or installed workflow sets it. | -| `WORKFLOW_DEFAULT_RETENTION_CANCELLED` | hub (`apps/hub`) | `1h` | How long a cancelled deployment keeps its capacity when no tenant or installed workflow sets it. | -| `CREDENTIAL_ENCRYPTION_KEY` | hub (`apps/hub`) | none (required) | 32-byte hex key that seals credential secrets and OAuth client secrets at rest in the database. The hub refuses to start without it. | -| `PRINCIPAL_KEY_ENCRYPTION_KEY` | hub (`apps/hub`) | none (required) | 32-byte hex key that seals per-principal signing keys at rest, separate from `CREDENTIAL_ENCRYPTION_KEY`. The hub refuses to start without it. | -| `SIDECAR_CREDENTIAL_ENCRYPTION_KEY` | sidecar (`apps/sidecar`) | none (required) | 32-byte hex key that seals credential material (inference apiKeys) at rest under `SIDECAR_DATA_DIR`. The sidecar refuses to start without it. | -| `SIDECAR_ADAPTER_MANIFEST` | hub (`apps/hub`), sidecar (`apps/sidecar`) | unset (no custom adapters) | JSON array of `{provider, specifier, export}` custom inference adapters. Each provider key starts with a lowercase letter or digit and contains only lowercase letters, digits, `.`, `_` and `-`. The hub forwards it to every provisioned sidecar; the sidecar loads it over the built-in registry. An empty string is treated as unset. The hub and the sidecar refuse to start on an invalid value. | -| `SIDECAR_CACHE_DIR` | sidecar (`apps/sidecar`) | `/cache/tarballs` | Directory for the tool-package tarball cache. | -| `SIDECAR_CACHE_MAX_BYTES` | sidecar (`apps/sidecar`) | 10 GiB | Target total size of cached tarballs and extracted trees. Entries in use can exceed it until they are released. | -| `SIDECAR_MAX_ACTIVE_DEPLOYMENTS` | sidecar (`apps/sidecar`) | 128 | Positive integer active-deployment limit, advertised to the Hub. Read at startup. | -| `SIDECAR_MAX_RETAINED_DEPLOYMENTS` | sidecar (`apps/sidecar`) | 256 | Non-negative integer retained-record limit; zero refuses retention. Combined capacity cannot exceed 2048 records. | -| `SIDECAR_REGISTRY_MAX_TARBALL_BYTES` | sidecar (`apps/sidecar`) | 10 MiB | Per-tarball cap enforced when pulling from upstream tool registries. | -| `SIDECAR_TOOL_REGISTRIES` | sidecar (`apps/sidecar`) | public npmjs | JSON array of `{name, url, auth?}` tool registries. Unset the variable to use npmjs; do not set it to an empty string. | -| `SIDECAR_LATENCY_BENCH_FILE` | sidecar (`apps/sidecar`) | unset (disabled) | When set to a file path, appends a per-operation latency line to that file. A dev latency-benchmarking hook; leave unset in normal use. | -| `SIDECAR_REPACK_EVERY_MESSAGES` | sidecar (`apps/sidecar`) | unset (disabled) | Forces a git repack of the workflow-run repo every N messages (positive integer). A repack-cadence A/B toggle; leave unset in normal use. | -| `CONSUMED_RETENTION_MS` | sidecar (`apps/sidecar`) | 24h | Retention horizon in milliseconds (positive integer) for the workflow-run consumed-dedup index. | -| `CHILD_READY_TIMEOUT_MS` | sidecar (`apps/sidecar`) | 30s | Milliseconds (positive integer) the supervisor waits for a spawned workflow-process child to signal ready before killing it and rejecting the spawn. | +| Variable | Read by | Default | Purpose | +| -------------------------------------- | ------------------------------------------ | ----------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `HUB_URL` | `bin/seed` | `http://localhost:3000` | Base URL `bin/seed` targets when seeding via the hub API. | +| `HUB_API_ORIGIN` | hub (`apps/hub`) | `BETTER_AUTH_BASE_URL` | The http or https origin a sidecar reaches the hub REST API at. A workflow run's hub token is pinned to it, so set it when sidecars reach the hub at an address other than its public URL (a remote or containerized sidecar cannot reach `localhost`). The value must be an origin: the hub refuses to start on any other value, including one with a path. When the value comes from `BETTER_AUTH_BASE_URL` and carries more than an origin (a path, credentials, a query or a fragment), the hub uses its origin and logs a warning. | +| `HUB_MAX_TARBALL_BYTES` | hub (`apps/hub`) | 10 MiB | Per-tarball cap for packages uploaded to the package registry. | +| `DB_STATEMENT_TIMEOUT_MS` | hub (`apps/hub`), db scripts (`bin/`) | 60000 (60s) | Positive integer milliseconds, up to 2147483647. Cancels SQL statements exceeding this duration, including lock waits. | +| `PG_SCHEMA` | hub (`apps/hub`) | unset | Pins the hub to a postgres schema. Integration-test-only; leave unset normally. | +| `WORKFLOW_DEFAULT_MAX_LIFETIME` | hub (`apps/hub`) | `7d` | Maximum deployment lifetime when no tenant or installed workflow sets one. See `docs/workflow-lifecycle-policy.md`. | +| `WORKFLOW_DEFAULT_MAX_DISCONNECTED` | hub (`apps/hub`) | `15m` | How long a deployment's sidecar may stay disconnected before the Hub fails it, when no tenant or installed workflow sets it. | +| `WORKFLOW_DEFAULT_RETENTION_COMPLETED` | hub (`apps/hub`) | `30m` | How long a completed deployment keeps its capacity when no tenant or installed workflow sets it. | +| `WORKFLOW_DEFAULT_RETENTION_FAILED` | hub (`apps/hub`) | `24h` | How long a failed deployment keeps its capacity for inspection when no tenant or installed workflow sets it. | +| `WORKFLOW_DEFAULT_RETENTION_CANCELLED` | hub (`apps/hub`) | `1h` | How long a cancelled deployment keeps its capacity when no tenant or installed workflow sets it. | +| `CREDENTIAL_ENCRYPTION_KEY` | hub (`apps/hub`) | none (required) | 32-byte hex key that seals credential secrets and OAuth client secrets at rest in the database. The hub refuses to start without it. | +| `PRINCIPAL_KEY_ENCRYPTION_KEY` | hub (`apps/hub`) | none (required) | 32-byte hex key that seals per-principal signing keys at rest, separate from `CREDENTIAL_ENCRYPTION_KEY`. The hub refuses to start without it. | +| `SIDECAR_CREDENTIAL_ENCRYPTION_KEY` | sidecar (`apps/sidecar`) | none (required) | 32-byte hex key that seals credential material (inference apiKeys) at rest under `SIDECAR_DATA_DIR`. The sidecar refuses to start without it. | +| `SIDECAR_ADAPTER_MANIFEST` | hub (`apps/hub`), sidecar (`apps/sidecar`) | unset (no custom adapters) | JSON array of `{provider, specifier, export}` custom inference adapters. Each provider key starts with a lowercase letter or digit and contains only lowercase letters, digits, `.`, `_` and `-`. The hub forwards it to every provisioned sidecar; the sidecar loads it over the built-in registry. An empty string is treated as unset. The hub and the sidecar refuse to start on an invalid value. | +| `SIDECAR_CACHE_DIR` | sidecar (`apps/sidecar`) | `/cache/tarballs` | Directory for the tool-package tarball cache. | +| `SIDECAR_CACHE_MAX_BYTES` | sidecar (`apps/sidecar`) | 10 GiB | Target total size of cached tarballs and extracted trees. Entries in use can exceed it until they are released. | +| `SIDECAR_MAX_ACTIVE_DEPLOYMENTS` | sidecar (`apps/sidecar`) | 128 | Positive integer active-deployment limit, advertised to the Hub. Read at startup. | +| `SIDECAR_MAX_RETAINED_DEPLOYMENTS` | sidecar (`apps/sidecar`) | 256 | Non-negative integer retained-record limit; zero refuses retention. Combined capacity cannot exceed 2048 records. | +| `SIDECAR_REGISTRY_MAX_TARBALL_BYTES` | sidecar (`apps/sidecar`) | 10 MiB | Per-tarball cap enforced when pulling from upstream tool registries. | +| `SIDECAR_TOOL_REGISTRIES` | sidecar (`apps/sidecar`) | public npmjs | JSON array of `{name, url, auth?}` tool registries. Unset the variable to use npmjs; do not set it to an empty string. | +| `SIDECAR_LATENCY_BENCH_FILE` | sidecar (`apps/sidecar`) | unset (disabled) | When set to a file path, appends a per-operation latency line to that file. A dev latency-benchmarking hook; leave unset in normal use. | +| `SIDECAR_REPACK_EVERY_MESSAGES` | sidecar (`apps/sidecar`) | unset (disabled) | Forces a git repack of the workflow-run repo every N messages (positive integer). A repack-cadence A/B toggle; leave unset in normal use. | +| `CONSUMED_RETENTION_MS` | sidecar (`apps/sidecar`) | 24h | Retention horizon in milliseconds (positive integer) for the workflow-run consumed-dedup index. | +| `CHILD_READY_TIMEOUT_MS` | sidecar (`apps/sidecar`) | 30s | Milliseconds (positive integer) the supervisor waits for a spawned workflow-process child to signal ready before killing it and rejecting the spawn. | ### Database Users @@ -116,11 +112,10 @@ The Hub does not start or trust an ambient sidecar. Workflow probing and executi Options: -| Flag | Effect | -| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--seed` | Seed the database after the hub is ready | -| `--no-admin-ui` | Skip the admin UI dev server | -| `--no-publish-builtins` | Skip publishing the built-in tool packages. Incompatible with `--seed` (errors out — the seed pins the built-ins, so the launch would fail with `tarball.missing`). | +| Flag | Effect | +| --------------- | ---------------------------------------- | +| `--seed` | Seed the database after the hub is ready | +| `--no-admin-ui` | Skip the admin UI dev server | Default ports: hub on 3000 and admin UI on 5173. Provisioned sidecars connect outbound to `ws://localhost:3000/api/sidecars/ws` unless the provisioner is given another Hub WebSocket URL. diff --git a/LAYOUT.md b/LAYOUT.md index 24f0d41c2..395ab2a42 100644 --- a/LAYOUT.md +++ b/LAYOUT.md @@ -83,7 +83,7 @@ Server-only packages that compose the portable core and environment-specific imp | Package | Purpose | | ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `@intx/hub-agent` | Sidecar-side orchestrator (`createSidecarOrchestrator`, the core seam `apps/sidecar` wires together). Wires the package-side pieces of the sidecar runtime — the per-agent key and repo stores, a session manager, and the hub WebSocket link — into a single start/close handle and forwards a spawned child's verified inference events back to the hub. In-process harness construction and agent provisioning are retired: every agent runs as a supervised workflow-process child on the workflow-run substrate (see `@intx/workflow-host`). | -| `@intx/tool-packaging` | Hub-side closure resolver and sidecar-side loader for the npm-distributed tool-package format. Resolves a manifest's dependency closure, materializes tarballs into a content-addressable cache, and lays out a per-instance nested `node_modules/` so a tool package and its transitives resolve bare-specifier imports against the manifest closure. Used by both `apps/hub` and `apps/sidecar`. | +| `@intx/tool-packaging` | Hub-side closure resolver and sidecar-side closure materialization for the npm-distributed tool-package format. Resolves a manifest's dependency closure, materializes tarballs into a content-addressable cache, and lays out a per-instance nested `node_modules/` so a tool package and its transitives resolve bare-specifier imports against the manifest closure. Used by both `apps/hub` and `apps/sidecar`. | | `@intx/workflow-host` | Supervisor and child orchestration for running agents as supervised workflow-process children: the dispatch loop, inbox claim-check, the `SubprocessSpawner`/`SubprocessHandle` spawn seam, the per-step step-invoker, and recycle/drain. It also holds the deploy program, the child substrate factory, the probe protocol, durable conversation state, and the per-run grants reader. Consumes the `@intx/hub-sessions/substrate` repo primitives and takes its spawner, mail bus, and step invoker as injected bindings. Intended to stay host-agnostic so an alternate host reuses it with a different tool/agent runtime — see the open item under Dependency Rules. Its `./local` subpath (the step invoker, step tool builders, authorize closure, and local credentials snapshot) imports no Node or Bun module and bundles for a browser, so `runLocal` callers run definitions outside a sidecar. | | `@intx/workflow-deploy` | Deploy-time validation and derivation over the workflow-run substrate: the capability walk and operator-approval gate (`walkCapabilities`, `createApprovalSetGate`), the child grant cap (`collectDeclaredResources`, `filterGrantsToDeclaredResources`), the runtime grant projections of the walk (`runtimeGrantRules`, `credentialUseGrantRules`), pure deployment-address and workflow-run repo-id derivation (`deriveRunAddress`, `deriveWorkflowRunRepoId`), and per-step inference-source pinning (`pickStepInferenceSource`). An agent-runtime package (it depends on `@intx/agent`, `@intx/authz`, and `@intx/workflow`) consumed by `apps/sidecar`, `@intx/hub-sessions`, and `@intx/workflow-host` (its `./local` entry) on the host side and reached into by `@intx/hub-api` on the control-plane side — one of the enumerated crossings in the Dependency Rules. | diff --git a/Makefile b/Makefile index cfb48e9bd..8cb420e42 100644 --- a/Makefile +++ b/Makefile @@ -125,12 +125,6 @@ format: FORCE docs: FORCE $(BUN) bin/gen-api-docs.ts -builtins: FORCE - $(BUN) run bin/build-builtins.ts - -publish-builtins: builtins - $(BUN) bin/publish-tool-packages.ts --registry workspace-builtins --from dist/builtins - clean: rm -f .env-checked .eslintcache find . -type f -name tsconfig.tsbuildinfo -a ! -path '*/node_modules/*' | xargs rm -f @@ -142,5 +136,5 @@ clean: include .env-checked -.PHONY: all build build-admin-ui lint test test-unit test-workflow test-core test-load test-e2e test-e2e-run test-storage-browser verify-tool-load format docs clean builtins publish-builtins +.PHONY: all build build-admin-ui lint test test-unit test-workflow test-core test-load test-e2e test-e2e-run test-storage-browser verify-tool-load format docs clean FORCE: diff --git a/apps/hub/src/server.ts b/apps/hub/src/server.ts index 0dc8834f5..4cccbda17 100644 --- a/apps/hub/src/server.ts +++ b/apps/hub/src/server.ts @@ -55,7 +55,6 @@ import { recoverSenderDeploy, DEFAULT_SIDECAR_ALLOCATION_CONCURRENCY, pushCredentialReconcile, - WORKSPACE_BUILTINS_REGISTRY, type SidecarLookups, type SidecarProvisioner, type SidecarProvisionerChooser, @@ -185,10 +184,10 @@ export async function createHubServer({ ), }); - // 10 MiB is the production cap for tool-package tarballs uploaded via - // the package-registry PUT endpoint. The npm registry's own per-tarball - // soft cap is several times this, but the substrate's tool packages are - // the curated subset the operator vets; an upload pushing past 10 MiB + // 10 MiB is the production cap for tarballs uploaded via the + // package-registry PUT endpoint. The npm registry's own per-tarball + // soft cap is several times this, but these uploads are the curated + // subset the operator vets; an upload pushing past 10 MiB // is far more likely to be misuse than a legitimate build. The // HUB_MAX_TARBALL_BYTES env var lets an operator opt into a different // cap without a code change. @@ -404,16 +403,6 @@ export async function createHubServer({ toolPackageRegistries: { httpRegistries, defaultRegistry: "npmjs", - // The `workspace-builtins` package-registry asset hosts the - // three in-tree tool packages (`@intx/tools-mail`, - // `@intx/tools-posix`, `@intx/tools-lsp`). Routing the `@intx` - // scope through it keeps an agent's pin set readable - // (`{ name: "@intx/tools-mail" }`) without forcing every pin to - // carry an explicit `registry` field. Operators who shadow this - // asset at a child tenancy with their own `workspace-builtins` - // asset get the closer-scope win for free, since the session - // service builds the per-launch registry map leaf-to-root. - scopeRouting: [{ scope: "@intx", registry: WORKSPACE_BUILTINS_REGISTRY }], }, }); diff --git a/apps/sidecar/src/index.ts b/apps/sidecar/src/index.ts index 6d8e09c3b..60b5a77dd 100644 --- a/apps/sidecar/src/index.ts +++ b/apps/sidecar/src/index.ts @@ -471,8 +471,8 @@ const buildHarness = createDefaultHarnessBuilder({ adapters }); // lays out a probe frame's frozen closure under a per-probe scratch dir // (rooted here so it shares the sidecar data dir's lifecycle); the executor // spawns the one-shot child that evaluates the workflow entry against it. -// The cache/registry caps are the same boot-edge-resolved values the tool -// loader uses. +// The cache/registry caps are the same boot-edge-resolved values the deploy +// path's closure apply uses. // The probe delivers source assets inline in one frame (see // `WorkflowProbeRequestFrame`). A git-sourced asset that grows past the shared // inline-payload cap is the signal to move the probe's asset delivery to the diff --git a/apps/sidecar/src/sidecar-materialization-config.ts b/apps/sidecar/src/sidecar-materialization-config.ts index 2e81de87f..cdf9e564e 100644 --- a/apps/sidecar/src/sidecar-materialization-config.ts +++ b/apps/sidecar/src/sidecar-materialization-config.ts @@ -1,4 +1,4 @@ -// Process edge for the tool-package materialization boundary. +// Process edge for the workflow closure materialization boundary. // // `parseToolRegistries` and `hostPlatform` in `@intx/tool-packaging` // own the `SIDECAR_TOOL_REGISTRIES` rules and the npm `os`/`cpu` diff --git a/bin/build-builtins.ts b/bin/build-builtins.ts deleted file mode 100644 index 1eda0b118..000000000 --- a/bin/build-builtins.ts +++ /dev/null @@ -1,423 +0,0 @@ -// Pack the workspace's built-in tool packages into npm-style tarballs. -// -// The artifacts land under `dist/builtins/`: -// -// dist/builtins/-.tgz # one per package -// -// The filename shape matches the `-.tgz` pattern -// that the `package-registry` asset KindHandler accepts under -// `tarballs/`, so `bin/publish-tool-packages.ts` can upload the files -// directly without renaming. -// -// The packing logic is deterministic (sorted entries, epoch mtime, -// portable tar headers) so the SRI integrity is reproducible across -// build runs and machines. The asset-substrate publish path will -// consume these tarballs and their integrities; this script is the -// boundary that produces the bytes. -// -// Run via `make builtins`. The script is idempotent: re-running -// overwrites any existing artifacts. - -import { promises as fs } from "node:fs"; -import path from "node:path"; -import * as tar from "tar"; -import ssri from "ssri"; -import { type } from "arktype"; - -import { PackageJSON } from "@intx/types/package-json"; - -interface BuiltinSpec { - /** npm package name as it appears in `package.json#name`. */ - name: string; - /** Workspace-relative path to the package root. */ - packageDir: string; -} - -// The set of built-in tool packages this binary ships with. Adding a -// new built-in means appending here, declaring `interchange.tools` in -// the package, and pinning the package on whichever agent definition -// wants it. -const BUILTINS: BuiltinSpec[] = [ - { name: "@intx/tools-mail", packageDir: "packages/tools-mail" }, - { name: "@intx/tools-posix", packageDir: "packages/tools-posix" }, - { name: "@intx/tools-lsp", packageDir: "packages/tools-lsp" }, -]; - -interface BuiltBuiltin { - /** Package name as it appears in the manifest. */ - name: string; - /** Pinned version. */ - version: string; - /** SRI integrity of the produced tarball. */ - integrity: string; - /** Path to the tarball, relative to the repo root. */ - tarballPath: string; -} - -const REPO_ROOT = path.resolve(import.meta.dir, ".."); -const OUT_DIR = path.join(REPO_ROOT, "dist", "builtins"); - -// npm's package-name rules, mirrored from the REST boundary's -// ToolPackagePinName: lowercase, optional `@scope/` prefix with both -// halves non-empty and starting with a URL-safe character. We reject -// malformed scoped names (`@foo/`, `@/bar`, `@scope`) here so the -// produced tarballs never carry a shape the resolver would reject. -const NPM_PACKAGE_NAME_PATTERN = - /^(?:@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*$/; - -function assertValidPackageName(pkgName: string): void { - if (!NPM_PACKAGE_NAME_PATTERN.test(pkgName)) { - throw new Error( - `${pkgName} is not a valid npm package name (expected lowercase, optional \`@scope/\` prefix, URL-safe characters)`, - ); - } -} - -/** - * Build the filename-safe form of a package name. Scoped names are - * flattened with `@scope-tail` rather than stripped to the tail alone: - * two scoped packages whose tails collide (`@scope-a/widget` and - * `@scope-b/widget`) would otherwise overwrite each other in - * `dist/builtins/`. The leading `@` and internal `-` both pass - * `TARBALL_FILENAME_PATTERN`, so the resulting name remains a valid - * tarball entry for the `package-registry` substrate. - */ -function tarballBaseName(pkgName: string): string { - assertValidPackageName(pkgName); - if (!pkgName.startsWith("@")) return pkgName; - const slash = pkgName.indexOf("/"); - // The pattern above guarantees a `/` in any name starting with `@`, - // and that both halves are non-empty; reassert the narrowing here so - // the slice cannot silently produce an empty tail. - if (slash === -1 || slash === pkgName.length - 1) { - throw new Error(`scoped package name missing trailing segment: ${pkgName}`); - } - return `${pkgName.slice(0, slash)}-${pkgName.slice(slash + 1)}`; -} - -async function readPackageJSON(packageDir: string): Promise { - const absPkgDir = path.join(REPO_ROOT, packageDir); - const raw = await fs.readFile(path.join(absPkgDir, "package.json"), "utf-8"); - const parsed: unknown = JSON.parse(raw); - const validated = PackageJSON(parsed); - if (validated instanceof type.errors) { - throw new Error( - `${packageDir}/package.json did not match expected shape: ${validated.summary}`, - ); - } - return validated; -} - -async function packBuiltin( - spec: BuiltinSpec, - pkg: PackageJSON, -): Promise { - const tarballName = `${tarballBaseName(spec.name)}-${pkg.version}.tgz`; - const stagingDir = path.join( - OUT_DIR, - ".staging", - `${spec.name.replace("/", "_")}-${pkg.version}`, - ); - const packageStaging = path.join(stagingDir, "package"); - await fs.rm(stagingDir, { recursive: true, force: true }); - await fs.mkdir(packageStaging, { recursive: true }); - - // Copy the package's source tree into the staging area. We exclude - // node_modules, test files, and tsbuildinfo cache to keep tarballs - // lean and reproducible. - const absPkgDir = path.join(REPO_ROOT, spec.packageDir); - await copyPackageTree(absPkgDir, packageStaging); - - // Transpile the package's `interchange.tools` entry into a self- - // contained JS bundle so the produced tarball runs on bare Node: - // the entry's source form depends on Bun's runtime TypeScript - // support and on workspace-relative imports (`@intx/agent`, - // `@intx/types`, `arktype` via the bun catalog). Bundling produces - // a single ESM file the loader can `import()` from any runtime, and - // lets the packed `package.json` drop the workspace-style - // `dependencies` the npm resolver cannot satisfy. The workspace - // `interchange.tools` value is the *packed* output path so the - // published tarball and the workspace file agree; the source file - // the bundler reads is the same path under `src/` with `.ts` in - // place of `.js`. - const outRel = pkg.interchange?.tools; - if (outRel === undefined) { - throw new Error( - `${spec.name} package.json has no interchange.tools field — it cannot be a built-in tool package`, - ); - } - const sourceRel = deriveSourceFromOutput(outRel); - await bundleInterchangeEntry({ - absPkgDir, - sourceRel, - outRel, - packageStaging, - }); - - // Rewrite the packed `package.json` to (a) point at the bundled - // entry and (b) drop the `workspace:` / `catalog:` dependency specs - // the closure resolver cannot resolve. Everything those specs - // referred to has been inlined into the bundle, so the published - // tarball needs no transitive closure of its own. - const packedPkgJson: Record = { ...pkg }; - // Override `tools` with the bundled entry path but preserve the rest of the - // interchange block (notably `credentials`) -- reconstructing it with only - // `tools` would silently drop a package's static credential declarations. - packedPkgJson.interchange = { ...pkg.interchange, tools: outRel }; - delete packedPkgJson.dependencies; - delete packedPkgJson.devDependencies; - delete packedPkgJson.optionalDependencies; - // No current built-in declares peerDependencies, but a future one - // might — and a peer dep specified as `workspace:` or `catalog:` - // would survive into the published tarball and fail to resolve at - // sidecar apply time, exactly the failure mode the rest of these - // deletions exist to prevent. - delete packedPkgJson.peerDependencies; - delete packedPkgJson.exports; - await fs.writeFile( - path.join(packageStaging, "package.json"), - JSON.stringify(packedPkgJson, null, 2), - ); - - // Walk the staging tree, sort, and hand the file list to `tar` in a - // stable order so the tarball bytes are deterministic across build - // runs and machines. Combined with `mtime: epoch` (zero entry - // mtimes) and `portable: true` (strip uid/gid/uname/gname), this - // makes the SRI integrity reproducible — hub and sidecar built - // from the same workspace yield bit-identical artifacts even when - // built separately. - const tarballPath = path.join(OUT_DIR, tarballName); - const tarEntries = await listFilesSorted(stagingDir, "package"); - // Normalize the staged tree's mode bits before packing. `tar.create` - // reads each entry's mode from `fs.stat`, so two builders with - // different umasks would otherwise produce non-byte-identical - // archives despite the rest of the determinism setup. 0o755 for - // directories and 0o644 for regular files is the canonical npm-pack - // shape. - await normalizeStagingModes(stagingDir, tarEntries); - // `mtime: new Date(0)` zeroes every entry's mtime so the artifact - // is byte-identical across runs. `portable: true` strips uid/gid/ - // uname/gname for the same reason. - // - // `noDirRecurse: true` is required because `tarEntries` already - // contains every directory AND every file under ``. Without - // it, tar would re-pack each directory's contents on top of the - // explicit per-file entries, emitting the same file two or three - // times in the same archive (once for itself, once per ancestor - // directory entry). - const createOpts: tar.TarOptionsWithAliasesAsyncFile = { - cwd: stagingDir, - gzip: true, - file: tarballPath, - portable: true, - mtime: new Date(0), - noDirRecurse: true, - }; - await tar.create(createOpts, tarEntries); - - await fs.rm(stagingDir, { recursive: true, force: true }); - - const bytes = await fs.readFile(tarballPath); - const integrity = ssri.fromData(bytes, { algorithms: ["sha512"] }).toString(); - - return { - name: spec.name, - version: pkg.version, - integrity, - tarballPath: path.relative(REPO_ROOT, tarballPath), - }; -} - -// Set every entry in `entries` to canonical mode bits (0o755 for -// directories, 0o644 for regular files). Symlinks are left alone: -// POSIX `chmod` follows the link and would silently rewrite the -// target's mode, which is never what we want during a packing pass. -async function normalizeStagingModes( - cwd: string, - entries: string[], -): Promise { - for (const rel of entries) { - const abs = path.join(cwd, rel); - const stat = await fs.lstat(abs); - if (stat.isSymbolicLink()) continue; - if (stat.isDirectory()) { - await fs.chmod(abs, 0o755); - } else if (stat.isFile()) { - await fs.chmod(abs, 0o644); - } - } -} - -// Recursively list `/`'s files (and the directories along -// the way) as paths relative to `cwd`, sorted lexically. Used to hand -// `tar.create` a deterministic entry order rather than letting it -// walk in whatever order `fs.readdir` returns. -async function listFilesSorted(cwd: string, root: string): Promise { - const acc: string[] = []; - async function walk(rel: string): Promise { - const abs = path.join(cwd, rel); - const entries = await fs.readdir(abs, { withFileTypes: true }); - entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); - for (const entry of entries) { - const childRel = path.join(rel, entry.name); - acc.push(childRel); - if (entry.isDirectory()) { - await walk(childRel); - } - } - } - acc.push(root); - await walk(root); - return acc; -} - -// The denylist below is hand-curated for the workspace's built-in -// packages, NOT a substitute for `npm pack`'s respect for -// `package.json#files` / `.npmignore`. External tool packages -// published to a registry should pack via the npm-team tooling -// directly; this script only services the in-tree built-ins whose -// shape we control. -// -// DRIFT RISK: when a future built-in introduces a new artifact dir -// (e.g. `coverage/`, `.cache/`, a generated stub directory), this -// list needs to grow to keep the packed tarball clean. The long- -// term answer is a migration to `npm pack`-style traversal that -// honors `package.json#files` and `.npmignore`; until that lands, -// every new artifact dir under a built-in needs an entry here. -async function copyPackageTree(src: string, dest: string): Promise { - const entries = await fs.readdir(src, { withFileTypes: true }); - for (const entry of entries) { - if ( - entry.name === "node_modules" || - entry.name === "tsconfig.tsbuildinfo" || - entry.name === "dist" || - entry.name.endsWith(".test.ts") - ) { - continue; - } - const srcPath = path.join(src, entry.name); - const destPath = path.join(dest, entry.name); - if (entry.isDirectory()) { - await fs.mkdir(destPath, { recursive: true }); - await copyPackageTree(srcPath, destPath); - } else if (entry.isFile()) { - await fs.copyFile(srcPath, destPath); - } - } -} - -/** - * Bundle `/` and write the result under the - * staging package at `outRel`. Bundling is configured so workspace - * imports (`@intx/agent`, `@intx/types`, the catalog-resolved - * `arktype`) are inlined; the produced JS file therefore needs - * nothing from `node_modules/` at load time. Node-builtins - * (`node:fs`, `node:path`, etc.) are left external because every - * Node-compatible runtime ships them. - */ -async function bundleInterchangeEntry(args: { - absPkgDir: string; - sourceRel: string; - outRel: string; - packageStaging: string; -}): Promise { - const entryAbs = path.resolve(args.absPkgDir, args.sourceRel); - // Build-time symmetry with the sidecar loader's containment check: - // a `..`-bearing `sourceRel` slipped through some future refactor - // would otherwise let the bundler read a file outside the package - // tree. The published tarball is a redistribution boundary; refuse - // to pack a bundle the loader would later reject for the same - // reason. - const pkgDirAbs = path.resolve(args.absPkgDir); - const pkgPrefix = pkgDirAbs.endsWith(path.sep) - ? pkgDirAbs - : pkgDirAbs + path.sep; - if (entryAbs !== pkgDirAbs && !entryAbs.startsWith(pkgPrefix)) { - throw new Error( - `bundleInterchangeEntry: sourceRel ${JSON.stringify(args.sourceRel)} resolves to ${JSON.stringify(entryAbs)} which escapes the package directory ${JSON.stringify(pkgDirAbs)}`, - ); - } - await fs.access(entryAbs); - const outAbs = path.resolve(args.packageStaging, args.outRel); - await fs.mkdir(path.dirname(outAbs), { recursive: true }); - - const result = await Bun.build({ - entrypoints: [entryAbs], - outdir: path.dirname(outAbs), - naming: path.basename(outAbs), - target: "node", - format: "esm", - // Resolve the bundled workspace deps (`@intx/*`) to their TypeScript - // source via the `intx-src` exports condition; no `dist` exists during - // the workspace build. - conditions: ["intx-src"], - minify: false, - sourcemap: "none", - }); - if (!result.success) { - const messages = result.logs - .map((log) => (log instanceof Error ? log.message : String(log))) - .join("\n"); - throw new Error( - `Bun.build failed for ${args.sourceRel}:\n${messages || "(no diagnostics)"}`, - ); - } -} - -/** - * Map `./dist/.js` → `./src/.ts`. The workspace stores - * tool-package entries as `.ts` under `src/`; the packed tarball - * carries the bundled `.js` under `dist/`. Keeping `interchange.tools` - * pointed at the packed path means the build script must derive the - * source path it actually feeds to the bundler, which is what this - * helper does. - */ -function deriveSourceFromOutput(outRel: string): string { - const normalized = outRel.startsWith("./") ? outRel.slice(2) : outRel; - if (!normalized.startsWith("dist/")) { - throw new Error( - `interchange.tools is "${outRel}"; expected a "./dist/.js" path`, - ); - } - if (!normalized.endsWith(".js")) { - throw new Error( - `interchange.tools is "${outRel}"; expected a ".js" suffix on the packed output`, - ); - } - const stem = normalized.slice("dist/".length, -".js".length); - return `./src/${stem}.ts`; -} - -async function main(): Promise { - await fs.rm(OUT_DIR, { recursive: true, force: true }); - await fs.mkdir(OUT_DIR, { recursive: true }); - - // Two BUILTINS entries that pack to the same tarball filename would - // silently overwrite each other in OUT_DIR — the second pack would - // win and the first would never reach the registry. Surface the - // collision at build time rather than letting the operator chase a - // missing tarball at publish time. `tarballBaseName` already includes - // the scope, so a same-tail/different-scope pair is fine; only true - // duplicates (`-.tgz` already produced) trip this. - const seenFilenames = new Set(); - for (const spec of BUILTINS) { - const pkg = await readPackageJSON(spec.packageDir); - const expected = `${tarballBaseName(spec.name)}-${pkg.version}.tgz`; - if (seenFilenames.has(expected)) { - throw new Error( - `built-in tarball filename collision: ${expected} would be produced by more than one BUILTINS entry`, - ); - } - seenFilenames.add(expected); - // The interchange.tools presence check lives inside packBuiltin - // so the failure surfaces with the same wording from both call - // sites and there is one canonical place to update when the - // contract changes. - const entry = await packBuiltin(spec, pkg); - process.stdout.write( - ` ${entry.name}@${entry.version} → ${entry.tarballPath} (${entry.integrity})\n`, - ); - } -} - -await main(); diff --git a/bin/dev.ts b/bin/dev.ts index a909fb4eb..7e4529e2a 100755 --- a/bin/dev.ts +++ b/bin/dev.ts @@ -15,17 +15,11 @@ // bun bin/dev.ts # start hub + admin-ui // bun bin/dev.ts --seed # also seed the database after hub is ready // bun bin/dev.ts --no-admin-ui # skip the admin UI dev server -// bun bin/dev.ts --no-publish-builtins # skip the built-in tool-package publish import { $, type ProcessPromise, type ProcessOutput } from "zx"; import { readFileSync, existsSync } from "node:fs"; import { resolve } from "node:path"; -import { - PUBLISH_SEED_DEFAULTS, - publishToolPackages, -} from "./lib/publish-tool-packages"; -import { WORKSPACE_BUILTINS_REGISTRY } from "@intx/hub-sessions"; import { waitForHTTP } from "@intx/test-harness/http"; $.verbose = false; @@ -48,26 +42,12 @@ function hasStderr(err: unknown): err is { stderr: string } { const args = new Set(process.argv.slice(2)); const wantSeed = args.delete("--seed"); const skipAdminUI = args.delete("--no-admin-ui"); -const skipPublishBuiltins = args.delete("--no-publish-builtins"); if (args.size > 0) { console.error(`Unknown flags: ${[...args].join(", ")}`); process.exit(1); } -// The seed pins three built-in tool packages (`@intx/tools-*`); their -// tarballs live in the workspace-builtins package-registry asset and -// are published by the build/publish step above. Seeding without -// publishing produces agent definitions whose launch immediately -// fails with `tarball.missing` because the registry asset is empty. -// Fail loudly here rather than after a successful seed run. -if (skipPublishBuiltins && wantSeed) { - console.error( - "--no-publish-builtins is incompatible with --seed: the seed creates agent definitions that pin the workspace built-ins; without a publish the launch would fail with tarball.missing. Either drop --no-publish-builtins or drop --seed.", - ); - process.exit(1); -} - // --------------------------------------------------------------------------- // Env file loading // --------------------------------------------------------------------------- @@ -270,84 +250,6 @@ try { console.log("Hub is ready."); -// -- Step 3a: Publish built-in tool packages -- -// -// Built-ins ride the same asset-substrate path as any operator-published -// package-registry asset. Build the tarballs deterministically, then -// publish them into the workspace-builtins registry on the dev tenant. -// The publish step authenticates as the seeded admin and is idempotent: -// it sign-ups-or-signs-in, ensures the tenant exists, ensures the asset -// row exists, and PUTs every tarball under `dist/builtins/`. -// -// Runs before `--seed` so that when the seed wires up agent pins those -// pins reference tarballs that are already in the registry. The seed -// step itself remains the canonical place for agent/role/credential -// fixtures. - -if (!skipPublishBuiltins) { - console.log("Building built-in tool-package tarballs..."); - try { - const build = await $({ - cwd: ROOT, - env: { ...process.env, ...sharedEnv }, - })`bun run --conditions=intx-src bin/build-builtins.ts`; - for (const line of build.stdout.split("\n")) { - if (line) console.log(`\x1b[90m[builtins]\x1b[0m ${line}`); - } - for (const line of build.stderr.split("\n")) { - if (line) console.error(`\x1b[90m[builtins]\x1b[0m ${line}`); - } - } catch (err) { - console.error("Built-in tarball build failed:"); - if (hasStderr(err)) { - console.error(err.stderr); - } else { - console.error(err instanceof Error ? err.message : String(err)); - } - await shutdown(1); - } - - console.log("Publishing built-in tool packages to the hub..."); - // Dev orchestration owns the publish parameters: it knows the seed's - // admin identity and tenant slug, and the publish entry point itself - // is a boundary that takes a fully-populated PublishOptions. Defaults - // are resolved here, at the edge, rather than inside the publish - // function. Env overrides let an operator point dev.ts at a non-seed - // hub without editing the script. - const adminEmailEnv = process.env["HUB_ADMIN_EMAIL"]; - const adminPasswordEnv = process.env["HUB_ADMIN_PASSWORD"]; - const tenantSlugEnv = process.env["HUB_TENANT_SLUG"]; - const tenantNameEnv = process.env["HUB_TENANT_NAME"]; - try { - await publishToolPackages({ - hubURL, - adminEmail: - adminEmailEnv === undefined || adminEmailEnv === "" - ? PUBLISH_SEED_DEFAULTS.adminEmail - : adminEmailEnv, - adminPassword: - adminPasswordEnv === undefined || adminPasswordEnv === "" - ? PUBLISH_SEED_DEFAULTS.adminPassword - : adminPasswordEnv, - tenantSlug: - tenantSlugEnv === undefined || tenantSlugEnv === "" - ? PUBLISH_SEED_DEFAULTS.tenantSlug - : tenantSlugEnv, - tenantName: - tenantNameEnv === undefined || tenantNameEnv === "" - ? PUBLISH_SEED_DEFAULTS.tenantName - : tenantNameEnv, - registryName: WORKSPACE_BUILTINS_REGISTRY, - fromDir: resolve(ROOT, "dist/builtins"), - }); - } catch (err) { - console.error("Publishing built-in tool packages failed:"); - console.error(err instanceof Error ? err.message : String(err)); - await shutdown(1); - } - console.log("Built-in tool packages published."); -} - // -- Step 3: Seed (optional) -- if (wantSeed) { diff --git a/bin/gen-api-docs.ts b/bin/gen-api-docs.ts index 67d5623b8..f69c0ea6e 100755 --- a/bin/gen-api-docs.ts +++ b/bin/gen-api-docs.ts @@ -18,7 +18,7 @@ const repoRoot = resolve(import.meta.dirname ?? ".", ".."); // --------------------------------------------------------------------------- // Predicates appear in arktype schemas whenever a `.narrow` callback is -// attached (e.g. ToolPackagePinArray's duplicate-name guard). Those rules +// attached (e.g. CreateGrant's exactly-one-target guard). Those rules // are runtime invariants that have no JSON-Schema analogue, so the // generator falls back to the predicate's base shape and keeps the // OpenAPI document focused on the wire shape. Validation still runs at diff --git a/bin/lib/publish-tool-packages.ts b/bin/lib/publish-tool-packages.ts deleted file mode 100644 index b91aa1db9..000000000 --- a/bin/lib/publish-tool-packages.ts +++ /dev/null @@ -1,442 +0,0 @@ -/* eslint-disable no-console */ - -// The library half of the tool-package publisher: authenticate against the -// hub, resolve the tenant, ensure the `package-registry` asset, and PUT -// every `*.tgz` under a directory into it. -// -// Workflow (driven by `publishToolPackages`): -// 1. Authenticate against the hub (sign in as the bootstrap admin). -// 2. Resolve the target tenant by slug. -// 3. Find (or create) a `package-registry` asset named . -// 4. PUT every `*.tgz` under into the asset's tarballs/. -// -// This lives in `bin/lib` rather than in the entry point because the entry -// point `bin/publish-tool-packages.ts` shares a basename with the -// `bin/publish-tool-packages` launcher wrapper; an importer using -// `./publish-tool-packages` would resolve to the extensionless bash wrapper, -// not the `.ts`. A `bin/lib` module has no such twin, so both the CLI -// entry point and `bin/dev.ts`'s in-process publish path import the -// publisher from here. -// -// Auth pattern: the hub's REST endpoints are session-gated; this reuses the -// same sign-in flow that `bin/seed.ts` uses, since there is no pre-existing -// bearer-token convention for human-facing assets. The admin credentials -// default to the seed's `alice@example.com` / `password123`; in dev -// orchestration `bin/dev.ts` and the CLI share these defaults. Operators -// running this against a non-dev hub set `HUB_ADMIN_EMAIL` and -// `HUB_ADMIN_PASSWORD`. -// -// Publishing is idempotent: re-running uploads every tarball under -// ``, overwriting same-name entries in the registry. - -import { promises as fs } from "node:fs"; -import path from "node:path"; -import { type, type Type } from "arktype"; - -import { - AssetResponse, - AssetWithOriginResponse, - PrincipalSummary, - TenantResponse, - paginatedSchema, -} from "@intx/types"; - -// `Headers#getSetCookie` (used by mergeSetCookies below) requires Bun -// or Node >= 18. Fail loudly with a runtime-shape message so an -// operator running this on an unsupported runtime gets an actionable -// error instead of an inscrutable "headers.getSetCookie is not a -// function" deep in the auth flow. The check runs at module load so it -// fires for every importer — the CLI, bin/dev.ts, and any future one. -assertSupportedRuntime(); - -function assertSupportedRuntime(): void { - if (typeof Bun !== "undefined") return; - const versionString = process.versions.node; - const major = Number.parseInt(versionString.split(".")[0] ?? "0", 10); - if (Number.isFinite(major) && major >= 18) return; - throw new Error( - `publish-tool-packages requires Bun (preferred) or Node >= 18; this process is running Node ${versionString}`, - ); -} - -const TarballPutResponse = type({ - commit: "string", - integrity: "string", -}); - -const AuthResponse = type({ "user?": { id: "string" } }); - -export type PublishOptions = { - hubURL: string; - adminEmail: string; - adminPassword: string; - tenantSlug: string; - tenantName: string; - registryName: string; - fromDir: string; -}; - -// The dev seed's publishing identity and target tenant. Both callers — -// `bin/dev.ts`'s in-process publish and the CLI entry point — resolve these -// from the environment, defaulting to the seed, so the defaults live in one -// place and the two edges cannot authenticate as different identities. -export const PUBLISH_SEED_DEFAULTS = { - adminEmail: "alice@example.com", - adminPassword: "password123", - tenantSlug: "acme", - tenantName: "Acme Corp", -} as const; - -type CookieJar = string[]; - -type ApiResult = { - status: number; - data: unknown; - cookies: CookieJar; -}; - -function parseSchema( - schema: T, - data: unknown, - label: string, -): T["infer"] { - const result = schema(data); - if (result instanceof type.errors) { - throw new Error( - `publish-tool-packages: validation failed for ${label}: ${result.summary}`, - ); - } - return result; -} - -function mergeSetCookies(prev: CookieJar, setCookies: string[]): CookieJar { - const next = [...prev]; - for (const sc of setCookies) { - const name = sc.split("=")[0]; - if (name === undefined) continue; - const value = sc.split(";")[0]; - if (value === undefined) continue; - const idx = next.findIndex((c) => c.startsWith(`${name}=`)); - if (idx >= 0) { - next[idx] = value; - } else { - next.push(value); - } - } - return next; -} - -async function jsonApi( - hubURL: string, - method: string, - apiPath: string, - body: unknown, - cookies: CookieJar, -): Promise { - const headers: Record = { - "Content-Type": "application/json", - }; - if (cookies.length > 0) { - headers["Cookie"] = cookies.join("; "); - } - let res: Response; - try { - res = await fetch(`${hubURL}${apiPath}`, { - method, - headers, - body: body === undefined ? undefined : JSON.stringify(body), - redirect: "manual", - }); - } catch (cause) { - throw new Error( - `publish-tool-packages: network error on ${method} ${apiPath}`, - { cause }, - ); - } - const nextCookies = mergeSetCookies(cookies, res.headers.getSetCookie()); - let data: unknown = null; - const ct = res.headers.get("content-type") ?? ""; - if (ct.includes("json")) { - data = await res.json(); - } - return { status: res.status, data, cookies: nextCookies }; -} - -async function putTarball( - hubURL: string, - tenantId: string, - assetId: string, - filename: string, - bytes: Uint8Array, - cookies: CookieJar, -): Promise<{ commit: string; integrity: string }> { - const headers: Record = { - "Content-Type": "application/octet-stream", - }; - if (cookies.length > 0) { - headers["Cookie"] = cookies.join("; "); - } - const url = `${hubURL}/api/tenants/${tenantId}/assets/${assetId}/tarballs/${filename}`; - // Retry once on a 5xx response. The hub's tarball upload path is - // idempotent (the asset's tip is keyed by integrity, not by - // request id), so a single immediate retry is safe and matches the - // honest-error-surfaces pattern: transient 5xx survives, persistent - // failures still surface. No exponential backoff — this is a dev - // workflow tool, not a hot-path producer. - let res: Response | undefined; - for (let attempt = 0; attempt < 2; attempt++) { - try { - res = await fetch(url, { - method: "PUT", - headers, - body: bytes, - redirect: "manual", - }); - } catch (cause) { - throw new Error( - `publish-tool-packages: network error uploading ${filename}`, - { cause }, - ); - } - if (res.status < 500 || attempt === 1) break; - // Drain the response so the connection can be reused for the - // retry, then loop. The first-attempt body is not useful — only - // the second attempt's outcome flows downstream. - await res.text(); - } - if (res === undefined) { - throw new Error( - `publish-tool-packages: upload for ${filename} produced no response`, - ); - } - if (res.status !== 200) { - const text = await res.text(); - throw new Error( - `publish-tool-packages: upload failed for ${filename}: ${String(res.status)} ${text}`, - ); - } - const data: unknown = await res.json(); - return parseSchema(TarballPutResponse, data, `PUT ${filename}`); -} - -async function authenticate( - hubURL: string, - email: string, - password: string, -): Promise { - const signUp = await jsonApi( - hubURL, - "POST", - "/api/auth/sign-up/email", - { name: "Publish Admin", email, password }, - [], - ); - // better-auth's sign-up returns 200 on success (with the new - // session's cookies) and 422 UNPROCESSABLE_ENTITY with body - // `{ code: "USER_ALREADY_EXISTS_USE_ANOTHER_EMAIL", ... }` when the - // address is already registered. Anything else is a hub-side fault - // that must not silently fall through to sign-in; that fall-through - // would mask a real misconfiguration as an authentication failure. - if (signUp.status === 200) { - parseSchema(AuthResponse, signUp.data, "sign-up response"); - return signUp.cookies; - } - if (signUp.status !== 422) { - throw new Error( - `publish-tool-packages: unexpected sign-up response for ${email}: ${String(signUp.status)} ${JSON.stringify(signUp.data)}`, - ); - } - const signIn = await jsonApi( - hubURL, - "POST", - "/api/auth/sign-in/email", - { email, password }, - [], - ); - if (signIn.status !== 200) { - throw new Error( - `publish-tool-packages: authentication failed for ${email}: sign-up=${String(signUp.status)} ${JSON.stringify(signUp.data)} sign-in=${String(signIn.status)} ${JSON.stringify(signIn.data)}`, - ); - } - parseSchema(AuthResponse, signIn.data, "sign-in response"); - return signIn.cookies; -} - -async function ensureTenant( - hubURL: string, - cookies: CookieJar, - slug: string, - name: string, -): Promise { - const create = await jsonApi( - hubURL, - "POST", - "/api/tenants", - { name, slug }, - cookies, - ); - if (create.status === 201) { - return parseSchema(TenantResponse, create.data, "create tenant response") - .id; - } - // 409 is the documented "tenant already exists" response: fall - // through to the slug lookup. Anything else (400 validation, 403 - // permission, 5xx) is surfaced explicitly so a hub-side regression - // does not silently degrade into a misleading "tenant slug not - // visible" error from the fallback path. - if (create.status !== 409) { - throw new Error( - `publish-tool-packages: failed to create tenant ${slug}: ${String(create.status)} ${JSON.stringify(create.data)}`, - ); - } - const list = await jsonApi( - hubURL, - "GET", - "/api/me/principals", - undefined, - cookies, - ); - if (list.status !== 200) { - throw new Error( - `publish-tool-packages: failed to look up tenant by slug ${slug}: ${String(list.status)} ${JSON.stringify(list.data)}`, - ); - } - const principals = parseSchema( - paginatedSchema(PrincipalSummary), - list.data, - "me/principals response", - ); - const match = principals.data.find((p) => p.tenantSlug === slug); - if (match === undefined) { - throw new Error( - `publish-tool-packages: tenant slug ${slug} not visible to authenticated principal`, - ); - } - return match.tenantId; -} - -async function ensureRegistryAsset( - hubURL: string, - cookies: CookieJar, - tenantId: string, - registryName: string, -): Promise { - const list = await jsonApi( - hubURL, - "GET", - `/api/tenants/${tenantId}/assets?kind=package-registry&inherited=false`, - undefined, - cookies, - ); - if (list.status !== 200) { - throw new Error( - `publish-tool-packages: failed to list assets: ${String(list.status)} ${JSON.stringify(list.data)}`, - ); - } - const rows = parseSchema( - AssetWithOriginResponse.array(), - list.data, - "list assets response", - ); - const existing = rows.find((r) => r.name === registryName); - if (existing !== undefined) { - return existing.id; - } - const create = await jsonApi( - hubURL, - "POST", - `/api/tenants/${tenantId}/assets`, - { kind: "package-registry", name: registryName }, - cookies, - ); - if (create.status !== 201) { - throw new Error( - `publish-tool-packages: failed to create asset ${registryName}: ${String(create.status)} ${JSON.stringify(create.data)}`, - ); - } - return parseSchema(AssetResponse, create.data, "create asset response").id; -} - -async function listTarballs(fromDir: string): Promise { - let entries: string[]; - try { - entries = await fs.readdir(fromDir); - } catch (cause) { - throw new Error( - `publish-tool-packages: cannot read --from directory ${fromDir}`, - { cause }, - ); - } - const tarballs = entries.filter((f) => f.endsWith(".tgz")).sort(); - if (tarballs.length === 0) { - throw new Error( - `publish-tool-packages: no *.tgz files found under ${fromDir}`, - ); - } - return tarballs; -} - -/** - * Publish every tarball in `opts.fromDir` to the registry. Returns one - * record per uploaded file so callers (including `bin/dev.ts`) can log - * a summary without re-deriving the data. - */ -export async function publishToolPackages(opts: PublishOptions): Promise< - { - filename: string; - commit: string; - integrity: string; - }[] -> { - console.log(`[publish] authenticating as ${opts.adminEmail}`); - const cookies = await authenticate( - opts.hubURL, - opts.adminEmail, - opts.adminPassword, - ); - - console.log(`[publish] resolving tenant ${opts.tenantSlug}`); - const tenantId = await ensureTenant( - opts.hubURL, - cookies, - opts.tenantSlug, - opts.tenantName, - ); - console.log(`[publish] tenant ${opts.tenantSlug} -> ${tenantId}`); - - console.log( - `[publish] ensuring package-registry asset ${opts.registryName} on ${opts.tenantSlug}`, - ); - const assetId = await ensureRegistryAsset( - opts.hubURL, - cookies, - tenantId, - opts.registryName, - ); - console.log(`[publish] asset ${opts.registryName} -> ${assetId}`); - - const tarballs = await listTarballs(opts.fromDir); - console.log( - `[publish] uploading ${String(tarballs.length)} tarball(s) from ${opts.fromDir}`, - ); - - const summaries: { filename: string; commit: string; integrity: string }[] = - []; - for (const filename of tarballs) { - const abs = path.join(opts.fromDir, filename); - const bytes = await fs.readFile(abs); - const result = await putTarball( - opts.hubURL, - tenantId, - assetId, - filename, - new Uint8Array(bytes), - cookies, - ); - console.log( - `[publish] ${filename} commit=${result.commit} integrity=${result.integrity}`, - ); - summaries.push({ filename, ...result }); - } - return summaries; -} diff --git a/bin/publish-tool-packages b/bin/publish-tool-packages deleted file mode 100755 index c21f0e2e2..000000000 --- a/bin/publish-tool-packages +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -# shellcheck shell=bash -# -# Run publish-tool-packages.ts with the intx-src resolution condition so `@intx/*` imports -# resolve to TypeScript source in-repo. The dev loop builds no `dist`; -# without the condition the published `default` target (compiled `dist`) -# would be selected and fail to resolve. All arguments are forwarded. - -set -euo pipefail -exec bun --conditions=intx-src "$(dirname -- "${BASH_SOURCE[0]}")/publish-tool-packages.ts" "$@" diff --git a/bin/publish-tool-packages.ts b/bin/publish-tool-packages.ts deleted file mode 100644 index c8a8ac456..000000000 --- a/bin/publish-tool-packages.ts +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env bun -/* eslint-disable no-console */ - -// Command-line front end for the tool-package publisher. Parses flags and -// environment, then delegates to `publishToolPackages` in -// `bin/lib/publish-tool-packages.ts` (which `bin/dev.ts` also calls in its -// in-process publish path). See that module for the publish workflow and -// the auth pattern. - -import path from "node:path"; -import { parseArgs } from "node:util"; - -import { WORKSPACE_BUILTINS_REGISTRY } from "@intx/hub-sessions"; - -import { - PUBLISH_SEED_DEFAULTS, - publishToolPackages, -} from "./lib/publish-tool-packages"; - -function requireEnv(name: string): string { - const value = process.env[name]; - if (value === undefined || value === "") { - throw new Error( - `publish-tool-packages: required environment variable ${name} is not set`, - ); - } - return value; -} - -function optionalEnv(name: string, fallback: string): string { - const value = process.env[name]; - if (value === undefined || value === "") return fallback; - return value; -} - -async function runCLI(): Promise { - const { values } = parseArgs({ - options: { - registry: { type: "string", default: WORKSPACE_BUILTINS_REGISTRY }, - from: { type: "string", default: "dist/builtins" }, - tenant: { type: "string" }, - "tenant-name": { type: "string" }, - help: { type: "boolean", default: false }, - }, - strict: true, - }); - - if (values.help) { - console.log( - "Usage: bin/publish-tool-packages [--registry ] [--from ] [--tenant ] [--tenant-name ]", - ); - console.log(""); - console.log("Environment:"); - console.log(" HUB_URL hub base URL (required)"); - console.log( - " HUB_ADMIN_EMAIL admin email for sign-in (default: alice@example.com)", - ); - console.log(" HUB_ADMIN_PASSWORD admin password (default: password123)"); - console.log(" HUB_TENANT_SLUG target tenant slug (default: acme)"); - console.log(" HUB_TENANT_NAME target tenant display name"); - return; - } - - const hubURL = requireEnv("HUB_URL"); - // Warn whenever the admin credentials still match the dev seed. - // The tenant slug is a routing concern (the wrong slug fails the - // launch loudly upstream), but default admin credentials let any - // operator authenticated against a seeded hub act as the - // publishing admin — that risk does not depend on whether the - // operator picked a custom tenant. Either default credential is - // enough to fire the warning. - const usingDefaultEmail = process.env.HUB_ADMIN_EMAIL === undefined; - const usingDefaultPassword = process.env.HUB_ADMIN_PASSWORD === undefined; - if (usingDefaultEmail || usingDefaultPassword) { - console.warn( - "[publish] running with dev seed credentials — set HUB_ADMIN_EMAIL and HUB_ADMIN_PASSWORD to override", - ); - } - const adminEmail = optionalEnv( - "HUB_ADMIN_EMAIL", - PUBLISH_SEED_DEFAULTS.adminEmail, - ); - const adminPassword = optionalEnv( - "HUB_ADMIN_PASSWORD", - PUBLISH_SEED_DEFAULTS.adminPassword, - ); - const tenantSlug = - values.tenant ?? - optionalEnv("HUB_TENANT_SLUG", PUBLISH_SEED_DEFAULTS.tenantSlug); - const tenantName = - values["tenant-name"] ?? - optionalEnv("HUB_TENANT_NAME", PUBLISH_SEED_DEFAULTS.tenantName); - // parseArgs above declares defaults for `registry` and `from`, so - // `values.registry` and `values.from` are always strings here. - const registryName = values.registry; - const fromRaw = values.from; - const fromDir = path.isAbsolute(fromRaw) - ? fromRaw - : path.resolve(process.cwd(), fromRaw); - - await publishToolPackages({ - hubURL, - adminEmail, - adminPassword, - tenantSlug, - tenantName, - registryName, - fromDir, - }); -} - -if (import.meta.main) { - try { - await runCLI(); - } catch (err) { - if (err instanceof Error) { - console.error(err.message); - if (err.cause !== undefined) { - console.error(` cause: ${String(err.cause)}`); - } - } else { - console.error(String(err)); - } - process.exit(1); - } -} diff --git a/bin/seed.ts b/bin/seed.ts index 4966c926a..e36b66f88 100755 --- a/bin/seed.ts +++ b/bin/seed.ts @@ -30,10 +30,7 @@ import { import { eq } from "drizzle-orm"; import { createDB } from "@intx/db"; import { asset } from "@intx/db/schema"; -import { - WORKSPACE_BUILTINS_REGISTRY, - ensureWorkflowDefinitionForAsset, -} from "@intx/hub-sessions"; +import { ensureWorkflowDefinitionForAsset } from "@intx/hub-sessions"; import { catalogModels, catalogProviders } from "@intx/inference-catalog"; import { computeLiveDefinitionHash } from "@intx/workflow"; @@ -313,38 +310,6 @@ const widgetsTenantId = await ensureTenant( ); log(` Widgets tenant ID: ${widgetsTenantId}`); -// -- Ensure the workspace-builtins package-registry asset exists -- -// -// Agent pins below reference `@intx/tools-*` packages, which the hub's -// scope-routing config maps to the `workspace-builtins` registry. The -// asset row must exist on the Acme tenant so the session-service -// resolver can find it at launch time. `bin/publish-tool-packages.ts` -// (run from `bin/dev.ts` before seed) populates the tarballs; this -// step is a safety net for re-runs of seed against a hub that already -// has the asset row created. - -log("Ensuring workspace-builtins package-registry asset..."); - -const { status: regStatus, data: regData } = await api( - "POST", - `/api/tenants/${acmeTenantId}/assets`, - { kind: "package-registry", name: WORKSPACE_BUILTINS_REGISTRY }, - aliceCookies, -); -if (regStatus === 201 || regStatus === 409) { - log( - regStatus === 201 - ? " Created workspace-builtins asset" - : " workspace-builtins asset already exists", - ); -} else { - process.stderr.write( - `[seed] FAIL ensure workspace-builtins: expected 201 or 409, got ${String(regStatus)}\n`, - ); - process.stderr.write(`[seed] ${JSON.stringify(regData)}\n`); - process.exit(1); -} - // -- Invite Bob to Acme -- log("Inviting Bob to Acme..."); diff --git a/bin/verify-tool-load.ts b/bin/verify-tool-load.ts index 5b29c61f0..fa4903f2f 100644 --- a/bin/verify-tool-load.ts +++ b/bin/verify-tool-load.ts @@ -4,7 +4,7 @@ // Load smoke test for the published tool packages. // // The three `@intx/tools-*` packages ship a compiled `dist/sidecar-bundle.js` -// that the interchange sidecar loads as a tool package. This proves that +// that a workflow's closure imports. This proves that // bundle — and the real `@intx/*` dependency closure it imports — installs // and loads from npm-style tarballs, exactly as a consumer would get it: // @@ -18,11 +18,11 @@ // 4. Import each compiled `sidecar-bundle.js` in a plain `bun` subprocess // that does NOT carry `--conditions=intx-src`, so `@intx/*` resolves // through `default` -> `dist` (the compiled output), exactly as a -// consumer or the production sidecar loader does — never the +// consumer or the workflow child does — never the // repo-internal source condition. Assert each exposes its tool or // plugin factory. // -// The interchange tool loader runs under Bun, so the load is exercised +// The workflow child runs under Bun, so the load is exercised // under Bun to mirror the real runtime. // // This is build- and network-heavy (it emits dist and installs from npm), diff --git a/bun.lock b/bun.lock index 7de82bb66..a3a423c36 100644 --- a/bun.lock +++ b/bun.lock @@ -555,7 +555,6 @@ "name": "@intx/tool-packaging", "version": "0.5.0", "dependencies": { - "@intx/agent": "workspace:*", "@intx/log": "workspace:*", "@intx/storage-isogit": "workspace:*", "@intx/types": "workspace:*", @@ -564,13 +563,13 @@ "npm-package-arg": "^12.0.2", "npm-pick-manifest": "^10.0.0", "npm-registry-fetch": "^19.0.0", - "semver": "catalog:", + "semver": "^7.7.2", "ssri": "catalog:", "tar": "catalog:", }, "devDependencies": { "@types/npm-package-arg": "^6.1.4", - "@types/semver": "catalog:", + "@types/semver": "^7.7.1", "@types/ssri": "catalog:", }, }, @@ -622,10 +621,6 @@ "version": "0.5.0", "dependencies": { "arktype": "catalog:", - "semver": "catalog:", - }, - "devDependencies": { - "@types/semver": "catalog:", }, }, "packages/workflow": { @@ -700,7 +695,6 @@ }, }, "catalog": { - "@types/semver": "^7.7.1", "@types/ssri": "^7.1.5", "arktype": "^2.1.29", "better-auth": "^1.4.18", @@ -708,7 +702,6 @@ "hono": "^4.11.9", "isomorphic-git": "^1.44.3", "postgres": "^3.4.8", - "semver": "^7.7.2", "ssri": "^12.0.0", "tar": "^7.5.1", }, diff --git a/docs/API.md b/docs/API.md index 7e162a8b9..a6d70e061 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1472,7 +1472,7 @@ Source: packages/types/src/common.ts `{ action: string, resource: string, consumer?: string >= 1 }` Source: packages/types/src/grants.ts -**consumer**: The workflow consumer to evaluate as, such as `action:` or `tool:`. A grant scoped to a consumer by a `{ tool }` condition matches only that consumer; without it, such a grant matches nothing. +**consumer**: The workflow consumer to evaluate as, such as `action:` or `tool:`. A grant scoped to a consumer by a `{ tool }` condition matches only that consumer; without it, such a grant matches nothing. ### EvaluateResult `{ effect: "allow" | "ask" | "deny", matchingGrants: { action: string, effect: "allow" | "ask" | "deny", id: string, origin: "creator" | "invoker" | "role" | "system", resource: string, specificity?: number }[] }` diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 240b7a2f6..19695fe5d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -95,7 +95,7 @@ The model provider list is not fixed for the agent's lifetime. The control plane The agent is unaware of which model provider is serving a given inference call. **Tools** -The harness consumes a single `ToolRunner` interface; the host (e.g. the sidecar) composes that runner from one or more tool packages — `@intx/tools-mail`, `@intx/tools-posix`, `@intx/tools-lsp`, plus any operator-supplied additions — and merges them through a generic name-collision-checked merger before passing the result to the harness. Each tool package receives the host services it needs (e.g. the bound `MessageTransport`) through a typed runtime-capability resolver, so packages are not statically wired to any one host. Local tools run within the agent's runtime — file system access, code execution, network requests, or custom tools the operator composes in. Remote tools are discovered through the control plane and invoke offerings exposed by other agents or services on the Interchange network. From the agent's perspective, local and remote tools share the same interface; the harness handles protocol negotiation, request routing, and wallet-based payment transparently. All tool invocations are subject to authorization policies. +The harness consumes a single `ToolRunner` interface; `createAgent` builds that runner from the tool factories the workflow's agent lists, such as `@intx/tools-mail` and `@intx/tools-posix`, and throws `DuplicateToolError` on a name collision. Each tool package receives the host services it needs (e.g. the bound `MessageTransport`) through a typed runtime-capability resolver, so packages are not statically wired to any one host. Local tools run within the agent's runtime — file system access, code execution, network requests, or custom tools the operator composes in. Remote tools are discovered through the control plane and invoke offerings exposed by other agents or services on the Interchange network. From the agent's perspective, local and remote tools share the same interface; the harness handles protocol negotiation, request routing, and wallet-based payment transparently. All tool invocations are subject to authorization policies. **Local Data** The harness manages persistent state on behalf of the agent. Conversation context, pending operations, and audit records are stored in a git repository — but this is an implementation detail invisible to the agent. The agent interacts with the outside world exclusively through tools and messages; it has no direct access to the storage layer, the git history, or any harness metadata. Data is isolated per-agent, except between agents that share a stock sidecar. Credentials and authorization grants are managed by the harness and explicitly hidden from the agent (see Trust Boundary below). diff --git a/docs/AUTH.md b/docs/AUTH.md index 2a1d6b546..6944610e1 100644 --- a/docs/AUTH.md +++ b/docs/AUTH.md @@ -91,7 +91,7 @@ Request to /api/me/... ### Workflow run tokens -A workflow run calls the tenant routes with a run token instead of a session. A run token is an opaque bearer of the form `itx_run_`, held in the `run_token` table as a SHA-256 hash and as a copy sealed with the credential cipher. Each token belongs to one run principal and one consumer: a tool package (`tool:`) or an action handler (`action:`). +A workflow run calls the tenant routes with a run token instead of a session. A run token is an opaque bearer of the form `itx_run_`, held in the `run_token` table as a SHA-256 hash and as a copy sealed with the credential cipher. Each token belongs to one run principal and one consumer: a tool factory (`tool:`) or an action handler (`action:`). ``` Request to /api/tenants/:tenantId/... with Authorization: Bearer itx_run_... @@ -207,7 +207,7 @@ The hub-approved `approvedWireHash` is persisted on the sidecar's deployment rec A code-sourced workflow's package code is installed and **evaluated on the sidecar** — untrusted, author-controlled code runs there in an airlocked child to produce the inert projection and the capability walk. So it matters that **the sidecar is not trusted to decrypt credentials or to determine which code runs.** Those are the two things that grant a workflow real power, and both are anchored hub-side: -**Credentials are decrypted and delivered by the hub.** The secrets a consumer (a tool package or a workflow action handler) uses to reach an external system are held encrypted in the tenant vault and decrypted **hub-side** (`buildCredentialDelivery`). The sidecar cannot decrypt the vault. It keeps the deliveries it received in each run record, so a restart can restore the run, and it seals every secret there under its own credential cipher, so no credential is plaintext at rest. The hub decides _which_ credentials to deliver from its **own** content-addressed workflow asset plus the operator's approval — never from anything the sidecar supplies — and each delivered credential grant is scoped `credential:{id}/use` to a specific consumer, resolving only a **tenant-owned** credential. A consumer whose secret the hub did not send is inert. +**Credentials are decrypted and delivered by the hub.** The secrets a consumer (a tool factory or a workflow action handler) uses to reach an external system are held encrypted in the tenant vault and decrypted **hub-side** (`buildCredentialDelivery`). The sidecar cannot decrypt the vault. It keeps the deliveries it received in each run record, so a restart can restore the run, and it seals every secret there under its own credential cipher, so no credential is plaintext at rest. The hub decides _which_ credentials to deliver from its **own** content-addressed workflow asset plus the operator's approval — never from anything the sidecar supplies — and each delivered credential grant is scoped `credential:{id}/use` to a specific consumer, resolving only a **tenant-owned** credential. A consumer whose secret the hub did not send is inert. **The code that runs is the code the operator's approval was bound to.** The dependency closure is hub-resolved and SRI-pinned; at every load boundary the child re-materializes it, re-projects, and re-checks the projection's content hash against the hub-approved hash, failing closed on any mismatch (see _Definition Approval and Re-Verification_ above). A sidecar cannot substitute different code without failing this barrier. The long-lived sidecar process never evaluates the code itself: each workflow child reports the projection it verified, and the supervisor checks that report against the same hash before it uses it, because the child runs author code and could misreport it. @@ -222,7 +222,7 @@ The fourth path is different in kind from the first three and is worth stating p The one decision has two representations, and the difference is layering rather than policy. Persisted, it is a flat list of approval items — `FrozenApprovalBundle.approvedGrants`, each entry either a grant string or a requirement record — because a stored row owes compatibility to rows already written, and a flat list is append-friendly and order-independent. In memory it is `ApprovalSet`, a struct holding the grant strings in a `Set` and the requirements in a list, because the two kinds are tested by two different operations: a string by exact membership, a record only by structural comparison. `approvalSetFromItems` is the single boundary between the two forms, and it is also where each requirement is parsed, so the structural comparison at the gate can trust the records it receives. Neither form is a second approval channel; both are the same one operator decision. -Even so, a forged requirement **delegates** authority rather than fabricating it: `resolveGrantMaterialization` resolves every requirement against the creator's or the invoker's own grants and rejects `insufficient_grants` on a miss. The ceiling is whatever that party already holds — which is why a wildcard requirement is the maximal case and not a universal bypass. Note that a wildcard row is nonetheless stronger than any walk-derived row: the walk emits bare `tool:` grants, which are inert against a pinned tool's namespaced `tool::` gate, while a `*` resource matches that gate. Step scoping, not the capability ceiling, is what an unapproved wildcard would defeat. +Even so, a forged requirement **delegates** authority rather than fabricating it: `resolveGrantMaterialization` resolves every requirement against the creator's or the invoker's own grants and rejects `insufficient_grants` on a miss. The ceiling is whatever that party already holds — which is why a wildcard requirement is the maximal case and not a universal bypass. Note that a wildcard row is nonetheless stronger than any walk-derived row: the walk emits one `tool:` grant per tool the definition declares, while a `*` resource matches any tool. Step scoping, not the capability ceiling, is what an unapproved wildcard would defeat. None of the four escalate privilege beyond that delegation, because forging the advisory list yields **no new credential material** (hub-delivered; the sidecar cipher only seals what the hub delivered) and **no new code** (closure-pinned). The list's _accuracy_ still matters — it is the single artifact the operator's one decision is made against, so an under-stated list could win an approval whose true grant implications the operator did not see. That is **informed-consent** integrity, worth keeping honest, not a privilege-escalation boundary. @@ -341,7 +341,7 @@ A standing decision changes the effect of an existing grant whose resource is th The `conditions` JSONB field constrains when a grant applies. Each key names a condition evaluator in the hub's condition registry. Two evaluators are registered: - `{ "time_window": { "after": "09:00", "before": "17:00", "timezone": "America/Los_Angeles" } }` -- time-based access. `timezone` is required and is an IANA zone name or a UTC offset. `after` and `before` are `HH:MM` in 24-hour format and must differ; `after` later than `before` gives a window that crosses midnight. No other key is allowed. -- `{ "tool": "tool:" }` -- a consumer-scoped grant, which applies only when the named credential consumer makes the request. The value is a non-empty consumer identity. +- `{ "tool": "tool:" }` -- a consumer-scoped grant, which applies only when the named credential consumer makes the request. The value is a non-empty consumer identity. Spending limits and threshold-based escalation are planned conditions with no evaluator yet: diff --git a/docs/CREDENTIALS.md b/docs/CREDENTIALS.md index c6d7c57a0..53392f236 100644 --- a/docs/CREDENTIALS.md +++ b/docs/CREDENTIALS.md @@ -75,16 +75,16 @@ A credential requirement specifies: - **Source**: whose credential to use -- the tenant's organizational credential, the agent creator's personal credential, or the invoking user's personal credential - **Name**: an optional tiebreaker when multiple credentials match the same provider, source, and scope requirements -The source field maps to the architecture document's three-source model. A definition binds a consumer's declared credential handle to a concrete provider and source through a **credential binding** (`credentialBindings` on the definition). The consumer is a tool package or a workflow action handler. At launch the control plane resolves each binding to a specific tenant-owned credential and materializes a consumer-scoped `credential:{id}` / `use` grant on the agent's principal, authorized by tenant ownership within the hierarchy — the walk-up resolution already proves it, so the grant is stamped `origin = 'system'` directly and scoped to the consumer by a `{ tool }` condition, not delegated from a creator or invoker. The resolved secret is then **delivered** to the consumer over the sidecar control channel (decrypted hub-side at launch, never put in the spawn environment, and written to disk only in the sidecar's run record, sealed under the sidecar's credential cipher) and **gated at runtime by a per-consumer capability**: a tool resolves its declared handle to an origin-pinned mediated credential only if its own package holds the `credential:{id}` / `use` grant, and an action handler resolves a handle bound to `action:` only if the action step's grants hold that grant for `action:`. The scope is per handler: steps that share a handler share the handles their declarations name. The secret is read fresh on each use, so a rotation or a named revoke reaches an already-shaped handle. An action handler's capability is built for each invocation and disposed when the handler settles; an action that declares a handle with no binding fails before its handler runs. The handler reaches the capability only inside `perform`'s `run`, so every authenticated call that `run` awaits is gated by the effect grant and recorded in the ledger; the scope and its handles refuse use once `run` settles. General provider/scope requirement resolution outside the binding mechanism is not yet built. +The source field maps to the architecture document's three-source model. A definition binds a consumer's declared credential handle to a concrete provider and source through a **credential binding** (`credentialBindings` on the definition). The consumer is a tool factory or a workflow action handler. At launch the control plane resolves each binding to a specific tenant-owned credential and materializes a consumer-scoped `credential:{id}` / `use` grant on the agent's principal, authorized by tenant ownership within the hierarchy — the walk-up resolution already proves it, so the grant is stamped `origin = 'system'` directly and scoped to the consumer by a `{ tool }` condition, not delegated from a creator or invoker. The resolved secret is then **delivered** to the consumer over the sidecar control channel (decrypted hub-side at launch, never put in the spawn environment, and written to disk only in the sidecar's run record, sealed under the sidecar's credential cipher) and **gated at runtime by a per-consumer capability**: a tool resolves its declared handle to an origin-pinned mediated credential only if its own factory holds the `credential:{id}` / `use` grant, and an action handler resolves a handle bound to `action:` only if the action step's grants hold that grant for `action:`. The scope is per handler: steps that share a handler share the handles their declarations name. The secret is read fresh on each use, so a rotation or a named revoke reaches an already-shaped handle. An action handler's capability is built for each invocation and disposed when the handler settles; an action that declares a handle with no binding fails before its handler runs. The handler reaches the capability only inside `perform`'s `run`, so every authenticated call that `run` awaits is gated by the effect grant and recorded in the ledger; the scope and its handles refuse use once `run` settles. General provider/scope requirement resolution outside the binding mechanism is not yet built. A binding names exactly one consumer: -- `package`: a tool package, given as the tool factory's bundle id. The stamped `{ tool }` condition is `tool:` plus that value. +- `package`: a tool factory, given as its bundle id. The stamped `{ tool }` condition is `tool:` plus that value. - `action`: a workflow action handler, given as the handler's export name in the workflow's `interchange.actions` module. The stamped `{ tool }` condition is `action:` plus that value. -The condition key is `tool` for both kinds. The prefix of the value names the kind, so a tool package and an action handler with the same name never share a grant. +The condition key is `tool` for both kinds. The prefix of the value names the kind, so a tool factory and an action handler with the same name never share a grant. -A tool declares the handles it resolves in `defineTool({ ..., credentials: [{ handle, scopes? }, ...] })`. The declaration is part of the approved projection and its hash. The check works per tool package, which is also the consumer a binding names: the declarations of every factory with that package id form one set, and every tool of the package can resolve every handle bound to it. A package none of whose factories declares credentials is not checked: its bindings may name any handle. A pinned package declares in its `package.json` `interchange.credentials`; that list covers all of its factories. A factory in it that also declares through `defineTool` may declare only handles the list declares, with the same scopes, or the loader refuses the package. So a package whose factories use different handles lists them all in `interchange.credentials`, and each factory declares its own. +A tool declares the handles it resolves in `defineTool({ ..., credentials: [{ handle, scopes? }, ...] })`. The declaration is part of the approved projection and its hash. The check works per factory id, which is also the consumer a binding names: the declarations of every factory with that id form one set, and every tool of those factories can resolve every handle bound to it. An id none of whose factories declares credentials is not checked: its bindings may name any handle. An action declares the handles its handler may resolve: `action({ handler, credentials: [handle, ...], effect: { requires: [capability, ...] } })`. The handles resolve only inside `perform`'s `run`, so `action()` refuses `credentials` without an `effect.requires` capability. The declaration is part of the approved projection and its hash. The bindings for one handler may name only the union of the handles its action steps declare. @@ -104,7 +104,7 @@ The check runs at install and approve, not when a deployment respawns, so a depl ## Resolution at Launch Time -The control plane resolves each **credential binding** at launch by the algorithm below, materializes the resulting authorization grant, and delivers the resolved secret to the harness (step 7). The harness gates use at runtime through a per-consumer capability that enforces the materialized grant's consumer scope: each tool package and each action handler receives a capability scoped to its own consumer identity, so one consumer's grant cannot authorize another's resolve. General provider/scope requirement resolution outside the binding mechanism is not yet built. +The control plane resolves each **credential binding** at launch by the algorithm below, materializes the resulting authorization grant, and delivers the resolved secret to the harness (step 7). The harness gates use at runtime through a per-consumer capability that enforces the materialized grant's consumer scope: each tool factory and each action handler receives a capability scoped to its own consumer identity, so one consumer's grant cannot authorize another's resolve. General provider/scope requirement resolution outside the binding mechanism is not yet built. When the control plane launches an agent into a harness, it processes each credential binding: @@ -118,7 +118,7 @@ When the control plane launches an agent into a harness, it processes each crede Step 5 is the ownership boundary: a tenant's own credentials are usable by ownership, with no personal grant to hold. Principal-owned credential use — whether owner-delegated or invoker-_brought_ (an invoker supplying their own credential material for an allowed provider) — remains Planned / Not Yet Implemented. -The whole tool-credential rail is exercised end-to-end by `tests/workflow-deploy/single-step-credential-tool.test.ts`: a real credential-consuming tool package is pinned into a deployed step, its credential delivered on the deploy frame (and, in a second case, over the live rotation push), resolved through the per-consumer capability, and used to authenticate an http request to a mock origin — asserting the exact delivered secret arrives as a bearer, that a rotated secret supersedes it, and that a run lacking the `credential:{id}` / `use` grant fails the resolve closed so nothing reaches the origin. +The whole tool-credential rail is exercised end-to-end by `tests/workflow-deploy/single-step-credential-tool.test.ts`: a real credential-consuming tool runs in a deployed step, its credential delivered on the deploy frame (and, in a second case, over the live rotation push), resolved through the per-consumer capability, and used to authenticate an http request to a mock origin — asserting the exact delivered secret arrives as a bearer, that a rotated secret supersedes it, and that a run lacking the `credential:{id}` / `use` grant fails the resolve closed so nothing reaches the origin. The action-credential rail is exercised end-to-end by `tests/workflow-deploy/action-step-credential.test.ts`. A top-level action, an action inside a childWorkflow body and an action inside a top-level loop body each resolve their declared handle and authenticate to a mock origin with the exact delivered secret. The test also asserts that deploy stamps each binding with `action:`, that each body run's grants keep only the use grant for the body's own handler, that a handler no binding names cannot resolve the handle, that a run without the use grant fails closed, and that deploy refuses a declared handle with no binding. @@ -130,9 +130,9 @@ A local run (`runLocal` in `@intx/workflow`) gives action handlers credentials t ### Credential-Use Authorization -Credential use — whether by the inference layer (a tenant-catalog model key) or through a credential binding (a tool package or an action handler) — is authorized the same way: by **tenant ownership within the hierarchy**. A tenant-owned credential (`principalId IS NULL`) is usable by the tenant that owns it and by any descendant tenant, resolved through the same ancestor walk-up that credential, provider, and OAuth resolution already run on. There is no role grant, no per-principal grant, and no seeded authority: a tenant may use what it (or an ancestor) owns, and descendants inherit. The check is fail-closed — a credential that is not tenant-owned, or not reachable in the launching tenant's chain, withholds — and runs at deploy. A deployment's sources are fixed at deploy, so a later catalog change does not reach a running deployment. The deployment paths (the reconnect reconcile, the rotation push, and the deploy-start re-check) re-resolve a credential they already delivered by its id and check only its status. +Credential use — whether by the inference layer (a tenant-catalog model key) or through a credential binding (a tool factory or an action handler) — is authorized the same way: by **tenant ownership within the hierarchy**. A tenant-owned credential (`principalId IS NULL`) is usable by the tenant that owns it and by any descendant tenant, resolved through the same ancestor walk-up that credential, provider, and OAuth resolution already run on. There is no role grant, no per-principal grant, and no seeded authority: a tenant may use what it (or an ancestor) owns, and descendants inherit. The check is fail-closed — a credential that is not tenant-owned, or not reachable in the launching tenant's chain, withholds — and runs at deploy. A deployment's sources are fixed at deploy, so a later catalog change does not reach a running deployment. The deployment paths (the reconnect reconcile, the rotation push, and the deploy-start re-check) re-resolve a credential they already delivered by its id and check only its status. -One ownership rule, two transports. The inference gate (`buildSource`) and the credential-binding path resolve ownership through the same rule — a tenant-owned credential reachable in the launching tenant's ancestor chain — and neither consults a creator, role, or seeded grant. They differ only in what a resolved credential feeds. The inference gate emits the secret straight into a launchable source: its consumer is the harness itself, so no durable grant is needed. A credential binding delivers the secret to its consumer (a tool package or an action handler) and stamps a `credential:{id}` / `use` grant carrying a `{ tool }` condition, which the runtime per-consumer gate reads to decide which consumer may resolve the handle. Only that runtime gate needs a durable grant; ownership itself is settled at resolution and never re-checked. Because ownership is the authority, an agent definition needs no creator to launch a tenant-owned inference source. +One ownership rule, two transports. The inference gate (`buildSource`) and the credential-binding path resolve ownership through the same rule — a tenant-owned credential reachable in the launching tenant's ancestor chain — and neither consults a creator, role, or seeded grant. They differ only in what a resolved credential feeds. The inference gate emits the secret straight into a launchable source: its consumer is the harness itself, so no durable grant is needed. A credential binding delivers the secret to its consumer (a tool factory or an action handler) and stamps a `credential:{id}` / `use` grant carrying a `{ tool }` condition, which the runtime per-consumer gate reads to decide which consumer may resolve the handle. Only that runtime gate needs a durable grant; ownership itself is settled at resolution and never re-checked. Because ownership is the authority, an agent definition needs no creator to launch a tenant-owned inference source. A **principal-owned** credential is not a tenant resource: it is not usable through a shared catalog offering or a credential binding today, and it fails closed. Owner-delegated use of a principal-owned credential is Planned / Not Yet Implemented. diff --git a/docs/HARNESS_DESIGN.md b/docs/HARNESS_DESIGN.md index 53ae2e9a4..c645d8ddd 100644 --- a/docs/HARNESS_DESIGN.md +++ b/docs/HARNESS_DESIGN.md @@ -56,7 +56,7 @@ apps/sidecar/ │ ├── index.ts # Entry point: wires the stores, the harness builder, and the hub link │ ├── config.ts # Boundary readers for the sidecar's env-config inputs │ ├── default-harness.ts # HarnessBuilder source-admission seam (canBuildSource) the deploy router consults before spawning -│ ├── sidecar-materialization-config.ts # Shared registry-map and host-platform resolution for tool materialization +│ ├── sidecar-materialization-config.ts # Shared registry-map and host-platform resolution for closure materialization │ ├── signing-keypair.ts # Loads or mints the sidecar's on-disk Ed25519 signing keypair │ ├── workflow-child-bindings.ts # Closes the child factory over the grant cap │ ├── workflow-child-spawner.ts # Bun spawner and resolved path for bin/workflow-child diff --git a/docs/MESSAGE.md b/docs/MESSAGE.md index d2ed649e4..129d3ce77 100644 --- a/docs/MESSAGE.md +++ b/docs/MESSAGE.md @@ -668,7 +668,7 @@ If UIDVALIDITY has changed (mailbox was recreated), the transport signals a full ## Mail Tools -The agent interacts with the message transport through tools provided by the `@intx/tools-mail` package, composed into the runtime by the sidecar alongside other tool packages. These tools are what the inference layer presents to the model. They map to the transport interface operations. +The agent interacts with the message transport through tools provided by the `@intx/tools-mail` package, which a workflow agent lists in its `toolFactories` alongside other tool packages. These tools are what the inference layer presents to the model. They map to the transport interface operations. Every tool below returns `{ error: string, code: string }` on failure. The `MailToolErrorCode` union in `@intx/tools-mail` is the authoritative list of codes; the per-tool lists below name the subset each tool can produce. Two codes belong to the runner rather than to any one tool and are absent from those lists: `unknown_tool` for a call naming a tool the runner does not provide, and `internal_error` for a handler that throws instead of returning a result. `mail.wait` names `internal_error` in its own list because it also returns that code directly. diff --git a/docs/PUBLISHING.md b/docs/PUBLISHING.md index 539314d06..4e4c9f93c 100644 --- a/docs/PUBLISHING.md +++ b/docs/PUBLISHING.md @@ -130,9 +130,10 @@ unpublished `0.0.0`. ## Tool packages The `@intx/tools-*` packages double as loadable interchange tool packages: -a sidecar configured with a public-npm tool registry pulls one and its -`@intx/*` dependency closure and loads the compiled `dist/sidecar-bundle.js` -it ships. They are ordinary compiled packages — the closure resolver +a workflow package can depend on one: the Hub resolves it and its `@intx/*` +closure from a public-npm registry, the sidecar lays the frozen closure out, +and the workflow child imports the compiled `dist/sidecar-bundle.js` it ships. +They are ordinary compiled packages — the closure resolver fetches the real dependency graph from npm — with no special bundling. `make verify-tool-load` proves a tool package and its packed closure install and load with the factories intact. diff --git a/docs/WORKFLOW_AUTHORING.md b/docs/WORKFLOW_AUTHORING.md index b2e21f43b..2881387dc 100644 --- a/docs/WORKFLOW_AUTHORING.md +++ b/docs/WORKFLOW_AUTHORING.md @@ -94,10 +94,9 @@ Where it lives today: the same two loaders, plus `loadWorkflowDirectorRegistryFromClosure`, `packages/workflow-host/src/workflow-definition-loader.ts` -**4.** Five of the `interchange.*` fields name a module -- `tools`, +**4.** The five `interchange.*` fields each name a module -- `tools`, `workflow`, `directors`, `loops` and `actions` -- and all five may name the -SAME module. The sixth field, `credentials`, is a declaration array, not a -module ref. Every deployed loop fixture points `workflow`, `loops` and +SAME module. Every deployed loop fixture points `workflow`, `loops` and `actions` at one bundled entry. Every field is optional, and an omission is not caught at deployment: per entry 3, an absent `loops` or `actions` fails closed only when a ref is resolved, and an absent `directors` falls back to @@ -228,7 +227,7 @@ const invokeStep = createWorkflowStepInvoker({ const env = { ...stepEnvBase(stepId), capabilities }; attachStepTools(env, { factories: req.agent.toolFactories.map((factory) => ({ - packageName: factory.id, + factoryId: factory.id, declaredCredentials: [], factory, })), @@ -590,10 +589,10 @@ the approved projection and its hash. Deploy refuses a declared handle that no root `credentialBindings` entry names for the tool, and a binding that names a handle the tool does not declare. The reserved hub handle `interchange.hub` needs no binding; the hub binds it itself. The check works -per tool package: the factories that share a package id share one set of -declarations, and every tool of the package can resolve every handle bound -to it. A package whose factories declare nothing is not held to its -bindings. A credential failure reaches the author in one of three ways: +per factory id: the factories that share an id share one set of declarations, +and every tool they provide can resolve every handle bound to that id. An id +whose factories declare nothing is not held to its bindings. A credential +failure reaches the author in one of three ways: - At install and approve, the probe gate refuses the definition with `credential_bindings_invalid` and names each problem. diff --git a/docs/unified-execution-host-design.md b/docs/unified-execution-host-design.md index d54b84af9..c169928f6 100644 --- a/docs/unified-execution-host-design.md +++ b/docs/unified-execution-host-design.md @@ -354,7 +354,7 @@ workflows.** Where it lives, when it tears down, how recycle/drain interact: space, owned by the run-loop, not in the supervisor. - **When it's built.** Lazily on the first `trigger.fired` for the address (or eagerly at spawn for a hot path; lazy is the default to avoid paying - tool materialization before any mail arrives). + tool construction before any mail arrives). - **When it's torn down.** On `agent.undeploy` (deployment teardown), on child `shutdown`, and on recycle. A long-lived agent is _not_ torn down between messages — that is the entire point of warm-keep. @@ -557,8 +557,8 @@ constraint of Decision 1. Because tools materialize and execute _in the child_ (§3d), whatever boundary the child runs inside must contain the child's **entire tool runtime**: -- The tool-package tarballs the agent pins — or network egress sufficient to - fetch them from the configured registries at materialization time. +- The tarballs of the workflow's frozen closure — or network egress sufficient + to fetch them from the configured registries at materialization time. - The tarball cache (`createTarballCache`) and the tool instance dir, on a filesystem the child can write. - The LSP server binaries (and any other external tool binaries) the plugin @@ -813,9 +813,7 @@ oci-container`. A per-node declaration equal to or weaker than the current rung `host-subprocess` on a child node; the floor is monotonic going down a rung). - **Workflow-level defaulting.** Unchanged from Decision 2: `defineWorkflow` normalizes an absent workflow-level `isolation` to - `{ granularity: "per-tenant" }`. A single agent deploys as a single-step - workflow folded from its code source, which carries the workflow-level field - from the agent's declared policy. + `{ granularity: "per-tenant" }`. - **Validation.** `defineWorkflow`'s `normalize` validates every `isolation` occurrence — workflow-level and per-node — against the allowed `granularity` / `sandbox` sets, throwing a structured error on an unknown @@ -1161,10 +1159,8 @@ Every surviving "trivial"-named symbol is renamed to reflect that the single-step identity path is the canonical single-agent deploy, not a "trivial" special case — or deleted where the in-process branch it served is gone: -- The one-step definition is built from the folded code source rather than a - `HarnessConfig` wrapper; `buildSingleStepAgentDefinition` - (`packages/workflow-deploy/src/orchestrator.ts`) assembles the lone step's - agent definition. +- The one-step definition is what the workflow package evaluates to; no + `HarnessConfig` wrapper builds it. - `buildTrivialApprovalSet`: deleted. The single-step approval set is built inline over `ApprovalSet` from `@intx/workflow-deploy` in `packages/hub-sessions/src/session-service.ts`; it was not renamed to a @@ -1216,10 +1212,10 @@ buildEnv, agentFactory: createAgent })`. Tools empty for now. ### Phase 2 — Tool materialization in the child (RISKIEST PHASE) -- **Changes.** Move `materializeToolPackages` logic + `@intx/tool-packaging` - loader + plugin/LSP chain into the sidecar's child-side wiring, injected via - the step-invoker's `buildEnv`/`agentFactory`. Root tarball cache + tool - instance dir per step/agent. +- **Changes.** Build each step's tools from its agent's evaluated factories + (`sourceStepToolFactories`) and the plugin/LSP chain in the child-side wiring, + injected via the step-invoker's `buildEnv`/`agentFactory`. The sidecar applies + the frozen closure the child evaluates (`applyFrozenWorkflowClosure`). - **Verification.** Spawned-child integration test: an agent with a posix tool invokes it; the result lands in `{ turn }`. A second test with the LSP plugin confirms the subprocess spawns and tears down with the agent. @@ -1863,8 +1859,6 @@ are explicitly **not** a go-live gate for INTR-209. `WorkflowRunWorkflowProcessPrincipal`. - Identity: `packages/workflow-deploy/src/addresses.ts` (`deriveWorkflowRunRepoId`); - `packages/workflow-deploy/src/orchestrator.ts` - (`buildSingleStepAgentDefinition`); `packages/types/src/agent-address.ts` (`formatRunAddress`, `parseRunAddress`, `isRunAddress`); `packages/db/src/sidecar-allocation-store.ts` (initialization key record); @@ -1872,6 +1866,6 @@ are explicitly **not** a go-live gate for INTR-209. (`resolveRoutableAddress`); sidecar reconnect identity and hosted bindings in `packages/hub-sessions/src/ws/sidecar-token-authenticator.ts`. - Transport interface: `packages/types/src/runtime-core.ts` (`MessageTransport`). -- Tool packaging: `@intx/tool-packaging` (`createToolLoader`, `applyAtomic`, - `createTarballCache`). +- Tool packaging: `@intx/tool-packaging` (`applyFrozenWorkflowClosure`, + `materializeClosure`, `createTarballCache`). - INTR-209 fixture: `tests/workflow-deploy/multistep-signal.test.ts`. diff --git a/package.json b/package.json index 8e419edfb..4445f045e 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,6 @@ "tests/lib" ], "catalog": { - "@types/semver": "^7.7.1", "@types/ssri": "^7.1.5", "arktype": "^2.1.29", "better-auth": "^1.4.18", @@ -19,7 +18,6 @@ "hono": "^4.11.9", "isomorphic-git": "^1.44.3", "postgres": "^3.4.8", - "semver": "^7.7.2", "ssri": "^12.0.0", "tar": "^7.5.1" }, diff --git a/packages/agent/CONVENTIONS.md b/packages/agent/CONVENTIONS.md index 082f2b430..52f1d3adc 100644 --- a/packages/agent/CONVENTIONS.md +++ b/packages/agent/CONVENTIONS.md @@ -6,26 +6,9 @@ consumes anything that fits these shapes; this file fixes the shapes. ## Tool-package convention -A package that contributes tools to an agent declares its tool entry in -`package.json` via the `interchange.tools` field: - -```json -{ - "name": "@vendor/my-tools", - "version": "1.2.3", - "interchange": { - "tools": "./dist/tools.js" - } -} -``` - -The path resolves against the package root. The module it names is the -**tool entry module**. - -### Tool entry module shape - -The tool entry module's **named exports** are `AnnotatedToolFactory` -values, built via `defineTool({ id, requires?, definitions, factory })`: +A package that contributes tools exports `AnnotatedToolFactory` values, +built via `defineTool({ id, requires?, definitions, factory })`. A workflow +definition imports them and lists them in an agent's `toolFactories`: ```ts import { defineTool } from "@intx/agent"; @@ -58,10 +41,6 @@ A package may export one or many factories. Each factory's `id` must be package-namespaced (`@vendor/pkg/name` or `pkg/name`); `defineTool` enforces this via `validateNamespacedId`. -The default export is not consumed. Any non-`AnnotatedToolFactory` -named export is ignored by the loader. The loader rejects an entry -module that has no `AnnotatedToolFactory` named exports. - ### Per-instance isolation A factory is invoked **once per agent instance**, with an env scoped to @@ -79,31 +58,10 @@ This means two agent instances on one sidecar can load the same the module graph itself is shared. The factory is the isolation boundary; the module is not. -### Tool-name namespacing - -The loader synthesizes the model-facing tool name as -`:`. Package authors write bare tool names inside -the `ToolDefinition` (e.g. `read_file`); the loader prefixes with the -bundle's `id`. +### Tool names -Grants in the existing authz system match the matching shape: -`tool:/`. The grant evaluator does not need to -know about packages. - -The model-facing form uses `:` and the grant form uses `/` on -purpose: the model never sees the `tool:` resource prefix the grant -evaluator works with, so reusing `:` in both would either force the -model-facing form to also carry the `tool:` prefix (verbose, leaks -authz internals) or have the grant form drop its `tool:` discriminator -(loses the prefix that lets the evaluator route resources by kind). -The two-character split keeps each form unambiguous within its own -layer. - -### Audit provenance - -Every tool invocation is tagged with the providing bundle's `id`. The -package does not need to participate; the loader records provenance -against the bundle it dispatched to. +A tool's model-facing name is the bare name in its `ToolDefinition` +(e.g. `read_file`), and grants match it as `tool:`. ## Env requirements @@ -112,8 +70,7 @@ reads at construction time, beyond `BaseEnv`'s core fields. The agent runtime's `validateEnv` (in `@intx/agent`'s `env-validation.ts`) asserts every declared key is present on the env before the factory is invoked. A missing key raises `AgentEnvError` at agent-construction -time, which the loader's atomic-apply layer surfaces as a -`factory.construct.failed` deploy-apply error. +time. The keys may name either: @@ -133,10 +90,23 @@ value. Some tool packages contribute extra capabilities to other tool packages (e.g. LSP's diagnostics middleware decorates posix's edit tools) -rather than producing a self-contained `ToolBundle`. These export -`AnnotatedPluginFactory` values, built via `definePlugin`. The loader -collects them and surfaces their results to tool factories via -`env.plugins` before any tool factory is invoked. +rather than producing a self-contained `ToolBundle`. These declare an +entry in `package.json` via the `interchange.tools` field, and its named +exports are `AnnotatedPluginFactory` values, built via `definePlugin`: + +```json +{ + "name": "@vendor/my-plugin", + "version": "1.2.3", + "interchange": { + "tools": "./dist/tools.js" + } +} +``` + +The workflow child imports the entry of each package an agent lists in +`plugins` and surfaces the results to tool factories via `env.plugins` +before any tool factory is invoked. Tool packages that consume plugins read `env.plugins` and filter by shape — the agent runtime does not interpret the plugin's return @@ -150,9 +120,9 @@ plugin factories as opaque: it accepts whatever the host hands to via `env.plugins`. Hosts MAY invoke plugin factories one at a time and re-feed each factory the prior results on `env.plugins`, producing a chain in which each successive plugin sees its -predecessors. The default sidecar harness in this repository does -exactly that (`apps/sidecar/src/default-harness.ts`), so packages -authored against the in-tree sidecar can assume the chained shape. +predecessors. The workflow child in this repository does exactly that +(`packages/workflow-host/src/child/step-tools.ts`), so packages authored +against the in-tree sidecar can assume the chained shape. Package authors that intend to consume sibling plugin results must inspect `env.plugins` at construction and fail loudly with a clear @@ -167,6 +137,5 @@ this convention. ## Versioning Changes to the entry-module shape (what `interchange.tools` may -export) are breaking. A tool package's declared `interchange.tools` -module must remain compatible with the loader version it ships -against. The loader version is the `@intx/agent` major it depends on. +export) are breaking. A package's declared `interchange.tools` module +must remain compatible with the `@intx/agent` major it depends on. diff --git a/packages/agent/src/definition.ts b/packages/agent/src/definition.ts index 0db3239b5..88d232ec1 100644 --- a/packages/agent/src/definition.ts +++ b/packages/agent/src/definition.ts @@ -12,8 +12,6 @@ // the authorize callback, the audit sink, the directors registry) // lives in the env supplied at `createAgent` time. -import type { ToolPackagePin } from "@intx/types/tool-packages"; - import type { AnnotatedToolFactory } from "./tool"; import type { BaseEnv } from "./env"; import type { DirectorRef } from "./director-types"; @@ -87,11 +85,6 @@ export interface AgentDefinition { * encoded JSON in a tag value. */ readonly tags?: Readonly>; - /** - * Tool-package pins declared for this agent, carried on the definition so a - * folded workflow asset is self-contained. - */ - readonly toolPackagePins?: readonly ToolPackagePin[]; } // Type-level helper for computing the intersection of env requirements diff --git a/packages/agent/src/director.ts b/packages/agent/src/director.ts index fcd448d10..f44ce104b 100644 --- a/packages/agent/src/director.ts +++ b/packages/agent/src/director.ts @@ -148,11 +148,9 @@ export function validateDirectorConfig( * (which carry only `id` and `requires`); without it, any tool-factory * export from a directors-entry module would be accepted as a director. * - * Shared by the tool-package loader (`@intx/tool-packaging`) and the - * workflow-closure director loader (`@intx/workflow-host`) so both accept - * and reject exactly the same shapes -- one accept/reject rule the - * approval-time probe and the runtime cannot drift apart on. Two copies of - * "is this a valid director" would be a silent congruence hole. + * The workflow-closure director loader (`@intx/workflow-host`) uses this + * for both the approval-time probe and the runtime, so the two cannot drift + * apart on what a valid director is. */ export function isAnnotatedDirectorFactory( value: unknown, diff --git a/packages/agent/src/namespace.ts b/packages/agent/src/namespace.ts index ad3e1b751..83081deb4 100644 --- a/packages/agent/src/namespace.ts +++ b/packages/agent/src/namespace.ts @@ -46,9 +46,8 @@ export function validateNamespacedId(id: string): void { /** * Whether a director id sits under `packageName`'s own namespace - * (`/`). The workflow-host closure loader and the - * tool-packaging loader share this predicate; each site keeps its own - * throw when it is false. + * (`/`). The workflow-host closure loader throws + * when it is false. */ export function isOwnedDirectorId(id: string, packageName: string): boolean { return id.startsWith(`${packageName}/`); diff --git a/packages/agent/src/tool.ts b/packages/agent/src/tool.ts index f523dc2f4..0c06d9adf 100644 --- a/packages/agent/src/tool.ts +++ b/packages/agent/src/tool.ts @@ -257,7 +257,7 @@ export type AnnotatedToolFactory = * contributes so callers (e.g. the deploy-time capability walk) can * enumerate them without instantiating the factory. * - `credentials` declares the credential handles the factory's tools - * resolve. Handles follow the package-manifest grammar and are unique; + * resolve. Handles follow the `ToolCredentialHandle` grammar and are unique; * an invalid list throws. Deploy refuses a declared handle that no * binding names for this factory, except the reserved hub handle * `interchange.hub`, which deploy binds itself. @@ -390,8 +390,7 @@ export function isToolPluginInstance( * own shape. * * `id` must be package-namespaced — same rule as `defineTool` — so - * audit provenance threads through plugins as cleanly as through - * tools. + * independently authored plugins cannot collide on an id. */ export function definePlugin< Result extends object, diff --git a/packages/authz/src/tool-consumer.test.ts b/packages/authz/src/tool-consumer.test.ts index 53773ee8f..d88d1fb99 100644 --- a/packages/authz/src/tool-consumer.test.ts +++ b/packages/authz/src/tool-consumer.test.ts @@ -89,7 +89,7 @@ describe("toolConsumerEvaluator", () => { describe("consumer identities", () => { const base = { handle: "gh", provider: "github", locator: "tenant" } as const; - test("a tool binding names its package's tool consumer", () => { + test("a tool binding names its factory's tool consumer", () => { expect( credentialBindingConsumer({ ...base, package: "@intx/tools-github" }), ).toBe("tool:@intx/tools-github"); @@ -101,7 +101,7 @@ describe("consumer identities", () => { ); }); - test("a tool package and an action handler with one name stay disjoint", () => { + test("a tool factory and an action handler with one name stay disjoint", () => { expect(actionConsumer("deploy")).not.toBe(toolConsumer("deploy")); }); diff --git a/packages/authz/src/tool-consumer.ts b/packages/authz/src/tool-consumer.ts index 13f0f2b59..384685fe7 100644 --- a/packages/authz/src/tool-consumer.ts +++ b/packages/authz/src/tool-consumer.ts @@ -3,7 +3,7 @@ // Restricts a grant to a specific credential consumer. The grant's // conditions object should contain: // -// { tool: "tool:" } // a tool package +// { tool: "tool:" } // a tool factory // { tool: "action:" } // a workflow action handler // { tool: "" } // a reserved built-in consumer identity // @@ -26,22 +26,22 @@ import { createConditionEvaluator } from "./condition-evaluator"; import type { ConditionRegistry } from "./types"; /** - * Build the consumer identity for a tool package. This is the value stamped + * Build the consumer identity for a tool factory. This is the value stamped * onto a `credential:{id}` / `use` grant's `{ tool }` condition at launch, and * the identical string the runtime gate supplies as `ctx.consumer` when a tool - * in that package uses the credential. Launch and gate MUST derive a consumer + * in that factory uses the credential. Launch and gate MUST derive a consumer * identity the same way, through the builders in this module, rather than - * inline the format. The granularity is package-level: every tool in a package - * shares one consumer identity. + * inline the format. The granularity is the factory: every tool in one + * `defineTool` bundle shares one consumer identity. */ -export function toolConsumer(packageName: string): string { - return `tool:${packageName}`; +export function toolConsumer(factoryId: string): string { + return `tool:${factoryId}`; } /** * Build the consumer identity for a workflow action handler, keyed by the * handler's export name. The `action:` prefix keeps it disjoint from every - * tool consumer, so a tool package and an action handler with the same name + * tool consumer, so a tool factory and an action handler with the same name * never share a grant. Every action step that runs the handler shares one * consumer identity. */ @@ -50,7 +50,7 @@ export function actionConsumer(handler: string): string { } /** - * The consumer identity a credential binding names: its tool package or its + * The consumer identity a credential binding names: its tool factory or its * action handler. */ export function credentialBindingConsumer(binding: CredentialBinding): string { diff --git a/packages/authz/src/tool-namespace-invariants.test.ts b/packages/authz/src/tool-namespace-invariants.test.ts deleted file mode 100644 index fbec9e4e2..000000000 --- a/packages/authz/src/tool-namespace-invariants.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -// How a wildcard grant row and a bare `tool:` row each fare against a -// PINNED tool's runtime gate. -// -// A pinned tool package is loaded through the namespacing loader, which -// rewrites every definition name to `:`. The workflow child -// gates each call on `tool:`, so the query it issues for a pinned -// tool is `tool::`. The deploy-time capability walk, by -// contrast, reads inline `agent.toolFactories`, which carry no bundle context, -// so the rows it contributes are bare `tool:`. -// -// That asymmetry is what makes the namespaced form a step-scoping mechanism: -// a bare row cannot address a pinned tool's gate. These cases pin what a -// wildcard row does to that scoping. - -import { describe, test, expect } from "bun:test"; - -import { evaluateGrants } from "./evaluate"; -import type { GrantRule } from "./types"; - -const BUNDLE_ID = "tp_pinned_bundle"; -const TOOL_NAME = "search"; -const NAMESPACED_GATE = `tool:${BUNDLE_ID}:${TOOL_NAME}`; -const BARE_GATE = `tool:${TOOL_NAME}`; -const INVOKE = "invoke"; - -function grant( - overrides: Partial & - Pick, -): GrantRule { - return { - id: `grt_${Math.random().toString(36).slice(2, 10)}`, - origin: "creator", - conditions: null, - expiresAt: null, - roleId: null, - principalId: null, - ...overrides, - }; -} - -describe("a wildcard row reaches a pinned tool's namespaced gate", () => { - test("*/* resolves to allow for tool::/invoke", async () => { - const result = await evaluateGrants( - [grant({ resource: "*", action: "*", effect: "allow" })], - NAMESPACED_GATE, - INVOKE, - { conditions: "pessimistic" }, - ); - expect(result.effect).toBe("allow"); - }); - - test("the walk's bare tool: row does NOT reach that gate", async () => { - const result = await evaluateGrants( - [grant({ resource: BARE_GATE, action: INVOKE, effect: "allow" })], - NAMESPACED_GATE, - INVOKE, - { conditions: "pessimistic" }, - ); - expect(result.effect).toBe(null); - }); - - test("the bare row is operative against a bare, unpinned gate", async () => { - const result = await evaluateGrants( - [grant({ resource: BARE_GATE, action: INVOKE, effect: "allow" })], - BARE_GATE, - INVOKE, - { conditions: "pessimistic" }, - ); - expect(result.effect).toBe("allow"); - }); - - test("a wildcard row reaches a second bundle's tool the step never loaded", async () => { - const result = await evaluateGrants( - [grant({ resource: "*", action: "*", effect: "allow" })], - "tool:tp_other_bundle:transfer", - INVOKE, - { conditions: "pessimistic" }, - ); - expect(result.effect).toBe("allow"); - }); -}); diff --git a/packages/db/migrations/0123_drop_launch_spec_tool_package_pins.sql b/packages/db/migrations/0123_drop_launch_spec_tool_package_pins.sql new file mode 100644 index 000000000..19c784502 --- /dev/null +++ b/packages/db/migrations/0123_drop_launch_spec_tool_package_pins.sql @@ -0,0 +1 @@ +ALTER TABLE "workflow_run_launch_spec" DROP COLUMN "tool_package_pins"; \ No newline at end of file diff --git a/packages/db/migrations/meta/0123_snapshot.json b/packages/db/migrations/meta/0123_snapshot.json new file mode 100644 index 000000000..196d3bc19 --- /dev/null +++ b/packages/db/migrations/meta/0123_snapshot.json @@ -0,0 +1,4995 @@ +{ + "id": "96142fd0-7059-4b59-8b92-efac58488312", + "prevId": "6be781d3-bd69-4af1-bdb6-620b6abf9fce", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.federation_trust": { + "name": "federation_trust", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_tenant_id": { + "name": "target_tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "direction": { + "name": "direction", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "federation_trust_tenant_id_tenant_id_fk": { + "name": "federation_trust_tenant_id_tenant_id_fk", + "tableFrom": "federation_trust", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "federation_trust_target_tenant_id_tenant_id_fk": { + "name": "federation_trust_target_tenant_id_tenant_id_fk", + "tableFrom": "federation_trust", + "tableTo": "tenant", + "columnsFrom": ["target_tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "federation_trust_tenant_id_target_tenant_id_unique": { + "name": "federation_trust_tenant_id_target_tenant_id_unique", + "nullsNotDistinct": false, + "columns": ["tenant_id", "target_tenant_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tenant": { + "name": "tenant", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_id": { + "name": "parent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "tenant_domain_lower_idx": { + "name": "tenant_domain_lower_idx", + "columns": [ + { + "expression": "lower(\"domain\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tenant_parent_id_tenant_id_fk": { + "name": "tenant_parent_id_tenant_id_fk", + "tableFrom": "tenant", + "tableTo": "tenant", + "columnsFrom": ["parent_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "tenant_slug_unique": { + "name": "tenant_slug_unique", + "nullsNotDistinct": false, + "columns": ["slug"] + } + }, + "policies": {}, + "checkConstraints": { + "tenant_slug_dns_label_check": { + "name": "tenant_slug_dns_label_check", + "value": "\"tenant\".\"slug\" ~ '^[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?$' and length(\"tenant\".\"slug\") <= 63" + } + }, + "isRLSEnabled": false + }, + "public.principal": { + "name": "principal", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref_id": { + "name": "ref_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "principal_tenant_id_tenant_id_fk": { + "name": "principal_tenant_id_tenant_id_fk", + "tableFrom": "principal", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "principal_tenant_id_kind_ref_id_unique": { + "name": "principal_tenant_id_kind_ref_id_unique", + "nullsNotDistinct": false, + "columns": ["tenant_id", "kind", "ref_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.principal_key": { + "name": "principal_key", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key": { + "name": "private_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "principal_key_one_active": { + "name": "principal_key_one_active", + "columns": [ + { + "expression": "principal_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"principal_key\".\"status\" = 'active'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "principal_key_principal_id_principal_id_fk": { + "name": "principal_key_principal_id_principal_id_fk", + "tableFrom": "principal_key", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "principal_key_public_key_unique": { + "name": "principal_key_public_key_unique", + "nullsNotDistinct": false, + "columns": ["public_key"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_role": { + "name": "agent_role", + "schema": "", + "columns": { + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_id": { + "name": "role_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agent_role_agent_id_workflow_definition_id_fk": { + "name": "agent_role_agent_id_workflow_definition_id_fk", + "tableFrom": "agent_role", + "tableTo": "workflow_definition", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_role_role_id_role_id_fk": { + "name": "agent_role_role_id_role_id_fk", + "tableFrom": "agent_role", + "tableTo": "role", + "columnsFrom": ["role_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "agent_role_agent_id_role_id_pk": { + "name": "agent_role_agent_id_role_id_pk", + "columns": ["agent_id", "role_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.principal_role": { + "name": "principal_role", + "schema": "", + "columns": { + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_id": { + "name": "role_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "principal_role_principal_id_principal_id_fk": { + "name": "principal_role_principal_id_principal_id_fk", + "tableFrom": "principal_role", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "principal_role_role_id_role_id_fk": { + "name": "principal_role_role_id_role_id_fk", + "tableFrom": "principal_role", + "tableTo": "role", + "columnsFrom": ["role_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "principal_role_principal_id_role_id_pk": { + "name": "principal_role_principal_id_role_id_pk", + "columns": ["principal_id", "role_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.role": { + "name": "role", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_system": { + "name": "is_system", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "role_tenant_id_tenant_id_fk": { + "name": "role_tenant_id_tenant_id_fk", + "tableFrom": "role", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.grant": { + "name": "grant", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_id": { + "name": "role_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "effect": { + "name": "effect", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "conditions": { + "name": "conditions", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "grant_invoker_allow_principal_idx": { + "name": "grant_invoker_allow_principal_idx", + "columns": [ + { + "expression": "principal_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"grant\".\"origin\" = 'invoker' and \"grant\".\"effect\" = 'allow'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "grant_tenant_id_tenant_id_fk": { + "name": "grant_tenant_id_tenant_id_fk", + "tableFrom": "grant", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "grant_role_id_role_id_fk": { + "name": "grant_role_id_role_id_fk", + "tableFrom": "grant", + "tableTo": "role", + "columnsFrom": ["role_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "grant_principal_id_principal_id_fk": { + "name": "grant_principal_id_principal_id_fk", + "tableFrom": "grant", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "grant_target_exactly_one": { + "name": "grant_target_exactly_one", + "value": "num_nonnulls(\"grant\".\"principal_id\", \"grant\".\"role_id\") = 1" + } + }, + "isRLSEnabled": false + }, + "public.approval": { + "name": "approval", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "anchor_run_id": { + "name": "anchor_run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_address": { + "name": "agent_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "correlation_id": { + "name": "correlation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_definition": { + "name": "tool_definition", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "tool_arguments": { + "name": "tool_arguments", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "timeout_at": { + "name": "timeout_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "approval_tenant_status_idx": { + "name": "approval_tenant_status_idx", + "columns": [ + { + "expression": "tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "approval_anchor_run_idx": { + "name": "approval_anchor_run_idx", + "columns": [ + { + "expression": "anchor_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "approval_run_idx": { + "name": "approval_run_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "approval_tenant_id_tenant_id_fk": { + "name": "approval_tenant_id_tenant_id_fk", + "tableFrom": "approval", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "approval_anchor_run_id_workflow_run_id_fk": { + "name": "approval_anchor_run_id_workflow_run_id_fk", + "tableFrom": "approval", + "tableTo": "workflow_run", + "columnsFrom": ["anchor_run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "approval_run_id_workflow_run_id_fk": { + "name": "approval_run_id_workflow_run_id_fk", + "tableFrom": "approval", + "tableTo": "workflow_run", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "approval_correlation_id_unique": { + "name": "approval_correlation_id_unique", + "nullsNotDistinct": false, + "columns": ["correlation_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.signal_correlation": { + "name": "signal_correlation", + "schema": "", + "columns": { + "correlation_id": { + "name": "correlation_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "anchor_run_id": { + "name": "anchor_run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_address": { + "name": "agent_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "signal_name": { + "name": "signal_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "signal_id": { + "name": "signal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "signal_correlation_tenant_id_tenant_id_fk": { + "name": "signal_correlation_tenant_id_tenant_id_fk", + "tableFrom": "signal_correlation", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "signal_correlation_anchor_run_id_workflow_run_id_fk": { + "name": "signal_correlation_anchor_run_id_workflow_run_id_fk", + "tableFrom": "signal_correlation", + "tableTo": "workflow_run", + "columnsFrom": ["anchor_run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "signal_correlation_run_id_workflow_run_id_fk": { + "name": "signal_correlation_run_id_workflow_run_id_fk", + "tableFrom": "signal_correlation", + "tableTo": "workflow_run", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider": { + "name": "provider", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "plugin": { + "name": "plugin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "api_base_url": { + "name": "api_base_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_url": { + "name": "authorization_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_url": { + "name": "token_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_info_url": { + "name": "user_info_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "provider_tenant_id_tenant_id_fk": { + "name": "provider_tenant_id_tenant_id_fk", + "tableFrom": "provider", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "provider_tenant_name": { + "name": "provider_tenant_name", + "nullsNotDistinct": false, + "columns": ["tenant_id", "name"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret": { + "name": "client_secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_uris": { + "name": "redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "default_scopes": { + "name": "default_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "oauth_client_tenant_id_tenant_id_fk": { + "name": "oauth_client_tenant_id_tenant_id_fk", + "tableFrom": "oauth_client", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_client_provider_id_provider_id_fk": { + "name": "oauth_client_provider_id_provider_id_fk", + "tableFrom": "oauth_client", + "tableTo": "provider", + "columnsFrom": ["provider_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_client_tenant_provider": { + "name": "oauth_client_tenant_provider", + "nullsNotDistinct": false, + "columns": ["tenant_id", "provider_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.credential": { + "name": "credential", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oauth_client_id": { + "name": "oauth_client_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_version": { + "name": "secret_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "refresh_secret": { + "name": "refresh_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "credential_tenant_id_tenant_id_fk": { + "name": "credential_tenant_id_tenant_id_fk", + "tableFrom": "credential", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_principal_id_principal_id_fk": { + "name": "credential_principal_id_principal_id_fk", + "tableFrom": "credential", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "credential_provider_id_provider_id_fk": { + "name": "credential_provider_id_provider_id_fk", + "tableFrom": "credential", + "tableTo": "provider", + "columnsFrom": ["provider_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_oauth_client_id_oauth_client_id_fk": { + "name": "credential_oauth_client_id_oauth_client_id_fk", + "tableFrom": "credential", + "tableTo": "oauth_client", + "columnsFrom": ["oauth_client_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "credential_tenant_name": { + "name": "credential_tenant_name", + "nullsNotDistinct": false, + "columns": ["tenant_id", "name"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.asset": { + "name": "asset", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "creator_principal_id": { + "name": "creator_principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "asset_tenant_id_tenant_id_fk": { + "name": "asset_tenant_id_tenant_id_fk", + "tableFrom": "asset", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "asset_creator_principal_id_principal_id_fk": { + "name": "asset_creator_principal_id_principal_id_fk", + "tableFrom": "asset", + "tableTo": "principal", + "columnsFrom": ["creator_principal_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "asset_tenant_kind_name": { + "name": "asset_tenant_kind_name", + "nullsNotDistinct": false, + "columns": ["tenant_id", "kind", "name"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.transaction": { + "name": "transaction", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "wallet_id": { + "name": "wallet_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "direction": { + "name": "direction", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "currency": { + "name": "currency", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_id": { + "name": "recipient_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sender_id": { + "name": "sender_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "request_id": { + "name": "request_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "transaction_wallet_id_wallet_id_fk": { + "name": "transaction_wallet_id_wallet_id_fk", + "tableFrom": "transaction", + "tableTo": "wallet", + "columnsFrom": ["wallet_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "transaction_run_id_workflow_run_id_fk": { + "name": "transaction_run_id_workflow_run_id_fk", + "tableFrom": "transaction", + "tableTo": "workflow_run", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.wallet": { + "name": "wallet", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backend_type": { + "name": "backend_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "currency": { + "name": "currency", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "wallet_tenant_id_tenant_id_fk": { + "name": "wallet_tenant_id_tenant_id_fk", + "tableFrom": "wallet", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.offering": { + "name": "offering", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pricing": { + "name": "pricing", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "schema": { + "name": "schema", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "offering_agent_id_workflow_definition_id_fk": { + "name": "offering_agent_id_workflow_definition_id_fk", + "tableFrom": "offering", + "tableTo": "workflow_definition", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "offering_tenant_id_tenant_id_fk": { + "name": "offering_tenant_id_tenant_id_fk", + "tableFrom": "offering", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model": { + "name": "model", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "canonical_name": { + "name": "canonical_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "model_tenant_id_tenant_id_fk": { + "name": "model_tenant_id_tenant_id_fk", + "tableFrom": "model", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "model_tenant_canonical_name": { + "name": "model_tenant_canonical_name", + "nullsNotDistinct": false, + "columns": ["tenant_id", "canonical_name"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model_offering": { + "name": "model_offering", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "deployment_tags": { + "name": "deployment_tags", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "capabilities": { + "name": "capabilities", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "quirks": { + "name": "quirks", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "model_offering_tenant_id_tenant_id_fk": { + "name": "model_offering_tenant_id_tenant_id_fk", + "tableFrom": "model_offering", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "model_offering_model_id_model_id_fk": { + "name": "model_offering_model_id_model_id_fk", + "tableFrom": "model_offering", + "tableTo": "model", + "columnsFrom": ["model_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "model_offering_provider_id_model_provider_id_fk": { + "name": "model_offering_provider_id_model_provider_id_fk", + "tableFrom": "model_offering", + "tableTo": "model_provider", + "columnsFrom": ["provider_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "model_offering_tenant_model_provider": { + "name": "model_offering_tenant_model_provider", + "nullsNotDistinct": false, + "columns": ["tenant_id", "model_id", "provider_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model_pricing": { + "name": "model_pricing", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "offering_id": { + "name": "offering_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "currency": { + "name": "currency", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "input_token_price": { + "name": "input_token_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "output_token_price": { + "name": "output_token_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cache_read_token_price": { + "name": "cache_read_token_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cache_write_token_price": { + "name": "cache_write_token_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "thinking_token_price": { + "name": "thinking_token_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "per_request_fee": { + "name": "per_request_fee", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "per_image_fee": { + "name": "per_image_fee", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "per_audio_fee": { + "name": "per_audio_fee", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "effective_from": { + "name": "effective_from", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "model_pricing_tenant_id_tenant_id_fk": { + "name": "model_pricing_tenant_id_tenant_id_fk", + "tableFrom": "model_pricing", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "model_pricing_offering_id_model_offering_id_fk": { + "name": "model_pricing_offering_id_model_offering_id_fk", + "tableFrom": "model_pricing", + "tableTo": "model_offering", + "columnsFrom": ["offering_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "model_pricing_offering_currency_effective_from": { + "name": "model_pricing_offering_currency_effective_from", + "nullsNotDistinct": false, + "columns": ["offering_id", "currency", "effective_from"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model_provider": { + "name": "model_provider", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "plugin": { + "name": "plugin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wallet_id": { + "name": "wallet_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "model_provider_tenant_id_tenant_id_fk": { + "name": "model_provider_tenant_id_tenant_id_fk", + "tableFrom": "model_provider", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "model_provider_credential_id_credential_id_fk": { + "name": "model_provider_credential_id_credential_id_fk", + "tableFrom": "model_provider", + "tableTo": "credential", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "model_provider_wallet_id_wallet_id_fk": { + "name": "model_provider_wallet_id_wallet_id_fk", + "tableFrom": "model_provider", + "tableTo": "wallet", + "columnsFrom": ["wallet_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "model_provider_tenant_name": { + "name": "model_provider_tenant_name", + "nullsNotDistinct": false, + "columns": ["tenant_id", "name"] + } + }, + "policies": {}, + "checkConstraints": { + "model_provider_auth_xor": { + "name": "model_provider_auth_xor", + "value": "(\"model_provider\".\"credential_id\" is not null) <> (\"model_provider\".\"wallet_id\" is not null)" + } + }, + "isRLSEnabled": false + }, + "public.sidecar": { + "name": "sidecar", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_hash_sha256": { + "name": "token_hash_sha256", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "provisioner_id": { + "name": "provisioner_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provisioner_api_version": { + "name": "provisioner_api_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "provisioner_binding_fingerprint": { + "name": "provisioner_binding_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "adapter_manifest_digest": { + "name": "adapter_manifest_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "retired_at": { + "name": "retired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "reconnect_window_ended_at": { + "name": "reconnect_window_ended_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'online'" + }, + "last_heartbeat": { + "name": "last_heartbeat", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sidecar_token_hash_sha256_unique": { + "name": "sidecar_token_hash_sha256_unique", + "nullsNotDistinct": false, + "columns": ["token_hash_sha256"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sidecar_allocation": { + "name": "sidecar_allocation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "anchor_run_id": { + "name": "anchor_run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provisioner_id": { + "name": "provisioner_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provisioner_api_version": { + "name": "provisioner_api_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "provisioner_binding_fingerprint": { + "name": "provisioner_binding_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sidecar_id": { + "name": "sidecar_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "adapter_manifest_digest": { + "name": "adapter_manifest_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "generation": { + "name": "generation", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "ensure_accepted_generation": { + "name": "ensure_accepted_generation", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "external_ref": { + "name": "external_ref", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "reconciliation_lease_id": { + "name": "reconciliation_lease_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reconciliation_lease_expires_at": { + "name": "reconciliation_lease_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "initialization_lease_id": { + "name": "initialization_lease_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ensure_attempts": { + "name": "ensure_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "destroy_attempts": { + "name": "destroy_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "deployment_cleanup_confirmed": { + "name": "deployment_cleanup_confirmed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "provider_hold_released": { + "name": "provider_hold_released", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "connect_deadline": { + "name": "connect_deadline", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "max_disconnected_ms": { + "name": "max_disconnected_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "first_deploy_failed_at": { + "name": "first_deploy_failed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "failure_code": { + "name": "failure_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_message": { + "name": "failure_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sidecar_allocation_anchor_run_idx": { + "name": "sidecar_allocation_anchor_run_idx", + "columns": [ + { + "expression": "anchor_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sidecar_allocation_sidecar_idx": { + "name": "sidecar_allocation_sidecar_idx", + "columns": [ + { + "expression": "sidecar_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sidecar_allocation_reconciliation_idx": { + "name": "sidecar_allocation_reconciliation_idx", + "columns": [ + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"sidecar_allocation\".\"status\" in ('pending', 'provisioning', 'allocated', 'replacing', 'releasing') and \"sidecar_allocation\".\"next_attempt_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sidecar_allocation_dispatchable_anchor_idx": { + "name": "sidecar_allocation_dispatchable_anchor_idx", + "columns": [ + { + "expression": "anchor_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"sidecar_allocation\".\"status\" in ('pending', 'provisioning', 'allocated', 'replacing')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sidecar_allocation_anchor_run_id_workflow_run_id_fk": { + "name": "sidecar_allocation_anchor_run_id_workflow_run_id_fk", + "tableFrom": "sidecar_allocation", + "tableTo": "workflow_run", + "columnsFrom": ["anchor_run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sidecar_allocation_tenant_id_tenant_id_fk": { + "name": "sidecar_allocation_tenant_id_tenant_id_fk", + "tableFrom": "sidecar_allocation", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "sidecar_allocation_sidecar_id_sidecar_id_fk": { + "name": "sidecar_allocation_sidecar_id_sidecar_id_fk", + "tableFrom": "sidecar_allocation", + "tableTo": "sidecar", + "columnsFrom": ["sidecar_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "sidecar_allocation_status_check": { + "name": "sidecar_allocation_status_check", + "value": "\"sidecar_allocation\".\"status\" in ('pending', 'provisioning', 'allocated', 'replacing', 'releasing', 'destroy_failed', 'released', 'failed')" + }, + "sidecar_allocation_generation_check": { + "name": "sidecar_allocation_generation_check", + "value": "\"sidecar_allocation\".\"generation\" >= 0" + }, + "sidecar_allocation_max_disconnected_check": { + "name": "sidecar_allocation_max_disconnected_check", + "value": "\"sidecar_allocation\".\"max_disconnected_ms\" > 0" + }, + "sidecar_allocation_accepted_generation_check": { + "name": "sidecar_allocation_accepted_generation_check", + "value": "\"sidecar_allocation\".\"ensure_accepted_generation\" is null or \"sidecar_allocation\".\"ensure_accepted_generation\" <= \"sidecar_allocation\".\"generation\"" + }, + "sidecar_allocation_initialization_check": { + "name": "sidecar_allocation_initialization_check", + "value": "\"sidecar_allocation\".\"initialization_lease_id\" is null or (\"sidecar_allocation\".\"status\" = 'allocated' and \"sidecar_allocation\".\"ensure_accepted_generation\" is not null and \"sidecar_allocation\".\"ensure_accepted_generation\" = \"sidecar_allocation\".\"generation\")" + } + }, + "isRLSEnabled": false + }, + "public.agent_session": { + "name": "agent_session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "agent_session_tenant_id_tenant_id_fk": { + "name": "agent_session_tenant_id_tenant_id_fk", + "tableFrom": "agent_session", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_session_agent_id_workflow_definition_id_fk": { + "name": "agent_session_agent_id_workflow_definition_id_fk", + "tableFrom": "agent_session", + "tableTo": "workflow_definition", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "agent_session_principal_id_principal_id_fk": { + "name": "agent_session_principal_id_principal_id_fk", + "tableFrom": "agent_session", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session_asset": { + "name": "session_asset", + "schema": "", + "columns": { + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mount_path": { + "name": "mount_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "asset_pack_sha": { + "name": "asset_pack_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_commit_sha": { + "name": "source_commit_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "materialized_at": { + "name": "materialized_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "session_asset_pack_sha_idx": { + "name": "session_asset_pack_sha_idx", + "columns": [ + { + "expression": "asset_pack_sha", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "session_asset_instance_id_mount_path_pk": { + "name": "session_asset_instance_id_mount_path_pk", + "columns": ["instance_id", "mount_path"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.inference_turn": { + "name": "inference_turn", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "inference_turn_instance_id_started_at_idx": { + "name": "inference_turn_instance_id_started_at_idx", + "columns": [ + { + "expression": "instance_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "inference_turn_session_id_agent_session_id_fk": { + "name": "inference_turn_session_id_agent_session_id_fk", + "tableFrom": "inference_turn", + "tableTo": "agent_session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "inference_turn_tenant_id_tenant_id_fk": { + "name": "inference_turn_tenant_id_tenant_id_fk", + "tableFrom": "inference_turn", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session_mail": { + "name": "session_mail", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "direction": { + "name": "direction", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "raw": { + "name": "raw", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "session_mail_instance_id_created_at_idx": { + "name": "session_mail_instance_id_created_at_idx", + "columns": [ + { + "expression": "instance_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "session_mail_session_id_created_at_idx": { + "name": "session_mail_session_id_created_at_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_mail_session_id_agent_session_id_fk": { + "name": "session_mail_session_id_agent_session_id_fk", + "tableFrom": "session_mail", + "tableTo": "agent_session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_mail_tenant_id_tenant_id_fk": { + "name": "session_mail_tenant_id_tenant_id_fk", + "tableFrom": "session_mail", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.turn_part": { + "name": "turn_part", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ordinal": { + "name": "ordinal", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "turn_part_turn_id_inference_turn_id_fk": { + "name": "turn_part_turn_id_inference_turn_id_fk", + "tableFrom": "turn_part", + "tableTo": "inference_turn", + "columnsFrom": ["turn_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "turn_part_session_id_agent_session_id_fk": { + "name": "turn_part_session_id_agent_session_id_fk", + "tableFrom": "turn_part", + "tableTo": "agent_session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.git_token": { + "name": "git_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash_sha256": { + "name": "token_hash_sha256", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref_pattern": { + "name": "ref_pattern", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actions": { + "name": "actions", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "git_token_user_id_name_active_idx": { + "name": "git_token_user_id_name_active_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"git_token\".\"revoked_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "git_token_tenant_id_tenant_id_fk": { + "name": "git_token_tenant_id_tenant_id_fk", + "tableFrom": "git_token", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "git_token_user_id_user_id_fk": { + "name": "git_token_user_id_user_id_fk", + "tableFrom": "git_token", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "git_token_principal_id_principal_id_fk": { + "name": "git_token_principal_id_principal_id_fk", + "tableFrom": "git_token", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "git_token_token_hash_sha256_unique": { + "name": "git_token_token_hash_sha256_unique", + "nullsNotDistinct": false, + "columns": ["token_hash_sha256"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.run_token": { + "name": "run_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "consumer": { + "name": "consumer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash_sha256": { + "name": "token_hash_sha256", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "sealed_token": { + "name": "sealed_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "run_token_principal_consumer_idx": { + "name": "run_token_principal_consumer_idx", + "columns": [ + { + "expression": "principal_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "consumer", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "run_token_principal_id_principal_id_fk": { + "name": "run_token_principal_id_principal_id_fk", + "tableFrom": "run_token", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "run_token_token_hash_sha256_unique": { + "name": "run_token_token_hash_sha256_unique", + "nullsNotDistinct": false, + "columns": ["token_hash_sha256"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_definition": { + "name": "workflow_definition", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "creator_principal_id": { + "name": "creator_principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "asset_id": { + "name": "asset_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wire_hash": { + "name": "wire_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "grant_requirements": { + "name": "grant_requirements", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "model_requirements": { + "name": "model_requirements", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "credential_bindings": { + "name": "credential_bindings", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "lifecycle_policy": { + "name": "lifecycle_policy", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "current_version": { + "name": "current_version", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'1'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'deployed'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_definition_tenant_idx": { + "name": "workflow_definition_tenant_idx", + "columns": [ + { + "expression": "tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_definition_asset_wire_hash_idx": { + "name": "workflow_definition_asset_wire_hash_idx", + "columns": [ + { + "expression": "asset_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "wire_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_definition_tenant_id_tenant_id_fk": { + "name": "workflow_definition_tenant_id_tenant_id_fk", + "tableFrom": "workflow_definition", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_definition_creator_principal_id_principal_id_fk": { + "name": "workflow_definition_creator_principal_id_principal_id_fk", + "tableFrom": "workflow_definition", + "tableTo": "principal", + "columnsFrom": ["creator_principal_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "workflow_definition_asset_id_asset_id_fk": { + "name": "workflow_definition_asset_id_asset_id_fk", + "tableFrom": "workflow_definition", + "tableTo": "asset", + "columnsFrom": ["asset_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_definition_version": { + "name": "workflow_definition_version", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "definition_id": { + "name": "definition_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "approved_wire_hash": { + "name": "approved_wire_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "grant_snapshot": { + "name": "grant_snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_definition_version_definition_idx": { + "name": "workflow_definition_version_definition_idx", + "columns": [ + { + "expression": "definition_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_definition_version_definition_id_workflow_definition_id_fk": { + "name": "workflow_definition_version_definition_id_workflow_definition_id_fk", + "tableFrom": "workflow_definition_version", + "tableTo": "workflow_definition", + "columnsFrom": ["definition_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workflow_definition_version_definition_version": { + "name": "workflow_definition_version_definition_version", + "nullsNotDistinct": false, + "columns": ["definition_id", "version"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_run": { + "name": "workflow_run", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "definition_id": { + "name": "definition_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "anchor_run_id": { + "name": "anchor_run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sidecar_id": { + "name": "sidecar_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "kernel_id": { + "name": "kernel_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model_preferences": { + "name": "model_preferences", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "credential_refs": { + "name": "credential_refs", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "lifecycle_policy": { + "name": "lifecycle_policy", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "cancellation_requested_at": { + "name": "cancellation_requested_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "operator_cancellation_requested_at": { + "name": "operator_cancellation_requested_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "cancellation_reason": { + "name": "cancellation_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cancellation_deadline": { + "name": "cancellation_deadline", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "capacity_release_at": { + "name": "capacity_release_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "infrastructure_failed_at": { + "name": "infrastructure_failed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "failure_code": { + "name": "failure_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_message": { + "name": "failure_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "legacy_mail_accept": { + "name": "legacy_mail_accept", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workflow_run_definition_idx": { + "name": "workflow_run_definition_idx", + "columns": [ + { + "expression": "definition_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_run_address_idx": { + "name": "workflow_run_address_idx", + "columns": [ + { + "expression": "address", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_run\".\"address\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_run_anchor_idx": { + "name": "workflow_run_anchor_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_run\".\"id\" = \"workflow_run\".\"anchor_run_id\"", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_run_live_anchor_idx": { + "name": "workflow_run_live_anchor_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_run\".\"id\" = \"workflow_run\".\"anchor_run_id\" and \"workflow_run\".\"status\" in ('deployed', 'running')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_run_infrastructure_failed_idx": { + "name": "workflow_run_infrastructure_failed_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_run\".\"infrastructure_failed_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_run_sidecar_idx": { + "name": "workflow_run_sidecar_idx", + "columns": [ + { + "expression": "sidecar_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_run\".\"sidecar_id\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_run_definition_id_workflow_definition_id_fk": { + "name": "workflow_run_definition_id_workflow_definition_id_fk", + "tableFrom": "workflow_run", + "tableTo": "workflow_definition", + "columnsFrom": ["definition_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_run_anchor_run_id_workflow_run_id_fk": { + "name": "workflow_run_anchor_run_id_workflow_run_id_fk", + "tableFrom": "workflow_run", + "tableTo": "workflow_run", + "columnsFrom": ["anchor_run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_run_tenant_id_tenant_id_fk": { + "name": "workflow_run_tenant_id_tenant_id_fk", + "tableFrom": "workflow_run", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_run_principal_id_principal_id_fk": { + "name": "workflow_run_principal_id_principal_id_fk", + "tableFrom": "workflow_run", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workflow_run_sidecar_id_sidecar_id_fk": { + "name": "workflow_run_sidecar_id_sidecar_id_fk", + "tableFrom": "workflow_run", + "tableTo": "sidecar", + "columnsFrom": ["sidecar_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workflow_run_legacy_mail_accept_check": { + "name": "workflow_run_legacy_mail_accept_check", + "value": "\"workflow_run\".\"legacy_mail_accept\" is null or \"workflow_run\".\"legacy_mail_accept\" in ('invoker-correspondent', 'correspondent')" + } + }, + "isRLSEnabled": false + }, + "public.workflow_run_dispatch": { + "name": "workflow_run_dispatch", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "anchor_run_id": { + "name": "anchor_run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'mail'" + }, + "sender_address": { + "name": "sender_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "raw_message": { + "name": "raw_message", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "step_grants": { + "name": "step_grants", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "acknowledged_generation": { + "name": "acknowledged_generation", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "delivery_lease_id": { + "name": "delivery_lease_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "delivery_lease_expires_at": { + "name": "delivery_lease_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "failure_code": { + "name": "failure_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_message": { + "name": "failure_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "acknowledged_at": { + "name": "acknowledged_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "settled_at": { + "name": "settled_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workflow_run_dispatch_anchor_message_idx": { + "name": "workflow_run_dispatch_anchor_message_idx", + "columns": [ + { + "expression": "anchor_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_run_dispatch_delivery_idx": { + "name": "workflow_run_dispatch_delivery_idx", + "columns": [ + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_run_dispatch\".\"status\" = 'pending'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_run_dispatch_anchor_status_idx": { + "name": "workflow_run_dispatch_anchor_status_idx", + "columns": [ + { + "expression": "anchor_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_run_dispatch_unresolved_anchor_idx": { + "name": "workflow_run_dispatch_unresolved_anchor_idx", + "columns": [ + { + "expression": "anchor_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_run_dispatch\".\"status\" in ('pending', 'acknowledged') or (\"workflow_run_dispatch\".\"status\" = 'abandoned' and \"workflow_run_dispatch\".\"next_attempt_at\" is not null)", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_run_dispatch_anchor_run_id_workflow_run_id_fk": { + "name": "workflow_run_dispatch_anchor_run_id_workflow_run_id_fk", + "tableFrom": "workflow_run_dispatch", + "tableTo": "workflow_run", + "columnsFrom": ["anchor_run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workflow_run_dispatch_status_check": { + "name": "workflow_run_dispatch_status_check", + "value": "\"workflow_run_dispatch\".\"status\" in ('pending', 'acknowledged', 'abandoned', 'settled', 'failed')" + }, + "workflow_run_dispatch_kind_check": { + "name": "workflow_run_dispatch_kind_check", + "value": "\"workflow_run_dispatch\".\"kind\" in ('mail', 'signal')" + }, + "workflow_run_dispatch_attempt_count_check": { + "name": "workflow_run_dispatch_attempt_count_check", + "value": "\"workflow_run_dispatch\".\"attempt_count\" >= 0" + }, + "workflow_run_dispatch_acknowledged_generation_check": { + "name": "workflow_run_dispatch_acknowledged_generation_check", + "value": "\"workflow_run_dispatch\".\"acknowledged_generation\" is null or \"workflow_run_dispatch\".\"acknowledged_generation\" >= 0" + }, + "workflow_run_dispatch_acknowledged_state_check": { + "name": "workflow_run_dispatch_acknowledged_state_check", + "value": "\"workflow_run_dispatch\".\"status\" <> 'acknowledged' or \"workflow_run_dispatch\".\"acknowledged_generation\" is not null" + }, + "workflow_run_dispatch_pending_schedule_check": { + "name": "workflow_run_dispatch_pending_schedule_check", + "value": "\"workflow_run_dispatch\".\"status\" <> 'pending' or \"workflow_run_dispatch\".\"next_attempt_at\" is not null" + }, + "workflow_run_dispatch_mail_sender_check": { + "name": "workflow_run_dispatch_mail_sender_check", + "value": "\"workflow_run_dispatch\".\"kind\" <> 'mail' or \"workflow_run_dispatch\".\"status\" in ('abandoned', 'settled', 'failed') or \"workflow_run_dispatch\".\"sender_address\" is not null" + } + }, + "isRLSEnabled": false + }, + "public.workflow_pending_projection": { + "name": "workflow_pending_projection", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "anchor_run_id": { + "name": "anchor_run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "history_required": { + "name": "history_required", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + } + }, + "indexes": { + "workflow_pending_projection_anchor_idx": { + "name": "workflow_pending_projection_anchor_idx", + "columns": [ + { + "expression": "anchor_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_pending_projection_anchor_run_id_workflow_run_id_fk": { + "name": "workflow_pending_projection_anchor_run_id_workflow_run_id_fk", + "tableFrom": "workflow_pending_projection", + "tableTo": "workflow_run", + "columnsFrom": ["anchor_run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_run_launch_spec": { + "name": "workflow_run_launch_spec", + "schema": "", + "columns": { + "anchor_run_id": { + "name": "anchor_run_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "schema_version": { + "name": "schema_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_domain": { + "name": "deployment_domain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_authority_principal_id": { + "name": "source_authority_principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "frozen_approval_bundle": { + "name": "frozen_approval_bundle", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "definition_artifact_id": { + "name": "definition_artifact_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "grant_snapshot": { + "name": "grant_snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "source_offering_ids": { + "name": "source_offering_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "default_source_offering_id": { + "name": "default_source_offering_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workflow_run_launch_spec_anchor_run_id_workflow_run_id_fk": { + "name": "workflow_run_launch_spec_anchor_run_id_workflow_run_id_fk", + "tableFrom": "workflow_run_launch_spec", + "tableTo": "workflow_run", + "columnsFrom": ["anchor_run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_run_launch_spec_source_authority_principal_id_principal_id_fk": { + "name": "workflow_run_launch_spec_source_authority_principal_id_principal_id_fk", + "tableFrom": "workflow_run_launch_spec", + "tableTo": "principal", + "columnsFrom": ["source_authority_principal_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "workflow_run_launch_spec_definition_artifact_id_workflow_definition_artifact_id_fk": { + "name": "workflow_run_launch_spec_definition_artifact_id_workflow_definition_artifact_id_fk", + "tableFrom": "workflow_run_launch_spec", + "tableTo": "workflow_definition_artifact", + "columnsFrom": ["definition_artifact_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_run_execution": { + "name": "workflow_run_execution", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workflow_run_execution_run_id_id_idx": { + "name": "workflow_run_execution_run_id_id_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_run_execution_status_idx": { + "name": "workflow_run_execution_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_run_execution_run_id_workflow_run_id_fk": { + "name": "workflow_run_execution_run_id_workflow_run_id_fk", + "tableFrom": "workflow_run_execution", + "tableTo": "workflow_run", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_probe": { + "name": "workflow_probe", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "definition_asset_id": { + "name": "definition_asset_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "entry": { + "name": "entry", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pin": { + "name": "pin", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "provisioner_id": { + "name": "provisioner_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provisioner_api_version": { + "name": "provisioner_api_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "provisioner_binding_fingerprint": { + "name": "provisioner_binding_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sidecar_id": { + "name": "sidecar_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "adapter_manifest_digest": { + "name": "adapter_manifest_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "external_ref": { + "name": "external_ref", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "failure_code": { + "name": "failure_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_message": { + "name": "failure_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_probe_active_idx": { + "name": "workflow_probe_active_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_probe\".\"status\" in ('pending', 'provisioning', 'probing', 'releasing')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_probe_sidecar_idx": { + "name": "workflow_probe_sidecar_idx", + "columns": [ + { + "expression": "sidecar_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_probe_tenant_id_tenant_id_fk": { + "name": "workflow_probe_tenant_id_tenant_id_fk", + "tableFrom": "workflow_probe", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "workflow_probe_definition_asset_id_asset_id_fk": { + "name": "workflow_probe_definition_asset_id_asset_id_fk", + "tableFrom": "workflow_probe", + "tableTo": "asset", + "columnsFrom": ["definition_asset_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "workflow_probe_sidecar_id_sidecar_id_fk": { + "name": "workflow_probe_sidecar_id_sidecar_id_fk", + "tableFrom": "workflow_probe", + "tableTo": "sidecar", + "columnsFrom": ["sidecar_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workflow_probe_status_check": { + "name": "workflow_probe_status_check", + "value": "\"workflow_probe\".\"status\" in ('pending', 'provisioning', 'probing', 'releasing', 'succeeded', 'failed')" + }, + "workflow_probe_succeeded_result_check": { + "name": "workflow_probe_succeeded_result_check", + "value": "\"workflow_probe\".\"status\" <> 'succeeded' or \"workflow_probe\".\"result\" is not null" + }, + "workflow_probe_generation_check": { + "name": "workflow_probe_generation_check", + "value": "\"workflow_probe\".\"generation\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.workflow_definition_artifact": { + "name": "workflow_definition_artifact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "revision_id": { + "name": "revision_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_definition_artifact_revision_idx": { + "name": "workflow_definition_artifact_revision_idx", + "columns": [ + { + "expression": "revision_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_definition_artifact_revision_id_workflow_definition_revision_id_fk": { + "name": "workflow_definition_artifact_revision_id_workflow_definition_revision_id_fk", + "tableFrom": "workflow_definition_artifact", + "tableTo": "workflow_definition_revision", + "columnsFrom": ["revision_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workflow_definition_artifact_state_check": { + "name": "workflow_definition_artifact_state_check", + "value": "(\"workflow_definition_artifact\".\"result\" is null and \"workflow_definition_artifact\".\"lease_owner\" is not null and \"workflow_definition_artifact\".\"lease_expires_at\" is not null) or (\"workflow_definition_artifact\".\"result\" is not null and \"workflow_definition_artifact\".\"lease_owner\" is null and \"workflow_definition_artifact\".\"lease_expires_at\" is null)" + } + }, + "isRLSEnabled": false + }, + "public.workflow_definition_revision": { + "name": "workflow_definition_revision", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "asset_id": { + "name": "asset_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "asset_commit_sha": { + "name": "asset_commit_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "entry": { + "name": "entry", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "closure": { + "name": "closure", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workflow_definition_revision_tenant_id_tenant_id_fk": { + "name": "workflow_definition_revision_tenant_id_tenant_id_fk", + "tableFrom": "workflow_definition_revision", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "workflow_definition_revision_asset_id_asset_id_fk": { + "name": "workflow_definition_revision_asset_id_asset_id_fk", + "tableFrom": "workflow_definition_revision", + "tableTo": "asset", + "columnsFrom": ["asset_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/packages/db/migrations/meta/_journal.json b/packages/db/migrations/meta/_journal.json index 890b3f7a7..5ff4eae8d 100644 --- a/packages/db/migrations/meta/_journal.json +++ b/packages/db/migrations/meta/_journal.json @@ -855,6 +855,13 @@ "when": 1793842023286, "tag": "0122_drop_workflow_run_public_key", "breakpoints": true + }, + { + "idx": 123, + "version": "7", + "when": 1793915609889, + "tag": "0123_drop_launch_spec_tool_package_pins", + "breakpoints": true } ] } diff --git a/packages/db/src/parse-row.ts b/packages/db/src/parse-row.ts index ea0480f39..3f3f6eabb 100644 --- a/packages/db/src/parse-row.ts +++ b/packages/db/src/parse-row.ts @@ -21,7 +21,6 @@ import { import { WireGrantRule } from "@intx/types/grant-wire"; import { FrozenApprovalBundle } from "@intx/types/sidecar"; import { RepoAction } from "@intx/types/repo"; -import { ToolPackagePinArray } from "@intx/types/tool-packages"; import { workflowRunDispatchStatuses } from "./schema/workflow-run-dispatch"; import type { @@ -248,10 +247,6 @@ export function parseWorkflowRunLaunchSpecRow( ? null : GrantWalkSnapshot.assert(row.grantSnapshot), sourceOfferingIds, - toolPackagePins: - row.toolPackagePins !== null - ? ToolPackagePinArray.assert(row.toolPackagePins) - : null, }; } diff --git a/packages/db/src/schema/run-tokens.ts b/packages/db/src/schema/run-tokens.ts index 1acaa4a55..d20bb413f 100644 --- a/packages/db/src/schema/run-tokens.ts +++ b/packages/db/src/schema/run-tokens.ts @@ -4,7 +4,7 @@ import { bytea } from "./column-types"; import { principal } from "./principals"; // Bearer tokens a workflow run presents to the hub REST API as its run -// principal, one for each consumer (a tool package `tool:` or an +// principal, one for each consumer (a tool factory `tool:` or an // action handler `action:`). Opaque tokens of the form // `itx_run_`. // diff --git a/packages/db/src/schema/workflow-definitions.ts b/packages/db/src/schema/workflow-definitions.ts index c75516124..d4b4754c6 100644 --- a/packages/db/src/schema/workflow-definitions.ts +++ b/packages/db/src/schema/workflow-definitions.ts @@ -15,7 +15,7 @@ import { tenant } from "./tenants"; // workflow_definition is the first-class definition entity for the workflow // model: one row per deployable definition, with a version table alongside it. // -// The definition body (system prompt, context/model config, tool packages) is +// The definition body (system prompt, context/model config, tool factories) is // not stored on this row -- it lives in the `workflow`-kind asset the row // points at, and the run reads it back from there. `asset_id` is nullable at // the schema level, but a runnable definition points at a materialized asset. @@ -64,7 +64,7 @@ export const workflowDefinition = pgTable( // at parse time. modelRequirements: jsonb("model_requirements"), // Credential bindings manifest: maps a consumer's declared credential - // handle -- (package, handle) for a tool package or (action, handle) for an + // handle -- (package, handle) for a tool factory or (action, handle) for an // action handler -- to a concrete credential resolved fresh at launch (by // locator + provider + name against the tenant walk-up). // Validated as CredentialBinding[] at parse time. A binding is a request diff --git a/packages/db/src/schema/workflow-run-launch-spec.ts b/packages/db/src/schema/workflow-run-launch-spec.ts index b9e206114..3c1f3a90d 100644 --- a/packages/db/src/schema/workflow-run-launch-spec.ts +++ b/packages/db/src/schema/workflow-run-launch-spec.ts @@ -27,6 +27,5 @@ export const workflowRunLaunchSpec = pgTable("workflow_run_launch_spec", { grantSnapshot: jsonb("grant_snapshot"), sourceOfferingIds: jsonb("source_offering_ids").notNull(), defaultSourceOfferingId: text("default_source_offering_id").notNull(), - toolPackagePins: jsonb("tool_package_pins"), createdAt: timestamp("created_at").notNull().defaultNow(), }); diff --git a/packages/db/src/workflow-run-launch-spec-store.ts b/packages/db/src/workflow-run-launch-spec-store.ts index 60faf2c72..19702c942 100644 --- a/packages/db/src/workflow-run-launch-spec-store.ts +++ b/packages/db/src/workflow-run-launch-spec-store.ts @@ -18,7 +18,6 @@ export function createWorkflowRunLaunchSpecStore(db: DBHandle) { parseWorkflowRunLaunchSpecRow({ ...row, schemaVersion: row.schemaVersion ?? 1, - toolPackagePins: row.toolPackagePins ?? null, definitionArtifactId: row.definitionArtifactId ?? null, grantSnapshot: row.grantSnapshot ?? null, createdAt: row.createdAt ?? new Date(), diff --git a/packages/harness/src/credential-capability.ts b/packages/harness/src/credential-capability.ts index fd08e5cbd..b5d9e6117 100644 --- a/packages/harness/src/credential-capability.ts +++ b/packages/harness/src/credential-capability.ts @@ -1,5 +1,5 @@ // The consumer-gated `credentials` capability: the sub-registry a consumer (a -// tool package or an action handler) queries by its declared handle to obtain a mediated credential. It is the runtime +// tool factory or an action handler) queries by its declared handle to obtain a mediated credential. It is the runtime // gate that enforces the `{ tool }` condition on a materialized // `credential:{id}` / `use` grant -- the check the launch-time grant // materialization sets up but does not itself evaluate. @@ -43,8 +43,8 @@ export interface ResolvedCredentialBinding { } /** - * Reconcile a consumer's declared credential handles (a tool package's C5 - * `interchange.credentials`, or an action's `credentials`) against the handles + * Reconcile a consumer's declared credential handles (a tool's `defineTool` + * `credentials`, or an action's `credentials`) against the handles * a binding actually resolved for it. A declared handle with no binding is a * launch-blocking misconfiguration -- the consumer needs a credential the * definition never bound -- so this fails the launch loudly rather than letting @@ -68,7 +68,7 @@ export function reconcileDeclaredCredentials( export interface CredentialCapabilityDeps { /** - * The consumer identity this capability serves (`tool:` from + * The consumer identity this capability serves (`tool:` from * `toolConsumer`, or `action:` from `actionConsumer`). Gate 2 checks each grant's * `{ tool }` condition against this value; an empty identity fails closed. */ diff --git a/packages/harness/src/step-credential-capabilities.test.ts b/packages/harness/src/step-credential-capabilities.test.ts index 663672820..0f56f6af3 100644 --- a/packages/harness/src/step-credential-capabilities.test.ts +++ b/packages/harness/src/step-credential-capabilities.test.ts @@ -48,12 +48,12 @@ function grant( }; } -// The assembly reads only `packageName` and `declaredCredentials`. +// The assembly reads only `factoryId` and `declaredCredentials`. function fac( - packageName: string, + factoryId: string, declaredCredentials: ToolCredentialDeclaration[], ) { - return { packageName, declaredCredentials }; + return { factoryId, declaredCredentials }; } const ORIGIN = "https://api.example.com"; @@ -87,9 +87,9 @@ function useGrant(consumer: string): GrantRule { } describe("buildCredentialCapabilities", () => { - test("a credential granted to one package cannot be resolved by another (confused deputy)", async () => { + test("a credential granted to one factory cannot be resolved by another (confused deputy)", async () => { const track = trackingProvider(); - // The same credential is bound for both packages, but only pkg-a holds the + // The same credential is bound for both factories, but only pkg-a holds the // credential:c1/use grant scoped to its consumer. const cell = { current: { @@ -298,9 +298,9 @@ describe("buildCredentialCapabilities", () => { ).toThrow(/delivery is malformed/); }); - test("a package that declares and binds no credential gets no capability", () => { + test("a factory that declares and binds no credential gets no capability", () => { const track = trackingProvider(); - // A credential is delivered, but only for a different package's consumer. + // A credential is delivered, but only for a different factory's consumer. const cell = { current: { bindings: [ @@ -328,7 +328,7 @@ describe("buildCredentialCapabilities", () => { expect(caps.has("@intx/pkg-a")).toBe(false); }); - // The grants thunk must be read only when a package actually needs a + // The grants thunk must be read only when a factory actually needs a // capability. This pins the self-discovery-resume fix: a toolless resume // precedes the grants barrier, so resolving grants for it would fault on a // snapshot that is not present yet. The "no capability" test above passes a @@ -338,7 +338,7 @@ describe("buildCredentialCapabilities", () => { throw new Error("resolveGrants must not be called for a step needing none"); }; - test("a step whose packages need no capability never resolves grants", () => { + test("a step whose factories need no capability never resolves grants", () => { const track = trackingProvider(); const cell = { current: { bindings: [], materials: [] } }; expect(() => @@ -350,7 +350,7 @@ describe("buildCredentialCapabilities", () => { ).not.toThrow(); }); - test("grants are resolved once a package needs a capability", () => { + test("grants are resolved once a factory needs a capability", () => { const track = trackingProvider(); const cell = { current: { @@ -382,16 +382,16 @@ describe("buildCredentialCapabilities", () => { }); describe("buildCredentialCapabilities grants reads", () => { - test("a step reads its grants once, whatever the package order", () => { + test("a step reads its grants once, whatever the factory order", () => { const track = trackingProvider(); const cell = deliveryFor(track.provider.key, [ { consumer: toolConsumer("@intx/pkg-a"), handle: "cred" }, { consumer: toolConsumer("@intx/pkg-b"), handle: "cred" }, ]); - const packages = ["@intx/pkg-none", "@intx/pkg-a", "@intx/pkg-b"]; - const rotations = packages.map((_unused, i) => [ - ...packages.slice(i), - ...packages.slice(0, i), + const factoryIds = ["@intx/pkg-none", "@intx/pkg-a", "@intx/pkg-b"]; + const rotations = factoryIds.map((_unused, i) => [ + ...factoryIds.slice(i), + ...factoryIds.slice(0, i), ]); for (const order of rotations) { let reads = 0; diff --git a/packages/harness/src/step-credential-capabilities.ts b/packages/harness/src/step-credential-capabilities.ts index 7d50419e8..a76421657 100644 --- a/packages/harness/src/step-credential-capabilities.ts +++ b/packages/harness/src/step-credential-capabilities.ts @@ -1,14 +1,14 @@ // Per-consumer assembly of the runtime `credentials` capability for a step's // tool bundles. // -// A step can carry tool factories from several packages, and each package is a -// distinct credential consumer (`toolConsumer(packageName)`). Gate 2 -- the -// `{ tool }` condition on a `credential:{id}` / `use` grant -- is baked into -// each capability at construction, so a single capability shared across a -// step's packages would be a confused-deputy hole (package A's grant would -// authorize package B's resolve). This module therefore builds ONE capability -// per package, keyed by package name, and the bundle-invocation site layers -// each package's capability onto only that package's bundles. +// A step can carry several tool factories, and each factory is a distinct +// credential consumer (`toolConsumer(factoryId)`). Gate 2 -- the `{ tool }` +// condition on a `credential:{id}` / `use` grant -- is baked into each +// capability at construction, so a single capability shared across a step's +// factories would be a confused-deputy hole (factory A's grant would authorize +// factory B's resolve). This module therefore builds ONE capability per factory, +// keyed by factory id, and the bundle-invocation site layers each factory's +// capability onto only that factory's bundles. // // Two invariants this module owns, both fail-closed: // 1. Material is read from the LIVE delivery cell on every use, never @@ -19,8 +19,8 @@ // launch/first-resolve gate (Seam A memoizes it), so mid-deploy // revocation takes effect here, at the material, not at the gate. // 2. The declared-vs-bound reconcile runs per consumer against that -// consumer's OWN bound set, so package A's declaration can never be -// satisfied by a handle bound for package B. +// consumer's OWN bound set, so factory A's declaration can never be +// satisfied by a handle bound for factory B. import { toolConsumer } from "@intx/authz"; import type { @@ -73,30 +73,29 @@ export interface StepCredentialWiring { /** * Build the credential capabilities for a step's tool factories, one per - * distinct package, keyed by package name. + * distinct `factoryId`. * - * A package with neither a declared nor a bound credential gets no entry: its + * A factory with neither a declared nor a bound credential gets no entry: its * bundles keep the base capabilities bag untouched, so `resolve("credentials")` * fails closed as "not provided by host" rather than handing back an empty * sub-registry. Every returned capability owns a `dispose` the caller must run * on teardown. * - * The factory list is structural. The body reads only `packageName` and - * `declaredCredentials`. Importing `StepToolFactory` or `@intx/tool-packaging` - * is rejected because `bin/check-deps.ts` counts `import type`, and that - * dependency does not belong on this package. + * The factory list is structural. The body reads only `factoryId` and + * `declaredCredentials`, and the caller's factory type lives in + * `@intx/workflow-host`, which depends on this package. */ export function buildCredentialCapabilities( factories: readonly { - readonly packageName: string; + readonly factoryId: string; readonly declaredCredentials: readonly ToolCredentialDeclaration[]; }[], wiring: StepCredentialWiring, ): Map { - const byPackage = new Map(); - const seenPackages = new Set(); - // Resolve the step's grants at most once, and only if some package actually - // needs a capability -- a step with no credential-consuming package never + const byFactory = new Map(); + const seenFactories = new Set(); + // Resolve the step's grants at most once, and only if some factory actually + // needs a capability -- a step with no credential-consuming factory never // reads them (see StepCredentialWiring.resolveGrants). let grants: readonly GrantRule[] | undefined; const stepWiring: StepCredentialWiring = { @@ -105,20 +104,20 @@ export function buildCredentialCapabilities( }; for (const stf of factories) { - if (seenPackages.has(stf.packageName)) continue; - seenPackages.add(stf.packageName); + if (seenFactories.has(stf.factoryId)) continue; + seenFactories.add(stf.factoryId); const capability = buildConsumerCredentialCapability( - toolConsumer(stf.packageName), + toolConsumer(stf.factoryId), stf.declaredCredentials, stepWiring, ); if (capability !== undefined) { - byPackage.set(stf.packageName, capability); + byFactory.set(stf.factoryId, capability); } } - return byPackage; + return byFactory; } /** diff --git a/packages/hub-agent/src/source-asset-delivery.ts b/packages/hub-agent/src/source-asset-delivery.ts index 7c9cd077c..64e7d9095 100644 --- a/packages/hub-agent/src/source-asset-delivery.ts +++ b/packages/hub-agent/src/source-asset-delivery.ts @@ -6,7 +6,8 @@ // (`applyAssetPack`), keyed by an `assetId -> mountPath` map; and // - a source-format entry checks a subtree out of the git objects, so the // pack is indexed into a RETAINED `.git` (a "gitDir"), keyed by an -// `assetId -> gitDir` map the loader hands `materializeGitEntry`. +// `assetId -> gitDir` map the closure materializer hands +// `materializeGitEntry`. // One asset can be referenced both ways; a source-format workflow closure // references only source entries, so it produces only gitDirs. @@ -151,7 +152,7 @@ export const MAX_INLINE_ASSET_PAYLOAD_BYTES = 32 * 1024 * 1024; * Materialize a workflow closure's delivered source assets: for each asset, * check out plain tarball files under `assetRoot` (if the closure has tarball * entries for it) and/or index the pack into a gitDir under `gitDirRoot` (if it - * has source entries). Returns both maps for the loader. + * has source entries). Returns both maps for the closure materializer. */ export async function materializeWorkflowAssets(args: { assets: readonly WorkflowSourceAssetMount[]; diff --git a/packages/hub-api/src/context.ts b/packages/hub-api/src/context.ts index b4b0b9347..8d370bb88 100644 --- a/packages/hub-api/src/context.ts +++ b/packages/hub-api/src/context.ts @@ -36,7 +36,7 @@ export type TenantEnv = Env & { tenant: TenantRow; principal: PrincipalRow; /** - * The workflow consumer (`tool:` or `action:`) a run + * The workflow consumer (`tool:` or `action:`) a run * token speaks for. Unset for sessions and git tokens, so a grant scoped * to a consumer by a `{tool}` condition never authorizes them. */ diff --git a/packages/hub-sessions/README.md b/packages/hub-sessions/README.md index d12d6bcec..6b52836d8 100644 --- a/packages/hub-sessions/README.md +++ b/packages/hub-sessions/README.md @@ -82,8 +82,7 @@ for the authoritative type shapes. - Package registry: `packageRegistryKindHandler`, `packageRegistryAuthorize`, `asTarballEntry`, `validateTarballPackageJSON`, and the constants `TARBALLS_PREFIX`, - `TARBALL_FILENAME_PATTERN`, `REGISTRY_INDEX_PATH`, - `WORKSPACE_BUILTINS_REGISTRY`. + `TARBALL_FILENAME_PATTERN`, `REGISTRY_INDEX_PATH`. ### Workflow-run substrate diff --git a/packages/hub-sessions/src/index.ts b/packages/hub-sessions/src/index.ts index 55de54651..6623f4ccb 100644 --- a/packages/hub-sessions/src/index.ts +++ b/packages/hub-sessions/src/index.ts @@ -185,7 +185,6 @@ export { TARBALLS_PREFIX, TARBALL_FILENAME_PATTERN, REGISTRY_INDEX_PATH, - WORKSPACE_BUILTINS_REGISTRY, } from "./package-registry-kind"; export { workflowKindHandler, diff --git a/packages/hub-sessions/src/package-registry-kind.ts b/packages/hub-sessions/src/package-registry-kind.ts index aa650d3e8..2adf09231 100644 --- a/packages/hub-sessions/src/package-registry-kind.ts +++ b/packages/hub-sessions/src/package-registry-kind.ts @@ -29,30 +29,6 @@ const logger = getLogger(["hub-sessions", "package-registry-kind"]); export const TARBALLS_PREFIX = "tarballs/"; export const REGISTRY_INDEX_PATH = "package-registry.json"; -/** - * Canonical asset name for the workspace's bundled package-registry — - * the in-tree `@intx/tools-*` packages live in an asset of this name - * under the workspace's root tenant. The hub's scope-routing config - * maps `@intx` to this registry, the seed script ensures the asset - * row exists, and the publish-tool-packages CLI defaults its target - * registry to this name. The constant lives at one site so a rename - * does not have to chase three independent string literals. - * - * Callers: - * - `bin/dev.ts` — orchestrator default for the registry the dev - * stack publishes the built-ins into. - * - `bin/seed.ts` — seeder that pins the workspace-builtins into - * the registry asset at boot. - * - `bin/publish-tool-packages.ts` — CLI default for the - * `--registry` flag. - * - * No test asserts that the seed's pinned built-ins actually land - * under this exact name; a mismatch between the constant and a - * caller would surface at apply time as a `tarball.missing` - * structured failure, not as a build error. - */ -export const WORKSPACE_BUILTINS_REGISTRY = "workspace-builtins"; - /** * Filename rule for tarballs in the repo tree. Filenames must end in * `.tgz`, start with an alphanumeric / underscore / scope-marker @@ -209,7 +185,7 @@ export const packageRegistryKindHandler: KindHandler = { // resolver's AssetRegistrySource overwrite by fs.readdir order — // an undefined-behaviour outcome across filesystems. Reject the // collision at the substrate boundary so the registry asset's - // closure is unambiguous regardless of how the loader walks it. + // closure is unambiguous. const publishedNameVersions = new Set(); for (const child of tarballChildren) { const repoPath = `${TARBALLS_PREFIX}${child}`; diff --git a/packages/hub-sessions/src/session-service.ts b/packages/hub-sessions/src/session-service.ts index 814cb226d..d0679a57b 100644 --- a/packages/hub-sessions/src/session-service.ts +++ b/packages/hub-sessions/src/session-service.ts @@ -28,7 +28,7 @@ import type { import { DeploymentRejectedError } from "@intx/types/sidecar"; import type { CredentialCipher } from "@intx/types"; import type { HarnessConfig, InferenceSource } from "@intx/types/runtime"; -import { type RegistryConfig, type ScopeRoute } from "@intx/tool-packaging"; +import { type RegistryConfig } from "@intx/tool-packaging"; import type { SourceRefPin, WorkflowProjectionWithSources, @@ -205,17 +205,16 @@ export type SessionServiceDeps = { * The registries a source deploy resolves through. Required for a source * deploy. `httpRegistries` maps each registry name to its config. A registry * source names its registry, and a source-format asset's external - * dependencies use `defaultRegistry`. No deploy path reads `scopeRouting`. + * dependencies use `defaultRegistry`. */ toolPackageRegistries?: { /** * Registry identifier → registry config. The key is the - * identifier `scopeRouting` entries and manifest `registry` - * references point at; the value carries url plus optional auth. + * identifier manifest `registry` references point at; the value + * carries url plus optional auth. */ readonly httpRegistries: ReadonlyMap; readonly defaultRegistry: string; - readonly scopeRouting?: readonly ScopeRoute[]; }; /** * Load a run's Hub tokens. A prepared deploy of an anchor that already has diff --git a/packages/hub-sessions/src/workflow-allocation-service.ts b/packages/hub-sessions/src/workflow-allocation-service.ts index 4d0822e8f..ade856b84 100644 --- a/packages/hub-sessions/src/workflow-allocation-service.ts +++ b/packages/hub-sessions/src/workflow-allocation-service.ts @@ -36,7 +36,6 @@ import { type FrozenApprovalBundle, } from "@intx/types/sidecar"; import type { HarnessConfig } from "@intx/types/runtime"; -import type { ToolPackagePin } from "@intx/types/tool-packages"; import type { WorkflowDefinitionSource } from "@intx/types/workflow-sources"; import { approvalItemsFromSet, @@ -118,7 +117,6 @@ export type PrepareProvisionedWorkflowDeploymentArgs = { readonly sourceAuthorityPrincipalId: string; readonly sourceOfferingIds: readonly string[]; readonly defaultSourceOfferingId: string; - readonly toolPackagePins?: readonly ToolPackagePin[]; /** * State to start agent steps from, keyed by step id. Each key must name a * top-level agent step of the approved definition. @@ -509,9 +507,6 @@ export function createWorkflowAllocationService({ definitionArtifactId: approved.artifactId ?? null, sourceOfferingIds: [...request.sourceOfferingIds], defaultSourceOfferingId: request.defaultSourceOfferingId, - ...(request.toolPackagePins !== undefined - ? { toolPackagePins: [...request.toolPackagePins] } - : {}), createdAt, }, tx, diff --git a/packages/hub-sessions/src/workflow-closure-resolution.test.ts b/packages/hub-sessions/src/workflow-closure-resolution.test.ts index 5aba11e53..ad1f8582a 100644 --- a/packages/hub-sessions/src/workflow-closure-resolution.test.ts +++ b/packages/hub-sessions/src/workflow-closure-resolution.test.ts @@ -21,7 +21,6 @@ import * as tar from "tar"; import type { Packument, PackumentFetcher } from "@intx/tool-packaging"; import type { WorkflowDefinitionRegistrySource } from "@intx/types/workflow-sources"; -import { getToolPackageSourceContentIdentity } from "@intx/types/tool-packages"; import { resolveWorkflowClosure } from "./workflow-closure-resolution"; import type { SourceTreeReads } from "./workflow-source-closure"; @@ -205,9 +204,6 @@ describe("resolveWorkflowClosure", () => { const topEntry = byName.get(top.name); if (topEntry === undefined) throw new Error("missing top-level entry"); expect(topEntry.version).toBe(top.version); - expect(getToolPackageSourceContentIdentity(topEntry.source)).toBe( - top.integrity, - ); expect(topEntry.source).toEqual({ kind: "registry", registry: source.registry, @@ -217,9 +213,6 @@ describe("resolveWorkflowClosure", () => { const depEntry = byName.get(dep.name); if (depEntry === undefined) throw new Error("missing transitive entry"); expect(depEntry.version).toBe(dep.version); - expect(getToolPackageSourceContentIdentity(depEntry.source)).toBe( - dep.integrity, - ); expect(depEntry.source).toEqual({ kind: "registry", registry: source.registry, @@ -254,9 +247,7 @@ describe("resolveWorkflowClosure", () => { const onlyEntry = manifest.entries[0]; if (onlyEntry === undefined) throw new Error("missing entry"); expect(onlyEntry.version).toBe(only.version); - expect(getToolPackageSourceContentIdentity(onlyEntry.source)).toBe( - only.integrity, - ); + expect(onlyEntry.source).toMatchObject({ integrity: only.integrity }); }); test("resolves an asset-sourced workflow to a kind:asset entry", async () => { @@ -284,9 +275,6 @@ describe("resolveWorkflowClosure", () => { const entry = manifest.entries[0]; if (entry === undefined) throw new Error("missing entry"); expect(entry.version).toBe(top.version); - expect(getToolPackageSourceContentIdentity(entry.source)).toBe( - top.integrity, - ); expect(entry.source).toEqual({ kind: "asset", assetId: "asset_top", @@ -357,9 +345,10 @@ describe("resolveWorkflowClosure", () => { // The external dep is a registry entry with the tarball's SRI. const ext = byName.get(dep.name); expect(ext?.version).toBe(dep.version); - expect(ext?.source.kind).toBe("registry"); - if (ext === undefined) throw new Error("missing external entry"); - expect(getToolPackageSourceContentIdentity(ext.source)).toBe(dep.integrity); + expect(ext?.source).toMatchObject({ + kind: "registry", + integrity: dep.integrity, + }); }); test("fails loud when the asset does not publish a declared dependency", async () => { diff --git a/packages/hub-sessions/src/workflow-closure-resolution.ts b/packages/hub-sessions/src/workflow-closure-resolution.ts index f424d9b03..46ff445e9 100644 --- a/packages/hub-sessions/src/workflow-closure-resolution.ts +++ b/packages/hub-sessions/src/workflow-closure-resolution.ts @@ -68,14 +68,12 @@ export type ResolveWorkflowClosureRegistryArgs = { * Resolve a workflow definition's dependency closure against a single hub * `package-registry` asset. * - * The resolver is single-source by design: the asset is the sole and default - * registry, so a self-contained definition (its `@intx/*` imports bundled at - * author time) resolves to one `kind:"asset"` entry. A specifier the asset does - * not publish fails loud from `AssetRegistrySource` -- there is no silent - * fallback to npm. A definition that genuinely needs external npm dependencies - * would resolve through the mixed `AssetRegistrySource` + `HttpRegistrySource` - * map `session-service` already builds for tool packages; that path is not - * threaded here. + * The resolver is single-source by design: the asset is the sole registry, so a + * self-contained definition (its `@intx/*` imports bundled at author time) + * resolves to one `kind:"asset"` entry. A specifier the asset does not publish + * fails loud from `AssetRegistrySource` -- there is no silent fallback to npm. A + * definition that needs external npm dependencies cannot use this arm; the + * registry and asset-source arms resolve them. */ export type ResolveWorkflowClosureAssetTarballArgs = { /** An asset source whose `package.format` is `"tarball"`. */ @@ -165,7 +163,7 @@ export async function resolveWorkflowClosure( if (isAssetTarballArgs(args)) { // The assetId is the source name so every walker resolution error is - // traceable to the exact asset; it is the sole and default source. + // traceable to the exact asset. const name = args.source.assetId; const assetSource = new AssetRegistrySource({ name, @@ -173,10 +171,7 @@ export async function resolveWorkflowClosure( readBlob: args.readBlob, listBlobs: args.listBlobs, }); - const resolver = createClosureResolver({ - registries: new Map([[name, assetSource]]), - defaultRegistry: name, - }); + const resolver = createClosureResolver({ source: assetSource }); return resolver.resolveClosure([parsePin(args.pin)]); } @@ -187,10 +182,7 @@ export async function resolveWorkflowClosure( ? { fetchPackument: args.fetchPackument } : {}), }); - const resolver = createClosureResolver({ - registries: new Map([[registrySource.name, registrySource]]), - defaultRegistry: registrySource.name, - }); + const resolver = createClosureResolver({ source: registrySource }); return resolver.resolveClosure([parsePin(args.pin)]); } diff --git a/packages/hub-sessions/src/workflow-probe-cache.ts b/packages/hub-sessions/src/workflow-probe-cache.ts index 491764025..c7a19ceb6 100644 --- a/packages/hub-sessions/src/workflow-probe-cache.ts +++ b/packages/hub-sessions/src/workflow-probe-cache.ts @@ -8,7 +8,7 @@ import type { WorkflowProbeResult } from "./ws/sidecar-handler"; // Bump when probe semantics or the stored result shape changes. Probe workers // must use compatible evaluator code and environments within this version. -const WORKFLOW_PROBE_CACHE_VERSION = 4; +const WORKFLOW_PROBE_CACHE_VERSION = 5; export type WorkflowProbeLeaseTiming = { readonly leaseMs: number; diff --git a/packages/hub-sessions/src/workflow-source-closure.ts b/packages/hub-sessions/src/workflow-source-closure.ts index 2cc605f44..ee551b877 100644 --- a/packages/hub-sessions/src/workflow-source-closure.ts +++ b/packages/hub-sessions/src/workflow-source-closure.ts @@ -473,10 +473,7 @@ async function resolveExternalClosure( config: registryConfig, ...(fetchPackument !== undefined ? { fetchPackument } : {}), }); - const resolver = createClosureResolver({ - registries: new Map([[registryName, registrySource]]), - defaultRegistry: registryName, - }); + const resolver = createClosureResolver({ source: registrySource }); const rootPins: ToolPackagePin[] = [...pins].map(([name, version]) => ({ name, version, diff --git a/packages/hub-sessions/src/workflow-source-repo-kind.ts b/packages/hub-sessions/src/workflow-source-repo-kind.ts index 0d604a702..1b8fe08fa 100644 --- a/packages/hub-sessions/src/workflow-source-repo-kind.ts +++ b/packages/hub-sessions/src/workflow-source-repo-kind.ts @@ -5,10 +5,10 @@ import type { RepoKind } from "./repo-store"; /** * The repo kind that holds each format of an asset-sourced workflow * definition. A `source` codebase lives in a `workflow` asset; a packed - * `tarball` lives in a `package-registry` asset, the same kind that publishes - * tool packages. The table is keyed by the format union, so a new format does - * not compile until it names its kind, and the route that admits a deploy and - * the resolvers that pack the asset read the same answer. + * `tarball` lives in a `package-registry` asset. The table is keyed by the + * format union, so a new format does not compile until it names its kind, and + * the route that admits a deploy and the resolvers that pack the asset read the + * same answer. */ const REPO_KIND_BY_FORMAT: Record< WorkflowDefinitionAssetSource["package"]["format"], diff --git a/packages/inference/src/tool-name.ts b/packages/inference/src/tool-name.ts index 1dbd4b782..f09bf73aa 100644 --- a/packages/inference/src/tool-name.ts +++ b/packages/inference/src/tool-name.ts @@ -8,9 +8,8 @@ // wire-charset-safe form and back. // // Invariant: `decodeToolName(encodeToolName(x, ...)) === x`. It is -// load-bearing. Tool-call dispatch keys on the exact prefixed name in two -// places (the agent's `byName` map and the tool-package loader's per-bundle -// `nameMap`), so a name that does not round-trip lands as an `unknown tool` +// load-bearing. Tool-call dispatch keys on the exact name in the agent's +// `byName` map, so a name that does not round-trip lands as an `unknown tool` // with no error at the point of the fault. The `encode`/`decode` naming // advertises the invertibility on purpose: a `sanitize`-style name invites a // future lossy "cleanup" that would break dispatch. diff --git a/packages/tool-packaging/ASSET-LAYOUT.md b/packages/tool-packaging/ASSET-LAYOUT.md index 7a2b03b56..8a1b159fb 100644 --- a/packages/tool-packaging/ASSET-LAYOUT.md +++ b/packages/tool-packaging/ASSET-LAYOUT.md @@ -1,27 +1,25 @@ -# Tool-package asset layout +# Package-registry asset layout -Tool packages distributed through the asset substrate live in a +Packages distributed through the asset substrate live in a `package-registry` asset. The asset row sits on a tenant; the asset's backing git repo holds one tarball per name+version under `tarballs/`. ``` / └── tarballs/ - ├── @intx-tools-mail-0.1.2.tgz - ├── @intx-tools-posix-0.1.2.tgz - └── @intx-tools-lsp-0.1.2.tgz + ├── @acme-workflow-1.0.0.tgz + └── @acme-tools-1.2.3.tgz ``` -Filenames for scoped packages flatten the scope with a `-` separator -(`@scope/tail` → `@scope-tail`) so two scoped packages whose tails -collide remain distinguishable on disk. +A filename cannot hold a `/`, so a scoped package flattens its scope with +a `-` by convention (`@scope/tail` → `@scope-tail`), which keeps two scoped +packages whose tails collide distinguishable on disk. ## Asset shape - **Asset kind**: `package-registry`. - **Asset name**: lowercase-kebab, addressable by REST and resolvable - via the tenant-inheritance walker. Operators choose the name; the - workspace's built-ins ship under `workspace-builtins`. + via the tenant-inheritance walker. Operators choose the name. - **Contents**: `tarballs/-.tgz` files at the root of the `tarballs/` directory. Filenames must match the `-.tgz` pattern the `package-registry` kind @@ -30,7 +28,7 @@ collide remain distinguishable on disk. `PackageJSON` schema before the push is accepted. - **Mount path on the sidecar**: `source-assets//` (`workflowSourceAssetMountPath` in `@intx/types/workflow-sources`). Asset - packs land at `//`, so the sidecar's loader finds the + packs land at `//`, so the sidecar finds the tarballs at `/source-assets//tarballs/`. ## Where the manifest lives @@ -38,7 +36,7 @@ collide remain distinguishable on disk. The closure resolver walks a list of top-level packages, resolves transitive dependencies, and produces a `ToolPackageManifest` (validated by `@intx/types/tool-packages`). A deploy from source resolves its -workflow's closure this way and hands the manifest to the loader. +workflow's closure this way and hands the manifest to the sidecar. Manifest entries whose `source.kind` is `"asset"` carry the asset's id plus a `package` envelope. The envelope's `format` discriminates where @@ -50,93 +48,66 @@ tarball entry looks like: ```json { - "name": "@intx/tools-mail", - "version": "0.1.2", + "name": "@acme/workflow", + "version": "1.0.0", "source": { "kind": "asset", - "assetId": "ast_workspace_builtins", + "assetId": "ast_tool_registry", "package": { "format": "tarball", - "path": "tarballs/@intx-tools-mail-0.1.2.tgz", + "path": "tarballs/@acme-workflow-1.0.0.tgz", "integrity": "sha512-..." } } } ``` -The loader joins `source.package.path` against the materialized asset +The tarball fetcher joins `source.package.path` against the materialized asset root on the sidecar to locate the bytes. ## Mount table -The loader takes a `Map`, for example -`ast_workspace_builtins` to `source-assets/ast_workspace_builtins/`. It uses +Materialization takes a `Map`, for example +`ast_tool_registry` to `source-assets/ast_tool_registry/`. It uses the map to resolve every `kind: "asset"` entry — looking up the mount path by `source.assetId`, then joining it with `source.package.path` against the workspace's asset root. ## Per-instance apply state on the sidecar -Each apply materializes its resolved closure into a stable, per-deploy-id -directory under the agent's tool-package instance directory -(`/tool-packages/`). The directory is never renamed: +Each apply materializes a workflow deployment's frozen closure into a +fixed directory under the deployment's instance directory +(`/workflow-definition-closures//`): ``` -/tool-packages/ -├── active-deploy-id # commit switch; content "v1:" -├── active-deploy-id.dirty # present only after a degraded persist +/ └── packages/ - ├── / # store///... for the live deploy - └── / # the immediately-prior deploy, retained + └── store///... ``` -The loader builds `packages//store///` and -dynamic-imports each pinned package's `interchange.tools` entry from that -path. Because the path is never renamed, a package that resolves files -relative to its own on-disk location at run time — `import.meta.url`, -`require.resolve()`, a call-time `await import("./sibling.js")` — keeps -resolving for the life of the deploy. - -**Commit.** Staging a deploy directory does not make it live. The commit -is a single write of the `active-deploy-id` file naming the new deploy -id. Until that write lands, the instance is still running the previous -deploy, whose directory is untouched. There is no filesystem rename in -the apply path; the `active-deploy-id` write is the only atomic switch. - -**Retention.** A prelude sweep at the start of every apply removes every -`packages//` except the current deploy and the immediately-prior -one, bounding disk to ~2 closures. The prior deploy is retained as a -liveness window: a session still draining against it may perform a -call-time import into its tree. The next apply reaps it once the prior -session's harness has been torn down. - -**Crash safety.** Boot never reads a deploy directory — the harness -rebuilds from the current manifest into a fresh deploy id — so a -half-written deploy directory left by a crash is self-healing: the next -boot re-materializes and the prelude sweep reclaims the orphan. Only -`active-deploy-id` needs durability; it carries an fsync + dirty-marker -ladder owned by the sidecar's materialization layer. See -`packages/tool-packaging/src/atomic-apply.ts` for the full protocol. +The workflow child evaluates the closure from +`packages/store///`. Nothing moves the tree once it is laid +out, so a package that resolves files relative to its own on-disk location +at run time — `import.meta.url`, `require.resolve()`, a call-time +`await import("./sibling.js")` — keeps resolving for the life of the +deployment. + +**Retention.** A deployment keeps one closure on disk. An apply clears +`packages/` before it lays anything out. + +**Crash safety.** Restoring a deployment never reads a closure back. It +applies the closure again, which clears `packages/` first, so a +half-written directory left by a crash cannot reach a child. See `packages/tool-packaging/src/workflow-closure-apply.ts` +for the apply. ## Tarballs - Bytes are exactly what an npm registry would serve: the package tree rooted under `package/`, gzipped tar. -- Integrity is verified by the sidecar loader against the entry's +- Integrity is verified by the sidecar against the entry's `source.package.integrity` field before extraction. That SRI hash, in the closure the hub froze, is what the sidecar trusts; the asset repo's commit signatures are not verified at runtime. -- The workspace's bundled tarballs (`bin/build-builtins.ts`) emit a - packed `package.json` with `dependencies`, `devDependencies`, - `optionalDependencies`, and `exports` deleted: every workspace and - catalog dependency is inlined into a single bundled entry, and the - tarball carries no `exports` so it cannot be loaded by anything - other than the tool-package loader. A consumer attempting - `import "@intx/tools-mail"` from outside the loader would fail to - resolve a public subpath. Operators publishing their own packages - through the asset substrate can preserve `exports` if they want - standalone consumability; the workspace builtins deliberately do - not. ## REST surface @@ -147,9 +118,6 @@ ladder owned by the sidecar's materialization layer. See | List tarballs | `GET /api/tenants/:tid/assets/:assetId/tarballs` | | Delete a tarball | `DELETE /api/tenants/:tid/assets/:assetId/tarballs/:filename` | -`bin/publish-tool-packages.ts` is the reference REST client for this -surface. - ## Registry vs asset | Concern | Asset-sourced | Registry-sourced | diff --git a/packages/tool-packaging/README.md b/packages/tool-packaging/README.md index 2deeb4669..a24223e10 100644 --- a/packages/tool-packaging/README.md +++ b/packages/tool-packaging/README.md @@ -1,22 +1,22 @@ # @intx/tool-packaging -Hub-side closure resolver and sidecar-side loader for the npm-distributed +Hub-side closure resolver and sidecar-side closure materialization for the npm-distributed tool-package format. The contract below describes what the layered -pipeline actually guarantees end-to-end; downstream code (the apply -protocol, the session service, the deploy-flow integration tests) leans +pipeline actually guarantees end-to-end; downstream code (the session +service, the deploy-flow integration tests) leans on it as written. -## Loader resolution contract +## Closure resolution contract -A tool package's `interchange.tools` entry, and the JS body it pulls in -transitively, may `import` (or `require`) any bare specifier the -manifest's resolved closure satisfies. The loader builds a per-instance +A closure package's body, and the JS it pulls in transitively, may +`import` (or `require`) any bare specifier the manifest's resolved +closure satisfies. Materialization builds a per-instance nested `node_modules/` layout under the scratch directory so each package's direct dependencies are reachable through Node's standard ancestor walk; the layout composes recursively, so a transitive's own transitives are reachable from inside it. -Concretely the loader: +Concretely, materialization: 1. Walks every manifest entry and validates that each one is source-consistent: every entry the manifest says lives at a @@ -41,9 +41,8 @@ Concretely the loader: `store///` chosen for that requirer. Symlinks at the `node_modules/` boundary let Node's realpath-based resolver walk to the dep's own layout dir (which has its own `node_modules/`). -4. Dynamically imports each top-level entry's `interchange.tools` - module and collects the `AnnotatedToolFactory` / - `AnnotatedPluginFactory` values it exports. + +Nothing in the closure is imported here; the workflow child evaluates it. Properties that fall out of this layout: @@ -58,12 +57,10 @@ Properties that fall out of this layout: store entry (one symlink target per `(name, version)`); divergent versions live as separate store entries and Node's ancestor walk picks the right one for each requirer. -- **Runtime-agnostic tarballs.** The loader does no Bun-specific - resolution. A tool package whose `interchange.tools` entry is a - `.js` file with no workspace-only deps loads identically on Node - and Bun. The workspace's built-in tool packages bundle their entries - through `bin/build-builtins.ts` precisely so the produced tarballs - satisfy this property. +- **Runtime-agnostic tarballs.** The layout does no Bun-specific + resolution. A package whose entry is a `.js` file with no + workspace-only deps loads identically on Node + and Bun. ## Optional-dependency semantics @@ -74,8 +71,7 @@ fetch — the whole subtree under that edge is dropped from the manifest silently. A hard dep of an optional dep is part of that optional subtree; a failure there drops the parent too, instead of leaving a parent in the closure whose own hard requirement is missing. Failures -outside any optional subtree abort the walk and surface as a -`manifest.invalid` deploy-apply error. +outside any optional subtree abort the walk and fail the deploy. ## Asset modification race @@ -88,15 +84,10 @@ deploy. The sidecar still verifies each tarball's SRI before it extracts it. Tool packages may resolve files relative to their own on-disk location at run time — `import.meta.url`, `require.resolve()`, or an `await import(new URL("./sibling.js", import.meta.url))` evaluated long -after the module first loaded. The loader imports each top-level -package's `interchange.tools` entry from a path under -`/packages//` that is never renamed, so a -module's `import.meta.url` stays valid for the life of the deploy. - -The prior deploy's directory is retained across the next apply, so a -session still draining against an older deploy keeps resolving its own -files until a further apply supersedes it. See `ASSET-LAYOUT.md` for -the on-disk layout and retention rules. +after the module first loaded. A closure's packages run from a path +under `/packages/` that nothing moves, so a module's +`import.meta.url` stays valid for the life of the deployment. +See `ASSET-LAYOUT.md` for the on-disk layout and retention rules. ## Follow-up reference diff --git a/packages/tool-packaging/package.json b/packages/tool-packaging/package.json index 9b0bd9d4f..53a563641 100644 --- a/packages/tool-packaging/package.json +++ b/packages/tool-packaging/package.json @@ -1,6 +1,6 @@ { "name": "@intx/tool-packaging", - "description": "Resolver and loader for the npm-distributed Interchange tool-package format", + "description": "Resolver and closure materialization for the npm-distributed Interchange tool-package format", "version": "0.5.0", "license": "LGPL-2.1-only", "type": "module", @@ -12,7 +12,6 @@ } }, "dependencies": { - "@intx/agent": "workspace:*", "@intx/log": "workspace:*", "@intx/storage-isogit": "workspace:*", "@intx/types": "workspace:*", @@ -21,13 +20,13 @@ "npm-package-arg": "^12.0.2", "npm-pick-manifest": "^10.0.0", "npm-registry-fetch": "^19.0.0", - "semver": "catalog:", + "semver": "^7.7.2", "ssri": "catalog:", "tar": "catalog:" }, "devDependencies": { "@types/npm-package-arg": "^6.1.4", - "@types/semver": "catalog:", + "@types/semver": "^7.7.1", "@types/ssri": "catalog:" }, "files": [ diff --git a/packages/tool-packaging/src/atomic-apply.test.ts b/packages/tool-packaging/src/atomic-apply.test.ts deleted file mode 100644 index e9be09ab6..000000000 --- a/packages/tool-packaging/src/atomic-apply.test.ts +++ /dev/null @@ -1,384 +0,0 @@ -import { describe, test, expect, beforeEach, afterEach } from "bun:test"; -import { promises as fs } from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { type AnnotatedPluginFactory, definePlugin } from "@intx/agent"; -import type { DeployApplyErrorCategory } from "@intx/types/sidecar"; -import type { ToolPackageManifest } from "@intx/types/tool-packages"; - -import { applyAtomic } from "./atomic-apply"; -import { - type LoadedToolFactory, - type LoadedToolPackage, - type ToolLoader, - ToolLoaderError, -} from "./loader"; - -let scratch: string; -let instanceDir: string; -let assetRoot: string; - -beforeEach(async () => { - scratch = await fs.mkdtemp(path.join(os.tmpdir(), "tool-packaging-atomic-")); - instanceDir = path.join(scratch, "instance"); - assetRoot = path.join(scratch, "asset"); - await fs.mkdir(instanceDir, { recursive: true }); - await fs.mkdir(assetRoot, { recursive: true }); -}); - -afterEach(async () => { - await fs.rm(scratch, { recursive: true, force: true }); -}); - -function deployDir(id: string): string { - return path.join(instanceDir, "packages", id); -} - -async function seedDeploy(id: string, marker: string): Promise { - const dir = deployDir(id); - await fs.mkdir(dir, { recursive: true }); - const markerPath = path.join(dir, "marker"); - await fs.writeFile(markerPath, marker); - return markerPath; -} - -function fakeFactory(id: string): LoadedToolFactory { - const fn = () => ({ - definitions: [], - run: async () => ({ callId: "stub", content: "ok" }), - }); - return Object.assign(fn, { - id, - requires: [] as readonly string[], - definitions: [], - }); -} - -function fakePlugin(id: string): AnnotatedPluginFactory { - return definePlugin({ id, factory: () => ({}) }); -} - -function makeStubLoader( - impl: (args: { instanceScratchDir: string }) => Promise, -): ToolLoader { - return { - loadManifest: async (args) => { - // Write a sentinel file into the scratch dir so the test can - // observe what staged before the loader returned. - await fs.writeFile( - path.join(args.instanceScratchDir, "loader-sentinel"), - "ran", - ); - return impl({ instanceScratchDir: args.instanceScratchDir }); - }, - }; -} - -const minimalManifest: ToolPackageManifest = { - schemaVersion: "1", - topLevel: [], - entries: [], -}; - -async function dirExists(p: string): Promise { - try { - await fs.access(p); - return true; - } catch { - return false; - } -} - -describe("applyAtomic success", () => { - test("stages a per-deploy-id directory and returns its path and the new id", async () => { - const loader = makeStubLoader(async () => [ - { - name: "foo", - version: "1.0.0", - plugins: [], - factories: [fakeFactory("@foo/a")], - directors: [], - credentials: [], - }, - ]); - const result = await applyAtomic({ - manifest: minimalManifest, - loader, - instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - attemptId: "atp_1", - previousDeployId: "dpl_0", - newDeployId: "dpl_1", - }); - expect(result.status).toBe("ok"); - if (result.status !== "ok") return; - expect(result.activeDeployId).toBe("dpl_1"); - expect(result.deployDir).toBe(deployDir("dpl_1")); - expect(result.loaded).toHaveLength(1); - expect(await dirExists(deployDir("dpl_1"))).toBe(true); - // The loader staged into the deploy dir, not a swappable path. - expect( - await dirExists(path.join(deployDir("dpl_1"), "loader-sentinel")), - ).toBe(true); - // The protocol no longer renames; there is no active/pending tree. - expect(await dirExists(path.join(instanceDir, "active"))).toBe(false); - expect(await dirExists(path.join(instanceDir, "pending"))).toBe(false); - }); - - test("retains the previous deploy directory across the next apply", async () => { - const loader = makeStubLoader(async () => []); - await applyAtomic({ - manifest: minimalManifest, - loader, - instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - attemptId: "atp_1", - previousDeployId: "dpl_0", - newDeployId: "dpl_1", - }); - // Marker so we can confirm the dpl_1 tree survives the next apply. - await fs.writeFile(path.join(deployDir("dpl_1"), "marker"), "first"); - - await applyAtomic({ - manifest: minimalManifest, - loader, - instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - attemptId: "atp_2", - previousDeployId: "dpl_1", - newDeployId: "dpl_2", - }); - expect(await dirExists(deployDir("dpl_2"))).toBe(true); - // dpl_1 is the previous deploy and is retained untouched. - expect( - await fs.readFile(path.join(deployDir("dpl_1"), "marker"), "utf8"), - ).toBe("first"); - }); - - test("prelude sweep reaps every deploy except current and previous", async () => { - // Seed a stale deploy (two generations back) and the immediately - // prior deploy. The apply must reap the stale one and keep the - // prior one. - const stalePath = await seedDeploy("dpl_stale", "two-back"); - const prevPath = await seedDeploy("dpl_prev", "one-back"); - - const loader = makeStubLoader(async () => []); - const result = await applyAtomic({ - manifest: minimalManifest, - loader, - instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - attemptId: "atp_S", - previousDeployId: "dpl_prev", - newDeployId: "dpl_new", - }); - expect(result.status).toBe("ok"); - expect(await dirExists(deployDir("dpl_stale"))).toBe(false); - expect(await dirExists(stalePath)).toBe(false); - // previous and current survive. - expect(await fs.readFile(prevPath, "utf8")).toBe("one-back"); - expect(await dirExists(deployDir("dpl_new"))).toBe(true); - }); -}); - -describe("applyAtomic failure: every loader error category", () => { - const categories: DeployApplyErrorCategory[] = [ - "tarball.missing", - "integrity.mismatch", - "registry.fetch.failed", - "registry.unknown", - "registry.auth.failed", - "tarball.extract.failed", - "manifest.invalid", - "package.entry.missing", - "package.entry.invalid", - "factory.construct.failed", - ]; - - for (const category of categories) { - test(`category ${category}: deploy dir discarded, previous untouched, error carries previousDeployId`, async () => { - // Seed a prior deploy to confirm it is preserved. - const priorMarker = await seedDeploy("dpl_prior", "untouched"); - - const loader: ToolLoader = { - loadManifest: async () => { - throw new ToolLoaderError({ - category, - message: `induced failure: ${category}`, - package: { name: "p", version: "1.0.0" }, - }); - }, - }; - const result = await applyAtomic({ - manifest: minimalManifest, - loader, - instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - attemptId: "atp_X", - previousDeployId: "dpl_prior", - newDeployId: "dpl_attempted", - }); - expect(result.status).toBe("failed"); - if (result.status !== "failed") return; - expect(result.category).toBe(category); - expect(result.previousDeployId).toBe("dpl_prior"); - expect(result.attemptId).toBe("atp_X"); - expect(result.package).toEqual({ name: "p", version: "1.0.0" }); - - // Atomicity invariant: the prior deploy is unchanged. - expect(await fs.readFile(priorMarker, "utf8")).toBe("untouched"); - // The attempted deploy dir has been removed. - expect(await dirExists(deployDir("dpl_attempted"))).toBe(false); - }); - } -}); - -describe("applyAtomic failure: tool.name.duplicate", () => { - test("two factories with the same id across packages → tool.name.duplicate", async () => { - const priorMarker = await seedDeploy("dpl_prior", "untouched"); - const loader = makeStubLoader(async () => [ - { - name: "a", - version: "1.0.0", - plugins: [], - factories: [fakeFactory("@dup/x")], - directors: [], - credentials: [], - }, - { - name: "b", - version: "1.0.0", - plugins: [], - factories: [fakeFactory("@dup/x")], - directors: [], - credentials: [], - }, - ]); - const result = await applyAtomic({ - manifest: minimalManifest, - loader, - instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - attemptId: "atp_D", - previousDeployId: "dpl_prior", - newDeployId: "dpl_attempted", - }); - expect(result.status).toBe("failed"); - if (result.status !== "failed") return; - expect(result.category).toBe("tool.name.duplicate"); - expect(result.previousDeployId).toBe("dpl_prior"); - expect(await dirExists(deployDir("dpl_attempted"))).toBe(false); - expect(await fs.readFile(priorMarker, "utf8")).toBe("untouched"); - }); - - test("two plugins with the same id across packages → tool.name.duplicate", async () => { - const priorMarker = await seedDeploy("dpl_prior", "untouched"); - const loader = makeStubLoader(async () => [ - { - name: "a", - version: "1.0.0", - plugins: [fakePlugin("dup/plug")], - factories: [], - directors: [], - credentials: [], - }, - { - name: "b", - version: "2.0.0", - plugins: [fakePlugin("dup/plug")], - factories: [], - directors: [], - credentials: [], - }, - ]); - const result = await applyAtomic({ - manifest: minimalManifest, - loader, - instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - attemptId: "atp_DP", - previousDeployId: "dpl_prior", - newDeployId: "dpl_attempted", - }); - expect(result.status).toBe("failed"); - if (result.status !== "failed") return; - expect(result.category).toBe("tool.name.duplicate"); - expect(result.message).toContain("plugin factory id dup/plug"); - // The collision is attributed to the second package that carried - // the already-seen plugin id. - expect(result.package).toEqual({ name: "b", version: "2.0.0" }); - expect(result.previousDeployId).toBe("dpl_prior"); - expect(await dirExists(deployDir("dpl_attempted"))).toBe(false); - expect(await fs.readFile(priorMarker, "utf8")).toBe("untouched"); - }); -}); - -describe("applyAtomic failure: unexpected error shape", () => { - test("loader throwing a plain Error becomes factory.construct.failed", async () => { - const loader: ToolLoader = { - loadManifest: async () => { - throw new Error("something exploded"); - }, - }; - const result = await applyAtomic({ - manifest: minimalManifest, - loader, - instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - attemptId: "atp_U", - previousDeployId: "dpl_prior", - newDeployId: "dpl_attempted", - }); - expect(result.status).toBe("failed"); - if (result.status !== "failed") return; - expect(result.category).toBe("factory.construct.failed"); - expect(result.message).toContain("something exploded"); - expect(await dirExists(deployDir("dpl_attempted"))).toBe(false); - }); -}); - -describe("applyAtomic clears a stale deploy dir before staging", () => { - test("a leftover directory under the new deploy id does not leak into the staged tree", async () => { - // A crash mid-build (or uuid reuse) can leave a partial tree under - // the exact id this apply is about to build. The prelude must clear - // it so the loader stages into a clean directory. - await fs.mkdir(deployDir("dpl_new"), { recursive: true }); - await fs.writeFile(path.join(deployDir("dpl_new"), "stale"), "old"); - const loader = makeStubLoader(async () => []); - const result = await applyAtomic({ - manifest: minimalManifest, - loader, - instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - attemptId: "atp_L", - previousDeployId: "dpl_prior", - newDeployId: "dpl_new", - }); - expect(result.status).toBe("ok"); - expect(await dirExists(path.join(deployDir("dpl_new"), "stale"))).toBe( - false, - ); - expect( - await dirExists(path.join(deployDir("dpl_new"), "loader-sentinel")), - ).toBe(true); - }); -}); diff --git a/packages/tool-packaging/src/atomic-apply.ts b/packages/tool-packaging/src/atomic-apply.ts deleted file mode 100644 index d30f2cb61..000000000 --- a/packages/tool-packaging/src/atomic-apply.ts +++ /dev/null @@ -1,300 +0,0 @@ -// Apply protocol for a ToolPackageManifest. -// -// Each apply materializes its closure into a stable, per-deploy-id -// directory that is never moved: -// -// /packages//store///... -// -// The loader dynamic-imports each pinned package's `interchange.tools` -// module from an absolute path inside that deploy-id directory. Because -// the directory is never renamed, the URL Node keys its ESM module -// cache under stays valid for the life of the deploy: a tool package -// that uses `import.meta.url`, `require.resolve()`, or a call-time -// `await import("./sibling.js")` resolves against a path that still -// exists. (The earlier protocol staged under `pending/` and renamed -// `pending/` → `active/` after import; the renamed-away URL ENOENTed -// any late path resolution. This module exists to remove that swap.) -// -// This module does NOT commit. It stages, loads, validates, and -// returns the loaded packages plus the deploy-id directory. The commit -// point lives one layer up: the caller writes the instance's -// `active-deploy-id` file to name `newDeployId`. Until that write -// lands, the instance is still running `previousDeployId`, whose -// directory is left untouched here. The "atomic swap" is therefore the -// single `active-deploy-id` file write the caller performs, not a -// directory rename. -// -// Retention. A prelude sweep removes every `packages//` except -// `{newDeployId, previousDeployId}`. The keep set is invariant across -// the caller's commit: before the commit the live deploy is -// `previousDeployId`; after it the live deploy is `newDeployId` and -// `previousDeployId` is the retained prior tree. So a single sweep at -// the start of the apply both bounds disk to ~2 closures and preserves -// exactly the prior deploy as a liveness window for any session still -// draining against it. -// -// Retention invariant: the previous deploy's directory must survive -// until the next apply, because a session built against it may still -// perform a call-time `await import()` into its tree. The next apply's -// prelude reaps it. This is safe only because per-agent applies are -// serialized (the hub runs one apply per `agentAddress` at a time) and -// each successful apply replaces the running harness, so by the time -// apply N+2 reaps deploy N, generation-N's harness teardown has -// completed and no live code references deploy N's tree. -// -// Caller responsibilities (not handled here): -// -// - Writing `active-deploy-id` to commit the staged deploy. -// - Constructing the WebSocket frame from the returned error. -// - Writing the rejected manifest + error to the sidecar's git audit -// trail under `audit/rejected-applies//`. -// -// Crash safety. Boot never reads a deploy-id directory: the harness -// rebuilds by re-running the apply against the current manifest into a -// fresh deploy id. A half-written `packages//` left by a -// crash mid-build is therefore self-healing — the next boot -// re-materializes into a new id and the prelude sweep reclaims the -// orphan. Only `active-deploy-id` needs durability, and the caller owns -// that. Consequently this module fsyncs nothing. -// -// Cross-apply ESM module-identity: Node keys its ESM cache by resolved -// URL. Per-deploy-id directories already make every apply's import URL -// unique, so a reused `(name, version)` whose bytes changed across -// applies resolves to a distinct path. The loader additionally -// cache-busts each import URL with an `?integrity=` query string; -// that remains correct (and harmless) under per-deploy-id paths. - -import { promises as fs } from "node:fs"; -import path from "node:path"; - -import { getLogger } from "@intx/log"; -import type { DeployApplyErrorCategory } from "@intx/types/sidecar"; -import type { ToolPackageManifest } from "@intx/types/tool-packages"; - -import { - type LoadedToolPackage, - type ToolLoader, - ToolLoaderError, -} from "./loader"; - -const logger = getLogger(["sidecar", "tool-packaging", "atomic-apply"]); - -const PACKAGES_DIR = "packages"; - -export interface ApplyAtomicArgs { - readonly manifest: ToolPackageManifest; - readonly loader: ToolLoader; - readonly instanceDir: string; - readonly assetRoot: string; - /** - * Maps a `source.assetId` to a workspace-relative mount path. The - * loader uses this to resolve `kind: "asset"` manifest entries. - * Forwarded verbatim to `ToolLoader.loadManifest`. - */ - readonly assetMounts: ReadonlyMap; - /** - * Maps a source-format asset entry's `source.assetId` to an absolute - * indexed git directory. Forwarded verbatim to `ToolLoader.loadManifest`. - */ - readonly gitDirs: ReadonlyMap; - readonly attemptId: string; - /** - * The deploy id the instance is currently running. Retained on disk - * through this apply (its `packages//` tree is not - * swept) and carried back on a failure so the caller can confirm the - * prior deploy is unchanged. - */ - readonly previousDeployId: string; - /** New deploy id; on the caller's commit becomes the active deploy id. */ - readonly newDeployId: string; -} - -export interface ApplyAtomicSuccess { - readonly status: "ok"; - readonly activeDeployId: string; - /** Absolute path of the staged, never-renamed deploy directory. */ - readonly deployDir: string; - readonly loaded: readonly LoadedToolPackage[]; -} - -export interface ApplyAtomicFailure { - readonly status: "failed"; - readonly category: DeployApplyErrorCategory; - readonly message: string; - readonly package?: { readonly name: string; readonly version: string }; - /** - * The deploy id the instance is still running. The apply never wrote - * `active-deploy-id`, so a failure always leaves the prior deploy - * live: this equals the input `previousDeployId`. (The caller owns - * the commit and is the only layer that can advance the active id; - * the persist-degraded case it handles there carries its own - * inverted meaning, but that case does not originate in this module.) - */ - readonly previousDeployId: string; - readonly attemptId: string; - readonly occurredAt: string; -} - -export type ApplyAtomicResult = ApplyAtomicSuccess | ApplyAtomicFailure; - -/** - * Stage a tool-package manifest into a per-deploy-id directory under - * `instanceDir` and return the loaded packages. The orchestration is - * single-threaded per instance: the prelude sweep assumes exclusive - * write access to `/packages/`. The hub-side session - * manager already serializes applies per agent (one apply per - * `agentAddress` at a time); host-side callers that bypass that - * serialization must provide their own per-`instanceDir` lock, because - * the prelude sweep deletes sibling deploy directories and a racing - * apply could delete a directory the other just committed. - */ -export async function applyAtomic( - args: ApplyAtomicArgs, -): Promise { - const packagesDir = path.join(args.instanceDir, PACKAGES_DIR); - const deployDir = path.join(packagesDir, args.newDeployId); - - // Prelude sweep. Reclaim every prior deploy directory except the one - // we are about to build (`newDeployId`) and the one still live - // (`previousDeployId`). `previousDeployId` may be the "no prior - // deploy" sentinel, which simply matches no directory. The sweep is - // best-effort per stray: an EIO/EPERM reclaiming one old deploy is a - // disk-reclamation concern, not a correctness one, and must not fail - // an otherwise-valid apply — the next apply's prelude retries the - // reclaim. The deploy directory we then build, by contrast, must be - // a clean tree, so its removal+mkdir below propagate on failure. - const keep = new Set([args.newDeployId, args.previousDeployId]); - let existing: string[]; - try { - existing = await fs.readdir(packagesDir); - } catch (err) { - if (!isENOENT(err)) throw err; - existing = []; - } - await Promise.all( - existing - .filter((id) => !keep.has(id)) - .map(async (id) => { - try { - await fs.rm(path.join(packagesDir, id), { - recursive: true, - force: true, - }); - } catch (err) { - logger.warn`apply prelude sweep could not reclaim stale deploy ${id} under ${packagesDir}: ${err instanceof Error ? err.message : String(err)}; next apply will retry`; - } - }), - ); - - // A leftover directory under this exact `newDeployId` (a crash mid- - // build, or the astronomically unlikely uuid reuse) must be cleared - // before staging so the loader builds into a clean tree. - await fs.rm(deployDir, { recursive: true, force: true }); - await fs.mkdir(deployDir, { recursive: true }); - - let loaded: readonly LoadedToolPackage[]; - try { - loaded = await args.loader.loadManifest({ - manifest: args.manifest, - instanceScratchDir: deployDir, - assetRoot: args.assetRoot, - assetMounts: args.assetMounts, - gitDirs: args.gitDirs, - }); - } catch (err) { - await fs.rm(deployDir, { recursive: true, force: true }); - if (err instanceof ToolLoaderError) { - logger.warn`apply rejected (${err.category}) for attempt ${args.attemptId}; previous deploy ${args.previousDeployId} retained`; - const out: ApplyAtomicFailure = { - status: "failed", - category: err.category, - message: err.message, - ...(err.package !== undefined ? { package: err.package } : {}), - previousDeployId: args.previousDeployId, - attemptId: args.attemptId, - occurredAt: new Date().toISOString(), - }; - return out; - } - // Unknown error shape: surface as factory.construct.failed since - // that is the closest catch-all in the taxonomy. - logger.error`unexpected loader error for attempt ${args.attemptId}: ${err instanceof Error ? err.message : String(err)}`; - return { - status: "failed", - category: "factory.construct.failed", - message: err instanceof Error ? err.message : String(err), - previousDeployId: args.previousDeployId, - attemptId: args.attemptId, - occurredAt: new Date().toISOString(), - }; - } - - // Check for duplicate factory ids across loaded bundles. The plan's - // taxonomy distinguishes `tool.name.duplicate` from other - // categories, so this check fires after a successful load but before - // the caller commits. The loader has already prefixed each tool - // factory id with its bundle id by the time we see it here, so a - // collision in `factory.id` means two pinned packages shared a bundle - // id (the per-bundle prefix did not produce unique ids across the - // load). - // - // Plugin factories carry their own `id` (not bundle-prefixed) and - // are addressed by id at harness construction; a collision between - // two plugin ids would resolve to undefined behavior at the harness - // layer. Treat that as the same apply-time gate: tool and plugin - // id spaces are tracked separately so the operator-facing message - // points at the right surface, but neither admits a duplicate. - const toolIdsSeen = new Set(); - const pluginIdsSeen = new Set(); - for (const pkg of loaded) { - for (const factory of pkg.factories) { - if (toolIdsSeen.has(factory.id)) { - await fs.rm(deployDir, { recursive: true, force: true }); - const out: ApplyAtomicFailure = { - status: "failed", - category: "tool.name.duplicate", - message: `tool factory id ${factory.id} appears in more than one pinned package`, - package: { name: pkg.name, version: pkg.version }, - previousDeployId: args.previousDeployId, - attemptId: args.attemptId, - occurredAt: new Date().toISOString(), - }; - return out; - } - toolIdsSeen.add(factory.id); - } - for (const plugin of pkg.plugins) { - if (pluginIdsSeen.has(plugin.id)) { - await fs.rm(deployDir, { recursive: true, force: true }); - const out: ApplyAtomicFailure = { - status: "failed", - category: "tool.name.duplicate", - message: `plugin factory id ${plugin.id} appears in more than one pinned package`, - package: { name: pkg.name, version: pkg.version }, - previousDeployId: args.previousDeployId, - attemptId: args.attemptId, - occurredAt: new Date().toISOString(), - }; - return out; - } - pluginIdsSeen.add(plugin.id); - } - } - - logger.info`apply staged: attempt ${args.attemptId} ready as ${args.newDeployId} (caller commits via active-deploy-id)`; - return { - status: "ok", - activeDeployId: args.newDeployId, - deployDir, - loaded, - }; -} - -function isENOENT(err: unknown): boolean { - return ( - err !== null && - typeof err === "object" && - "code" in err && - (err as { code: unknown }).code === "ENOENT" - ); -} diff --git a/packages/tool-packaging/src/cache.test.ts b/packages/tool-packaging/src/cache.test.ts index 2431e28e3..795fcf769 100644 --- a/packages/tool-packaging/src/cache.test.ts +++ b/packages/tool-packaging/src/cache.test.ts @@ -157,7 +157,7 @@ describe("LRU eviction when over cap", () => { // A single entry bigger than the cap would, without protection, // be the only candidate the LRU sweep finds: the just-written // tarball + just-extracted tree would be deleted before the - // loader could read it back, and the next apply would re-fetch + // materialization could read it back, and the next apply would re-fetch // and re-evict — perpetual churn. The just-written entry is // exempt from the sweep that its own extractTarball triggers. const big = await packFixtureTarball(scratch, { @@ -439,8 +439,11 @@ describe("LRU eviction when over cap", () => { spy.mockRestore(); for (const [index, result] of results.entries()) { if (result.status === "fulfilled" && result.value !== undefined) { + // Only a cap eviction still has a reclaim waiting on this release. const removed = - index === 1 ? removalOf(result.value.dir) : undefined; + index === 1 && eviction === "cap" + ? removalOf(result.value.dir) + : undefined; result.value.release(); await removed; } @@ -596,8 +599,8 @@ describe("LRU eviction when over cap", () => { handle.release(); }); - test("extractTarball advances atime so loader-pattern hits beat the LRU sweep", async () => { - // The loader's hot path is `cache.has` + `cache.extractTarball` — + test("extractTarball advances atime so materialization-pattern hits beat the LRU sweep", async () => { + // Materialization's hot path is `cache.has` + `cache.extractTarball` — // `cache.get` is not in that flow. `cache.get` explicitly calls // `fs.utimes` so LRU ordering reflects access even on // noatime/relatime mounts; `extractTarball` must do the same or @@ -626,8 +629,8 @@ describe("LRU eviction when over cap", () => { (await cache.extractTarball(b.integrity)).release(); await new Promise((r) => setTimeout(r, 25)); - // Touch `a` ONLY through the loader-shaped path (has + extract). - // This is the access pattern `materialize` uses on a cache hit; + // Touch `a` ONLY through the materialization-shaped path (has + extract). + // This is the access pattern `materializeClosure` uses on a cache hit; // it must advance `a`'s atime so the next cap sweep treats `a` // as the most-recently-used. expect(await cache.has(a.integrity)).toBe(true); @@ -823,8 +826,8 @@ describe("extractTarball", () => { // another agent in the same sidecar process trips an // integrity-mismatch evict for the same integrity. The cache holds // a refcount per (integrity, extraction-dir) pair: `evict` deletes - // the tarball blob immediately so a subsequent `extractTarball` - // cannot reuse the on-disk extraction, but the physical removal of + // the tarball blob immediately and a subsequent `extractTarball` + // refuses the on-disk extraction, but the physical removal of // the extraction tree is deferred until every outstanding // `release` from `extractTarball` has fired. const cache = createTarballCache({ @@ -841,10 +844,8 @@ describe("extractTarball", () => { expect(await fs.readdir(reader.dir)).toContain("package.json"); // Evict races against the still-open reader. The tarball blob - // disappears immediately so a fresh `extractTarball` cannot - // short-circuit on the existing tree, but the tree itself must - // remain on disk so the in-flight reader's walk continues to - // succeed. + // disappears immediately, but the tree itself must remain on disk + // so the in-flight reader's walk continues to succeed. await cache.evict(integrity); expect(await cache.get(integrity)).toBeNull(); expect(await fs.readdir(reader.dir)).toContain("package.json"); @@ -873,6 +874,80 @@ describe("extractTarball", () => { expect(cleared).toBe(true); }); + test("extractTarball refuses an evicted extraction a reader still holds", async () => { + const cache = createTarballCache({ + rootDir: scratch, + maxBytes: 10_000_000, + }); + const { bytes, integrity } = await packFixtureTarball(scratch, { + "package.json": JSON.stringify({ name: "foo", version: "1.0.0" }), + }); + await cache.put(integrity, bytes); + const reader = await cache.extractTarball(integrity); + await cache.evict(integrity); + + // Re-fetched bytes do not make the tree still on disk reusable. + await cache.put(integrity, bytes); + await expect(cache.extractTarball(integrity)).rejects.toThrow( + /was evicted/, + ); + expect(await fs.readdir(reader.dir)).toContain("package.json"); + + // The refused call gave its reference back, so this release is the last + // one and the tree goes. + reader.release(); + for (let attempt = 0; attempt < 50; attempt++) { + const present = await fs.access(reader.dir).then( + () => true, + () => false, + ); + if (!present) break; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + + const fresh = await cache.extractTarball(integrity); + expect(await fs.readdir(fresh.dir)).toContain("package.json"); + fresh.release(); + }); + + test("a pin alone does not keep an evicted extraction from being replaced", async () => { + const cache = createTarballCache({ + rootDir: scratch, + maxBytes: 10_000_000, + }); + const { bytes, integrity } = await packFixtureTarball(scratch, { + "package.json": JSON.stringify({ name: "foo", version: "1.0.0" }), + }); + await cache.put(integrity, bytes); + const stale = await cache.extractTarball(integrity); + const marker = path.join(stale.dir, "poisoned"); + await fs.writeFile(marker, ""); + stale.release(); + + // The pin holds the entry through the evict, the way a materialization + // that hit a corrupt tarball does, but nothing is walking the tree. + const unpin = await cache.pin(integrity); + await cache.evict(integrity); + await cache.put(integrity, bytes); + + const first = await cache.extractTarball(integrity); + expect(await fs.readdir(first.dir)).toContain("package.json"); + expect(await fs.readdir(first.dir)).not.toContain("poisoned"); + await fs.writeFile(path.join(first.dir, "fresh"), ""); + // That tree came from verified bytes, so a second reader may share it. + const second = await cache.extractTarball(integrity); + expect(second.dir).toBe(first.dir); + + first.release(); + second.release(); + unpin(); + + // Replacing the tree was the reclaim, so the last release leaves it alone. + const next = await cache.extractTarball(integrity); + expect(await fs.readdir(next.dir)).toContain("fresh"); + next.release(); + }); + test("refcount tracks each handle in isolation across sequential acquire/release cycles", async () => { // Each extractTarball acquires the refcount; each handle.release() // decrements it. Cycling through acquire → release → acquire → diff --git a/packages/tool-packaging/src/cache.ts b/packages/tool-packaging/src/cache.ts index 3e3f96576..345a64e7e 100644 --- a/packages/tool-packaging/src/cache.ts +++ b/packages/tool-packaging/src/cache.ts @@ -11,10 +11,10 @@ // one entry (used after extraction discovers a corrupted file on disk). // // `maxBytes` covers both the tarball bytes and the size of the -// extracted directory tree the loader copies from. The extraction +// extracted directory tree materialization copies from. The extraction // tree dominates disk usage in practice (tarballs are gzip-compressed; // the unpacked tree is multiples larger), so the cap reflects the cost -// of holding one cache entry but excludes the loader's per-instance +// of holding one cache entry but excludes materialization's per-instance // copies. // // Integrity is verified on store and re-verified inside @@ -25,7 +25,7 @@ // // `extractTarball` is the second face of the same on-disk store: it // unpacks the tarball into a sibling `extracted/` directory keyed by -// the same integrity, so the per-instance loader can copy from a +// the same integrity, so per-instance materialization can copy from a // stable, deduplicated extraction without re-doing the tar work on // every apply. The unpack is gated by a per-integrity tmp-and-rename // dance with the same crash-safety properties as `put`. @@ -79,19 +79,17 @@ export interface TarballCache { put(integrity: string, bytes: Uint8Array): Promise; /** * Mark the cache entry for `integrity` as poisoned and remove its - * tarball bytes immediately so a subsequent `extractTarball` call - * cannot reuse the on-disk extraction. The extraction directory's - * physical reclaim is deferred until every in-flight reader released - * by `extractTarball` has dropped its reference, so an evict that - * races a concurrent `copyTree` walk of the same extraction does - * not pull the tree out from under the walk and surface as ENOENT. + * tarball bytes immediately. The extraction directory's physical + * reclaim is deferred until every in-flight reader released by + * `extractTarball` has dropped its reference, so an evict that races + * a concurrent `copyTree` walk of the same extraction does not pull + * the tree out from under the walk and surface as ENOENT. * - * Until every reader releases, the on-disk extraction is left in - * place but is no longer reachable via a fresh `extractTarball` - * (because the tarball blob is gone). Callers that need the bytes - * back must re-fetch and `put` them, which will trigger a fresh - * extraction into a new directory once the deferred reclaim - * completes. + * While a reader holds it, the on-disk extraction stays in place and + * `extractTarball` refuses it, even if the bytes were `put` back. + * Callers that need the bytes back must re-fetch and `put` them, then + * extract once no reader holds the tree, which removes it and unpacks + * afresh. */ evict(integrity: string): Promise; /** @@ -107,7 +105,8 @@ export interface TarballCache { * fully-populated directory and each get their own `release` handle. * * Throws if `integrity` is not present in the cache; callers must - * `put` (or otherwise materialize) the bytes first. + * `put` (or otherwise materialize) the bytes first. Also throws while + * a reader holds an extraction that `evict` poisoned. */ extractTarball(integrity: string): Promise<{ readonly dir: string; @@ -181,6 +180,12 @@ export function createTarballCache(config: TarballCacheConfig): TarballCache { // intra-process races between two agents on the same sidecar. const extractionRefcounts = new Map(); const pendingEvictions = new Set(); + // Pending entries `evict` poisoned. A cap sweep's pending entry is not here: + // its tree is good, and whoever pinned it may still use it. + const poisonedExtractions = new Set(); + // The extractions past the poison check, or handed out and not yet released. + // Unlike a pin, each of these can be walking the tree. + const extractionReaders = new Map(); // Entries a cap sweep could not evict because they were held. Their last // release requests another pass; releases in one burst share that pass. const heldOverCap = new Set(); @@ -209,6 +214,29 @@ export function createTarballCache(config: TarballCacheConfig): TarballCache { } } + function countExtractionReader(integrity: string, delta: 1 | -1): void { + const next = (extractionReaders.get(integrity) ?? 0) + delta; + if (next > 0) extractionReaders.set(integrity, next); + else extractionReaders.delete(integrity); + } + + // A pin only keeps the entry from being swept, so with no reader left the + // poisoned tree can go and the caller's re-fetched bytes can unpack. That is + // the reclaim the last release was deferred to run, so it no longer has to. + async function discardPoisonedExtraction(integrity: string): Promise { + if (extractionReaders.has(integrity)) { + throw new Error( + `extractTarball: the extraction for ${integrity} was evicted and a reader still holds it; retry once it is released`, + ); + } + await reclaimEntry(integrity, async () => { + if (!poisonedExtractions.has(integrity)) return; + await fs.rm(extractedDir(integrity), { recursive: true, force: true }); + poisonedExtractions.delete(integrity); + pendingEvictions.delete(integrity); + }); + } + function acquireExtraction(integrity: string): void { const next = (extractionRefcounts.get(integrity) ?? 0) + 1; extractionRefcounts.set(integrity, next); @@ -233,6 +261,7 @@ export function createTarballCache(config: TarballCacheConfig): TarballCache { if (heldOverCap.delete(integrity)) requestReleaseSweep(); if (pendingEvictions.has(integrity)) { pendingEvictions.delete(integrity); + poisonedExtractions.delete(integrity); const entryDirPath = entryDir(integrity); try { await reclaimEntry(integrity, async () => { @@ -387,7 +416,7 @@ export function createTarballCache(config: TarballCacheConfig): TarballCache { * * ACCOUNTING vs. DISK USAGE: `maxBytes` bounds the sum reported by * this walker, not the actual disk consumption of the cache plus - * its downstream consumers. The loader's per-instance store dirs + * its downstream consumers. Materialization's per-instance store dirs * contain independent copies that are outside this cache tree, so * the cap is a steady-state ceiling on the cache's own accounting, * not a sidecar-wide disk-usage limit. Concurrent vanishes during @@ -399,7 +428,7 @@ export function createTarballCache(config: TarballCacheConfig): TarballCache { * Symlinks (both file- and directory-targeted) are NOT traversed: * `lstat` here returns the link itself rather than its target, and * `dirent.isSymbolicLink()` is the entry-walk equivalent. An npm - * tarball that ships symlinks is preserved verbatim by the loader's + * tarball that ships symlinks is preserved verbatim by materialization's * copy-tree pass; charging the link's own size (zero in our * accounting) avoids both symlink-loop divergence and double-counting * the target through whatever path also names it directly. @@ -523,7 +552,7 @@ export function createTarballCache(config: TarballCacheConfig): TarballCache { heldOverCap.clear(); } else { // This warning may fire twice for the same integrity in one - // loader pass — once after `put` writes the bytes and once after + // materialization pass — once after `put` writes the bytes and once after // `extractTarball` unpacks them, since both invoke the sweep // with the same `justWritten` integrity. Operator log // aggregation should dedup on `integrity` if the noise becomes @@ -612,9 +641,7 @@ export function createTarballCache(config: TarballCacheConfig): TarballCache { // rebuildable. A crash between write and rename leaves an orphaned // .tmp file that `sweepOrphans` clears on the next boot; a crash // after rename but before the data hits disk forces a re-fetch on - // the next miss, validated by SRI. The persistence requirement - // that earns an fsync is the apply pipeline's active-deploy-id, - // not cache entries. + // the next miss, validated by SRI. await fs.rename(tmp, file); // No cap sweep here. `put` only stages the tarball bytes; the @@ -648,6 +675,7 @@ export function createTarballCache(config: TarballCacheConfig): TarballCache { const inFlight = extractionRefcounts.get(integrity) ?? 0; if (inFlight > 0) { pendingEvictions.add(integrity); + poisonedExtractions.add(integrity); logger.debug`deferring extraction reclaim for ${integrity}: ${String(inFlight)} reader(s) in flight`; return; } @@ -665,6 +693,10 @@ export function createTarballCache(config: TarballCacheConfig): TarballCache { async extractTarball(integrity) { const finalDir = extractedDir(integrity); + const releaseReader = (): void => { + countExtractionReader(integrity, -1); + void releaseExtraction(integrity); + }; // Handle factory. The acquire MUST have already happened by the // time we call this — the cache-hit path acquires before the @@ -680,10 +712,10 @@ export function createTarballCache(config: TarballCacheConfig): TarballCache { if (released) return; released = true; // Fire-and-forget: deferred reclaim runs asynchronously, - // but release() returns synchronously so the loader's + // but release() returns synchronously so materialization's // walk-complete site does not need to await. Reclaim // failures are logged inside releaseExtraction. - void releaseExtraction(integrity); + releaseReader(); }, }; }; @@ -699,13 +731,22 @@ export function createTarballCache(config: TarballCacheConfig): TarballCache { // refcount before falling through to the unpack path. acquireExtraction(integrity); await reclaims.get(integrity); + if (poisonedExtractions.has(integrity)) { + try { + await discardPoisonedExtraction(integrity); + } catch (err) { + void releaseExtraction(integrity); + throw err; + } + } + countExtractionReader(integrity, 1); try { const stat = await fs.stat(finalDir); if (stat.isDirectory()) { // Touch the tarball's atime so the cap-driven LRU sweep - // sees this access. The loader's hot path is `cache.has` + + // sees this access. Materialization's hot path is `cache.has` + // `cache.extractTarball`; `cache.get` is the only other - // entry point that calls `utimes`, and the loader does not + // entry point that calls `utimes`, and materialization does not // use it. Without this, hot integrities accessed only // through this path stay LRU-stale at their put time and // become preferential eviction targets on noatime/relatime @@ -715,7 +756,7 @@ export function createTarballCache(config: TarballCacheConfig): TarballCache { } } catch (err) { if (!isENOENT(err)) { - void releaseExtraction(integrity); + releaseReader(); throw err; } } @@ -731,7 +772,7 @@ export function createTarballCache(config: TarballCacheConfig): TarballCache { // tar extraction. The bytes only land in the cache after // passing the upload-time cap (`HUB_MAX_TARBALL_BYTES` on the // hub edge) for asset-sourced tarballs, or the fetch-time cap - // (`maxRegistryTarballBytes` in the loader's HTTP fetcher) + // (`maxRegistryTarballBytes` in the HTTP tarball fetcher) // for registry-sourced tarballs — both default to 10 MiB. At // those caps and the small concurrent-extraction count the // memory footprint is bounded; a streaming path (pipe @@ -754,7 +795,7 @@ export function createTarballCache(config: TarballCacheConfig): TarballCache { // block can unpack to syntactically-valid but semantically- // wrong content that then surfaces as a far-removed dynamic- // import failure). Re-hashing here turns that failure into a - // structured TarballIntegrityMismatchError the loader can + // structured TarballIntegrityMismatchError materialization can // react to. const matched = ssri.checkData(bytes, integrity); if (matched === false) { @@ -812,7 +853,7 @@ export function createTarballCache(config: TarballCacheConfig): TarballCache { } catch (err) { // Release the refcount on failure; the caller will not call // the handle's `release` because no handle was returned. - void releaseExtraction(integrity); + releaseReader(); throw err; } }, diff --git a/packages/tool-packaging/src/git-materialize.ts b/packages/tool-packaging/src/git-materialize.ts index cdef4c43b..fe798e3ff 100644 --- a/packages/tool-packaging/src/git-materialize.ts +++ b/packages/tool-packaging/src/git-materialize.ts @@ -20,7 +20,10 @@ import { } from "@intx/storage-isogit/node"; import type { ToolPackageAssetSourceTree } from "@intx/types/tool-packages"; -import { ToolLoaderError, describeError } from "./loader-internal"; +import { + ClosureMaterializationError, + describeError, +} from "./materialization-internal"; const logger = getLogger(["sidecar", "tool-packaging", "git-materialize"]); @@ -29,8 +32,8 @@ const logger = getLogger(["sidecar", "tool-packaging", "git-materialize"]); * `treeOid`, and write it into a fresh scratch directory under * `instanceScratchDir`. Returns the same `{ dir, release }` shape the tarball * path returns; `release` is a no-op because there is no content-addressed - * cache handle to hold — the scratch dir lives under the deploy directory the - * caller reclaims. + * cache handle to hold — the scratch dir lives under `instanceScratchDir`, which + * the caller reclaims. */ export async function materializeGitEntry(args: { tree: ToolPackageAssetSourceTree; @@ -54,10 +57,9 @@ export async function materializeGitEntry(args: { }); subtreeOid = oid; } catch (err) { - throw new ToolLoaderError({ + throw new ClosureMaterializationError({ category: "git.materialization.failed", message: `git subtree ${JSON.stringify(source.packageDir)} for ${name}@${version} could not be read at ${source.commitSha}: ${describeError(err)}`, - package: { name, version }, }); } @@ -65,10 +67,9 @@ export async function materializeGitEntry(args: { // same git objects, so a mismatch means the delivered pack diverges from // what was frozen; fail loud rather than materialize unverified bytes. if (subtreeOid !== source.treeOid) { - throw new ToolLoaderError({ + throw new ClosureMaterializationError({ category: "git.materialization.failed", message: `git subtree ${JSON.stringify(source.packageDir)} at ${source.commitSha} for ${name}@${version} hashed to tree ${subtreeOid}, not the frozen ${source.treeOid}`, - package: { name, version }, }); } @@ -88,10 +89,9 @@ export async function materializeGitEntry(args: { await fs.rm(dir, { recursive: true, force: true }).catch((rmErr) => { logger.warn`git subtree scratch cleanup failed at ${dir}: ${describeError(rmErr)}`; }); - throw new ToolLoaderError({ + throw new ClosureMaterializationError({ category: "git.materialization.failed", message: `writing git subtree for ${name}@${version} failed: ${describeError(err)}`, - package: { name, version }, }); } diff --git a/packages/tool-packaging/src/index.ts b/packages/tool-packaging/src/index.ts index a8a54a6d4..9e2416232 100644 --- a/packages/tool-packaging/src/index.ts +++ b/packages/tool-packaging/src/index.ts @@ -11,23 +11,17 @@ // Two surface areas: // // - Hub-side (deploy assembly): -// createClosureResolver({ registries, scopeRouting? }) +// createClosureResolver({ source }) // → resolveClosure(pins) → ToolPackageManifest // // - Sidecar-side (deploy apply): // createTarballCache({ rootDir, maxBytes }) — content-addressable // store keyed by SRI integrity. -// createToolLoader({ cache, registries, host, … }) — fetches, -// extracts, and dynamic-imports each pinned package. -// applyAtomic({ manifest, loader, … }) — per-deploy-id apply -// protocol that stages each deploy into its own never-renamed -// directory and maps every loader failure category onto an -// `ApplyAtomicFailure` the caller surfaces as a rejected apply. -// materializeToolPackages({ …, registries, host, assetRoot }) — -// validates a step manifest, applies it, and persists the active -// deploy id. parseToolRegistries(raw) and hostPlatform(os, cpu) -// are the registry-map and host-platform boundary that apply -// consumes. +// materializeClosure({ manifest, cache, fetchTarball, … }) — fetches, +// extracts, and lays out a closure without importing any of it. +// createTarballFetcher({ … }) — the production fetcher it takes. +// parseToolRegistries(raw) and hostPlatform(os, cpu) are the +// registry-map and host-platform boundary that apply consumes. // applyFrozenWorkflowClosure({ …, host }) — lays out a frozen // workflow closure for the workflow child to evaluate, importing // none of it. @@ -45,7 +39,6 @@ export { type PeerDependencyViolation, type RegistryConfig, type RegistrySource, - type ScopeRoute, AssetRegistrySource, HttpRegistrySource, ManifestInvalidError, @@ -62,28 +55,22 @@ export { export { type HostPlatform, - type LoadManifestArgs, - type LoadedDirectorFactory, - type LoadedToolFactory, - type LoadedToolPackage, - type LoaderConfig, type MaterializeClosureArgs, type MaterializeClosureResult, - type TarballFetcher, - type ToolLoader, - ToolLoaderError, - createToolLoader, materializeClosure, storeEntryDir, -} from "./loader"; +} from "./store-layout"; + +export { + type TarballFetcher, + type TarballFetcherConfig, + createTarballFetcher, +} from "./tarball-fetcher"; export { - type ApplyAtomicArgs, - type ApplyAtomicFailure, - type ApplyAtomicResult, - type ApplyAtomicSuccess, - applyAtomic, -} from "./atomic-apply"; + type ClosureMaterializationErrorCategory, + ClosureMaterializationError, +} from "./materialization-internal"; export { type ExtractPackageJSONOutcome, @@ -92,11 +79,6 @@ export { export { hostPlatform, parseToolRegistries } from "./materialization-config"; -export { - type StepToolFactory, - materializeToolPackages, -} from "./tool-materialization"; - export { type ApplyFrozenWorkflowClosureArgs, type AppliedWorkflowClosure, diff --git a/packages/tool-packaging/src/loader-internal.ts b/packages/tool-packaging/src/loader-internal.ts deleted file mode 100644 index b6874e238..000000000 --- a/packages/tool-packaging/src/loader-internal.ts +++ /dev/null @@ -1,81 +0,0 @@ -// Shared internals of the tool-package loader: the failure type, the fetch -// byte/time caps, and the small cross-cutting helpers. Split out so the -// registry-fetch and store-layout modules and `loader.ts` itself can all depend -// on them without an import cycle. `loader.ts` re-exports the public members -// (`ToolLoaderError`, the `DEFAULT_*` caps) so existing consumers are unaffected. - -import type { DeployApplyErrorCategory } from "@intx/types/sidecar"; - -/** - * Maximum bytes a single registry tarball fetch will read before aborting. The - * cap bounds the memory a malicious or misconfigured registry can force a - * deploy to buffer while the atomic-apply mirror replays it. - */ -export const DEFAULT_MAX_REGISTRY_TARBALL_BYTES = 10 * 1024 * 1024; - -/** - * Default deadline for a single HTTP-registry tarball fetch, covering both the - * request and the streamed body read. `readResponseWithLimit` consumes the body - * through a manual reader loop, so the byte cap bounds size but nothing bounds - * time: a registry that accepts the connection and then stalls mid-stream would - * block the fetch -- and the deploy's tool materialization awaiting it -- - * indefinitely. The deadline is generous so a legitimately large tarball on a - * slow link still completes within it. Callers that need a different bound pass - * `registryFetchTimeoutMs` to `createToolLoader`. - */ -export const DEFAULT_REGISTRY_FETCH_TIMEOUT_MS = 120 * 1000; - -/** - * A tool-loader failure carrying the atomic-apply error category (and, when the - * failure is attributable to a specific package, its name and version) so the - * apply layer can map every loader failure onto a `DeployApplyErrorCategory`. - */ -export class ToolLoaderError extends Error { - readonly category: DeployApplyErrorCategory; - readonly package: - | { readonly name: string; readonly version: string } - | undefined; - - constructor(opts: { - category: DeployApplyErrorCategory; - message: string; - package?: { name: string; version: string }; - }) { - super(opts.message); - this.name = "ToolLoaderError"; - this.category = opts.category; - this.package = opts.package; - } -} - -/** - * Whether a `platform`/`os`/`cpu`-style allowlist matches `host`. A list with a - * `!`-negated entry matches everything except the negated hosts; an unnegated - * list matches only its members. - */ -export function platformListMatches( - entries: readonly string[], - host: string, -): boolean { - const hasNegation = entries.some((e) => e.startsWith("!")); - if (hasNegation) { - return !entries.includes(`!${host}`); - } - return entries.includes(host); -} - -export function isEEXIST(err: unknown): boolean { - if (err === null || typeof err !== "object") return false; - if (!("code" in err)) return false; - return (err as { code: unknown }).code === "EEXIST"; -} - -export function isENOENT(err: unknown): boolean { - if (err === null || typeof err !== "object") return false; - if (!("code" in err)) return false; - return (err as { code: unknown }).code === "ENOENT"; -} - -export function describeError(err: unknown): string { - return err instanceof Error ? err.message : String(err); -} diff --git a/packages/tool-packaging/src/loader.ts b/packages/tool-packaging/src/loader.ts deleted file mode 100644 index 96b5dd2a2..000000000 --- a/packages/tool-packaging/src/loader.ts +++ /dev/null @@ -1,1012 +0,0 @@ -// eslint-disable-next-line @typescript-eslint/triple-slash-reference -- npm-team packages ship no types; declarations.d.ts must be visible to downstream typecheckers that import from this package's source. -/// -// Sidecar-side tool-package loader. -// -// Given a resolved `ToolPackageManifest`, the loader builds an -// npm-compatible nested `node_modules/` layout under the per-instance -// scratch directory so each top-level package and each transitive -// dependency can satisfy its own `require()` / `import` calls without -// help from the sidecar host. -// -// 1. Filters by host os/cpu metadata; mismatches are skipped with a -// debug log (`platform.mismatch.skipped`). -// 2. Materializes every remaining entry into the content-addressable -// cache: bytes are pulled from the entry's source on a miss and -// verified through `cache.put`; the bytes are then unpacked via -// `cache.extractTarball` so a single sha512 has a single extraction -// shared across instances. -// 3. Lays out each entry under `/store///` by -// copying the file tree from the cache extraction. Each layout -// directory gets its own `node_modules/` symlink to the -// sibling `store///` chosen for that requirer. -// Diamond dependencies share a single store entry; version -// conflicts coexist as separate store entries and Node's standard -// ancestor-walk resolves each requirer's deps to the version that -// satisfies its own range. -// 4. Reads each top-level package's unpacked `package.json`, resolves -// the `interchange.tools` entry path, and dynamic-import()s it. -// 5. Validates each named export is an `AnnotatedToolFactory` (a -// callable with `id: string` and `requires: readonly string[]`). -// -// Only entries listed in `manifest.topLevel` contribute tools; the -// loader still materializes every other entry (modulo platform -// filtering) because top-level packages reach them through Node's -// `node_modules/` resolution at apply time. -// -// Errors are surfaced as `ToolLoaderError` with a `category` matching -// one of the `DeployApplyErrorCategory` values. The atomic-apply layer -// catches these and translates them into wire-level frames. - -import { promises as fs } from "node:fs"; -import path from "node:path"; -import { pathToFileURL } from "node:url"; -import npmRegistryFetch from "npm-registry-fetch"; -import { type } from "arktype"; - -import type { - AnnotatedDirectorFactory, - AnnotatedPluginFactory, - AnnotatedToolFactory, - BaseEnv, -} from "@intx/agent"; -import { - isAnnotatedDirectorFactory, - isAnnotatedPluginFactory, - isOwnedDirectorId, - toolFactoryMeta, -} from "@intx/agent/authoring"; -import type { - ReadonlyToolCredentialDeclaration, - ToolCredentialDeclaration, -} from "@intx/types/package-json"; -import { ToolCredentialDeclarationArray } from "@intx/types/package-json"; -import type { ToolCall, ToolResult } from "@intx/types/runtime"; -import { getLogger } from "@intx/log"; -import type { - ToolPackageManifest, - ToolPackageManifestEntry, -} from "@intx/types/tool-packages"; -import { getToolPackageSourceContentIdentity } from "@intx/types/tool-packages"; - -import type { TarballCache } from "./cache"; -import { - DEFAULT_MAX_REGISTRY_TARBALL_BYTES, - DEFAULT_REGISTRY_FETCH_TIMEOUT_MS, - ToolLoaderError, - describeError, -} from "./loader-internal"; -import { - buildRegistryFetchOpts, - defaultTarballUrl, - readResponseWithLimit, -} from "./registry-fetch"; -import type { RegistryConfig } from "./resolver"; -import { materializeClosure, storeEntryDir } from "./store-layout"; - -// Re-export the members that moved to focused modules so existing `./loader` -// and package-root consumers keep resolving them here: the public failure type -// and fetch caps (now in `loader-internal`), the registry fetch helpers, and -// the closure materialization + store-layout entry points. -export { - ToolLoaderError, - DEFAULT_MAX_REGISTRY_TARBALL_BYTES, - DEFAULT_REGISTRY_FETCH_TIMEOUT_MS, - buildRegistryFetchOpts, - readResponseWithLimit, - materializeClosure, - storeEntryDir, -}; - -const logger = getLogger(["sidecar", "tool-packaging", "loader"]); - -/** - * Loaded factory shape. We re-export `AnnotatedToolFactory` so - * callers receive the canonical agent type without re-deriving it; the - * loader still performs the structural check at import time so a - * package emitting something that does not satisfy this shape is - * rejected with `package.entry.invalid`. - */ -export type LoadedToolFactory = AnnotatedToolFactory; - -/** - * Loaded director factory shape. Same erased-Config storage as the - * registry uses: the loader walks `interchange.directors` and surfaces - * every export the structural check accepts. Downstream consumers feed - * these into `createDirectorRegistry` alongside the built-in defaults. - */ -export type LoadedDirectorFactory = AnnotatedDirectorFactory; - -/** One pinned package after materialization and entry-module import. */ -export interface LoadedToolPackage { - readonly name: string; - readonly version: string; - readonly factories: readonly LoadedToolFactory[]; - /** - * Plugin factories the package's `interchange.tools` module exported - * (via `definePlugin`). Plugins are instantiated by the loader before - * tool factories are run; their results are passed via `env.plugins` - * to the tool factories that read them. - */ - readonly plugins: readonly AnnotatedPluginFactory[]; - /** - * Director factories the package's `interchange.directors` module - * exported (via `defineDirector`). The capability walk resolves - * `DirectorRef.id` against these at deploy time; the agent runtime - * resolves them again at instantiation. Empty when the package's - * `package.json` omits the `interchange.directors` field — the - * walker treats absence as a no-op so a tools-only package stays - * valid. - */ - readonly directors: readonly LoadedDirectorFactory[]; - /** - * The provider-backed credentials the package's tools statically - * declare via `interchange.credentials`. A declaration is advisory: it - * names a handle the agent definition binds to a concrete credential and - * the launch-time grant gate authorizes; it consents to nothing on its - * own. Surfaced here -- parsed from the SAME `package.json` the code - * loaded from -- so the declared set is the authoritative one (the loaded - * package's own) rather than a hub manifest that could drift. Empty when - * the package omits the field; a tools-only package stays valid. - */ - readonly credentials: readonly ToolCredentialDeclaration[]; -} - -export interface HostPlatform { - readonly os: string; - readonly cpu: string; -} - -export interface LoaderConfig { - readonly cache: TarballCache; - /** - * Registry identifier → registry config. The key is the same - * `registry` identifier manifest entries point at and the resolver - * keyed its own registries map under. The loader resolves each - * `kind: "registry"` entry by looking up this map. - */ - readonly registries: ReadonlyMap; - readonly host: HostPlatform; - /** - * Hard cap on the byte length of a tarball fetched from an HTTP - * registry. The default fetcher honors both the upstream - * `Content-Length` header (rejecting up front when the header value - * exceeds the cap) and the realized body byte count (aborting the - * read once the running total crosses the cap). Asset-sourced - * tarballs do not go through this path; their containment is the - * substrate's upload-time cap on the hub side. - * - * The default mirrors the hub's `HUB_MAX_TARBALL_BYTES` cap so a - * tarball legitimately accepted by the hub-side upload route is - * also legitimately fetchable from a registry mirror seeded from - * that hub. An operator pointing the sidecar at a third-party - * registry whose curated tarballs run larger should raise the cap - * explicitly rather than relying on the runtime to grow. - */ - readonly maxRegistryTarballBytes?: number; - /** - * Deadline in milliseconds for a single HTTP-registry tarball fetch, - * spanning the request and the streamed body read. A stalled registry - * cannot block the fetch -- and the deploy's tool materialization - * awaiting it -- past this bound. Defaults to - * `DEFAULT_REGISTRY_FETCH_TIMEOUT_MS`. Asset-sourced tarballs read from - * the local filesystem and are not subject to it. - */ - readonly registryFetchTimeoutMs?: number; - /** - * Test seam for tarball fetching. Production omits this and the - * loader uses npm-registry-fetch + filesystem reads. - */ - readonly fetchTarball?: TarballFetcher; - /** - * Test seam for dynamic import. Production omits this and the loader - * uses the native dynamic-import expression. The argument is the URL - * the loader hands to `import()`: a `file://` URL with an - * `integrity=` query string the loader appends to bust Node's - * ESM module cache across applies that swap bytes under the same - * `(name, version)` pair. - */ - readonly importModule?: (importUrl: string) => Promise; -} - -export type TarballFetcher = ( - entry: ToolPackageManifestEntry, - ctx: { - registries: ReadonlyMap; - assetRoot: string; - assetMounts: ReadonlyMap; - }, -) => Promise; - -export interface LoadManifestArgs { - readonly manifest: ToolPackageManifest; - readonly instanceScratchDir: string; - /** - * Filesystem root that `assetMounts` paths are joined against. Mount - * paths are workspace-relative; the loader - * resolves them against `assetRoot` to get the absolute tarball - * location for `kind: "asset"` entries. - */ - readonly assetRoot: string; - /** - * Maps a `source.assetId` from a manifest entry to a - * workspace-relative mount path. Empty map is valid when no entry - * sources from an asset. - */ - readonly assetMounts: ReadonlyMap; - /** - * Maps a source-format asset entry's `source.assetId` to an absolute path - * to an indexed git directory whose object database holds the pinned - * commit and its trees. The caller checks the delivered pack out into - * this directory once per asset before applying. Empty map is valid - * when no entry sources from a git subtree. - */ - readonly gitDirs: ReadonlyMap; -} - -export interface ToolLoader { - loadManifest(args: LoadManifestArgs): Promise; -} - -/** - * Arguments for `materializeClosure`, the eval-free materialization - * primitive `loadManifest` wraps. Everything phases 1-2 need is passed - * explicitly so the function stands alone without a `ToolLoader` - * instance: the closure's `cache`, `registries`, `host`, and resolved - * `fetchTarball` are threaded in directly. There is no import seam here - * because `materializeClosure` never imports author code — that first - * `import()` belongs to `loadManifest`'s phase-3 loop. - */ -export interface MaterializeClosureArgs { - readonly manifest: ToolPackageManifest; - readonly instanceScratchDir: string; - readonly assetRoot: string; - readonly assetMounts: ReadonlyMap; - /** - * Maps a source-format asset entry's `source.assetId` to an absolute - * indexed git directory the pinned subtree is read from. Empty map is valid - * when no entry sources from a git subtree. - */ - readonly gitDirs: ReadonlyMap; - readonly host: HostPlatform; - readonly cache: TarballCache; - /** - * Registry identifier → registry config, the same map - * `createToolLoader` keys its registries under. Used both to gate - * `kind: "registry"` entries against the sidecar config and as the - * `registries` context handed to `fetchTarball`. - */ - readonly registries: ReadonlyMap; - /** - * Resolved tarball fetcher. `createToolLoader` builds the default - * (npm-registry-fetch + filesystem) fetcher or honors a test seam and - * threads the result here; `materializeClosure` does not construct one - * of its own. - */ - readonly fetchTarball: TarballFetcher; -} - -/** - * Result of `materializeClosure`: the per-instance store directory the - * closure was laid out under (`/store`), plus the exact - * host-filtered entries that were laid out under it. A caller computes a - * specific package's directory with `storeEntryDir`, and iterates `entries` to - * load exactly what was materialized -- rather than re-deriving the host filter - * with a second predicate and risking the two drifting apart. - */ -export interface MaterializeClosureResult { - readonly storeDir: string; - readonly entries: readonly ToolPackageManifestEntry[]; -} - -export function createToolLoader(config: LoaderConfig): ToolLoader { - const registriesByName = config.registries; - const maxRegistryTarballBytes = - config.maxRegistryTarballBytes ?? DEFAULT_MAX_REGISTRY_TARBALL_BYTES; - if ( - !Number.isFinite(maxRegistryTarballBytes) || - maxRegistryTarballBytes <= 0 - ) { - throw new Error( - `createToolLoader: maxRegistryTarballBytes must be a positive finite number; got ${String(maxRegistryTarballBytes)}`, - ); - } - const registryFetchTimeoutMs = - config.registryFetchTimeoutMs ?? DEFAULT_REGISTRY_FETCH_TIMEOUT_MS; - if (!Number.isFinite(registryFetchTimeoutMs) || registryFetchTimeoutMs <= 0) { - throw new Error( - `createToolLoader: registryFetchTimeoutMs must be a positive finite number; got ${String(registryFetchTimeoutMs)}`, - ); - } - const fetchTarball = config.fetchTarball ?? makeDefaultTarballFetcher(); - const importModule = - config.importModule ?? ((u: string) => import(u) as Promise); - - async function loadTopLevel( - entry: ToolPackageManifestEntry, - pkgDir: string, - ): Promise { - const pkgJsonPath = path.join(pkgDir, "package.json"); - let pkgJsonRaw: string; - try { - pkgJsonRaw = await fs.readFile(pkgJsonPath, "utf8"); - } catch (err) { - throw new ToolLoaderError({ - category: "package.entry.invalid", - message: `package.json missing for ${entry.name}@${entry.version}: ${describeError(err)}`, - package: { name: entry.name, version: entry.version }, - }); - } - let pkgJson: unknown; - try { - pkgJson = JSON.parse(pkgJsonRaw); - } catch (err) { - throw new ToolLoaderError({ - category: "package.entry.invalid", - message: `malformed package.json in ${entry.name}@${entry.version}: ${describeError(err)}`, - package: { name: entry.name, version: entry.version }, - }); - } - // The pin is the name the operator approved, and the namespace check - // below is made against it. Reject a tarball whose own manifest - // disagrees so the pin and the manifest name are interchangeable - // for everything downstream. - const manifestName = readPackageName(pkgJson); - if (manifestName !== entry.name) { - throw new ToolLoaderError({ - category: "package.entry.invalid", - message: `${entry.name}@${entry.version} package.json declares name ${JSON.stringify(manifestName)}, which does not match the pinned package name`, - package: { name: entry.name, version: entry.version }, - }); - } - const toolsRel = readInterchangeEntry(pkgJson, "tools"); - if (toolsRel === null) { - throw new ToolLoaderError({ - category: "package.entry.missing", - message: `${entry.name}@${entry.version} package.json has no "interchange.tools" field`, - package: { name: entry.name, version: entry.version }, - }); - } - const toolsMod = await importInterchangeEntry({ - entry, - pkgDir, - entryRel: toolsRel, - field: "tools", - }); - const factories: LoadedToolFactory[] = []; - const plugins: AnnotatedPluginFactory[] = []; - for (const value of Object.values(toolsMod)) { - if (isAnnotatedPluginFactory(value)) { - plugins.push(value); - } else if (isAnnotatedToolFactory(value)) { - factories.push( - applyNamespacePrefix(value, { - name: entry.name, - version: entry.version, - }), - ); - } - } - if (factories.length === 0 && plugins.length === 0) { - throw new ToolLoaderError({ - category: "package.entry.invalid", - message: `${entry.name}@${entry.version} interchange.tools entry exported no AnnotatedToolFactory or AnnotatedPluginFactory values`, - package: { name: entry.name, version: entry.version }, - }); - } - - // Director walk: separate `package.json` field, separate dynamic - // import, separate structural validation. Absence is a no-op so a - // tools-only package stays valid; a directors-only package is not - // supported because the tools field's absence is already a hard - // error above. A package whose director-entry module exports - // nothing director-shaped is rejected the same way the tool entry - // would be. - const directors: LoadedDirectorFactory[] = []; - const directorsRel = readInterchangeEntry(pkgJson, "directors"); - if (directorsRel !== null) { - const directorsMod = await importInterchangeEntry({ - entry, - pkgDir, - entryRel: directorsRel, - field: "directors", - }); - for (const value of Object.values(directorsMod)) { - if (isAnnotatedDirectorFactory(value)) { - directors.push(value); - } - } - if (directors.length === 0) { - throw new ToolLoaderError({ - category: "package.entry.invalid", - message: `${entry.name}@${entry.version} interchange.directors entry exported no AnnotatedDirectorFactory values`, - package: { name: entry.name, version: entry.version }, - }); - } - // Every exported factory id must sit under the shipping package's own - // name: the operator approves `director:` only, so the id prefix is - // the only thing that ties the approved grant to the package whose code - // runs. Same rule as the workflow-host closure loader. - for (const director of directors) { - if (!isOwnedDirectorId(director.id, entry.name)) { - throw new ToolLoaderError({ - category: "package.entry.invalid", - message: `${entry.name}@${entry.version} interchange.directors entry exports director ${JSON.stringify(director.id)} outside the package's own namespace ${JSON.stringify(entry.name)}`, - package: { name: entry.name, version: entry.version }, - }); - } - } - } - - // Inline credential declarations: read + validate from the same - // package.json above. Runs after the tools/directors walks so a - // malformed `interchange.credentials` surfaces the same - // package.entry.invalid class as a bad tool/director entry. - const credentials = readInterchangeCredentials(pkgJson, entry); - // The manifest is the pinned package's declaration of record, for all of - // its factories. A factory may also declare through `defineTool`; it may - // then name only handles the manifest declares, with the same scopes, so - // a package whose factories use different handles declares each one. - for (const factory of factories) { - if (factory.credentials === undefined) continue; - // The factory comes from the package's module exports, so its - // declaration is untrusted until it passes the same validator. - const declared = ToolCredentialDeclarationArray(factory.credentials); - if (declared instanceof type.errors) { - throw new ToolLoaderError({ - category: "package.entry.invalid", - message: `${entry.name}@${entry.version} tool factory ${factory.id} credentials failed validation: ${declared.summary}`, - package: { name: entry.name, version: entry.version }, - }); - } - if (!declaresSubsetOf(declared, credentials)) { - throw new ToolLoaderError({ - category: "package.entry.invalid", - message: `${entry.name}@${entry.version} tool factory ${factory.id} declares credentials that the package's interchange.credentials does not`, - package: { name: entry.name, version: entry.version }, - }); - } - } - - return { - name: entry.name, - version: entry.version, - factories, - plugins, - directors, - credentials, - }; - } - - /** - * Resolve `entryRel` against `pkgDir`, enforce path-safety - * (`..`-traversal, absolute-path, and node_modules symlink-graph - * escapes), and dynamic-import the result. Centralized so the - * `interchange.tools` and `interchange.directors` walkers share one - * containment surface. - */ - async function importInterchangeEntry(args: { - entry: ToolPackageManifestEntry; - pkgDir: string; - entryRel: string; - field: "tools" | "directors"; - }): Promise { - const { entry, pkgDir, entryRel, field } = args; - const entryAbs = path.resolve(pkgDir, entryRel); - // `entryRel` originates from the tarball's `package.json` and - // crosses the trust boundary into the sidecar process. `..` or an - // absolute path inside `entryRel` would let a malicious tarball - // import any file the sidecar process can read. Confine the - // resolved import target to the package's own extraction - // directory and reject anything that escapes. - // - // The string-level check rejects `..` and absolute paths inside - // `entryRel`. It is not enough on its own: the per-instance - // scratch tree contains a `node_modules/` symlink graph the - // loader builds to satisfy nested resolution, and an - // `interchange.*` entry that traverses that graph would - // string-contain inside `pkgDir` but resolve via realpath to - // another package's code (or anywhere else the symlink target - // points). Re-check containment against the realpath so a - // tarball cannot reach another package's bytes through its own - // package directory's symlinks. - // - // The containment check assumes POSIX-shaped path separators on - // disk — the sidecar runs on Linux and macOS only; Windows - // path-separator handling (drive letters, mixed `/` and `\\`, - // case-insensitive comparison) is out of scope. - const containmentRoot = pkgDir.endsWith(path.sep) - ? pkgDir - : pkgDir + path.sep; - if (entryAbs !== pkgDir && !entryAbs.startsWith(containmentRoot)) { - throw new ToolLoaderError({ - category: "package.entry.invalid", - message: `${entry.name}@${entry.version} interchange.${field} entry path ${JSON.stringify(entryRel)} escapes the package directory`, - package: { name: entry.name, version: entry.version }, - }); - } - // Realpath the entry so a `node_modules/` symlink traversal - // inside `entryRel` does not let a tarball point at another - // package's bytes. The package directory itself is resolved the - // same way so the comparison is realpath-vs-realpath rather than - // realpath-vs-as-declared (the per-instance scratch tree may - // itself live under a symlinked tmpdir, notably on macOS where - // `/tmp` is a symlink to `/private/tmp`). - let realPkgDir: string; - let realEntryAbs: string; - try { - realPkgDir = await fs.realpath(pkgDir); - realEntryAbs = await fs.realpath(entryAbs); - } catch (err) { - throw new ToolLoaderError({ - category: "package.entry.invalid", - message: `${entry.name}@${entry.version} interchange.${field} entry path ${JSON.stringify(entryRel)} could not be resolved: ${describeError(err)}`, - package: { name: entry.name, version: entry.version }, - }); - } - const realContainmentRoot = realPkgDir.endsWith(path.sep) - ? realPkgDir - : realPkgDir + path.sep; - if ( - realEntryAbs !== realPkgDir && - !realEntryAbs.startsWith(realContainmentRoot) - ) { - throw new ToolLoaderError({ - category: "package.entry.invalid", - message: `${entry.name}@${entry.version} interchange.${field} entry path ${JSON.stringify(entryRel)} escapes the package extraction directory via a symlink`, - package: { name: entry.name, version: entry.version }, - }); - } - - // Cache-bust the ESM module cache by appending the entry's content - // identity as a query string. Node keys the ESM cache by resolved - // URL/path, not by content: a `(name, version)` pair whose bytes - // change across applies (an operator-recompiled built-in, a - // hot-fixed tarball republished under the same version) would - // otherwise resolve to the previously-imported module instance until - // the sidecar restarts. Same path with a different query is a - // distinct ESM cache entry, so the import reflects the bytes actually - // extracted for this apply. - const importUrl = `${pathToFileURL(entryAbs).href}?integrity=${encodeURIComponent(getToolPackageSourceContentIdentity(entry.source))}`; - let mod: unknown; - try { - mod = await importModule(importUrl); - } catch (err) { - throw new ToolLoaderError({ - category: "package.entry.invalid", - message: `dynamic import of ${entry.name}@${entry.version} interchange.${field} failed: ${describeError(err)}`, - package: { name: entry.name, version: entry.version }, - }); - } - if (mod === null || typeof mod !== "object") { - throw new ToolLoaderError({ - category: "package.entry.invalid", - message: `${entry.name}@${entry.version} interchange.${field} entry did not return an object`, - package: { name: entry.name, version: entry.version }, - }); - } - return mod; - } - - return { - async loadManifest(args) { - // Phases 1-2 (fetch + SRI-verify + extract into the cache, then - // resolve the closure ranges and lay out the per-instance store) - // are the eval-free materialization primitive. They import no - // author code; that first `import()` happens only in phase 3 - // below. `materializeClosure` returns the exact host-filtered - // entries it laid out, so phase 3 loads precisely those -- there is - // no second platform-filter pass that could drift from the first. - const { storeDir, entries } = await materializeClosure({ - manifest: args.manifest, - instanceScratchDir: args.instanceScratchDir, - assetRoot: args.assetRoot, - assetMounts: args.assetMounts, - gitDirs: args.gitDirs, - host: config.host, - cache: config.cache, - registries: config.registries, - fetchTarball, - }); - - const topLevelKeys = new Set( - args.manifest.topLevel.map((p) => `${p.name}@${p.version}`), - ); - - // 3. Load only the top-level packages; transitive entries exist - // for `node_modules/` satisfaction but do not contribute - // factories of their own. This is the first point at which - // author code is imported. - const loaded: LoadedToolPackage[] = []; - const coveredTopLevelKeys = new Set(); - for (const entry of entries) { - const key = `${entry.name}@${entry.version}`; - if (!topLevelKeys.has(key)) continue; - const pkgDir = storeEntryDir(storeDir, entry.name, entry.version); - loaded.push(await loadTopLevel(entry, pkgDir)); - coveredTopLevelKeys.add(key); - } - // Top-level pins the platform filter dropped contribute zero - // factories, which is a legitimate operator choice (e.g. an - // optionalDependencies-shaped opt-in for a single-platform - // helper). Surface it as a warn so an apply that produces no - // tools at all because every pin was platform-filtered out is - // diagnosable from the logs without re-reading the manifest. - const droppedTopLevelKeys: string[] = []; - for (const key of topLevelKeys) { - if (!coveredTopLevelKeys.has(key)) droppedTopLevelKeys.push(key); - } - if (droppedTopLevelKeys.length > 0) { - logger.warn`tool-package apply dropped top-level pins via platform filter on host os=${config.host.os} cpu=${config.host.cpu}: ${droppedTopLevelKeys.join(", ")}`; - } - return loaded; - }, - }; - - function makeDefaultTarballFetcher(): TarballFetcher { - return async (entry, ctx) => { - if (entry.source.kind === "asset") { - // A source-format asset is a git subtree materialized by checkout - // in the store layout, never fetched as a tarball. Reaching the - // tarball fetcher with one is a loader bug; fail loud. - if (entry.source.package.format !== "tarball") { - throw new ToolLoaderError({ - category: "git.materialization.failed", - message: `source-format asset entry ${entry.name}@${entry.version} reached the tarball fetcher`, - package: { name: entry.name, version: entry.version }, - }); - } - const tarballPath = entry.source.package.path; - // The mount lookup is guaranteed by `materialize`'s - // pre-fetch gate, but reassert here so the narrowing is - // visible to readers — the caller of fetchTarball has no - // structural guarantee it ran through that gate. - const mount = ctx.assetMounts.get(entry.source.assetId); - if (mount === undefined) { - throw new ToolLoaderError({ - category: "asset.mount.missing", - message: `default fetcher reached without a mount for assetId "${entry.source.assetId}"`, - package: { name: entry.name, version: entry.version }, - }); - } - // Both `mount` and `entry.source.package.path` originate from the - // hub and cross the trust boundary into the sidecar process. A `..` - // segment in either would let a malicious manifest read any - // file the sidecar can open. Resolve the join and assert the - // result still sits under `assetRoot` so a traversal attempt - // surfaces as a structured manifest rejection rather than a - // silent arbitrary read. - // - // Reject absolute mount paths up front: `path.resolve` would - // discard the assetRoot prefix when handed an absolute segment, - // letting an absolute mount escape the containment check that - // follows. Defense-in-depth for the (today-trusted) hub-side - // mount producer. - if (path.isAbsolute(mount)) { - throw new ToolLoaderError({ - category: "package.entry.invalid", - message: `assetMounts entry for ${entry.name}@${entry.version} is absolute (${JSON.stringify(mount)}); mounts must be assetRoot-relative`, - package: { name: entry.name, version: entry.version }, - }); - } - const mountAbs = path.resolve(ctx.assetRoot, mount); - const absPath = path.resolve(mountAbs, tarballPath); - const mountContainmentRoot = mountAbs.endsWith(path.sep) - ? mountAbs - : mountAbs + path.sep; - if (absPath !== mountAbs && !absPath.startsWith(mountContainmentRoot)) { - throw new ToolLoaderError({ - category: "package.entry.invalid", - message: `source.package.path for ${entry.name}@${entry.version} resolves to ${JSON.stringify(absPath)} which escapes the declared mount ${JSON.stringify(mountAbs)} (cross-mount traversal)`, - package: { name: entry.name, version: entry.version }, - }); - } - try { - return await fs.readFile(absPath); - } catch (err) { - throw new ToolLoaderError({ - category: "tarball.missing", - message: `asset-stored tarball for ${entry.name}@${entry.version} not present at ${absPath}: ${describeError(err)}`, - package: { name: entry.name, version: entry.version }, - }); - } - } - const registry = registriesByName.get(entry.source.registry); - if (registry === undefined) { - throw new ToolLoaderError({ - category: "registry.unknown", - message: `manifest references registry "${entry.source.registry}" which is not in the sidecar config`, - package: { name: entry.name, version: entry.version }, - }); - } - const tarballUrl = - entry.tarballUrl ?? - defaultTarballUrl(registry.url, entry.name, entry.version); - // Bound the whole fetch -- request and streamed body read -- so a - // stalled registry cannot block the awaiting deploy forever. - // npm-registry-fetch honors the signal for the request phase; - // readResponseWithLimit honors it for the manual body read. The - // timer spans both phases and is cleared only once the read settles. - const controller = new AbortController(); - const timer = setTimeout(() => { - controller.abort(); - }, registryFetchTimeoutMs); - try { - const res = await npmRegistryFetch(tarballUrl, { - ...buildRegistryFetchOpts(registry), - signal: controller.signal, - }); - if (res.status === 401 || res.status === 403) { - throw new ToolLoaderError({ - category: "registry.auth.failed", - message: `registry "${entry.source.registry}" rejected credentials for ${entry.name}@${entry.version} (HTTP ${String(res.status)})`, - package: { name: entry.name, version: entry.version }, - }); - } - if (!res.ok) { - throw new ToolLoaderError({ - category: "registry.fetch.failed", - message: `registry "${entry.source.registry}" returned HTTP ${String(res.status)} fetching ${entry.name}@${entry.version}`, - package: { name: entry.name, version: entry.version }, - }); - } - return await readResponseWithLimit( - res, - maxRegistryTarballBytes, - { - registry: entry.source.registry, - name: entry.name, - version: entry.version, - }, - controller.signal, - ); - } catch (err) { - if (err instanceof ToolLoaderError) throw err; - if (controller.signal.aborted) { - throw new ToolLoaderError({ - category: "registry.fetch.failed", - message: `registry "${entry.source.registry}" fetch for ${entry.name}@${entry.version} exceeded the ${String(registryFetchTimeoutMs)}ms timeout`, - package: { name: entry.name, version: entry.version }, - }); - } - throw new ToolLoaderError({ - category: "registry.fetch.failed", - message: `registry "${entry.source.registry}" fetch failed for ${entry.name}@${entry.version}: ${describeError(err)}`, - package: { name: entry.name, version: entry.version }, - }); - } finally { - clearTimeout(timer); - } - }; - } -} - -function readPackageName(pkgJson: unknown): string | undefined { - if (pkgJson === null || typeof pkgJson !== "object") return undefined; - if (!("name" in pkgJson)) return undefined; - const name = (pkgJson as { name: unknown }).name; - return typeof name === "string" ? name : undefined; -} - -function readInterchangeEntry( - pkgJson: unknown, - field: "tools" | "directors", -): string | null { - if (pkgJson === null || typeof pkgJson !== "object") return null; - if (!("interchange" in pkgJson)) return null; - const interchange = (pkgJson as { interchange: unknown }).interchange; - if (interchange === null || typeof interchange !== "object") return null; - // Branch on the field rather than dynamic index-access so each path - // narrows through a single-property shape — matches the pattern the - // surrounding helpers use to inspect package.json without widening - // through a `Record` assertion. - let value: unknown; - if (field === "tools") { - if (!("tools" in interchange)) return null; - value = (interchange as { tools: unknown }).tools; - } else { - if (!("directors" in interchange)) return null; - value = (interchange as { directors: unknown }).directors; - } - if (typeof value !== "string") return null; - return value; -} - -/** - * Whether every declaration in `declared` appears in `manifest` with the same - * scopes. Order, duplicate scopes and an empty scope list carry no meaning. - */ -function declaresSubsetOf( - declared: readonly ReadonlyToolCredentialDeclaration[], - manifest: readonly ReadonlyToolCredentialDeclaration[], -): boolean { - const scopesOf = (d: ReadonlyToolCredentialDeclaration) => - JSON.stringify([...new Set(d.scopes ?? [])].sort()); - const manifestScopes = new Map(manifest.map((d) => [d.handle, scopesOf(d)])); - return declared.every((d) => manifestScopes.get(d.handle) === scopesOf(d)); -} - -/** - * Read a package's inline `interchange.credentials` declarations from its - * parsed `package.json`. Unlike `tools`/`directors` (module-path fields - * `readInterchangeEntry` resolves and imports), `credentials` is inline - * data, so it is validated here against `ToolCredentialDeclarationArray` -- - * the same arktype the upload boundary enforces. A duplicate handle or a - * malformed entry is therefore rejected at load with parity to the push - * gate rather than collapsing silently downstream. Absence is a no-op: a - * tools-only package that declares no credentials returns an empty array - * and stays valid. - */ -function readInterchangeCredentials( - pkgJson: unknown, - entry: ToolPackageManifestEntry, -): readonly ToolCredentialDeclaration[] { - if (pkgJson === null || typeof pkgJson !== "object") return []; - if (!("interchange" in pkgJson)) return []; - const interchange = (pkgJson as { interchange: unknown }).interchange; - if (interchange === null || typeof interchange !== "object") return []; - if (!("credentials" in interchange)) return []; - const raw = (interchange as { credentials: unknown }).credentials; - const validated = ToolCredentialDeclarationArray(raw); - if (validated instanceof type.errors) { - throw new ToolLoaderError({ - category: "package.entry.invalid", - message: `${entry.name}@${entry.version} interchange.credentials failed validation: ${validated.summary}`, - package: { name: entry.name, version: entry.version }, - }); - } - return validated; -} - -/** - * Wrap a factory so the bundle it returns has its tool definitions - * prefixed by the bundle's `id`. Package authors write bare tool - * names; the model and the grant evaluator see - * `:`. Audit provenance recorded against the - * bundle id stays correct because the prefix is the bundle id. - */ -function applyNamespacePrefix( - factory: LoadedToolFactory, - pkg: { name: string; version: string }, -): LoadedToolFactory { - const prefix = `${factory.id}:`; - // Freeze the wrapper AND the requires array it points at so - // downstream consumers cannot mutate the `id`/`requires` metadata - // the namespacing depends on. A mutated `id` would skew audit-trail - // provenance away from the bundle the loader actually constructed; - // a mutated `requires` would let a wrapper accumulate unintended - // capability requests over its lifetime. Freezing the wrapper alone - // blocks reassigning `wrapped.requires`; freezing the array (after - // copying so the source factory's own `requires` is not also frozen - // as a side-effect) blocks the `push` / `splice` mutations that - // would otherwise grow the surface in place. - const frozenRequires = Object.freeze([...factory.requires]); - // The wrapped factory contributes prefixed tool names at runtime, so - // its static declaration must carry the same prefixed names to stay - // truthful for callers that enumerate `definitions` without invoking - // the factory. - const frozenDefinitions = Object.freeze( - factory.definitions.map((def) => ({ - ...def, - name: `${prefix}${def.name}`, - })), - ); - const wrapped: LoadedToolFactory = Object.freeze( - Object.assign( - (env: BaseEnv) => { - const bundle = factory(env); - // A definition whose raw name already starts with the bundle's - // prefix indicates the package author either double-prefixed or - // happened to choose a name that collides with the prefix shape. - // Either way silently passing it through would yield surprising - // results in the audit trail and grant evaluator — surface it. - // Build the prefixed-definition list and the prefixed→raw name - // map in a single pass so the name map is provably aligned with - // the array TypeScript already proved was the same length. - // Shape-check `bundle.definitions` before iterating: a factory - // that returns `definitions: null` (or omits the field) would - // otherwise yield a bare TypeError that the apply pipeline - // surfaces as `factory.construct.failed` instead of the more - // accurate `package.entry.invalid` (the bundle's shape is - // wrong, not its construction). - if (!Array.isArray(bundle.definitions)) { - throw new ToolLoaderError({ - category: "package.entry.invalid", - message: `bundle ${factory.id} returned a non-array \`definitions\` field; AnnotatedToolFactory bundles must produce an array of tool definitions`, - }); - } - // Surface intra-bundle name collisions BEFORE prefixing so two - // definitions named `search` would not silently collapse to a - // single `:search` entry in the name map. - // - // Timing note: this check runs at first factory invocation - // (agent construction at sidecar boot), NOT at apply time. The - // loader cannot read `bundle.definitions` without invoking the - // factory, and the `BaseEnv` the factory needs is constructed - // by the sidecar harness only after the apply commits. As a - // consequence, an intra-bundle duplicate surfaces on the - // runtime construct-failure channel rather than as an - // apply-error frame. The cross-bundle case (in atomic-apply.ts) - // catches the same category at apply time because it operates - // on `factory.id` metadata, which is available without invoking - // the factory. See the `tool.name.duplicate` category docstring - // on `DeployApplyErrorCategory` for the operator-facing - // contract this split honors. - const rawSeen = new Set(); - for (const def of bundle.definitions) { - if (rawSeen.has(def.name)) { - throw new ToolLoaderError({ - category: "tool.name.duplicate", - message: `bundle ${factory.id} exports two tool definitions named ${JSON.stringify(def.name)}; tool names must be unique within a bundle`, - package: pkg, - }); - } - rawSeen.add(def.name); - } - const nameMap = new Map(); - const prefixed = bundle.definitions.map((def) => { - if (def.name.startsWith(prefix)) { - throw new ToolLoaderError({ - category: "package.entry.invalid", - message: `tool definition name ${JSON.stringify(def.name)} already begins with bundle prefix ${JSON.stringify(prefix)}; raw definition names must not include the bundle id`, - package: pkg, - }); - } - const prefixedName = `${prefix}${def.name}`; - nameMap.set(prefixedName, def.name); - return { ...def, name: prefixedName }; - }); - return { - definitions: prefixed, - run: (call: ToolCall, signal: AbortSignal): Promise => { - const original = nameMap.get(call.name); - // nameMap holds every prefixed form this bundle minted; a - // miss means `call.name` is not one of those prefixed - // names. Forwarding the unprefixed name into the inner - // bundle would bypass the namespacing the wrapper exists - // to enforce — an unprefixed name that happened to match - // the bundle's raw tool name would run the tool — so - // return a structured unknown-tool error directly. - if (original === undefined) { - return Promise.resolve({ - callId: call.id, - content: `unknown tool: ${call.name}`, - isError: true, - }); - } - const inner: ToolCall = { ...call, name: original }; - return bundle.run(inner, signal); - }, - ...(bundle.dispose !== undefined ? { dispose: bundle.dispose } : {}), - }; - }, - { - ...toolFactoryMeta(factory), - requires: frozenRequires, - definitions: frozenDefinitions, - }, - ), - ); - return wrapped; -} - -function isAnnotatedToolFactory(value: unknown): value is LoadedToolFactory { - if (typeof value !== "function") return false; - // Plugin factories carry the same id/requires duck-shape — explicitly - // reject anything bearing the plugin marker so the predicate stands - // alone instead of relying on the loader's ordering at the call site. - if (isAnnotatedPluginFactory(value)) return false; - if (!("id" in value) || !("requires" in value)) return false; - // Director factories carry id/requires plus a callable `configSchema`; - // without this guard a director placed in `interchange.tools` would be - // silently classified as a tool and namespace-prefixed. Mirrors the - // discriminator `isAnnotatedDirectorFactory` uses against tool shapes. - if ("configSchema" in value) { - const configSchema = (value as { configSchema: unknown }).configSchema; - if (typeof configSchema === "function") return false; - } - const id = (value as { id: unknown }).id; - const requires = (value as { requires: unknown }).requires; - if (typeof id !== "string") return false; - if (!Array.isArray(requires)) return false; - return requires.every((r) => typeof r === "string"); -} diff --git a/packages/tool-packaging/src/materialization-config.ts b/packages/tool-packaging/src/materialization-config.ts index eaf9a4b1f..36638f6ca 100644 --- a/packages/tool-packaging/src/materialization-config.ts +++ b/packages/tool-packaging/src/materialization-config.ts @@ -7,13 +7,13 @@ import { type } from "arktype"; -import type { HostPlatform } from "./loader"; +import type { HostPlatform } from "./store-layout"; import type { RegistryConfig } from "./resolver"; // Boundary validator for the SIDECAR_TOOL_REGISTRIES env var. The // env-wire shape carries `name` alongside the registry config so an // operator can author the JSON as a flat array; the boundary collapses -// the array into a Map keyed by name before handing it to the loader. +// the array into a Map keyed by name before handing it to materialization. const RegistryConfigEnvEntry = type({ name: "string", url: "string", @@ -74,7 +74,7 @@ export function parseToolRegistries( // npm's `os` token namespace, mirrored from Node's `process.platform` // enum. Any value outside this set means the host is running a Node -// build the loader's platform filter would silently mis-route — a +// build materialization's platform filter would silently mis-route — a // pinned package whose `os` list excludes the host would not be // excluded if `process.platform` is a token npm has never heard of. // Validate at the boundary so an unknown platform fails the boot @@ -103,7 +103,7 @@ const KNOWN_PROCESS_PLATFORMS = new Set([ // npm's `cpu` token namespace, mirrored from Node's `process.arch` // enum. Same rationale as KNOWN_PROCESS_PLATFORMS — an unknown arch -// would mis-route the loader's filter without surfacing the gap. +// would mis-route materialization's filter without surfacing the gap. // Same upgrade tax applies: Node has added `loong64` and `riscv64` // in recent releases, and future arch additions will need to be // added here when the sidecar is rebuilt against them. @@ -137,7 +137,7 @@ function assertKnownHostArch(arch: string): void { } /** - * Resolve the host platform token pair the loader filters manifest + * Resolve the host platform token pair materialization filters manifest * entries against, asserting each token is one npm recognizes before * returning it. */ diff --git a/packages/tool-packaging/src/materialization-internal.ts b/packages/tool-packaging/src/materialization-internal.ts new file mode 100644 index 000000000..cb68492d3 --- /dev/null +++ b/packages/tool-packaging/src/materialization-internal.ts @@ -0,0 +1,104 @@ +// Shared internals of closure materialization: the failure type, the fetch +// byte/time caps, and the small cross-cutting helpers. Split out so the +// registry-fetch, tarball-fetcher and store-layout modules can all depend on +// them without an import cycle. + +/** + * Maximum bytes a single registry tarball fetch will read before aborting. The + * cap bounds the memory a malicious or misconfigured registry can force a + * deploy to buffer. + */ +export const DEFAULT_MAX_REGISTRY_TARBALL_BYTES = 10 * 1024 * 1024; + +/** + * Default deadline for a single HTTP-registry tarball fetch, covering both the + * request and the streamed body read. `readResponseWithLimit` consumes the body + * through a manual reader loop, so the byte cap bounds size but nothing bounds + * time: a registry that accepts the connection and then stalls mid-stream would + * block the fetch -- and the closure materialization awaiting it -- + * indefinitely. The deadline is generous so a legitimately large tarball on a + * slow link still completes within it. Callers that need a different bound pass + * `registryFetchTimeoutMs` to `createTarballFetcher`. + */ +export const DEFAULT_REGISTRY_FETCH_TIMEOUT_MS = 120 * 1000; + +/** + * Categories of closure materialization failure. Each value names one failure + * kind; a single category fires per failed attempt. + * + * - `tarball.missing`: a manifest entry's asset-sourced tarball is not present + * at the recorded path. + * - `asset.mount.missing`: a `kind: "asset"` manifest entry names an `assetId` + * the asset mount map does not cover. That is a mismatch between the assets + * the closure names and those delivered, not a missing file. + * - `integrity.mismatch`: fetched tarball bytes do not match the manifest's + * pinned SRI integrity. + * - `registry.fetch.failed`: the configured registry refused or dropped the + * request for a tarball. + * - `registry.unknown`: the manifest entry references a registry name not + * present in the sidecar's registry config. + * - `registry.auth.failed`: the registry rejected the sidecar's credentials. + * - `tarball.extract.failed`: reading, re-verifying, or unpacking the cached + * tarball failed. + * - `git.materialization.failed`: a git-sourced entry could not be materialized + * from its checked-out subtree, or reached the tarball fetcher. + * - `package.entry.invalid`: a closure entry could not be laid out: its + * package.json is malformed, a path escapes its root, its dependencies + * collide, or a hard dependency has no satisfying closure version. + */ +export type ClosureMaterializationErrorCategory = + | "tarball.missing" + | "asset.mount.missing" + | "integrity.mismatch" + | "registry.fetch.failed" + | "registry.unknown" + | "registry.auth.failed" + | "tarball.extract.failed" + | "git.materialization.failed" + | "package.entry.invalid"; + +/** A closure-materialization failure carrying its category. */ +export class ClosureMaterializationError extends Error { + readonly category: ClosureMaterializationErrorCategory; + + constructor(opts: { + category: ClosureMaterializationErrorCategory; + message: string; + }) { + super(opts.message); + this.name = "ClosureMaterializationError"; + this.category = opts.category; + } +} + +/** + * Whether a `platform`/`os`/`cpu`-style allowlist matches `host`. A list with a + * `!`-negated entry matches everything except the negated hosts; an unnegated + * list matches only its members. + */ +export function platformListMatches( + entries: readonly string[], + host: string, +): boolean { + const hasNegation = entries.some((e) => e.startsWith("!")); + if (hasNegation) { + return !entries.includes(`!${host}`); + } + return entries.includes(host); +} + +export function isEEXIST(err: unknown): boolean { + if (err === null || typeof err !== "object") return false; + if (!("code" in err)) return false; + return (err as { code: unknown }).code === "EEXIST"; +} + +export function isENOENT(err: unknown): boolean { + if (err === null || typeof err !== "object") return false; + if (!("code" in err)) return false; + return (err as { code: unknown }).code === "ENOENT"; +} + +export function describeError(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} diff --git a/packages/tool-packaging/src/registry-fetch.ts b/packages/tool-packaging/src/registry-fetch.ts index 16b062af6..3ffe4a4ed 100644 --- a/packages/tool-packaging/src/registry-fetch.ts +++ b/packages/tool-packaging/src/registry-fetch.ts @@ -1,10 +1,8 @@ -// Registry HTTP fetch helpers for the tool-package loader: building the +// Registry HTTP fetch helpers for closure materialization: building the // npm-registry-fetch options for a configured registry, deriving a default -// tarball URL, and reading a response body under a byte cap. Extracted from -// `loader.ts` so the fetch concern is isolated from author-code loading and -// store layout. +// tarball URL, and reading a response body under a byte cap. -import { ToolLoaderError } from "./loader-internal"; +import { ClosureMaterializationError } from "./materialization-internal"; import type { RegistryConfig } from "./resolver"; export function buildRegistryFetchOpts( @@ -118,33 +116,29 @@ export async function readResponseWithLimit( const declaredLengthRaw = res.headers.get("content-length"); if (declaredLengthRaw !== null) { if (!/^\d+$/.test(declaredLengthRaw)) { - throw new ToolLoaderError({ + throw new ClosureMaterializationError({ category: "registry.fetch.failed", message: `registry "${ctx.registry}" returned non-digit Content-Length ${JSON.stringify(declaredLengthRaw)} for ${ctx.name}@${ctx.version}`, - package: { name: ctx.name, version: ctx.version }, }); } const declaredLength = Number(declaredLengthRaw); if (!Number.isFinite(declaredLength) || declaredLength > maxBytes) { - throw new ToolLoaderError({ + throw new ClosureMaterializationError({ category: "registry.fetch.failed", message: `tarball for ${ctx.name}@${ctx.version} declares Content-Length ${declaredLengthRaw} which exceeds the ${String(maxBytes)}-byte cap`, - package: { name: ctx.name, version: ctx.version }, }); } } - const timeoutError = (): ToolLoaderError => - new ToolLoaderError({ + const timeoutError = (): ClosureMaterializationError => + new ClosureMaterializationError({ category: "registry.fetch.failed", message: `tarball read for ${ctx.name}@${ctx.version} exceeded the registry fetch timeout`, - package: { name: ctx.name, version: ctx.version }, }); - const capOverflowError = (): ToolLoaderError => - new ToolLoaderError({ + const capOverflowError = (): ClosureMaterializationError => + new ClosureMaterializationError({ category: "registry.fetch.failed", message: `tarball for ${ctx.name}@${ctx.version} streamed past the ${String(maxBytes)}-byte cap`, - package: { name: ctx.name, version: ctx.version }, }); // `res.body`'s declared web-`ReadableStream` type is a lie on the @@ -215,10 +209,9 @@ export async function readResponseWithLimit( for await (const chunk of body) { if (timedOut) throw timeoutError(); if (!(chunk instanceof Uint8Array)) { - throw new ToolLoaderError({ + throw new ClosureMaterializationError({ category: "registry.fetch.failed", message: `registry "${ctx.registry}" streamed a non-binary chunk fetching ${ctx.name}@${ctx.version}`, - package: { name: ctx.name, version: ctx.version }, }); } total += chunk.byteLength; @@ -230,16 +223,16 @@ export async function readResponseWithLimit( // race does not slip through as a successful read. if (timedOut) throw timeoutError(); } catch (err) { - if (timedOut && !(err instanceof ToolLoaderError)) throw timeoutError(); + if (timedOut && !(err instanceof ClosureMaterializationError)) + throw timeoutError(); throw err; } finally { signal?.removeEventListener("abort", onAbort); } } else { - throw new ToolLoaderError({ + throw new ClosureMaterializationError({ category: "registry.fetch.failed", message: `registry "${ctx.registry}" returned a response body of an unreadable shape for ${ctx.name}@${ctx.version}`, - package: { name: ctx.name, version: ctx.version }, }); } diff --git a/packages/tool-packaging/src/resolver.test.ts b/packages/tool-packaging/src/resolver.test.ts index de0d8dc58..e0a788861 100644 --- a/packages/tool-packaging/src/resolver.test.ts +++ b/packages/tool-packaging/src/resolver.test.ts @@ -4,13 +4,11 @@ import * as tar from "tar"; import { promises as fs } from "node:fs"; import os from "node:os"; import path from "node:path"; -import { getToolPackageSourceContentIdentity } from "@intx/types/tool-packages"; import { type Packument, type PackumentFetcher, type RegistryConfig, - type RegistrySource, AssetRegistrySource, HttpRegistrySource, ManifestInvalidError, @@ -29,7 +27,6 @@ function packument( peerDependenciesMeta?: Record; os?: string[]; cpu?: string[]; - credentials?: { handle: string; scopes?: string[] }[]; integrity?: string; } >, @@ -60,9 +57,6 @@ function packument( : {}), ...(body.os !== undefined ? { os: body.os } : {}), ...(body.cpu !== undefined ? { cpu: body.cpu } : {}), - ...(body.credentials !== undefined - ? { credentials: body.credentials } - : {}), }, ]), ), @@ -84,79 +78,6 @@ function httpSource( return new HttpRegistrySource({ name, config, fetchPackument }); } -function registriesOf( - ...sources: readonly RegistrySource[] -): Map { - const m = new Map(); - for (const src of sources) { - m.set(src.name, src); - } - return m; -} - -describe("createResolver construction", () => { - test("throws if registries map is empty", () => { - expect(() => - createClosureResolver({ - registries: new Map(), - defaultRegistry: "any", - }), - ).toThrow(/registries cannot be empty/); - }); - - test("throws if defaultRegistry is not present in the map", () => { - expect(() => - createClosureResolver({ - registries: registriesOf( - httpSource({ name: "a", url: "https://a.test" }, {}), - ), - defaultRegistry: "missing", - }), - ).toThrow(/defaultRegistry "missing" not present/); - }); - - test("throws if a scopeRouting entry omits the leading @", () => { - // A bare scope like "intx" never matches any package name's scope - // (parseScope only returns strings that start with "@"), so the - // route would silently never fire and the package would fall - // through to the default registry. createClosureResolver must - // reject this at config-load time rather than swallow the typo. - expect(() => - createClosureResolver({ - registries: registriesOf( - httpSource({ name: "npmjs", url: "https://npmjs.test" }, {}), - ), - defaultRegistry: "npmjs", - scopeRouting: [{ scope: "intx", registry: "npmjs" }], - }), - ).toThrow(/scopeRouting entry has invalid scope/); - }); - - test("throws if a scopeRouting entry uses uppercase characters", () => { - expect(() => - createClosureResolver({ - registries: registriesOf( - httpSource({ name: "npmjs", url: "https://npmjs.test" }, {}), - ), - defaultRegistry: "npmjs", - scopeRouting: [{ scope: "@INTX", registry: "npmjs" }], - }), - ).toThrow(/scopeRouting entry has invalid scope/); - }); - - test("accepts a well-formed npm scope", () => { - expect(() => - createClosureResolver({ - registries: registriesOf( - httpSource({ name: "npmjs", url: "https://npmjs.test" }, {}), - ), - defaultRegistry: "npmjs", - scopeRouting: [{ scope: "@intx", registry: "npmjs" }], - }), - ).not.toThrow(); - }); -}); - describe("resolveClosure", () => { test("resolves a single pin with no transitive deps", async () => { const src = httpSource( @@ -164,8 +85,7 @@ describe("resolveClosure", () => { { "tools-leaf": packument("tools-leaf", { "1.0.0": {} }) }, ); const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "npmjs", + source: src, }); const manifest = await resolver.resolveClosure([ { name: "tools-leaf", version: "1.0.0" }, @@ -197,8 +117,7 @@ describe("resolveClosure", () => { }, ); const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "npmjs", + source: src, }); const manifest = await resolver.resolveClosure([ { name: "tools-with-deps", version: "1.0.0" }, @@ -217,8 +136,7 @@ describe("resolveClosure", () => { }, ); const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "npmjs", + source: src, }); const manifest = await resolver.resolveClosure([ { name: "a", version: "1.0.0" }, @@ -249,8 +167,7 @@ describe("resolveClosure", () => { }, ); const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "npmjs", + source: src, }); const manifest = await resolver.resolveClosure([ { name: "watcher", version: "1.0.0" }, @@ -261,89 +178,6 @@ describe("resolveClosure", () => { expect(linux?.os).toEqual(["linux"]); }); - test("routes scoped packages to the matching registry", async () => { - const calls: string[] = []; - const npmjs = new HttpRegistrySource({ - name: "npmjs", - config: { url: "https://npmjs.test" }, - fetchPackument: async (name) => { - calls.push(`${name}@npmjs`); - if (name === "left-pad") { - return packument("left-pad", { "1.0.0": {} }); - } - throw new Error(`unexpected fetch: ${name}`); - }, - }); - const intxPrivate = new HttpRegistrySource({ - name: "intx-private", - config: { url: "https://intx-private.test" }, - fetchPackument: async (name) => { - calls.push(`${name}@intx-private`); - if (name === "@intx/tools-mail") { - return packument("@intx/tools-mail", { "1.0.0": {} }); - } - throw new Error(`unexpected fetch: ${name}`); - }, - }); - const resolver = createClosureResolver({ - registries: registriesOf(npmjs, intxPrivate), - defaultRegistry: "npmjs", - scopeRouting: [{ scope: "@intx", registry: "intx-private" }], - }); - await resolver.resolveClosure([ - { name: "@intx/tools-mail", version: "1.0.0" }, - { name: "left-pad", version: "1.0.0" }, - ]); - expect(calls).toContain("@intx/tools-mail@intx-private"); - expect(calls).toContain("left-pad@npmjs"); - }); - - test("keeps separate packument caches for registries whose names share a substring", async () => { - // Length-prefixed cache keys: regression for `${name}::${pkg}` that - // collided when one registry's name contained the separator. - const ambiguousCalls: string[] = []; - const baseCalls: string[] = []; - const ambiguous = new HttpRegistrySource({ - name: "intx::a", - config: { url: "https://ambiguous.test" }, - fetchPackument: async (name) => { - ambiguousCalls.push(name); - return packument(name, { "1.0.0": {} }); - }, - }); - const base = new HttpRegistrySource({ - name: "intx", - config: { url: "https://base.test" }, - fetchPackument: async (name) => { - baseCalls.push(name); - return packument(name, { "1.0.0": {} }); - }, - }); - const resolver = createClosureResolver({ - registries: registriesOf(ambiguous, base), - defaultRegistry: "intx", - scopeRouting: [{ scope: "@a", registry: "intx::a" }], - }); - await resolver.resolveClosure([ - { name: "@a/foo", version: "1.0.0" }, - { name: "foo", version: "1.0.0" }, - ]); - expect(ambiguousCalls).toContain("@a/foo"); - expect(baseCalls).toContain("foo"); - }); - - test("throws if scopeRouting references unknown registry", async () => { - const src = httpSource({ name: "npmjs", url: "https://r.test" }, {}); - const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "npmjs", - scopeRouting: [{ scope: "@intx", registry: "does-not-exist" }], - }); - await expect( - resolver.resolveClosure([{ name: "@intx/foo", version: "1.0.0" }]), - ).rejects.toThrow(/scopeRouting references unknown registry/); - }); - test("throws if a packument entry lacks dist.integrity", async () => { const src = new HttpRegistrySource({ name: "npmjs", @@ -360,8 +194,7 @@ describe("resolveClosure", () => { }), }); const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "npmjs", + source: src, }); await expect( resolver.resolveClosure([{ name: "broken", version: "1.0.0" }]), @@ -384,8 +217,7 @@ describe("resolveClosure peer-dependency validation", () => { }, ); const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "npmjs", + source: src, }); const manifest = await resolver.resolveClosure([ { name: "react-tools", version: "1.0.0" }, @@ -404,8 +236,7 @@ describe("resolveClosure peer-dependency validation", () => { }, ); const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "npmjs", + source: src, }); await expect( resolver.resolveClosure([{ name: "react-tools", version: "1.0.0" }]), @@ -425,8 +256,7 @@ describe("resolveClosure peer-dependency validation", () => { }, ); const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "npmjs", + source: src, }); const manifest = await resolver.resolveClosure([ { name: "react-tools", version: "1.0.0" }, @@ -449,8 +279,7 @@ describe("resolveClosure peer-dependency validation", () => { }, ); const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "npmjs", + source: src, }); await expect( resolver.resolveClosure([{ name: "react-tools", version: "1.0.0" }]), @@ -467,8 +296,7 @@ describe("resolveClosure peer-dependency validation", () => { }, ); const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "npmjs", + source: src, }); let caught: unknown; try { @@ -513,8 +341,7 @@ describe("resolveClosure top-level versions", () => { }, }); const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "r", + source: src, }); const manifest = await resolver.resolveClosure([ @@ -599,14 +426,11 @@ describe("resolveClosure top-level versions", () => { throw new Error(`no packument: ${name}`); }; const resolver = createClosureResolver({ - registries: registriesOf( - new HttpRegistrySource({ - name: "r", - config: { url: "https://r.test" }, - fetchPackument, - }), - ), - defaultRegistry: "r", + source: new HttpRegistrySource({ + name: "r", + config: { url: "https://r.test" }, + fetchPackument, + }), }); // Pin `shared` to the narrow `~1.2.0` range (>=1.2.0, <1.3.0) so @@ -634,89 +458,6 @@ describe("resolveClosure top-level versions", () => { }); }); -describe("resolveClosure credential declarations", () => { - test("harvests a top-level package's declaration onto the manifest", async () => { - const src = httpSource( - { name: "npmjs", url: "https://r.test" }, - { - "tools-root": packument("tools-root", { - "1.0.0": { credentials: [{ handle: "gh", scopes: ["repo"] }] }, - }), - }, - ); - const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "npmjs", - }); - const manifest = await resolver.resolveClosure([ - { name: "tools-root", version: "1.0.0" }, - ]); - expect(manifest.topLevel).toEqual([ - { - name: "tools-root", - version: "1.0.0", - credentials: [{ handle: "gh", scopes: ["repo"] }], - }, - ]); - }); - - test("a package with no declaration carries no credentials field", async () => { - const src = httpSource( - { name: "npmjs", url: "https://r.test" }, - { "tools-root": packument("tools-root", { "1.0.0": {} }) }, - ); - const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "npmjs", - }); - const manifest = await resolver.resolveClosure([ - { name: "tools-root", version: "1.0.0" }, - ]); - // Absent, not [] or null -- the field is omitted entirely. - expect(manifest.topLevel).toEqual([ - { name: "tools-root", version: "1.0.0" }, - ]); - expect(manifest.topLevel[0]).not.toHaveProperty("credentials"); - }); - - test("a transitive dependency's declaration does not surface on topLevel", async () => { - const src = httpSource( - { name: "npmjs", url: "https://r.test" }, - { - "tools-root": packument("tools-root", { - "1.0.0": { - dependencies: { "dep-lib": "1.0.0" }, - credentials: [{ handle: "root-cred" }], - }, - }), - "dep-lib": packument("dep-lib", { - "1.0.0": { credentials: [{ handle: "dep-cred" }] }, - }), - }, - ); - const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "npmjs", - }); - const manifest = await resolver.resolveClosure([ - { name: "tools-root", version: "1.0.0" }, - ]); - // Only the top-level pin's declaration is harvested; dep-lib is a - // transitive entry, not a topLevel contributor. - expect(manifest.topLevel).toEqual([ - { - name: "tools-root", - version: "1.0.0", - credentials: [{ handle: "root-cred" }], - }, - ]); - expect(manifest.entries.map((e) => e.name).sort()).toEqual([ - "dep-lib", - "tools-root", - ]); - }); -}); - describe("resolveClosure duplicate-pin gate", () => { test("rejects two pins with the same name but different ranges", async () => { const src = httpSource( @@ -729,8 +470,7 @@ describe("resolveClosure duplicate-pin gate", () => { }, ); const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "npmjs", + source: src, }); await expect( resolver.resolveClosure([ @@ -767,8 +507,7 @@ describe("resolveClosure optionalDependencies", () => { }, }); const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "r", + source: src, }); const manifest = await resolver.resolveClosure([ @@ -803,8 +542,7 @@ describe("resolveClosure optionalDependencies", () => { }, }); const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "r", + source: src, }); let caught: unknown; @@ -862,8 +600,7 @@ describe("resolveClosure optionalDependencies", () => { }, }); const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "r", + source: src, }); const manifest = await resolver.resolveClosure([ @@ -913,8 +650,7 @@ describe("resolveClosure optionalDependencies", () => { }, }); const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "r", + source: src, }); const manifest = await resolver.resolveClosure([ { name: "root", version: "1.0.0" }, @@ -964,8 +700,7 @@ describe("resolveClosure optionalDependencies", () => { }, }); const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "r", + source: src, }); const manifest = await resolver.resolveClosure([ { name: "A", version: "1.0.0" }, @@ -1050,14 +785,13 @@ describe("AssetRegistrySource", () => { { name: "tools-alpha", version: "1.0.0" }, ]); const src = new AssetRegistrySource({ - name: "workspace-builtins", - assetId: "asset_workspace_builtins", + name: "tool-registry", + assetId: "asset_tool_registry", readBlob: fixture.readBlob, listBlobs: fixture.listBlobs, }); const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "workspace-builtins", + source: src, }); const manifest = await resolver.resolveClosure([ { name: "tools-alpha", version: "1.0.0" }, @@ -1066,7 +800,7 @@ describe("AssetRegistrySource", () => { const entry = manifest.entries[0]; expect(entry?.source).toEqual({ kind: "asset", - assetId: "asset_workspace_builtins", + assetId: "asset_tool_registry", package: { format: "tarball", path: `tarballs/${fixture.tarballNames[0]}`, @@ -1074,44 +808,6 @@ describe("AssetRegistrySource", () => { }, }); expect(entry?.tarballUrl).toBeUndefined(); - expect(entry && getToolPackageSourceContentIdentity(entry.source)).toMatch( - /^sha512-/, - ); - }); - - test("harvests interchange.credentials from a tarball's package.json onto topLevel", async () => { - const fixture = await buildAssetFixture([ - { - name: "tools-alpha", - version: "1.0.0", - pkgJsonExtras: { - interchange: { - tools: "./dist/bundle.js", - credentials: [{ handle: "gh", scopes: ["repo"] }], - }, - }, - }, - ]); - const src = new AssetRegistrySource({ - name: "workspace-builtins", - assetId: "asset_workspace_builtins", - readBlob: fixture.readBlob, - listBlobs: fixture.listBlobs, - }); - const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "workspace-builtins", - }); - const manifest = await resolver.resolveClosure([ - { name: "tools-alpha", version: "1.0.0" }, - ]); - expect(manifest.topLevel).toEqual([ - { - name: "tools-alpha", - version: "1.0.0", - credentials: [{ handle: "gh", scopes: ["repo"] }], - }, - ]); }); test("computes per-tarball SRI from bytes — divergent contents yield divergent integrities", async () => { @@ -1126,16 +822,18 @@ describe("AssetRegistrySource", () => { listBlobs: fixture.listBlobs, }); const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "wb", + source: src, }); const manifest = await resolver.resolveClosure([ { name: "p", version: "1.0.0" }, { name: "q", version: "1.0.0" }, ]); - const integrities = manifest.entries.map((e) => - getToolPackageSourceContentIdentity(e.source), - ); + const integrities = manifest.entries.map((e) => { + if (e.source.kind !== "asset" || e.source.package.format !== "tarball") { + throw new Error(`expected an asset tarball entry for ${e.name}`); + } + return e.source.package.integrity; + }); expect(new Set(integrities).size).toBe(integrities.length); for (const integrity of integrities) { expect(integrity).toMatch(/^sha512-/); @@ -1158,146 +856,6 @@ describe("AssetRegistrySource", () => { } }); - test("mixes with an HTTP source for transitive deps the asset does not publish", async () => { - // Top-level pins live in the asset; transitive `left-pad` lives in - // the HTTP registry. The asset registry is the default so unscoped - // top-level pins resolve through it; the transitive dep then - // bounces to the HTTP registry because the asset does not publish - // it. This is the asset-publishes-its-own-closure shape: the asset - // is authoritative for the names it carries and the http registry - // covers any name the asset does not. - // - // The two-step fallback is not implemented at the resolver layer — - // the walker queries the source the routing rule names and treats - // a miss as a fatal closure-walk failure. To make `left-pad` route - // to the http registry, exercise the scopeRouting feature: `@np` - // is a scope that routes to npmjs, but unscoped names that the - // asset does not publish need an alternative. For this test we - // exercise the cleaner shape: the asset-registry version of - // `tools-mixed` declares the transitive dep via a scoped name so - // scope routing carries it to the http registry. - const fixture = await buildAssetFixture([ - { - name: "tools-mixed", - version: "1.0.0", - pkgJsonExtras: { dependencies: { "@np/left-pad": "^1.3.0" } }, - }, - ]); - const assetSrc = new AssetRegistrySource({ - name: "asset-reg", - assetId: "asset_mixed", - readBlob: fixture.readBlob, - listBlobs: fixture.listBlobs, - }); - const httpSrc = httpSource( - { name: "npmjs", url: "https://r.test" }, - { "@np/left-pad": packument("@np/left-pad", { "1.3.0": {} }) }, - ); - const resolver = createClosureResolver({ - registries: registriesOf(assetSrc, httpSrc), - defaultRegistry: "asset-reg", - scopeRouting: [{ scope: "@np", registry: "npmjs" }], - }); - const manifest = await resolver.resolveClosure([ - { name: "tools-mixed", version: "1.0.0" }, - ]); - expect(manifest.entries).toHaveLength(2); - const byName = new Map(manifest.entries.map((e) => [e.name, e])); - expect(byName.get("tools-mixed")?.source.kind).toBe("asset"); - expect(byName.get("@np/left-pad")?.source.kind).toBe("registry"); - }); - - test("inner-tenancy asset shadows http registry of the same name", async () => { - // The resolver does not itself implement tenant inheritance — the - // session service walks the chain and decides which source claims - // each registry name. The collision behavior at the resolver layer - // is simpler: the map only carries one entry per name, so whichever - // source the session service inserted under "shared" wins. This - // test locks in that the resolver respects that single-entry - // contract end-to-end. - const fixture = await buildAssetFixture([ - { name: "shadowed", version: "1.0.0" }, - ]); - const inner = new AssetRegistrySource({ - name: "shared", - assetId: "asset_inner", - readBlob: fixture.readBlob, - listBlobs: fixture.listBlobs, - }); - // Build a map directly so the test calls out that the asset source - // occupies the slot the http source would otherwise hold. - const registries = new Map([[inner.name, inner]]); - const resolver = createClosureResolver({ - registries, - defaultRegistry: "shared", - }); - const manifest = await resolver.resolveClosure([ - { name: "shadowed", version: "1.0.0" }, - ]); - expect(manifest.entries).toHaveLength(1); - expect(manifest.entries[0]?.source.kind).toBe("asset"); - }); - - test("asset registry wins over an http registry of the same name", async () => { - // Lock in the collision-resolution contract the session service - // enforces: when both an AssetRegistrySource and an - // HttpRegistrySource carry the same registry name, the asset - // entry is what the resolver consults. The session service - // populates its registry map by inserting assets first and - // skipping HTTP entries on name collision; this test replays the - // same composition and asserts the asset path produced the - // manifest entry. - const NAME = "shared-registry"; - const fixture = await buildAssetFixture([ - { name: "claimed", version: "1.0.0" }, - ]); - const assetSrc = new AssetRegistrySource({ - name: NAME, - assetId: "asset_winner", - readBlob: fixture.readBlob, - listBlobs: fixture.listBlobs, - }); - const httpSrc = new HttpRegistrySource({ - name: NAME, - config: { url: "https://shadowed.test" }, - fetchPackument: async () => { - throw new Error( - "http registry was consulted but the asset registry should have shadowed it", - ); - }, - }); - - // Replay the session-service composition: asset first, then HTTP - // only when the name is not already claimed. The Map is the - // resolver's view of the world; the policy lives in how this - // map is built. - const registries = new Map(); - for (const src of [assetSrc, httpSrc]) { - if (registries.has(src.name)) continue; - registries.set(src.name, src); - } - expect(registries.get(NAME)).toBe(assetSrc); - - const resolver = createClosureResolver({ - registries, - defaultRegistry: NAME, - }); - const manifest = await resolver.resolveClosure([ - { name: "claimed", version: "1.0.0" }, - ]); - expect(manifest.entries).toHaveLength(1); - const entry = manifest.entries[0]; - expect(entry?.source).toEqual({ - kind: "asset", - assetId: "asset_winner", - package: { - format: "tarball", - path: `tarballs/${fixture.tarballNames[0]}`, - integrity: expect.stringMatching(/^sha512-/), - }, - }); - }); - test("rejects the closure when a tarball's package.json fails validation", async () => { // Construct a tarball whose package.json is missing the required // `name` field; AssetRegistrySource must surface this via @@ -1330,8 +888,7 @@ describe("AssetRegistrySource", () => { Array.from(byPath.keys()).map((p) => p.slice("tarballs/".length)), }); const resolver = createClosureResolver({ - registries: registriesOf(src), - defaultRegistry: "broken", + source: src, }); await expect( resolver.resolveClosure([{ name: "anything", version: "1.0.0" }]), @@ -1360,7 +917,7 @@ describe("parsePin", () => { test("rejects a mixed-case package name", () => { // npm itself forbids uppercase in package names; the sidecar - // loader matches entries by their lowercase packument name, so a + // matches entries by their lowercase packument name, so a // mixed-case pin would silently fall off the matcher's key set. expect(() => parsePin("@INTX/tools-mail@1.0.0")).toThrow( /pin name must be lowercase/, diff --git a/packages/tool-packaging/src/resolver.ts b/packages/tool-packaging/src/resolver.ts index 7f1f12659..ad6bd43f4 100644 --- a/packages/tool-packaging/src/resolver.ts +++ b/packages/tool-packaging/src/resolver.ts @@ -2,7 +2,7 @@ /// // Hub-side npm closure resolver. // -// Walks a list of agent-pinned tool packages, resolves each pin against +// Walks a list of pinned root packages, resolves each pin against // a configured registry source (HTTP-backed `npm-registry-fetch` or an // in-process `package-registry` asset), recurses on transitive // `dependencies` and `optionalDependencies`, and produces a @@ -18,9 +18,6 @@ // `package-registry` asset via the asset service's in-process // read API. // - Version picking via npm-pick-manifest. -// - Scope routing: `@scope/foo` requests can be routed to a specific -// registry name from the scopeRouting table; everything else uses -// the registry named by `defaultRegistry`. // - Per-entry source materialization: the walker asks the // `RegistrySource` for the entry's `source` and `tarballUrl`. HTTP // sources emit `kind: "registry"` plus the picked tarball URL; @@ -40,10 +37,7 @@ import semver from "semver"; import ssri from "ssri"; import { getLogger } from "@intx/log"; -import type { - PackageJSON, - ToolCredentialDeclaration, -} from "@intx/types/package-json"; +import type { PackageJSON } from "@intx/types/package-json"; import { extractTarballPackageJSON } from "./package-json-extract"; @@ -61,20 +55,11 @@ import { * `auth.basic` mirror the npm-registry-fetch options for bearer / * basic auth. * - * The registry's identifier (the string `scopeRouting` entries and - * manifest `registry` references point at) is the slot it occupies - * in the registry map: in the hub-side resolver, the key of the - * `ReadonlyMap` passed to - * `createClosureResolver`; in the sidecar-side loader, the key of - * the `ReadonlyMap` on `LoaderConfig`. - * Keeping the name on the map slot rather than on the value - * eliminates the (until-now uninforced) "the map key must equal the + * The registry's identifier (the string manifest `registry` references + * point at) is its key in the `ReadonlyMap` + * passed to `materializeClosure`. Keeping the name on the map slot + * rather than on the value eliminates the "the map key must equal the * config's name" invariant. - * - * The default-registry decision is owned by `defaultRegistry` on - * `ClosureResolverConfig`, not by a per-registry flag; only one - * entry can be the default and centralizing the choice keeps that - * invariant in the config object. */ export interface RegistryConfig { readonly url: string; @@ -84,12 +69,6 @@ export interface RegistryConfig { }; } -/** Route packages in a given scope (e.g. `@intx`) to a named registry. */ -export interface ScopeRoute { - readonly scope: string; - readonly registry: string; -} - /** * One version manifest as it appears inside a packument. Fields are * non-readonly to match the npm-pick-manifest declaration; consumers @@ -108,12 +87,6 @@ export interface PackumentVersion { peerDependenciesMeta?: Record; os?: string[]; cpu?: string[]; - // Static provider-backed credential declarations from the package's - // `interchange.credentials`. The AssetRegistrySource populates this from the - // arktype-validated package.json. An HTTP registry only carries it when its - // packument serves the field (abbreviated packuments strip custom fields); - // an under-served declaration is fail-closed downstream, never a bypass. - credentials?: ToolCredentialDeclaration[]; } /** @@ -156,9 +129,8 @@ export interface MaterializedRef { * regardless of whether bytes live behind an HTTP registry or inside a * `package-registry` asset. * - * `name` is the identifier the source was registered under in the - * resolver's registry map. Diagnostics that need to print a - * registry's name read it from here; the impls are the source of + * `name` is the registry's identifier. Diagnostics that need to print + * a registry's name read it from here; the impls are the source of * truth and the interface guarantees it is always available. */ export interface RegistrySource { @@ -172,22 +144,9 @@ export interface RegistrySource { ): MaterializedRef; } -/** - * Configuration for `createClosureResolver`. - * - * `registries` is keyed by the same name the agent's pin set - * references via the `registry` field in `ToolPackagePin`-derived - * entries (today indirectly via scope routing; built-ins reach this - * map by the registry name the session service resolved them from). - * - * `defaultRegistry` names the entry in `registries` consulted for any - * package whose scope does not match a `scopeRouting` entry. The map - * must contain this key. - */ +/** Configuration for `createClosureResolver`: the registry a closure resolves through. */ export interface ClosureResolverConfig { - readonly registries: ReadonlyMap; - readonly defaultRegistry: string; - readonly scopeRouting?: readonly ScopeRoute[]; + readonly source: RegistrySource; } export interface ClosureResolver { @@ -207,18 +166,13 @@ export interface PeerDependencyViolation { /** * Thrown by `resolveClosure` when the resolved closure does not satisfy * one or more peer-dependency declarations, by the closure walker for - * duplicate-name pins, and by the hub-side session-launch path when a - * direct package-registry attachment overlaps a resolver-driven pin - * for the same asset. The deploy-assembly path maps every shape to - * the same `manifest.invalid` deploy-apply error category before the - * deploy ships. + * duplicate-name pins, and by `extractPackageJSON` for a tarball it + * cannot read. It fails the deploy before anything ships. * - * The category is intentionally broad: peer-dep violations and + * The one error type is intentionally broad: peer-dep violations and * duplicate-name pins are structurally distinct defects but both * indicate a closure the operator cannot ship without changing the - * pin set. Adding a per-shape category would expand the wire - * taxonomy for no operator-facing gain; readers parsing - * deploy-apply errors should look at the message for the structural + * pin. Readers should look at the message for the structural * distinction. */ export class ManifestInvalidError extends Error { @@ -263,8 +217,7 @@ export class HttpRegistrySource implements RegistrySource { readonly #fetchPackument: PackumentFetcher; constructor(args: { - /** Registry identifier. Becomes the map slot key in the - * resolver's registries map. */ + /** Registry identifier, recorded on the entries it resolves. */ readonly name: string; readonly config: RegistryConfig; /** Test seam: when omitted, the default `npm-registry-fetch` @@ -406,11 +359,6 @@ export class AssetRegistrySource implements RegistrySource { version: validated.version, dist: { tarball: repoPath, integrity }, ...readDependencyFields(extracted.raw), - // Read from the arktype-validated `parsed` (credentials is in-schema), - // not `raw`. Absent stays absent -- never defaulted to an empty array. - ...(validated.interchange?.credentials !== undefined - ? { credentials: validated.interchange.credentials } - : {}), }; if (packument === undefined) { byName.set(validated.name, { @@ -507,55 +455,10 @@ function isStringArray(value: unknown): value is string[] { return Array.isArray(value) && value.every((v) => typeof v === "string"); } -/** - * npm scope shape: `@` followed by a name character class that matches - * what npm itself accepts in `validate-npm-package-name`. We validate - * the routing entries here rather than during pin resolution because - * a typo (`intx` for `@intx`) silently never matches and the package - * falls through to the default registry — the exact failure mode the - * scope-routing feature exists to prevent. - */ -const NPM_SCOPE_RE = /^@[a-z0-9][a-z0-9-_]*$/; - export function createClosureResolver( config: ClosureResolverConfig, ): ClosureResolver { - if (config.registries.size === 0) { - throw new Error("createClosureResolver: registries cannot be empty"); - } - const maybeDefault = config.registries.get(config.defaultRegistry); - if (maybeDefault === undefined) { - throw new Error( - `createClosureResolver: defaultRegistry "${config.defaultRegistry}" not present in registries map`, - ); - } - if (config.scopeRouting !== undefined) { - for (const route of config.scopeRouting) { - if (!NPM_SCOPE_RE.test(route.scope)) { - throw new Error( - `createClosureResolver: scopeRouting entry has invalid scope ${JSON.stringify(route.scope)}; scopes must match ${NPM_SCOPE_RE.toString()} (e.g. "@intx")`, - ); - } - } - } - const defaultSource: RegistrySource = maybeDefault; - - function resolveSource(packageName: string): RegistrySource { - const scope = parseScope(packageName); - if (scope !== null && config.scopeRouting !== undefined) { - const route = config.scopeRouting.find((r) => r.scope === scope); - if (route !== undefined) { - const src = config.registries.get(route.registry); - if (src === undefined) { - throw new Error( - `scopeRouting references unknown registry "${route.registry}" for scope "${scope}"`, - ); - } - return src; - } - } - return defaultSource; - } + const { source } = config; return { async resolveClosure(pins) { @@ -586,24 +489,14 @@ export function createClosureResolver( // through an unrelated subtree, otherwise dropping the first // subtree would silently drop the dep for the second one too. // That correctness requirement would multiply packument requests - // against the upstream registry; this cache keeps each - // `(registry, name)` pair to one network round-trip per walk. + // against the upstream registry; this cache keeps each name to + // one network round-trip per walk. const packumentCache = new Map>(); - function fetchPackumentCached( - source: RegistrySource, - name: string, - ): Promise { - // Length-prefix the registry name so two registries whose names - // happen to overlap with the package-name separator (e.g. a - // hypothetical registry literally named `foo::bar`) cannot - // collide on the cache key with another registry. The prefix - // makes the encoding injective on `(registryName, packageName)` - // regardless of which characters appear in either string. - const key = `${source.name.length.toString()}:${source.name}:${name}`; - const existing = packumentCache.get(key); + function fetchPackumentCached(name: string): Promise { + const existing = packumentCache.get(name); if (existing !== undefined) return existing; const promise = source.fetchPackument(name); - packumentCache.set(key, promise); + packumentCache.set(name, promise); return promise; } // BFS so root-pin entries are visited before their transitive @@ -632,11 +525,10 @@ export function createClosureResolver( // but only after the round-trip; this catches them before. const requested = new Set(); // Track the resolved (concrete) version each top-level pin - // walked to, keyed by pin name. The sidecar's loader matches - // top-level entries by `${name}@${concrete-version}` to decide - // which packages contribute tool factories; emitting the pin's - // raw range here would cause every range-form pin to silently - // fail that lookup at apply time. Only the first arrival on + // walked to, keyed by pin name. The sidecar finds a top-level + // package in the store by `${name}@${concrete-version}`; emitting + // the pin's raw range here would make every range-form pin miss + // that lookup at apply time. Only the first arrival on // each pin name counts: a transitive dep that shares a // top-level name must not overwrite the top-level's resolved // version with its own. @@ -652,14 +544,6 @@ export function createClosureResolver( // top-level resolution. const topLevelNames = new Set(pins.map((p) => p.name)); const topLevelResolved = new Map(); - // Credential declarations harvested from each top-level package, - // captured alongside the resolved version. Only top-level pins - // contribute; a transitive dependency's declaration is never recorded. - const topLevelDeclarations = new Map< - string, - ToolCredentialDeclaration[] - >(); - // Optional-subtree bookkeeping. Subtree-tagged entries and peer // declarations stay in their per-subtree stash until the whole // subtree finishes successfully, at which point they merge into @@ -759,11 +643,10 @@ export function createClosureResolver( } requested.add(requestKey); - const source = resolveSource(next.name); let packument: Packument; let picked: PackumentVersion; try { - packument = await fetchPackumentCached(source, next.name); + packument = await fetchPackumentCached(next.name); picked = npmPickManifest(packument, next.range); } catch (err) { if (next.subtreeId !== null) { @@ -786,9 +669,6 @@ export function createClosureResolver( if (topLevelNames.has(next.name) && !topLevelResolved.has(next.name)) { topLevelResolved.set(next.name, picked.version); - if (picked.credentials !== undefined) { - topLevelDeclarations.set(next.name, picked.credentials); - } } const key = `${picked.name}@${picked.version}`; @@ -932,12 +812,7 @@ export function createClosureResolver( `resolver internal error: top-level pin ${p.name}@${p.version} did not resolve to a concrete version`, ); } - const credentials = topLevelDeclarations.get(p.name); - return { - name: p.name, - version: resolved, - ...(credentials !== undefined ? { credentials } : {}), - }; + return { name: p.name, version: resolved }; }), entries: Array.from(entries.values()), }; @@ -973,26 +848,6 @@ function checkPeerDependencies( return violations; } -function parseScope(packageName: string): string | null { - if (!packageName.startsWith("@")) return null; - const slash = packageName.indexOf("/"); - if (slash === -1) return null; - // npm package names are case-insensitive at the registry layer but - // npm's own validator forbids uppercase. Scope-routing lookups are - // gated on NPM_SCOPE_RE which rejects uppercase, so an uppercase - // scope in a package name would silently fail to match any routing - // entry. Lowercase the lookup so a `@INTX/foo` import still routes - // through a `@intx` routing entry rather than falling through to - // the default registry. - // - // Pin-time validation rejects uppercase top-level names, but - // transitive names sourced from registry packuments are not gated - // against the same rule. The lowercasing here is the only defense - // against a misconfigured registry returning mixed-case names that - // would otherwise mis-route at fetch time. - return packageName.slice(0, slash).toLowerCase(); -} - /** * Categorize a closure-walk resolution failure as transport or * structural so the poison-reason log can route accordingly. Inputs @@ -1108,15 +963,8 @@ async function extractPackageJSON( * any-version range; the resolver then picks whatever the registry * advertises as latest at deploy-assembly time. Operators who care * about reproducibility of tool-package closures should pin to a - * concrete range (`^1.2.3`) — `*` lets the closure shift under the - * agent without any change to the pin set. - * - * NOTE: `*` is special-cased in two places — here and inside the - * `ToolPackagePinArray` narrow at `@intx/types/tool-packages`. The - * sites live in separate packages by design (resolver vs. wire-type - * validation) and cannot import each other; any new magic-range - * additions need to be made at both call sites to keep the - * REST-boundary validator and the resolver in agreement. + * concrete range (`^1.2.3`) — `*` lets the closure shift between + * deployments without any change to the pin. */ export function parsePin(spec: string): ToolPackagePin { const parsed = npmPackageArg(spec); @@ -1125,7 +973,7 @@ export function parsePin(spec: string): ToolPackagePin { } // npm itself rejects uppercase in package names; packuments arrive // lowercased, so a mixed-case pin would self-resolve through - // `topLevelResolved` and then silently fail the sidecar loader's + // `topLevelResolved` and then silently fail the sidecar's // `${name}@${version}` lookup against the lowercase entry the // packument produced. Reject loudly at the boundary rather than // lowercasing the input — pin sets that mix cases are a bug in the diff --git a/packages/tool-packaging/src/loader.test.ts b/packages/tool-packaging/src/store-layout.test.ts similarity index 51% rename from packages/tool-packaging/src/loader.test.ts rename to packages/tool-packaging/src/store-layout.test.ts index 9ae1f541c..2a78fd4b5 100644 --- a/packages/tool-packaging/src/loader.test.ts +++ b/packages/tool-packaging/src/store-layout.test.ts @@ -3,26 +3,27 @@ import { promises as fs } from "node:fs"; import os from "node:os"; import path from "node:path"; import { Readable } from "node:stream"; +import { pathToFileURL } from "node:url"; import * as tar from "tar"; import ssri from "ssri"; import npmRegistryFetch from "npm-registry-fetch"; -import { applyAtomic } from "./atomic-apply"; import { createTarballCache } from "./cache"; import { - type LoadedDirectorFactory, - type LoadedToolFactory, - type TarballFetcher, - ToolLoaderError, + type ClosureMaterializationErrorCategory, + ClosureMaterializationError, +} from "./materialization-internal"; +import { buildRegistryFetchOpts, - createToolLoader, - materializeClosure, readResponseWithLimit, +} from "./registry-fetch"; +import { + type MaterializeClosureArgs, + materializeClosure, storeEntryDir, -} from "./loader"; -import type { DeployApplyErrorCategory } from "@intx/types/sidecar"; +} from "./store-layout"; +import { type TarballFetcher, createTarballFetcher } from "./tarball-fetcher"; import type { ToolPackageManifest } from "@intx/types/tool-packages"; -import { getToolPackageSourceContentIdentity } from "@intx/types/tool-packages"; let scratchRoot: string; let cacheDir: string; @@ -32,7 +33,7 @@ let fixtureSourceRoot: string; beforeEach(async () => { scratchRoot = await fs.mkdtemp( - path.join(os.tmpdir(), "tool-packaging-loader-"), + path.join(os.tmpdir(), "tool-packaging-store-layout-"), ); cacheDir = path.join(scratchRoot, "cache"); instanceDir = path.join(scratchRoot, "instance"); @@ -53,11 +54,6 @@ interface FixtureSpec { version: string; interchangeToolsRelPath?: string | null; // null means omit the field entryModuleSource?: string; // emitted at the interchange.tools path - interchangeDirectorsRelPath?: string | null; // null means omit the field - directorsModuleSource?: string; // emitted at the interchange.directors path - // Inline `interchange.credentials` value, written verbatim into - // package.json. `unknown` so malformed-shape cases can be exercised. - interchangeCredentials?: unknown; extraFiles?: Record; dependencies?: Record; optionalDependencies?: Record; @@ -87,15 +83,6 @@ async function packFixture( if (spec.interchangeToolsRelPath !== null) { interchangeField.tools = spec.interchangeToolsRelPath ?? "./tools.js"; } - if ( - spec.interchangeDirectorsRelPath !== null && - spec.interchangeDirectorsRelPath !== undefined - ) { - interchangeField.directors = spec.interchangeDirectorsRelPath; - } - if (spec.interchangeCredentials !== undefined) { - interchangeField.credentials = spec.interchangeCredentials; - } if (Object.keys(interchangeField).length > 0) { pkgJson.interchange = interchangeField; } @@ -119,19 +106,6 @@ async function packFixture( await fs.writeFile(entryAbs, spec.entryModuleSource); } - if ( - spec.directorsModuleSource !== undefined && - spec.interchangeDirectorsRelPath !== undefined && - spec.interchangeDirectorsRelPath !== null - ) { - const directorsAbs = path.resolve( - packageDir, - spec.interchangeDirectorsRelPath, - ); - await fs.mkdir(path.dirname(directorsAbs), { recursive: true }); - await fs.writeFile(directorsAbs, spec.directorsModuleSource); - } - for (const [rel, contents] of Object.entries(spec.extraFiles ?? {})) { const abs = path.join(packageDir, rel); await fs.mkdir(path.dirname(abs), { recursive: true }); @@ -152,106 +126,47 @@ async function packFixture( return { bytes, integrity }; } -function makeFakeFactory( - id: string, - requires: readonly string[] = [], -): LoadedToolFactory { - const fn = () => ({ - definitions: [], - run: async () => ({ - callId: "stub", - content: "ok", - }), - }); - return Object.assign(fn, { id, requires, definitions: [] }); +async function isLaidOut( + storeDir: string, + name: string, + version: string, +): Promise { + const manifest = path.join( + storeEntryDir(storeDir, name, version), + "package.json", + ); + return fs.access(manifest).then( + () => true, + () => false, + ); } -/** - * Build a director-factory-shaped stub the loader's structural check - * accepts. `configSchema` is a callable arktype-shaped validator (the - * real `defineDirector` requires this); the body returns a director - * stub adequate for the loader's structural narrow without exercising - * the runtime director protocol. - */ -function makeFakeDirectorFactory( - id: string, - requires: readonly string[] = [], -): LoadedDirectorFactory { - const fn = () => { - throw new Error( - `stub director factory ${id} should not be invoked in loader tests`, - ); +// Binds what a test holds fixed around `materializeClosure`'s per-call layout +// arguments, and gives it the production fetcher unless it supplies its own. +function createMaterializer( + config: Pick & + Partial>, +) { + const fetchTarball = config.fetchTarball ?? createTarballFetcher(); + return { + materialize: ( + args: Pick< + MaterializeClosureArgs, + | "manifest" + | "instanceScratchDir" + | "assetRoot" + | "assetMounts" + | "gitDirs" + >, + ) => materializeClosure({ ...args, ...config, fetchTarball }), }; - const configSchema = (value: unknown) => value; - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- LoadedDirectorFactory carries an erased Config (`unknown`); the structural fields the loader's predicate checks (id, requires, configSchema) are satisfied below, and the loader does not invoke the factory body in these tests. - return Object.assign(fn, { - id, - requires, - configSchema, - }) as unknown as LoadedDirectorFactory; } -describe("createToolLoader", () => { - test("loads a single top-level package via injected fetcher and importer", async () => { - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); - const fixture = await packFixture({ - name: "tools-leaf", - version: "1.0.0", - entryModuleSource: "// stub; importer is faked", - }); - - const fakeFactory = makeFakeFactory("@vendor/leaf/main", ["env.thing"]); - const fetchTarball: TarballFetcher = async (entry) => { - expect(entry.name).toBe("tools-leaf"); - return fixture.bytes; - }; - const loader = createToolLoader({ - cache, - registries: new Map([["npmjs", { url: "https://r.test" }]]), - host: { os: "linux", cpu: "x64" }, - fetchTarball, - importModule: async () => ({ main: fakeFactory }), - }); - - const manifest: ToolPackageManifest = { - schemaVersion: "1", - topLevel: [{ name: "tools-leaf", version: "1.0.0" }], - entries: [ - { - name: "tools-leaf", - version: "1.0.0", - source: { - kind: "registry", - registry: "npmjs", - integrity: fixture.integrity, - }, - }, - ], - }; - - const loaded = await loader.loadManifest({ - manifest, - instanceScratchDir: instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - }); - expect(loaded).toHaveLength(1); - expect(loaded[0]?.name).toBe("tools-leaf"); - expect(loaded[0]?.factories).toHaveLength(1); - expect(loaded[0]?.factories[0]?.id).toBe("@vendor/leaf/main"); - // A package that declares no credentials surfaces an empty array, - // not a missing field -- the reconcile downstream reads it verbatim. - expect(loaded[0]?.credentials).toEqual([]); - }); - +describe("materializeClosure layout", () => { test("lays out a workspace: protocol dependency against its closure member", async () => { // A source-workspace member depends on another member via `workspace:*`. // That protocol is not semver, so before it is normalized the range - // resolver rejects the layout with "no satisfying version"; the loader must + // resolver rejects the layout with "no satisfying version"; materialization must // instead match the one closure entry of that name. const cache = createTarballCache({ rootDir: cacheDir, @@ -260,7 +175,7 @@ describe("createToolLoader", () => { const app = await packFixture({ name: "@wf/app", version: "1.0.0", - entryModuleSource: "// stub; importer is faked", + entryModuleSource: "// stub", dependencies: { "@wf/lib": "workspace:*" }, }); const lib = await packFixture({ @@ -275,12 +190,11 @@ describe("createToolLoader", () => { if (entry.name === "@wf/lib") return lib.bytes; throw new Error(`unexpected fetch ${entry.name}`); }; - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map([["npmjs", { url: "https://r.test" }]]), host: { os: "linux", cpu: "x64" }, fetchTarball, - importModule: async () => ({ main: makeFakeFactory("@wf/app/main") }), }); const manifest: ToolPackageManifest = { @@ -310,15 +224,15 @@ describe("createToolLoader", () => { // Before the workspace: normalization this rejected with // `package.entry.invalid`; it now lays out cleanly. - const loaded = await loader.loadManifest({ + const { storeDir } = await materializer.materialize({ manifest, instanceScratchDir: instanceDir, assetRoot, assetMounts: new Map(), gitDirs: new Map(), }); - expect(loaded).toHaveLength(1); - expect(loaded[0]?.name).toBe("@wf/app"); + expect(await isLaidOut(storeDir, "@wf/app", "1.0.0")).toBe(true); + expect(await isLaidOut(storeDir, "@wf/lib", "1.0.0")).toBe(true); }); test("lays out a catalog: protocol dependency against its closure member", async () => { @@ -333,7 +247,7 @@ describe("createToolLoader", () => { const app = await packFixture({ name: "@wf/catalog-app", version: "1.0.0", - entryModuleSource: "// stub; importer is faked", + entryModuleSource: "// stub", dependencies: { "left-pad": "catalog:" }, }); const leftPad = await packFixture({ @@ -348,14 +262,11 @@ describe("createToolLoader", () => { if (entry.name === "left-pad") return leftPad.bytes; throw new Error(`unexpected fetch ${entry.name}`); }; - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map([["npmjs", { url: "https://r.test" }]]), host: { os: "linux", cpu: "x64" }, fetchTarball, - importModule: async () => ({ - main: makeFakeFactory("@wf/catalog-app/main"), - }), }); const manifest: ToolPackageManifest = { @@ -383,15 +294,14 @@ describe("createToolLoader", () => { ], }; - const loaded = await loader.loadManifest({ + const { storeDir } = await materializer.materialize({ manifest, instanceScratchDir: instanceDir, assetRoot, assetMounts: new Map(), gitDirs: new Map(), }); - expect(loaded).toHaveLength(1); - expect(loaded[0]?.name).toBe("@wf/catalog-app"); + expect(await isLaidOut(storeDir, "@wf/catalog-app", "1.0.0")).toBe(true); }); test("materializes package files independently from the cache extraction", async () => { @@ -404,17 +314,14 @@ describe("createToolLoader", () => { version: "1.0.0", entryModuleSource: "original contents", }); - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map([["npmjs", { url: "https://r.test" }]]), host: { os: "linux", cpu: "x64" }, fetchTarball: async () => fixture.bytes, - importModule: async () => ({ - main: makeFakeFactory("@vendor/copied/main"), - }), }); - await loader.loadManifest({ + await materializer.materialize({ manifest: { schemaVersion: "1", topLevel: [{ name: "copied-tool", version: "1.0.0" }], @@ -457,240 +364,10 @@ describe("createToolLoader", () => { } }); - test("surfaces interchange.credentials declarations on the loaded package", async () => { - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); - const fixture = await packFixture({ - name: "tools-creds", - version: "1.0.0", - entryModuleSource: "// stub; importer is faked", - interchangeCredentials: [ - { handle: "openai", scopes: ["chat"] }, - { handle: "stripe" }, - ], - }); - - const fakeFactory = makeFakeFactory("@vendor/creds/main", ["env.thing"]); - const loader = createToolLoader({ - cache, - registries: new Map([["npmjs", { url: "https://r.test" }]]), - host: { os: "linux", cpu: "x64" }, - fetchTarball: async () => fixture.bytes, - importModule: async () => ({ main: fakeFactory }), - }); - - const loaded = await loader.loadManifest({ - manifest: { - schemaVersion: "1", - topLevel: [{ name: "tools-creds", version: "1.0.0" }], - entries: [ - { - name: "tools-creds", - version: "1.0.0", - source: { - kind: "registry", - registry: "npmjs", - integrity: fixture.integrity, - }, - }, - ], - }, - instanceScratchDir: instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - }); - - expect(loaded[0]?.credentials).toEqual([ - { handle: "openai", scopes: ["chat"] }, - { handle: "stripe" }, - ]); - }); - - type Declarations = readonly { - readonly handle: string; - readonly scopes?: readonly string[]; - }[]; - - // Load one package whose manifest declares `manifest` and whose only - // factory declares `factory` through `defineTool` (undefined: nothing). - // `factory` is whatever the package's module exports as the factory's - // `credentials`, so a test can hand the loader a malformed declaration. - async function loadWithDeclarations( - manifest: Declarations | undefined, - factory: unknown, - ) { - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); - const fixture = await packFixture({ - name: "tools-creds", - version: "1.0.0", - entryModuleSource: "// stub; importer is faked", - ...(manifest !== undefined ? { interchangeCredentials: manifest } : {}), - }); - const loader = createToolLoader({ - cache, - registries: new Map([["npmjs", { url: "https://r.test" }]]), - host: { os: "linux", cpu: "x64" }, - fetchTarball: async () => fixture.bytes, - importModule: async () => ({ - main: Object.assign( - makeFakeFactory("@vendor/creds/main", ["env.thing"]), - factory === undefined ? {} : { credentials: factory }, - ), - }), - }); - return loader.loadManifest({ - manifest: { - schemaVersion: "1", - topLevel: [{ name: "tools-creds", version: "1.0.0" }], - entries: [ - { - name: "tools-creds", - version: "1.0.0", - source: { - kind: "registry", - registry: "npmjs", - integrity: fixture.integrity, - }, - }, - ], - }, - instanceScratchDir: instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - }); - } - - const MANIFEST: Declarations = [ - { handle: "openai", scopes: ["chat", "embed"] }, - { handle: "stripe" }, - ]; - - for (const [label, factory] of [ - ["declares nothing", undefined], - [ - "declares one of the manifest's handles", - [{ handle: "openai", scopes: ["chat", "embed"] }], - ], - [ - "declares the same handles in another order", - [{ handle: "stripe" }, { handle: "openai", scopes: ["chat", "embed"] }], - ], - [ - "lists scopes in another order or more than once", - [ - { handle: "openai", scopes: ["embed", "chat", "chat"] }, - { handle: "stripe", scopes: [] }, - ], - ], - ] as const) { - test(`accepts a factory that ${label}`, async () => { - const loaded = await loadWithDeclarations(MANIFEST, factory); - expect(loaded[0]?.credentials).toEqual([ - { handle: "openai", scopes: ["chat", "embed"] }, - { handle: "stripe" }, - ]); - }); - } - - for (const [label, manifest, factory] of [ - [ - "declares different scopes", - MANIFEST, - [{ handle: "openai", scopes: ["chat"] }, { handle: "stripe" }], - ], - [ - "declares handles the manifest does not", - undefined, - [{ handle: "stripe" }], - ], - ] as const) { - test(`rejects a factory that ${label}`, async () => { - await expect(loadWithDeclarations(manifest, factory)).rejects.toThrow( - /that the package's interchange.credentials does not/, - ); - }); - } - - for (const [label, factory] of [ - ["null", null], - ["a string", "openai"], - ["a list with a number", [1]], - ["a handle that is not a string", [{ handle: 1 }]], - ] as const) { - test(`rejects a factory whose credentials are ${label}`, async () => { - await expect(loadWithDeclarations(MANIFEST, factory)).rejects.toThrow( - /tool factory @vendor\/creds\/main credentials failed validation/, - ); - }); - } - - test("rejects interchange.credentials with a duplicate handle", async () => { - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); - const fixture = await packFixture({ - name: "tools-dupcred", - version: "1.0.0", - entryModuleSource: "// stub; importer is faked", - interchangeCredentials: [{ handle: "openai" }, { handle: "openai" }], - }); - - const loader = createToolLoader({ - cache, - registries: new Map([["npmjs", { url: "https://r.test" }]]), - host: { os: "linux", cpu: "x64" }, - fetchTarball: async () => fixture.bytes, - importModule: async () => ({ - main: makeFakeFactory("@vendor/dupcred/main", ["env.thing"]), - }), - }); - - await expect( - loader.loadManifest({ - manifest: { - schemaVersion: "1", - topLevel: [{ name: "tools-dupcred", version: "1.0.0" }], - entries: [ - { - name: "tools-dupcred", - version: "1.0.0", - source: { - kind: "registry", - registry: "npmjs", - integrity: fixture.integrity, - }, - }, - ], - }, - instanceScratchDir: instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - }), - ).rejects.toThrow(/duplicate credential handles/); - }); - test("rejects a non-positive registryFetchTimeoutMs", () => { - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); - expect(() => - createToolLoader({ - cache, - registries: new Map([["npmjs", { url: "https://r.test" }]]), - host: { os: "linux", cpu: "x64" }, - registryFetchTimeoutMs: 0, - }), - ).toThrow(/registryFetchTimeoutMs must be a positive finite number/); + expect(() => createTarballFetcher({ registryFetchTimeoutMs: 0 })).toThrow( + /registryFetchTimeoutMs must be a positive finite number/, + ); }); test("skips entries whose os does not match the host", async () => { @@ -715,14 +392,11 @@ describe("createToolLoader", () => { ? matching.bytes : nonMatching.bytes; }; - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map([["npmjs", { url: "https://r.test" }]]), host: { os: "linux", cpu: "x64" }, fetchTarball, - importModule: async () => ({ - main: makeFakeFactory("@watcher/linux/main"), - }), }); const manifest: ToolPackageManifest = { @@ -755,15 +429,15 @@ describe("createToolLoader", () => { ], }; - const loaded = await loader.loadManifest({ + const { storeDir } = await materializer.materialize({ manifest, instanceScratchDir: instanceDir, assetRoot, assetMounts: new Map(), gitDirs: new Map(), }); - expect(loaded).toHaveLength(1); - expect(loaded[0]?.name).toBe("watcher-linux"); + expect(await isLaidOut(storeDir, "watcher-linux", "1.0.0")).toBe(true); + expect(await isLaidOut(storeDir, "watcher-darwin", "1.0.0")).toBe(false); expect(fetched).toEqual(["watcher-linux@1.0.0"]); }); @@ -805,7 +479,7 @@ describe("createToolLoader", () => { rootDir: cacheDir, maxBytes: 10_000_000, }); - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map([["npmjs", { url: "https://r.test" }]]), host: { os: "linux", cpu: "x64" }, @@ -814,7 +488,6 @@ describe("createToolLoader", () => { if (f === undefined) throw new Error(`no fixture: ${entry.name}`); return f.bytes; }, - importModule: async () => ({ main: makeFakeFactory("@vendor/x/main") }), }); const manifest: ToolPackageManifest = { schemaVersion: "1", @@ -868,15 +541,17 @@ describe("createToolLoader", () => { ], }; - const loaded = await loader.loadManifest({ + const { storeDir } = await materializer.materialize({ manifest, instanceScratchDir: instanceDir, assetRoot, assetMounts: new Map(), gitDirs: new Map(), }); - const names = loaded.map((p) => p.name).sort(); - expect(names).toEqual(["allow-bare", "allow-negate"]); + expect(await isLaidOut(storeDir, "allow-bare", "1.0.0")).toBe(true); + expect(await isLaidOut(storeDir, "allow-negate", "1.0.0")).toBe(true); + expect(await isLaidOut(storeDir, "deny-bare", "1.0.0")).toBe(false); + expect(await isLaidOut(storeDir, "deny-negate", "1.0.0")).toBe(false); }); test("uses cache hit and skips the fetcher", async () => { @@ -895,12 +570,11 @@ describe("createToolLoader", () => { fetcherCalls += 1; throw new Error("fetcher should not be called on cache hit"); }; - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map([["npmjs", { url: "https://r.test" }]]), host: { os: "linux", cpu: "x64" }, fetchTarball, - importModule: async () => ({ main: makeFakeFactory("@c/main") }), }); const manifest: ToolPackageManifest = { schemaVersion: "1", @@ -917,7 +591,7 @@ describe("createToolLoader", () => { }, ], }; - await loader.loadManifest({ + await materializer.materialize({ manifest, instanceScratchDir: instanceDir, assetRoot, @@ -926,379 +600,54 @@ describe("createToolLoader", () => { }); expect(fetcherCalls).toBe(0); }); +}); - test("cache-busts the import URL with the entry integrity so reapplies with new bytes import distinct modules", async () => { - // Same (name, version), different bytes — Node's ESM cache keys - // by resolved URL, so without the integrity query string both - // applies would share the first apply's module. Track the URLs - // the loader hands to import(): they must (a) carry the - // `integrity=` query string and (b) differ between the two - // applies whose integrities differ. +describe("materialization error categories", () => { + async function expectCategory( + args: { + manifest: ToolPackageManifest; + fetchTarball: TarballFetcher; + }, + category: ClosureMaterializationErrorCategory, + ): Promise { const cache = createTarballCache({ rootDir: cacheDir, maxBytes: 10_000_000, }); - const fixtureA = await packFixture({ - name: "swappable", - version: "1.0.0", - entryModuleSource: "// bytes A", - }); - const fixtureB = await packFixture({ - name: "swappable", - version: "1.0.0", - entryModuleSource: "// bytes B (distinct from A)", - }); - expect(fixtureA.integrity).not.toBe(fixtureB.integrity); - - const importUrls: string[] = []; - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map([["npmjs", { url: "https://r.test" }]]), host: { os: "linux", cpu: "x64" }, - fetchTarball: async (entry) => - getToolPackageSourceContentIdentity(entry.source) === fixtureA.integrity - ? fixtureA.bytes - : fixtureB.bytes, - importModule: async (importUrl) => { - importUrls.push(importUrl); - return { main: makeFakeFactory("@swappable/main") }; - }, - }); - - const makeManifest = (integrity: string): ToolPackageManifest => ({ - schemaVersion: "1", - topLevel: [{ name: "swappable", version: "1.0.0" }], - entries: [ - { - name: "swappable", - version: "1.0.0", - source: { kind: "registry", registry: "npmjs", integrity }, - }, - ], - }); - - const instanceDirA = path.join(scratchRoot, "instance-a"); - const instanceDirB = path.join(scratchRoot, "instance-b"); - await fs.mkdir(instanceDirA, { recursive: true }); - await fs.mkdir(instanceDirB, { recursive: true }); - - await loader.loadManifest({ - manifest: makeManifest(fixtureA.integrity), - instanceScratchDir: instanceDirA, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - }); - await loader.loadManifest({ - manifest: makeManifest(fixtureB.integrity), - instanceScratchDir: instanceDirB, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), + fetchTarball: args.fetchTarball, }); - - expect(importUrls).toHaveLength(2); - const [urlA, urlB] = importUrls; - if (urlA === undefined || urlB === undefined) { - throw new Error("expected two import URLs"); + let caught: unknown; + try { + await materializer.materialize({ + manifest: args.manifest, + instanceScratchDir: instanceDir, + assetRoot, + assetMounts: new Map(), + gitDirs: new Map(), + }); + } catch (err) { + caught = err; } - expect(urlA.startsWith("file://")).toBeTruthy(); - expect(urlB.startsWith("file://")).toBeTruthy(); - expect(urlA).toContain( - `integrity=${encodeURIComponent(fixtureA.integrity)}`, - ); - expect(urlB).toContain( - `integrity=${encodeURIComponent(fixtureB.integrity)}`, - ); - expect(urlA).not.toBe(urlB); - }); + if (!(caught instanceof ClosureMaterializationError)) { + throw new Error( + `expected ClosureMaterializationError, got ${caught === undefined ? "no error" : String(caught)}`, + ); + } + expect(caught.category).toBe(category); + } - test("loader prefixes tool definitions with the bundle id", async () => { - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); + test("integrity.mismatch when fetched bytes do not match", async () => { const fixture = await packFixture({ - name: "tools-prefix", + name: "i", version: "1.0.0", entryModuleSource: "", }); - // The factory exports a definition named "search". The loader - // should expose it as "@vendor/prefix/main:search" to the model. - const factory = Object.assign( - () => ({ - definitions: [{ name: "search", description: "test", inputSchema: {} }], - run: async (call: { id: string; name: string }) => ({ - callId: call.id, - content: `ran ${call.name}`, - }), - }), - { - id: "@vendor/prefix/main", - requires: [] as readonly string[], - definitions: [{ name: "search" }], - }, - ); - const loader = createToolLoader({ - cache, - registries: new Map([["npmjs", { url: "https://r.test" }]]), - host: { os: "linux", cpu: "x64" }, - fetchTarball: async () => fixture.bytes, - importModule: async () => ({ main: factory }), - }); - const manifest: ToolPackageManifest = { - schemaVersion: "1", - topLevel: [{ name: "tools-prefix", version: "1.0.0" }], - entries: [ - { - name: "tools-prefix", - version: "1.0.0", - source: { - kind: "registry", - registry: "npmjs", - integrity: fixture.integrity, - }, - }, - ], - }; - const loaded = await loader.loadManifest({ - manifest, - instanceScratchDir: instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - }); - const loadedFactory = loaded[0]?.factories[0]; - if (loadedFactory === undefined) { - throw new Error("expected at least one factory"); - } - // Stub env; the factory does not read anything from it in this - // test, but BaseEnv must satisfy the type at the call site. - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- BaseEnv satisfied by the {} cast for this stub; the factory does not read env at runtime. - const bundle = loadedFactory({} as Parameters[0]); - expect(bundle.definitions[0]?.name).toBe("@vendor/prefix/main:search"); - // Invoking via the prefixed name should round-trip to the - // original handler. - const result = await bundle.run( - { - id: "c1", - name: "@vendor/prefix/main:search", - arguments: {}, - }, - new AbortController().signal, - ); - expect(result.content).toBe("ran search"); - }); - - test("loader rejects bundles that export duplicate-named tool definitions", async () => { - // Two definitions with the same name in the same bundle would - // silently collapse to one entry in the wrapper's name map and - // leave one tool unreachable. The wrapper surfaces the collision - // as tool.name.duplicate when the factory is invoked. - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); - const fixture = await packFixture({ - name: "tools-dup", - version: "1.0.0", - entryModuleSource: "", - }); - const factory = Object.assign( - () => ({ - definitions: [ - { name: "search", description: "first", inputSchema: {} }, - { name: "search", description: "second", inputSchema: {} }, - ], - run: async (call: { id: string; name: string }) => ({ - callId: call.id, - content: `ran ${call.name}`, - }), - }), - { - id: "@vendor/dup/main", - requires: [] as readonly string[], - definitions: [{ name: "search" }, { name: "search" }], - }, - ); - const loader = createToolLoader({ - cache, - registries: new Map([["npmjs", { url: "https://r.test" }]]), - host: { os: "linux", cpu: "x64" }, - fetchTarball: async () => fixture.bytes, - importModule: async () => ({ main: factory }), - }); - const loaded = await loader.loadManifest({ - manifest: { - schemaVersion: "1", - topLevel: [{ name: "tools-dup", version: "1.0.0" }], - entries: [ - { - name: "tools-dup", - version: "1.0.0", - source: { - kind: "registry", - registry: "npmjs", - integrity: fixture.integrity, - }, - }, - ], - }, - instanceScratchDir: instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - }); - const loadedFactory = loaded[0]?.factories[0]; - if (loadedFactory === undefined) { - throw new Error("expected at least one factory"); - } - let caught: unknown; - try { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- BaseEnv satisfied by the {} cast for this stub; the factory does not read env at runtime. - loadedFactory({} as Parameters[0]); - } catch (err) { - caught = err; - } - if (!(caught instanceof ToolLoaderError)) { - throw new Error(`expected ToolLoaderError, got ${String(caught)}`); - } - expect(caught.category).toBe("tool.name.duplicate"); - expect(caught.message).toMatch(/two tool definitions named/); - }); - - test("loader rejects bundles whose `definitions` field is not an array", async () => { - // A factory that returns `definitions: null` (or omits the field - // entirely) would otherwise yield a bare TypeError when the - // wrapper iterates `bundle.definitions`. The apply pipeline would - // route that through the unknown-shape catch-all - // (`factory.construct.failed`), which is wrong: the shape of the - // returned bundle is the violated invariant, so the failure - // belongs in `package.entry.invalid`. The wrapper shape-checks - // before iterating to keep the operator-facing taxonomy honest. - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); - const fixture = await packFixture({ - name: "tools-malformed", - version: "1.0.0", - entryModuleSource: "", - }); - const factory = Object.assign( - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- the test exercises the wrapper's defensive shape-check, which catches malformed bundles whose runtime shape disagrees with the type. Suppressing the type system on the violating field is the point. - (): any => ({ - // Deliberately violate the bundle shape — `definitions` must - // be an array per the `AnnotatedToolFactory` contract. - definitions: null, - run: async (call: { id: string; name: string }) => ({ - callId: call.id, - content: `ran ${call.name}`, - }), - }), - { - id: "@vendor/malformed/main", - requires: [] as readonly string[], - definitions: [{ name: "search" }], - }, - ); - const loader = createToolLoader({ - cache, - registries: new Map([["npmjs", { url: "https://r.test" }]]), - host: { os: "linux", cpu: "x64" }, - fetchTarball: async () => fixture.bytes, - importModule: async () => ({ main: factory }), - }); - const loaded = await loader.loadManifest({ - manifest: { - schemaVersion: "1", - topLevel: [{ name: "tools-malformed", version: "1.0.0" }], - entries: [ - { - name: "tools-malformed", - version: "1.0.0", - source: { - kind: "registry", - registry: "npmjs", - integrity: fixture.integrity, - }, - }, - ], - }, - instanceScratchDir: instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - }); - const loadedFactory = loaded[0]?.factories[0]; - if (loadedFactory === undefined) { - throw new Error("expected at least one factory"); - } - let caught: unknown; - try { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- BaseEnv satisfied by the {} cast for this stub; the factory does not read env at runtime. - loadedFactory({} as Parameters[0]); - } catch (err) { - caught = err; - } - if (!(caught instanceof ToolLoaderError)) { - throw new Error(`expected ToolLoaderError, got ${String(caught)}`); - } - expect(caught.category).toBe("package.entry.invalid"); - expect(caught.message).toMatch(/non-array `definitions`/); - }); -}); - -describe("loader error categories", () => { - async function expectCategory( - args: { - manifest: ToolPackageManifest; - fetchTarball: TarballFetcher; - importModule?: (p: string) => Promise; - }, - category: DeployApplyErrorCategory, - ): Promise { - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); - const loader = createToolLoader({ - cache, - registries: new Map([["npmjs", { url: "https://r.test" }]]), - host: { os: "linux", cpu: "x64" }, - fetchTarball: args.fetchTarball, - ...(args.importModule !== undefined - ? { importModule: args.importModule } - : {}), - }); - let caught: unknown; - try { - await loader.loadManifest({ - manifest: args.manifest, - instanceScratchDir: instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - }); - } catch (err) { - caught = err; - } - if (!(caught instanceof ToolLoaderError)) { - throw new Error( - `expected ToolLoaderError, got ${caught === undefined ? "no error" : String(caught)}`, - ); - } - expect(caught.category).toBe(category); - } - - test("integrity.mismatch when fetched bytes do not match", async () => { - const fixture = await packFixture({ - name: "i", - version: "1.0.0", - entryModuleSource: "", - }); - const wrongBytes = new TextEncoder().encode("not the right bytes"); - await expectCategory( + const wrongBytes = new TextEncoder().encode("not the right bytes"); + await expectCategory( { manifest: { schemaVersion: "1", @@ -1321,7 +670,7 @@ describe("loader error categories", () => { ); }); - test("git.materialization.failed when a git-sourced entry reaches the loader", async () => { + test("git.materialization.failed when a git-sourced entry has no gitDir", async () => { await expectCategory( { manifest: { @@ -1360,7 +709,7 @@ describe("loader error categories", () => { // bytes we control. Stuff the cache directly via a put with // matched integrity, then point the manifest at it. Extraction // fails inside `tar.extract` (not at the integrity re-check), so - // the loader must NOT evict — a sibling agent in the same process + // materialization must NOT evict — a sibling agent in the same process // may be copying from the same extraction tree at this moment. const cache = createTarballCache({ rootDir: cacheDir, @@ -1371,16 +720,15 @@ describe("loader error categories", () => { .fromData(garbage, { algorithms: ["sha512"] }) .toString(); await cache.put(integrity, garbage); - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map([["npmjs", { url: "https://r.test" }]]), host: { os: "linux", cpu: "x64" }, fetchTarball: async () => garbage, - importModule: async () => ({}), }); let caught: unknown; try { - await loader.loadManifest({ + await materializer.materialize({ manifest: { schemaVersion: "1", topLevel: [{ name: "g", version: "1.0.0" }], @@ -1400,8 +748,10 @@ describe("loader error categories", () => { } catch (err) { caught = err; } - if (!(caught instanceof ToolLoaderError)) { - throw new Error(`expected ToolLoaderError, got ${String(caught)}`); + if (!(caught instanceof ClosureMaterializationError)) { + throw new Error( + `expected ClosureMaterializationError, got ${String(caught)}`, + ); } expect(caught.category).toBe("tarball.extract.failed"); // The integrity check at extract time passed; the failure was a @@ -1440,16 +790,15 @@ describe("loader error categories", () => { await realCache.evict(integrity); }, }; - const loader = createToolLoader({ + const materializer = createMaterializer({ cache: wrappedCache, registries: new Map([["npmjs", { url: "https://r.test" }]]), host: { os: "linux", cpu: "x64" }, fetchTarball: async () => fixture.bytes, - importModule: async () => ({}), }); let caught: unknown; try { - await loader.loadManifest({ + await materializer.materialize({ manifest: { schemaVersion: "1", topLevel: [{ name: "t", version: "1.0.0" }], @@ -1473,8 +822,10 @@ describe("loader error categories", () => { } catch (err) { caught = err; } - if (!(caught instanceof ToolLoaderError)) { - throw new Error(`expected ToolLoaderError, got ${String(caught)}`); + if (!(caught instanceof ClosureMaterializationError)) { + throw new Error( + `expected ClosureMaterializationError, got ${String(caught)}`, + ); } expect(caught.category).toBe("tarball.extract.failed"); expect(evictCalls).toEqual([]); @@ -1511,16 +862,15 @@ describe("loader error categories", () => { await realCache.evict(integrity); }, }; - const loader = createToolLoader({ + const materializer = createMaterializer({ cache: wrappedCache, registries: new Map([["npmjs", { url: "https://r.test" }]]), host: { os: "linux", cpu: "x64" }, fetchTarball: async () => fixture.bytes, - importModule: async () => ({}), }); let caught: unknown; try { - await loader.loadManifest({ + await materializer.materialize({ manifest: { schemaVersion: "1", topLevel: [{ name: "m", version: "1.0.0" }], @@ -1544,8 +894,10 @@ describe("loader error categories", () => { } catch (err) { caught = err; } - if (!(caught instanceof ToolLoaderError)) { - throw new Error(`expected ToolLoaderError, got ${String(caught)}`); + if (!(caught instanceof ClosureMaterializationError)) { + throw new Error( + `expected ClosureMaterializationError, got ${String(caught)}`, + ); } expect(caught.category).toBe("tarball.extract.failed"); expect(evictCalls).toEqual([fixture.integrity]); @@ -1575,7 +927,7 @@ describe("loader error categories", () => { }, ], }, - // Default fetcher used; no override; the loader resolves + // Default fetcher used; no override; materialization resolves // registry "ghost" against config and fails before fetching. fetchTarball: async () => { throw new Error("unreachable"); @@ -1587,9 +939,9 @@ describe("loader error categories", () => { test("package.entry.invalid when the package.json bytes are malformed JSON", async () => { // Hand-pack a tarball whose package.json contains invalid JSON so - // the loader's JSON.parse throws. The wrapper translates the + // materialization's JSON.parse throws. The wrapper translates the // SyntaxError into the package.entry.invalid category rather than - // letting it bubble out as the catch-all factory.construct.failed. + // letting it bubble out as a raw SyntaxError. const stagingDir = path.join(scratchRoot, "bad-json-staging"); const pkgDir = path.join(stagingDir, "package"); await fs.mkdir(pkgDir, { recursive: true }); @@ -1607,7 +959,7 @@ describe("loader error categories", () => { rootDir: cacheDir, maxBytes: 10_000_000, }); - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map([["npmjs", { url: "https://r.test" }]]), host: { os: "linux", cpu: "x64" }, @@ -1615,7 +967,7 @@ describe("loader error categories", () => { }); let caught: unknown; try { - await loader.loadManifest({ + await materializer.materialize({ manifest: { schemaVersion: "1", topLevel: [{ name: "bad-json", version: "1.0.0" }], @@ -1635,8 +987,8 @@ describe("loader error categories", () => { } catch (err) { caught = err; } - expect(caught).toBeInstanceOf(ToolLoaderError); - if (caught instanceof ToolLoaderError) { + expect(caught).toBeInstanceOf(ClosureMaterializationError); + if (caught instanceof ClosureMaterializationError) { expect(caught.category).toBe("package.entry.invalid"); expect(caught.message).toMatch(/malformed package\.json/); } @@ -1672,7 +1024,7 @@ describe("loader error categories", () => { rootDir: cacheDir, maxBytes: 10_000_000, }); - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map([["npmjs", { url: "https://r.test" }]]), host: { os: "linux", cpu: "x64" }, @@ -1680,7 +1032,7 @@ describe("loader error categories", () => { }); let caught: unknown; try { - await loader.loadManifest({ + await materializer.materialize({ manifest: { schemaVersion: "1", topLevel: [{ name: "deps-array", version: "1.0.0" }], @@ -1700,8 +1052,8 @@ describe("loader error categories", () => { } catch (err) { caught = err; } - expect(caught).toBeInstanceOf(ToolLoaderError); - if (caught instanceof ToolLoaderError) { + expect(caught).toBeInstanceOf(ClosureMaterializationError); + if (caught instanceof ClosureMaterializationError) { expect(caught.category).toBe("package.entry.invalid"); expect(caught.message).toMatch( /package\.json#dependencies.*must be an object map/, @@ -1734,7 +1086,7 @@ describe("loader error categories", () => { rootDir: cacheDir, maxBytes: 10_000_000, }); - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map([["npmjs", { url: "https://r.test" }]]), host: { os: "linux", cpu: "x64" }, @@ -1742,7 +1094,7 @@ describe("loader error categories", () => { }); let caught: unknown; try { - await loader.loadManifest({ + await materializer.materialize({ manifest: { schemaVersion: "1", topLevel: [{ name: "opt-deps-array", version: "1.0.0" }], @@ -1762,8 +1114,8 @@ describe("loader error categories", () => { } catch (err) { caught = err; } - expect(caught).toBeInstanceOf(ToolLoaderError); - if (caught instanceof ToolLoaderError) { + expect(caught).toBeInstanceOf(ClosureMaterializationError); + if (caught instanceof ClosureMaterializationError) { expect(caught.category).toBe("package.entry.invalid"); expect(caught.message).toMatch( /package\.json#optionalDependencies.*must be an object map/, @@ -1776,14 +1128,14 @@ describe("loader error categories", () => { rootDir: cacheDir, maxBytes: 10_000_000, }); - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map([["npmjs", { url: "https://r.test" }]]), host: { os: "linux", cpu: "x64" }, }); let caught: unknown; try { - await loader.loadManifest({ + await materializer.materialize({ manifest: { schemaVersion: "1", topLevel: [{ name: "m", version: "1.0.0" }], @@ -1811,8 +1163,10 @@ describe("loader error categories", () => { } catch (err) { caught = err; } - if (!(caught instanceof ToolLoaderError)) { - throw new Error(`expected ToolLoaderError, got ${String(caught)}`); + if (!(caught instanceof ClosureMaterializationError)) { + throw new Error( + `expected ClosureMaterializationError, got ${String(caught)}`, + ); } expect(caught.category).toBe("tarball.missing"); }); @@ -1826,14 +1180,14 @@ describe("loader error categories", () => { rootDir: cacheDir, maxBytes: 10_000_000, }); - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map([["npmjs", { url: "https://r.test" }]]), host: { os: "linux", cpu: "x64" }, }); let caught: unknown; try { - await loader.loadManifest({ + await materializer.materialize({ manifest: { schemaVersion: "1", topLevel: [{ name: "m", version: "1.0.0" }], @@ -1861,8 +1215,10 @@ describe("loader error categories", () => { } catch (err) { caught = err; } - if (!(caught instanceof ToolLoaderError)) { - throw new Error(`expected ToolLoaderError, got ${String(caught)}`); + if (!(caught instanceof ClosureMaterializationError)) { + throw new Error( + `expected ClosureMaterializationError, got ${String(caught)}`, + ); } expect(caught.category).toBe("package.entry.invalid"); expect(caught.message).toMatch(/escapes the declared mount/); @@ -1873,7 +1229,7 @@ describe("loader error categories", () => { rootDir: cacheDir, maxBytes: 10_000_000, }); - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map([["npmjs", { url: "https://r.test" }]]), host: { os: "linux", cpu: "x64" }, @@ -1885,7 +1241,7 @@ describe("loader error categories", () => { }); let caught: unknown; try { - await loader.loadManifest({ + await materializer.materialize({ manifest: { schemaVersion: "1", topLevel: [{ name: "m", version: "1.0.0" }], @@ -1913,236 +1269,40 @@ describe("loader error categories", () => { } catch (err) { caught = err; } - if (!(caught instanceof ToolLoaderError)) { - throw new Error(`expected ToolLoaderError, got ${String(caught)}`); + if (!(caught instanceof ClosureMaterializationError)) { + throw new Error( + `expected ClosureMaterializationError, got ${String(caught)}`, + ); } expect(caught.category).toBe("asset.mount.missing"); - }); - - test("package.entry.missing when package.json has no interchange.tools field", async () => { - const fixture = await packFixture({ - name: "pem", - version: "1.0.0", - interchangeToolsRelPath: null, - }); - await expectCategory( - { - manifest: { - schemaVersion: "1", - topLevel: [{ name: "pem", version: "1.0.0" }], - entries: [ - { - name: "pem", - version: "1.0.0", - source: { - kind: "registry", - registry: "npmjs", - integrity: fixture.integrity, - }, - }, - ], - }, - fetchTarball: async () => fixture.bytes, - }, - "package.entry.missing", - ); - }); - - test("package.entry.invalid when import throws", async () => { - const fixture = await packFixture({ - name: "pi", - version: "1.0.0", - entryModuleSource: "", - }); - await expectCategory( - { - manifest: { - schemaVersion: "1", - topLevel: [{ name: "pi", version: "1.0.0" }], - entries: [ - { - name: "pi", - version: "1.0.0", - source: { - kind: "registry", - registry: "npmjs", - integrity: fixture.integrity, - }, - }, - ], - }, - fetchTarball: async () => fixture.bytes, - importModule: async () => { - throw new Error("boom"); - }, - }, - "package.entry.invalid", - ); - }); - - test("package.entry.invalid when entry exports nothing factory-shaped", async () => { - const fixture = await packFixture({ - name: "ps", - version: "1.0.0", - entryModuleSource: "", - }); - await expectCategory( - { - manifest: { - schemaVersion: "1", - topLevel: [{ name: "ps", version: "1.0.0" }], - entries: [ - { - name: "ps", - version: "1.0.0", - source: { - kind: "registry", - registry: "npmjs", - integrity: fixture.integrity, - }, - }, - ], - }, - fetchTarball: async () => fixture.bytes, - importModule: async () => ({ random: 42, alsoRandom: "hello" }), - }, - "package.entry.invalid", - ); + expect(caught.message).toContain("m@1.0.0"); }); }); describe("transitive resolution contract", () => { // Locks the closure-satisfies-itself contract: a top-level package // whose body imports a transitive that lives in the manifest closure - // resolves through the per-instance node_modules layout the loader + // resolves through the per-instance node_modules layout materialization // builds, without help from the sidecar host's own node_modules. - test("a top-level package importing a closure transitive resolves through the loader's node_modules layout", async () => { + test("scoped packages land under /@scope//", async () => { const cache = createTarballCache({ rootDir: cacheDir, maxBytes: 10_000_000, }); - - // The transitive dep ships a JS module the top-level can import by - // its bare specifier. The dep is a fully-formed npm package: it - // has a `main` so Node knows where to load from. The loader is - // expected to materialize it under - // `/store/@closure-scope/transitive-dep/1.0.0/` and to - // place a symlink at `.../top/1.0.0/node_modules/@closure-scope/transitive-dep` - // pointing at it. - const depEntry = `export const greeting = "hello from transitive";`; - const depPkg = await packFixture({ - name: "@closure-scope/transitive-dep", - version: "1.0.0", - interchangeToolsRelPath: null, - main: "./index.js", - type: "module", - extraFiles: { - "index.js": depEntry, - }, + const scopedPkg = await packFixture({ + name: "@scope/only", + version: "2.3.4", + interchangeToolsRelPath: "./entry.js", + entryModuleSource: ` +export const factory = Object.assign( + () => ({ definitions: [], run: async () => ({ callId: "x", content: "" }) }), + { id: "@scope/only", requires: [], definitions: [] }, +); +`, }); + await cache.put(scopedPkg.integrity, scopedPkg.bytes); - // The top-level imports the dep by name. The loaded module both - // re-exports a tool factory (so the loader accepts it as a valid - // entry) and proves the import succeeded by carrying the dep's - // greeting in a marker the test can read back. - const topEntry = ` -import { greeting } from "@closure-scope/transitive-dep"; -export const factory = Object.assign( - () => ({ - definitions: [{ name: "echo", description: "", inputSchema: {} }], - run: async (call) => ({ callId: call.id, content: greeting }), - }), - { id: "closure/top", requires: [], definitions: [{ name: "echo" }] }, -); -`; - const topPkg = await packFixture({ - name: "@closure-scope/top", - version: "1.0.0", - interchangeToolsRelPath: "./index.js", - entryModuleSource: topEntry, - type: "module", - dependencies: { "@closure-scope/transitive-dep": "^1.0.0" }, - }); - - await cache.put(topPkg.integrity, topPkg.bytes); - await cache.put(depPkg.integrity, depPkg.bytes); - - const loader = createToolLoader({ - cache, - registries: new Map([["test", { url: "https://example.test" }]]), - host: { os: process.platform, cpu: process.arch }, - fetchTarball: async (entry) => { - if (entry.name === "@closure-scope/top") return topPkg.bytes; - if (entry.name === "@closure-scope/transitive-dep") return depPkg.bytes; - throw new Error(`unexpected fetch: ${entry.name}`); - }, - }); - - const manifest: ToolPackageManifest = { - schemaVersion: "1", - topLevel: [{ name: "@closure-scope/top", version: "1.0.0" }], - entries: [ - { - name: "@closure-scope/top", - version: "1.0.0", - source: { - kind: "registry", - registry: "test", - integrity: topPkg.integrity, - }, - }, - { - name: "@closure-scope/transitive-dep", - version: "1.0.0", - source: { - kind: "registry", - registry: "test", - integrity: depPkg.integrity, - }, - }, - ], - }; - - const loaded = await loader.loadManifest({ - manifest, - instanceScratchDir: instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - }); - expect(loaded).toHaveLength(1); - const factory = loaded[0]?.factories[0]; - if (factory === undefined) throw new Error("expected one factory"); - // The factory's body returns the transitive's greeting, proving - // the bare-specifier import composed end-to-end. - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- BaseEnv is satisfied by an empty object for this stub; the factory does not read env at runtime. - const bundle = factory({} as Parameters[0]); - const result = await bundle.run( - { id: "c1", name: "closure/top:echo", arguments: {} }, - new AbortController().signal, - ); - expect(result.content).toBe("hello from transitive"); - }); - - test("scoped packages land under /@scope//", async () => { - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); - const scopedPkg = await packFixture({ - name: "@scope/only", - version: "2.3.4", - interchangeToolsRelPath: "./entry.js", - entryModuleSource: ` -export const factory = Object.assign( - () => ({ definitions: [], run: async () => ({ callId: "x", content: "" }) }), - { id: "@scope/only", requires: [], definitions: [] }, -); -`, - }); - await cache.put(scopedPkg.integrity, scopedPkg.bytes); - - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map([["test", { url: "https://example.test" }]]), host: { os: process.platform, cpu: process.arch }, @@ -2165,15 +1325,13 @@ export const factory = Object.assign( ], }; - const loaded = await loader.loadManifest({ + await materializer.materialize({ manifest, instanceScratchDir: instanceDir, assetRoot, assetMounts: new Map(), gitDirs: new Map(), }); - expect(loaded).toHaveLength(1); - expect(loaded[0]?.name).toBe("@scope/only"); // The store dir keeps npm's `@scope/name` shape so the scope // segment is a real directory, not a slash baked into the name. @@ -2237,16 +1395,7 @@ export const seenLodash = major; entryModuleSource: ` import { major as topSawLodash } from "lodash"; import { seenLodash } from "lib-a"; -export const factory = Object.assign( - () => ({ - definitions: [{ name: "echo", description: "", inputSchema: {} }], - run: async (call) => ({ - callId: call.id, - content: JSON.stringify({ top: topSawLodash, libA: seenLodash }), - }), - }), - { id: "vt/main", requires: [], definitions: [{ name: "echo" }] }, -); +export const seen = { top: topSawLodash, libA: seenLodash }; `, dependencies: { lodash: "^3.0.0", "lib-a": "^1.0.0" }, }); @@ -2255,7 +1404,7 @@ export const factory = Object.assign( await cache.put(libA.integrity, libA.bytes); await cache.put(topPkg.integrity, topPkg.bytes); - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map([["test", { url: "https://example.test" }]]), host: { os: process.platform, cpu: process.arch }, @@ -2313,30 +1462,22 @@ export const factory = Object.assign( ], }; - const loaded = await loader.loadManifest({ + const { storeDir } = await materializer.materialize({ manifest, instanceScratchDir: instanceDir, assetRoot, assetMounts: new Map(), gitDirs: new Map(), }); - const factory = loaded[0]?.factories[0]; - if (factory === undefined) throw new Error("expected one factory"); - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- BaseEnv stub - const bundle = factory({} as Parameters[0]); - const result = await bundle.run( - { id: "c1", name: "vt/main:echo", arguments: {} }, - new AbortController().signal, - ); + const entryUrl = pathToFileURL( + path.join(storeEntryDir(storeDir, "version-top", "1.0.0"), "index.js"), + ).href; + const entry: unknown = await import(entryUrl); // The top saw lodash major 3 (the version satisfying its own // range); lib-a saw lodash major 4 (the version satisfying its // own range). Both must be true simultaneously, which is only // possible with nested node_modules entries. - if (typeof result.content !== "string") { - throw new Error("expected stringified content"); - } - const parsed: unknown = JSON.parse(result.content); - expect(parsed).toEqual({ top: 3, libA: 4 }); + expect(entry).toMatchObject({ seen: { top: 3, libA: 4 } }); // Verify both lodash versions actually exist in the store at // their nested positions, not hoisted under a single one. @@ -2365,7 +1506,7 @@ export const factory = Object.assign( test("three requirers with overlapping ranges land on the first-arrival pick", async () => { // Closure carries three versions of `dep` (1.0.0, 1.5.0, 2.0.0) // and three requirers, each declaring a range that more than one - // closure version satisfies. The loader's first-arrival BFS from + // closure version satisfies. Materialization's first-arrival BFS from // the top-level pin records its pick for each (name, range), so // later requirers see the same answer even though semver alone // would return a different "max satisfying" version. This locks @@ -2394,7 +1535,7 @@ export const factory = Object.assign( const dep2 = await packDep("2.0.0"); // Three requirers under the top-level package. The top requires - // them in declaration order so the loader's BFS visits them in + // them in declaration order so materialization's BFS visits them in // that order; rA arrives first with range "^1" and records // dep->1.5.0 in the resolution map (max-satisfying inside the // closure for "^1" is 1.5.0). rB ("^1.0") and rC ("^1.5") both @@ -2454,7 +1595,7 @@ export const factory = Object.assign( await cache.put(fx.integrity, fx.bytes); } - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map([["test", { url: "https://example.test" }]]), host: { os: process.platform, cpu: process.arch }, @@ -2542,7 +1683,7 @@ export const factory = Object.assign( ], }; - await loader.loadManifest({ + await materializer.materialize({ manifest, instanceScratchDir: instanceDir, assetRoot, @@ -2552,7 +1693,7 @@ export const factory = Object.assign( // Every requirer's node_modules/dep should resolve to the same // store slot — the first-arrival pick of 1.5.0 for ranges that - // all satisfy it. If the loader re-ran maxSatisfying per requirer + // all satisfy it. If materialization re-ran maxSatisfying per requirer // without the recording layer, all three would still pick 1.5.0 // in this exact scenario, but the test's point is the // recorded-pick path: a future change that drifts requirer C onto @@ -2575,84 +1716,6 @@ export const factory = Object.assign( }); describe("entry-path containment", () => { - test("rejects an interchange.tools entry that escapes the package dir", async () => { - // Stage a tarball whose package.json points interchange.tools at - // a file outside the extracted package directory. The loader's - // path-resolve step must reject the escape rather than letting - // the dynamic import target arbitrary sidecar-readable files. - const sinkPath = path.join(scratchRoot, "loaded-from-outside.mjs"); - await fs.writeFile( - sinkPath, - `export const mod = Object.assign(() => ({ - definitions: [], - run: async () => ({ callId: "x", content: "ok" }), - }), { id: "external/sink", requires: [], definitions: [] });`, - ); - - const stagingDir = path.join(scratchRoot, "evil-staging"); - const pkgDir = path.join(stagingDir, "package"); - await fs.mkdir(pkgDir, { recursive: true }); - const layoutDir = path.join(instanceDir, "store", "evil", "1.0.0"); - const escapeRel = path.relative(layoutDir, sinkPath); - await fs.writeFile( - path.join(pkgDir, "package.json"), - JSON.stringify({ - name: "evil", - version: "1.0.0", - interchange: { tools: escapeRel }, - }), - ); - const tarballPath = path.join(stagingDir, "evil.tgz"); - await tar.create({ cwd: stagingDir, gzip: true, file: tarballPath }, [ - "package", - ]); - const finalBytes = await fs.readFile(tarballPath); - const integrity = ssri - .fromData(finalBytes, { algorithms: ["sha512"] }) - .toString(); - - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); - const loader = createToolLoader({ - cache, - registries: new Map([["npmjs", { url: "https://r.test" }]]), - host: { os: "linux", cpu: "x64" }, - fetchTarball: async () => finalBytes, - }); - - const manifest: ToolPackageManifest = { - schemaVersion: "1", - topLevel: [{ name: "evil", version: "1.0.0" }], - entries: [ - { - name: "evil", - version: "1.0.0", - source: { kind: "registry", registry: "npmjs", integrity }, - }, - ], - }; - - let caught: unknown; - try { - await loader.loadManifest({ - manifest, - instanceScratchDir: instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - }); - } catch (err) { - caught = err; - } - expect(caught).toBeInstanceOf(ToolLoaderError); - if (caught instanceof ToolLoaderError) { - expect(caught.category).toBe("package.entry.invalid"); - expect(caught.message).toMatch(/escapes the package directory/); - } - }); - test("rejects a tarball symlink whose target escapes the extraction root", async () => { // node-tar strips escape-pointing symlinks at extraction time, so // the layout-build guard is defense-in-depth: corrupt cache state @@ -2675,7 +1738,7 @@ describe("entry-path containment", () => { }); await cache.put(fixture.integrity, fixture.bytes); // Force the extraction to materialize once, then plant an - // escape-pointing symlink inside it before the loader's + // escape-pointing symlink inside it before materialization's // layout pass picks it up. const extractedHandle = await cache.extractTarball(fixture.integrity); const extractedRoot = extractedHandle.dir; @@ -2685,7 +1748,7 @@ describe("entry-path containment", () => { ); extractedHandle.release(); - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map([["npmjs", { url: "https://r.test" }]]), host: { os: "linux", cpu: "x64" }, @@ -2710,7 +1773,7 @@ describe("entry-path containment", () => { let caught: unknown; try { - await loader.loadManifest({ + await materializer.materialize({ manifest, instanceScratchDir: instanceDir, assetRoot, @@ -2720,12 +1783,13 @@ describe("entry-path containment", () => { } catch (err) { caught = err; } - expect(caught).toBeInstanceOf(ToolLoaderError); - if (caught instanceof ToolLoaderError) { + expect(caught).toBeInstanceOf(ClosureMaterializationError); + if (caught instanceof ClosureMaterializationError) { expect(caught.category).toBe("package.entry.invalid"); expect(caught.message).toMatch( /escapes the package extraction directory/, ); + expect(caught.message).toContain("evil-symlink@1.0.0"); } }); @@ -2770,7 +1834,7 @@ describe("entry-path containment", () => { ); await fs.symlink("inner/escape", path.join(extractedRoot, "aaa-jump")); - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map([["npmjs", { url: "https://r.test" }]]), host: { os: "linux", cpu: "x64" }, @@ -2795,7 +1859,7 @@ describe("entry-path containment", () => { let caught: unknown; try { - await loader.loadManifest({ + await materializer.materialize({ manifest, instanceScratchDir: instanceDir, assetRoot, @@ -2805,8 +1869,8 @@ describe("entry-path containment", () => { } catch (err) { caught = err; } - expect(caught).toBeInstanceOf(ToolLoaderError); - if (caught instanceof ToolLoaderError) { + expect(caught).toBeInstanceOf(ClosureMaterializationError); + if (caught instanceof ClosureMaterializationError) { expect(caught.category).toBe("package.entry.invalid"); expect(caught.message).toMatch( /escapes the package extraction directory/, @@ -2820,7 +1884,7 @@ describe("entry-path containment", () => { // relative symlink within the extraction root cannot escape (the // target name is fixed; even if the target is later created, it // would land at the same inside-the-root path). Plant a - // dangling-but-contained symlink and verify the load succeeds. + // dangling-but-contained symlink and verify the layout succeeds. const fixture = await packFixture({ name: "ok-dangling", version: "1.0.0", @@ -2842,7 +1906,7 @@ export const main = Object.assign( // Relative symlink whose literal target string resolves to a path // inside the extraction root, but the target does not exist on - // disk. realpath ENOENTs; the loader should fall back to the + // disk. realpath ENOENTs; materialization should fall back to the // literal-path containment check and accept. await fs.symlink( "missing-sibling.txt", @@ -2850,7 +1914,7 @@ export const main = Object.assign( ); extractedHandle.release(); - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map([["npmjs", { url: "https://r.test" }]]), host: { os: "linux", cpu: "x64" }, @@ -2873,15 +1937,14 @@ export const main = Object.assign( ], }; - const loaded = await loader.loadManifest({ + const { storeDir } = await materializer.materialize({ manifest, instanceScratchDir: instanceDir, assetRoot, assetMounts: new Map(), gitDirs: new Map(), }); - expect(loaded).toHaveLength(1); - expect(loaded[0]?.name).toBe("ok-dangling"); + expect(await isLaidOut(storeDir, "ok-dangling", "1.0.0")).toBe(true); }); test("rejects a dangling symlink whose literal target escapes the extraction root", async () => { @@ -2913,7 +1976,7 @@ export const main = Object.assign( ); extractedHandle.release(); - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map([["npmjs", { url: "https://r.test" }]]), host: { os: "linux", cpu: "x64" }, @@ -2938,7 +2001,7 @@ export const main = Object.assign( let caught: unknown; try { - await loader.loadManifest({ + await materializer.materialize({ manifest, instanceScratchDir: instanceDir, assetRoot, @@ -2948,8 +2011,8 @@ export const main = Object.assign( } catch (err) { caught = err; } - expect(caught).toBeInstanceOf(ToolLoaderError); - if (caught instanceof ToolLoaderError) { + expect(caught).toBeInstanceOf(ClosureMaterializationError); + if (caught instanceof ClosureMaterializationError) { expect(caught.category).toBe("package.entry.invalid"); expect(caught.message).toMatch( /escapes the package extraction directory/, @@ -2984,7 +2047,7 @@ export const main = Object.assign( rootDir: cacheDir, maxBytes: 10_000_000, }); - const loader = createToolLoader({ + const materializer = createMaterializer({ cache, registries: new Map(), host: { os: process.platform, cpu: process.arch }, @@ -3012,7 +2075,7 @@ export const main = Object.assign( let caught: unknown; try { - await loader.loadManifest({ + await materializer.materialize({ manifest, instanceScratchDir: instanceDir, assetRoot, @@ -3022,118 +2085,14 @@ export const main = Object.assign( } catch (err) { caught = err; } - expect(caught).toBeInstanceOf(ToolLoaderError); - if (caught instanceof ToolLoaderError) { + expect(caught).toBeInstanceOf(ClosureMaterializationError); + if (caught instanceof ClosureMaterializationError) { expect(caught.category).toBe("package.entry.invalid"); expect(caught.message).toMatch(/cross-mount traversal/); } }); }); -describe("interchange.tools containment under symlink graph", () => { - test("rejects an interchange.tools entry that traverses a node_modules symlink to another package", async () => { - // The per-instance scratch tree contains a `node_modules/` symlink - // graph the loader builds for nested resolution. An - // interchange.tools entry that string-contains inside the top - // package's directory but realpath-resolves into a transitive - // dependency's bytes (or anywhere else the symlink target points) - // would bypass the string-level containment check entirely. - // - // Build a real two-package closure (top + transitive dep), then - // point the top's interchange.tools at a path that walks through - // its own `node_modules//...` symlink. The string check - // accepts the path because it starts with the package directory; - // the realpath check must reject it because the resolved target - // lives under the dep's store entry. - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); - - const depEntry = `export const x = 1;`; - const depPkg = await packFixture({ - name: "evil-dep", - version: "1.0.0", - interchangeToolsRelPath: null, - main: "./index.js", - type: "module", - extraFiles: { "index.js": depEntry }, - }); - - const topPkg = await packFixture({ - name: "evil-top", - version: "1.0.0", - // The entry path strings-inside the top directory but - // realpath-walks through `node_modules/evil-dep` into the dep's - // store entry. Leave entryModuleSource undefined so the tarball - // ships no `node_modules/` of its own; the loader's layout pass - // will then build `node_modules/evil-dep` as a symlink to the - // dep's store entry, and the realpath check should fire when - // the entry path is resolved. - interchangeToolsRelPath: "./node_modules/evil-dep/index.js", - type: "module", - dependencies: { "evil-dep": "^1.0.0" }, - }); - - await cache.put(topPkg.integrity, topPkg.bytes); - await cache.put(depPkg.integrity, depPkg.bytes); - - const loader = createToolLoader({ - cache, - registries: new Map([["test", { url: "https://example.test" }]]), - host: { os: process.platform, cpu: process.arch }, - fetchTarball: async (entry) => { - if (entry.name === "evil-top") return topPkg.bytes; - if (entry.name === "evil-dep") return depPkg.bytes; - throw new Error(`unexpected fetch: ${entry.name}`); - }, - }); - - const manifest: ToolPackageManifest = { - schemaVersion: "1", - topLevel: [{ name: "evil-top", version: "1.0.0" }], - entries: [ - { - name: "evil-top", - version: "1.0.0", - source: { - kind: "registry", - registry: "test", - integrity: topPkg.integrity, - }, - }, - { - name: "evil-dep", - version: "1.0.0", - source: { - kind: "registry", - registry: "test", - integrity: depPkg.integrity, - }, - }, - ], - }; - - let caught: unknown; - try { - await loader.loadManifest({ - manifest, - instanceScratchDir: instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - }); - } catch (err) { - caught = err; - } - expect(caught).toBeInstanceOf(ToolLoaderError); - if (caught instanceof ToolLoaderError) { - expect(caught.category).toBe("package.entry.invalid"); - expect(caught.message).toMatch(/symlink/); - } - }); -}); - describe("buildRegistryFetchOpts", () => { test("passes the basic-auth password verbatim to forceAuth", () => { // `npm-registry-fetch` base64-encodes `:` itself when @@ -3176,8 +2135,8 @@ describe("readResponseWithLimit", () => { } catch (err) { caught = err; } - expect(caught).toBeInstanceOf(ToolLoaderError); - if (caught instanceof ToolLoaderError) { + expect(caught).toBeInstanceOf(ClosureMaterializationError); + if (caught instanceof ClosureMaterializationError) { expect(caught.category).toBe("registry.fetch.failed"); expect(caught.message).toMatch(/exceeds the/); } @@ -3195,8 +2154,8 @@ describe("readResponseWithLimit", () => { } catch (err) { caught = err; } - expect(caught).toBeInstanceOf(ToolLoaderError); - if (caught instanceof ToolLoaderError) { + expect(caught).toBeInstanceOf(ClosureMaterializationError); + if (caught instanceof ClosureMaterializationError) { expect(caught.category).toBe("registry.fetch.failed"); expect(caught.message).toMatch(/non-digit Content-Length/); } @@ -3221,8 +2180,8 @@ describe("readResponseWithLimit", () => { } catch (err) { caught = err; } - expect(caught).toBeInstanceOf(ToolLoaderError); - if (caught instanceof ToolLoaderError) { + expect(caught).toBeInstanceOf(ClosureMaterializationError); + if (caught instanceof ClosureMaterializationError) { expect(caught.category).toBe("registry.fetch.failed"); expect(caught.message).toMatch(/streamed past/); } @@ -3259,8 +2218,8 @@ describe("readResponseWithLimit", () => { } finally { clearTimeout(timer); } - expect(caught).toBeInstanceOf(ToolLoaderError); - if (caught instanceof ToolLoaderError) { + expect(caught).toBeInstanceOf(ClosureMaterializationError); + if (caught instanceof ClosureMaterializationError) { expect(caught.category).toBe("registry.fetch.failed"); expect(caught.message).toMatch(/exceeded the registry fetch timeout/); } @@ -3329,8 +2288,8 @@ describe("readResponseWithLimit", () => { } catch (err) { caught = err; } - expect(caught).toBeInstanceOf(ToolLoaderError); - if (caught instanceof ToolLoaderError) { + expect(caught).toBeInstanceOf(ClosureMaterializationError); + if (caught instanceof ClosureMaterializationError) { expect(caught.category).toBe("registry.fetch.failed"); expect(caught.message).toMatch(/streamed past/); } @@ -3356,584 +2315,36 @@ describe("readResponseWithLimit", () => { } finally { clearTimeout(timer); } - expect(caught).toBeInstanceOf(ToolLoaderError); - if (caught instanceof ToolLoaderError) { + expect(caught).toBeInstanceOf(ClosureMaterializationError); + if (caught instanceof ClosureMaterializationError) { expect(caught.category).toBe("registry.fetch.failed"); expect(caught.message).toMatch(/exceeded the registry fetch timeout/); } }); }); -describe("interchange.directors walker", () => { - test("loads director factories from an interchange.directors entry alongside tool factories", async () => { +describe("materializeClosure", () => { + test("lays out a resolvable store dir without importing any author code", async () => { const cache = createTarballCache({ rootDir: cacheDir, maxBytes: 10_000_000, }); - const fixture = await packFixture({ - name: "tools-and-directors", - version: "1.0.0", - entryModuleSource: "// stub; importer is faked", - interchangeDirectorsRelPath: "./directors.js", - directorsModuleSource: "// stub; importer is faked", - }); - - const toolsModule = { main: makeFakeFactory("@vendor/main") }; - const directorsModule = { - planner: makeFakeDirectorFactory("tools-and-directors/planner"), - }; - const loader = createToolLoader({ - cache, - registries: new Map([["npmjs", { url: "https://r.test" }]]), - host: { os: "linux", cpu: "x64" }, - fetchTarball: async () => fixture.bytes, - importModule: async (importUrl) => { - // Match on the entry basename rather than the substring - // "directors" because the per-instance scratch tree's mkdtemp - // suffix can happen to contain that substring. - if (importUrl.includes("directors.js")) return directorsModule; - return toolsModule; - }, - }); - - const loaded = await loader.loadManifest({ - manifest: { - schemaVersion: "1", - topLevel: [{ name: "tools-and-directors", version: "1.0.0" }], - entries: [ - { - name: "tools-and-directors", - version: "1.0.0", - source: { - kind: "registry", - registry: "npmjs", - integrity: fixture.integrity, - }, - }, - ], - }, - instanceScratchDir: instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - }); - expect(loaded).toHaveLength(1); - expect(loaded[0]?.factories).toHaveLength(1); - expect(loaded[0]?.factories[0]?.id).toBe("@vendor/main"); - expect(loaded[0]?.directors).toHaveLength(1); - expect(loaded[0]?.directors[0]?.id).toBe("tools-and-directors/planner"); - }); + // The top-level entry's module has an import-time side effect: if it + // were ever imported, it would write a sentinel file. materializeClosure + // is the eval-free primitive — it must lay out the store without + // running this module, so the sentinel must not exist afterward. + const sentinelPath = path.join(scratchRoot, "author-code-ran.marker"); - test("rejects a director declared outside the shipping package's namespace", async () => { - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); - const fixture = await packFixture({ - name: "tools-and-directors", + const depPkg = await packFixture({ + name: "@closure-scope/transitive-dep", version: "1.0.0", - entryModuleSource: "// stub; importer is faked", - interchangeDirectorsRelPath: "./directors.js", - directorsModuleSource: "// stub; importer is faked", - }); - - const toolsModule = { - main: makeFakeFactory("@vendor/main"), - }; - const directorsModule = { - planner: makeFakeDirectorFactory("@vendor/other/planner"), - }; - const loader = createToolLoader({ - cache, - registries: new Map([["npmjs", { url: "https://r.test" }]]), - host: { os: "linux", cpu: "x64" }, - fetchTarball: async () => fixture.bytes, - importModule: async (url) => - url.includes("directors.js") ? directorsModule : toolsModule, - }); - - let caught: unknown; - try { - await loader.loadManifest({ - manifest: { - schemaVersion: "1", - topLevel: [{ name: "tools-and-directors", version: "1.0.0" }], - entries: [ - { - name: "tools-and-directors", - version: "1.0.0", - source: { - kind: "registry", - registry: "npmjs", - integrity: fixture.integrity, - }, - }, - ], - }, - instanceScratchDir: instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - }); - } catch (err) { - caught = err; - } - expect(caught).toBeInstanceOf(ToolLoaderError); - if (caught instanceof ToolLoaderError) { - expect(caught.category).toBe("package.entry.invalid"); - expect(caught.message).toMatch(/outside the package's own namespace/); - } - }); - - test("rejects a tarball whose manifest name differs from its pin", async () => { - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); - const fixture = await packFixture({ - name: "tools-and-directors", - version: "1.0.0", - entryModuleSource: "// stub; importer is faked", - }); - const loader = createToolLoader({ - cache, - registries: new Map([["npmjs", { url: "https://r.test" }]]), - host: { os: "linux", cpu: "x64" }, - fetchTarball: async () => fixture.bytes, - importModule: async () => ({ main: makeFakeFactory("@vendor/main") }), - }); - - let caught: unknown; - try { - await loader.loadManifest({ - manifest: { - schemaVersion: "1", - topLevel: [{ name: "renamed-pin", version: "1.0.0" }], - entries: [ - { - name: "renamed-pin", - version: "1.0.0", - source: { - kind: "registry", - registry: "npmjs", - integrity: fixture.integrity, - }, - }, - ], - }, - instanceScratchDir: instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - }); - } catch (err) { - caught = err; - } - expect(caught).toBeInstanceOf(ToolLoaderError); - if (caught instanceof ToolLoaderError) { - expect(caught.category).toBe("package.entry.invalid"); - expect(caught.message).toMatch(/does not match the pinned package name/); - } - }); - - test("ignores a director-shaped export in the interchange.tools entry", async () => { - // The tools walker's predicate must reject director-shaped values so - // a director placed in `interchange.tools` is not silently - // classified as a tool and namespace-prefixed. Mirrors the - // discriminator the directors walker uses against tool shapes. - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); - const fixture = await packFixture({ - name: "tools-with-stray-director", - version: "1.0.0", - entryModuleSource: "// stub; importer is faked", - }); - - const toolsModule = { - tool: makeFakeFactory("@vendor/tool"), - strayDirector: makeFakeDirectorFactory("@vendor/director"), - }; - const loader = createToolLoader({ - cache, - registries: new Map([["npmjs", { url: "https://r.test" }]]), - host: { os: "linux", cpu: "x64" }, - fetchTarball: async () => fixture.bytes, - importModule: async () => toolsModule, - }); - - const loaded = await loader.loadManifest({ - manifest: { - schemaVersion: "1", - topLevel: [{ name: "tools-with-stray-director", version: "1.0.0" }], - entries: [ - { - name: "tools-with-stray-director", - version: "1.0.0", - source: { - kind: "registry", - registry: "npmjs", - integrity: fixture.integrity, - }, - }, - ], - }, - instanceScratchDir: instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - }); - - expect(loaded).toHaveLength(1); - expect(loaded[0]?.factories).toHaveLength(1); - expect(loaded[0]?.factories[0]?.id).toBe("@vendor/tool"); - expect(loaded[0]?.directors).toEqual([]); - }); - - test("treats a missing interchange.directors field as a no-op", async () => { - // Locks the absence-is-fine contract: a tools-only package must - // still load. The directors walker contributes an empty list. - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); - const fixture = await packFixture({ - name: "tools-only", - version: "1.0.0", - entryModuleSource: "// stub", - }); - const loader = createToolLoader({ - cache, - registries: new Map([["npmjs", { url: "https://r.test" }]]), - host: { os: "linux", cpu: "x64" }, - fetchTarball: async () => fixture.bytes, - importModule: async () => ({ - main: makeFakeFactory("@vendor/tools-only"), - }), - }); - - const loaded = await loader.loadManifest({ - manifest: { - schemaVersion: "1", - topLevel: [{ name: "tools-only", version: "1.0.0" }], - entries: [ - { - name: "tools-only", - version: "1.0.0", - source: { - kind: "registry", - registry: "npmjs", - integrity: fixture.integrity, - }, - }, - ], - }, - instanceScratchDir: instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - }); - - expect(loaded).toHaveLength(1); - expect(loaded[0]?.directors).toEqual([]); - }); - - test("package.entry.invalid when the package.json bytes are malformed JSON", async () => { - // Cross-cuts the directors walker: the package-json read happens - // once for both walkers, so a malformed package.json blocks both. - // Pinned here to document the shared failure surface. - const stagingDir = path.join(scratchRoot, "bad-json-dir-staging"); - const pkgDir = path.join(stagingDir, "package"); - await fs.mkdir(pkgDir, { recursive: true }); - await fs.writeFile(path.join(pkgDir, "package.json"), "{ not valid json"); - const tarballPath = path.join(stagingDir, "bad.tgz"); - await tar.create({ cwd: stagingDir, gzip: true, file: tarballPath }, [ - "package", - ]); - const bytes = await fs.readFile(tarballPath); - const integrity = ssri - .fromData(bytes, { algorithms: ["sha512"] }) - .toString(); - - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); - const loader = createToolLoader({ - cache, - registries: new Map([["npmjs", { url: "https://r.test" }]]), - host: { os: "linux", cpu: "x64" }, - fetchTarball: async () => bytes, - }); - let caught: unknown; - try { - await loader.loadManifest({ - manifest: { - schemaVersion: "1", - topLevel: [{ name: "bad-json-dir", version: "1.0.0" }], - entries: [ - { - name: "bad-json-dir", - version: "1.0.0", - source: { kind: "registry", registry: "npmjs", integrity }, - }, - ], - }, - instanceScratchDir: instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - }); - } catch (err) { - caught = err; - } - expect(caught).toBeInstanceOf(ToolLoaderError); - if (caught instanceof ToolLoaderError) { - expect(caught.category).toBe("package.entry.invalid"); - expect(caught.message).toMatch(/malformed package\.json/); - } - }); - - test("rejects an interchange.directors entry path that escapes the package dir", async () => { - const sinkPath = path.join(scratchRoot, "loaded-directors-outside.mjs"); - await fs.writeFile(sinkPath, "export const x = 1;"); - - const stagingDir = path.join(scratchRoot, "evil-dir-staging"); - const pkgDir = path.join(stagingDir, "package"); - await fs.mkdir(pkgDir, { recursive: true }); - const layoutDir = path.join(instanceDir, "store", "evil-dir", "1.0.0"); - const escapeRel = path.relative(layoutDir, sinkPath); - await fs.writeFile( - path.join(pkgDir, "package.json"), - JSON.stringify({ - name: "evil-dir", - version: "1.0.0", - interchange: { - tools: "./tools.js", - directors: escapeRel, - }, - }), - ); - await fs.writeFile(path.join(pkgDir, "tools.js"), ""); - const tarballPath = path.join(stagingDir, "evil-dir.tgz"); - await tar.create({ cwd: stagingDir, gzip: true, file: tarballPath }, [ - "package", - ]); - const finalBytes = await fs.readFile(tarballPath); - const integrity = ssri - .fromData(finalBytes, { algorithms: ["sha512"] }) - .toString(); - - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); - const loader = createToolLoader({ - cache, - registries: new Map([["npmjs", { url: "https://r.test" }]]), - host: { os: "linux", cpu: "x64" }, - fetchTarball: async () => finalBytes, - importModule: async () => ({ - main: makeFakeFactory("@vendor/tools-stub"), - }), - }); - - let caught: unknown; - try { - await loader.loadManifest({ - manifest: { - schemaVersion: "1", - topLevel: [{ name: "evil-dir", version: "1.0.0" }], - entries: [ - { - name: "evil-dir", - version: "1.0.0", - source: { kind: "registry", registry: "npmjs", integrity }, - }, - ], - }, - instanceScratchDir: instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - }); - } catch (err) { - caught = err; - } - expect(caught).toBeInstanceOf(ToolLoaderError); - if (caught instanceof ToolLoaderError) { - expect(caught.category).toBe("package.entry.invalid"); - expect(caught.message).toMatch(/interchange\.directors/); - expect(caught.message).toMatch(/escapes the package directory/); - } - }); - - test("rejects an interchange.directors target reached via a symlink that escapes the extraction root", async () => { - const sinkPath = path.join(scratchRoot, "directors-symlink-sink.mjs"); - await fs.writeFile(sinkPath, "export const x = 1;"); - - const fixture = await packFixture({ - name: "evil-dir-symlink", - version: "1.0.0", - entryModuleSource: ` -export const main = Object.assign( - () => ({ definitions: [], run: async () => ({ callId: "x", content: "" }) }), - { id: "@vendor/evil-dir-symlink", requires: [], definitions: [] }, -); -`, - interchangeDirectorsRelPath: "./directors-link.js", - }); - - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); - await cache.put(fixture.integrity, fixture.bytes); - const extractedHandle = await cache.extractTarball(fixture.integrity); - const extractedRoot = extractedHandle.dir; - // Plant a symlink at the directors-entry name whose realpath - // resolves outside the package extraction directory. - await fs.symlink( - path.relative(extractedRoot, sinkPath), - path.join(extractedRoot, "directors-link.js"), - ); - extractedHandle.release(); - - const loader = createToolLoader({ - cache, - registries: new Map([["npmjs", { url: "https://r.test" }]]), - host: { os: "linux", cpu: "x64" }, - fetchTarball: async () => fixture.bytes, - importModule: async () => ({ - main: makeFakeFactory("@vendor/evil-dir-symlink"), - }), - }); - - let caught: unknown; - try { - await loader.loadManifest({ - manifest: { - schemaVersion: "1", - topLevel: [{ name: "evil-dir-symlink", version: "1.0.0" }], - entries: [ - { - name: "evil-dir-symlink", - version: "1.0.0", - source: { - kind: "registry", - registry: "npmjs", - integrity: fixture.integrity, - }, - }, - ], - }, - instanceScratchDir: instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - }); - } catch (err) { - caught = err; - } - // The symlink fires `copyTree`'s containment guard at layout - // time, before either walker runs. The directors walker's own - // realpath containment check is the second line of defense and - // shares the same operator-facing category; the test asserts the - // taxonomy mapping, not which layer produced the message. - expect(caught).toBeInstanceOf(ToolLoaderError); - if (caught instanceof ToolLoaderError) { - expect(caught.category).toBe("package.entry.invalid"); - expect(caught.message).toMatch( - /escapes the package extraction directory/, - ); - } - }); - - test("package.entry.invalid when the directors module import throws", async () => { - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); - const fixture = await packFixture({ - name: "dir-import-fail", - version: "1.0.0", - entryModuleSource: "// stub", - interchangeDirectorsRelPath: "./directors.js", - directorsModuleSource: "// stub", - }); - const loader = createToolLoader({ - cache, - registries: new Map([["npmjs", { url: "https://r.test" }]]), - host: { os: "linux", cpu: "x64" }, - fetchTarball: async () => fixture.bytes, - importModule: async (importUrl) => { - if (importUrl.includes("directors.js")) { - throw new Error("director module boom"); - } - return { main: makeFakeFactory("@vendor/dir-import-fail") }; - }, - }); - - let caught: unknown; - try { - await loader.loadManifest({ - manifest: { - schemaVersion: "1", - topLevel: [{ name: "dir-import-fail", version: "1.0.0" }], - entries: [ - { - name: "dir-import-fail", - version: "1.0.0", - source: { - kind: "registry", - registry: "npmjs", - integrity: fixture.integrity, - }, - }, - ], - }, - instanceScratchDir: instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - }); - } catch (err) { - caught = err; - } - expect(caught).toBeInstanceOf(ToolLoaderError); - if (caught instanceof ToolLoaderError) { - expect(caught.category).toBe("package.entry.invalid"); - expect(caught.message).toMatch( - /dynamic import of dir-import-fail@1\.0\.0 interchange\.directors failed/, - ); - } - }); -}); - -describe("materializeClosure", () => { - test("lays out a resolvable store dir without importing any author code", async () => { - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); - - // The top-level entry's module has an import-time side effect: if it - // were ever imported, it would write a sentinel file. materializeClosure - // is the eval-free primitive — it must lay out the store without - // running this module, so the sentinel must not exist afterward. - const sentinelPath = path.join(scratchRoot, "author-code-ran.marker"); - - const depPkg = await packFixture({ - name: "@closure-scope/transitive-dep", - version: "1.0.0", - interchangeToolsRelPath: null, - main: "./index.js", - type: "module", - extraFiles: { - "index.js": `export const greeting = "hello from transitive";`, - }, + interchangeToolsRelPath: null, + main: "./index.js", + type: "module", + extraFiles: { + "index.js": `export const greeting = "hello from transitive";`, + }, }); const topEntry = ` @@ -4050,159 +2461,3 @@ export const factory = Object.assign( expect(sentinelExists).toBe(false); }); }); - -// End-to-end coverage for the per-deploy-id apply layout: a tool -// package whose `interchange.tools` module performs a CALL-TIME -// `await import("./other.js")` must resolve that sibling against the -// on-disk path it was loaded from. Because each apply materializes into -// a stable `packages//` directory that is never renamed, the -// loaded module's URL stays valid across subsequent applies for as long -// as its deploy directory is retained — and stops resolving once the -// prelude sweep reaps it. These tests drive the real native importer -// (no `importModule` seam) so the dynamic import hits disk. -describe("applyAtomic per-deploy-id late-import liveness", () => { - // A factory whose tool `run` lazily imports a sibling module and - // returns its marker. The import is call-time, not load-time, so it - // resolves against the deploy directory only when the tool runs — - // after any number of intervening applies. - const lateImportEntry = ` -export const factory = Object.assign( - () => ({ - definitions: [{ name: "probe", description: "", inputSchema: {} }], - run: async (call) => { - const m = await import("./other.js"); - return { callId: call.id, content: m.marker }; - }, - }), - { id: "late/main", requires: [], definitions: [{ name: "probe" }] }, -); -`; - - async function buildLateImportLoader(): Promise<{ - loader: ReturnType; - manifest: ToolPackageManifest; - }> { - const cache = createTarballCache({ - rootDir: cacheDir, - maxBytes: 10_000_000, - }); - const pkg = await packFixture({ - name: "@late/pkg", - version: "1.0.0", - interchangeToolsRelPath: "./index.js", - entryModuleSource: lateImportEntry, - type: "module", - extraFiles: { "other.js": `export const marker = "late-import-ok";` }, - }); - await cache.put(pkg.integrity, pkg.bytes); - const loader = createToolLoader({ - cache, - registries: new Map([["test", { url: "https://example.test" }]]), - host: { os: process.platform, cpu: process.arch }, - fetchTarball: async () => pkg.bytes, - }); - const manifest: ToolPackageManifest = { - schemaVersion: "1", - topLevel: [{ name: "@late/pkg", version: "1.0.0" }], - entries: [ - { - name: "@late/pkg", - version: "1.0.0", - source: { - kind: "registry", - registry: "test", - integrity: pkg.integrity, - }, - }, - ], - }; - return { loader, manifest }; - } - - function applyArgs( - loader: ReturnType, - manifest: ToolPackageManifest, - previousDeployId: string, - newDeployId: string, - ) { - return { - manifest, - loader, - instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - attemptId: `atp-${newDeployId}`, - previousDeployId, - newDeployId, - }; - } - - // Invoke the loaded factory's `probe` tool, whose `run` performs the - // call-time sibling import. The namespaced tool name carries the - // bundle-id prefix the loader applied. - function runProbe(factory: LoadedToolFactory): Promise<{ content: unknown }> { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- BaseEnv is satisfied by an empty object for this stub; the factory does not read env, only the call-time sibling import. - const bundle = factory({} as Parameters[0]); - return bundle.run( - { id: "c1", name: "late/main:probe", arguments: {} }, - new AbortController().signal, - ); - } - - async function deployDirExists(id: string): Promise { - try { - await fs.stat(path.join(instanceDir, "packages", id)); - return true; - } catch { - return false; - } - } - - test("a retained previous deploy still resolves a call-time sibling import", async () => { - const { loader, manifest } = await buildLateImportLoader(); - - const first = await applyAtomic(applyArgs(loader, manifest, "none", "d1")); - expect(first.status).toBe("ok"); - if (first.status !== "ok") return; - const d1Factory = first.loaded[0]?.factories[0]; - if (d1Factory === undefined) throw new Error("expected a loaded factory"); - - // A second apply makes d1 the retained previous deploy. The prelude - // keeps {d2, d1}, so d1's tree survives. - const second = await applyAtomic(applyArgs(loader, manifest, "d1", "d2")); - expect(second.status).toBe("ok"); - expect(await deployDirExists("d1")).toBe(true); - - // d1's tool runs for the first time AFTER the second apply; its - // call-time `import("./other.js")` resolves against the still-present - // packages/d1/ tree. - const result = await runProbe(d1Factory); - expect(result.content).toBe("late-import-ok"); - }); - - test("a reaped deploy can no longer resolve a call-time sibling import", async () => { - const { loader, manifest } = await buildLateImportLoader(); - - const first = await applyAtomic(applyArgs(loader, manifest, "none", "d1")); - expect(first.status).toBe("ok"); - if (first.status !== "ok") return; - const d1Factory = first.loaded[0]?.factories[0]; - if (d1Factory === undefined) throw new Error("expected a loaded factory"); - - // d2 retains d1; d3 then reaps d1 (keep-set becomes {d3, d2}). - const second = await applyAtomic(applyArgs(loader, manifest, "d1", "d2")); - expect(second.status).toBe("ok"); - const third = await applyAtomic(applyArgs(loader, manifest, "d2", "d3")); - expect(third.status).toBe("ok"); - - // d1's tree is gone, bounding disk to the two retained deploys. - expect(await deployDirExists("d1")).toBe(false); - - // d1's tool runs for the FIRST time only now — its sibling module - // was never imported before the reap, so the ESM cache cannot mask - // the missing file. The call-time import fails against the absent - // packages/d1/ tree. - await expect(runProbe(d1Factory)).rejects.toThrow(); - }); -}); diff --git a/packages/tool-packaging/src/store-layout.ts b/packages/tool-packaging/src/store-layout.ts index 6ebfc084c..8d816fe95 100644 --- a/packages/tool-packaging/src/store-layout.ts +++ b/packages/tool-packaging/src/store-layout.ts @@ -1,38 +1,92 @@ -// Closure materialization and per-instance store layout for the tool-package -// loader: fetching + SRI-verifying + extracting each platform-matching entry -// into the content-addressable cache, resolving dependency ranges, and -// copying the closure into a per-instance store the loader then imports -// author code from. Extracted from `loader.ts` so this eval-free -// materialization concern is isolated from author-code loading. `loader.ts` -// re-exports `materializeClosure` and `storeEntryDir` for existing consumers. +// Closure materialization and per-instance store layout: fetching + +// SRI-verifying + extracting each platform-matching entry into the +// content-addressable cache, resolving dependency ranges, and copying the +// closure into a per-instance store. Nothing here imports author code. import { constants as fsConstants, promises as fs } from "node:fs"; import path from "node:path"; import semver from "semver"; import { getLogger } from "@intx/log"; -import type { ToolPackageManifestEntry } from "@intx/types/tool-packages"; +import type { + ToolPackageManifest, + ToolPackageManifestEntry, +} from "@intx/types/tool-packages"; -import { TarballIntegrityMismatchError } from "./cache"; +import { TarballIntegrityMismatchError, type TarballCache } from "./cache"; import { materializeGitEntry } from "./git-materialize"; -import type { - HostPlatform, - MaterializeClosureArgs, - MaterializeClosureResult, -} from "./loader"; import { - ToolLoaderError, + ClosureMaterializationError, describeError, isEEXIST, isENOENT, platformListMatches, -} from "./loader-internal"; +} from "./materialization-internal"; +import type { RegistryConfig } from "./resolver"; +import type { TarballFetcher } from "./tarball-fetcher"; const logger = getLogger(["sidecar", "tool-packaging", "store-layout"]); +export interface HostPlatform { + readonly os: string; + readonly cpu: string; +} + +/** + * Arguments for `materializeClosure`. Everything it needs is passed + * explicitly: the closure's `cache`, `registries`, `host`, and resolved + * `fetchTarball` are threaded in directly. + */ +export interface MaterializeClosureArgs { + readonly manifest: ToolPackageManifest; + readonly instanceScratchDir: string; + /** + * Filesystem root that `assetMounts` paths are joined against. Mount + * paths are workspace-relative; the fetcher resolves them against + * `assetRoot` to get the absolute tarball location for `kind: "asset"` + * entries. + */ + readonly assetRoot: string; + /** + * Maps a `source.assetId` from a manifest entry to a + * workspace-relative mount path. Empty map is valid when no entry + * sources from an asset. + */ + readonly assetMounts: ReadonlyMap; + /** + * Maps a source-format asset entry's `source.assetId` to an absolute + * indexed git directory whose object database holds the pinned commit and + * its trees. Empty map is valid when no entry sources from a git subtree. + */ + readonly gitDirs: ReadonlyMap; + readonly host: HostPlatform; + readonly cache: TarballCache; + /** + * Registry identifier → registry config. The key is the `registry` + * identifier manifest entries point at. Used both to gate + * `kind: "registry"` entries against the sidecar config and as the + * `registries` context handed to `fetchTarball`. + */ + readonly registries: ReadonlyMap; + /** + * Resolved tarball fetcher, normally from `createTarballFetcher`. + * `materializeClosure` does not construct one of its own. + */ + readonly fetchTarball: TarballFetcher; +} + +/** + * Result of `materializeClosure`: the per-instance store directory the + * closure was laid out under (`/store`). A caller computes + * a specific package's directory with `storeEntryDir`. + */ +export interface MaterializeClosureResult { + readonly storeDir: string; +} + /** * Lay out a resolved manifest closure into the per-instance store - * WITHOUT importing any author code. This is phases 1-2 of the loader: + * WITHOUT importing any author code: * * 1. Fetch + SRI-verify + extract each platform-matching entry into * the content-addressable cache. @@ -41,16 +95,14 @@ const logger = getLogger(["sidecar", "tool-packaging", "store-layout"]); * and symlink each direct dep into that entry's `node_modules/` so * Node's ancestor walk resolves bare-specifier imports. * - * Returns the `storeDir` the closure was laid out under. The first real - * `import()` of author code is NOT here — it belongs to `loadManifest`'s - * phase-3 loop — so a caller (e.g. an install-time probe) can stage the - * frozen closure into a package directory the loader consumes without - * executing author code on its host. + * Returns the `storeDir` the closure was laid out under, so a caller (e.g. + * an install-time probe) can stage the frozen closure into a package + * directory without executing author code on its host. * * Only entries whose `os`/`cpu` match `host` are materialized; the rest * are skipped with a `platform.mismatch.skipped` debug log. Errors - * surface as `ToolLoaderError` with a `category` matching the - * corresponding `DeployApplyErrorCategory`. + * surface as `ClosureMaterializationError` with a `category` matching the + * corresponding `ClosureMaterializationErrorCategory`. */ export async function materializeClosure( args: MaterializeClosureArgs, @@ -71,20 +123,18 @@ export async function materializeClosure( let integrity: string; if (source.kind === "registry") { if (!args.registries.has(source.registry)) { - throw new ToolLoaderError({ + throw new ClosureMaterializationError({ category: "registry.unknown", - message: `manifest references registry "${source.registry}" which is not in the sidecar config`, - package: { name: entry.name, version: entry.version }, + message: `manifest entry ${entry.name}@${entry.version} references registry "${source.registry}" which is not in the sidecar config`, }); } integrity = source.integrity; } else if (source.package.format === "source") { const gitDir = args.gitDirs.get(source.assetId); if (gitDir === undefined) { - throw new ToolLoaderError({ + throw new ClosureMaterializationError({ category: "git.materialization.failed", - message: `manifest entry references git assetId "${source.assetId}" which is not in the git-dirs map`, - package: { name: entry.name, version: entry.version }, + message: `manifest entry ${entry.name}@${entry.version} references git assetId "${source.assetId}" which is not in the git-dirs map`, }); } return materializeGitEntry({ @@ -96,10 +146,9 @@ export async function materializeClosure( }); } else { if (!args.assetMounts.has(source.assetId)) { - throw new ToolLoaderError({ + throw new ClosureMaterializationError({ category: "asset.mount.missing", - message: `manifest entry references assetId "${source.assetId}" which is not in the asset-mounts map`, - package: { name: entry.name, version: entry.version }, + message: `manifest entry ${entry.name}@${entry.version} references assetId "${source.assetId}" which is not in the asset-mounts map`, }); } integrity = source.package.integrity; @@ -136,10 +185,9 @@ export async function materializeClosure( await args.cache.put(integrity, bytes); } catch (err) { if (err instanceof TarballIntegrityMismatchError) { - throw new ToolLoaderError({ + throw new ClosureMaterializationError({ category: "integrity.mismatch", message: `bytes for ${entry.name}@${entry.version} did not match pinned integrity`, - package: { name: entry.name, version: entry.version }, }); } throw err; @@ -161,10 +209,9 @@ export async function materializeClosure( if (err instanceof TarballIntegrityMismatchError) { await args.cache.evict(integrity); } - throw new ToolLoaderError({ + throw new ClosureMaterializationError({ category: "tarball.extract.failed", message: `tar extraction failed for ${entry.name}@${entry.version}: ${describeError(err)}`, - package: { name: entry.name, version: entry.version }, }); } } @@ -182,9 +229,8 @@ export async function materializeClosure( // pass below has finished walking every dir to copy files out. // Releases are aggregated and drained in a `finally` so an error // mid-layout still hands the cache its references back. They are - // drained once the layout is built: the phase-3 import loop reads - // from the copied store, not from these extraction trees, so - // the extraction handles are a phase-1-2 concern only. + // drained once the layout is built: nothing reads these extraction + // trees after that. const extractionByEntry = new Map(); const entriesByNameVersion = new Map(); const releases: (() => void)[] = []; @@ -222,16 +268,12 @@ export async function materializeClosure( } } - return { storeDir, entries: filtered }; + return { storeDir }; } /** * True when `entry` may run on `host`: its `os`/`cpu` platform lists (if - * present) accept the host per npm's allow/block-list semantics. Pure — - * the single source of truth for the host-match decision, shared by the - * logging filter `passesPlatformFilter` and `loadManifest`'s phase-3 - * re-selection (which must not re-emit the mismatch debug logs - * `materializeClosure` already wrote). + * present) accept the host per npm's allow/block-list semantics. */ function entryMatchesHost( entry: ToolPackageManifestEntry, @@ -273,10 +315,9 @@ export function storeEntryDir( name: string, version: string, ): string { - // `@scope/name` carries a slash that, taken naively, would push the - // package's contents one directory deeper than `loadTopLevel` - // expects. Mirror npm's on-disk shape: `node_modules/@scope/name/`, - // so a scoped entry's dir is `/@scope/name//`. + // `@scope/name` carries a slash, so a scoped entry's dir sits one level + // deeper, mirroring npm's on-disk shape `node_modules/@scope/name/`: + // `/@scope/name//`. return path.join(storeDir, name, version); } @@ -309,7 +350,7 @@ async function buildStoreLayout(args: BuildStoreLayoutArgs): Promise { } const layoutDir = storeEntryDir(args.storeDir, entry.name, entry.version); await fs.mkdir(path.dirname(layoutDir), { recursive: true }); - await copyTree(extraction, layoutDir); + await copyTree(extraction, layoutDir, key); } for (const entry of args.filtered) { @@ -334,10 +375,9 @@ async function buildStoreLayout(args: BuildStoreLayoutArgs): Promise { logger.debug`optional.dropped.skipped: ${entry.name}@${entry.version} optional dep ${dep.name}@${dep.range} has no satisfying version in the closure (likely platform-filtered out)`; continue; } - throw new ToolLoaderError({ + throw new ClosureMaterializationError({ category: "package.entry.invalid", message: `${entry.name}@${entry.version} depends on ${dep.name}@${dep.range} but the manifest closure has no satisfying version; the resolver was expected to include it`, - package: { name: entry.name, version: entry.version }, }); } const target = storeEntryDir(args.storeDir, dep.name, pickedVersion); @@ -352,14 +392,10 @@ async function buildStoreLayout(args: BuildStoreLayoutArgs): Promise { if (!isEEXIST(err)) throw err; const existing = await fs.readlink(symlinkPath); if (existing !== relativeTarget) { - // A symlink collision inside the loader's per-package - // layout pass is a loader-layer invariant violation, not an - // unknown error shape — route it through the same structured - // envelope every other loader failure uses so atomic-apply - // surfaces it as `package.entry.invalid` instead of falling - // back to the unknown-shape catch-all (`factory.construct. - // failed`). - throw new ToolLoaderError({ + // A symlink collision inside the per-package layout pass is a + // layout invariant violation, not an unknown error shape, so it + // carries the same structured envelope as every other failure here. + throw new ClosureMaterializationError({ category: "package.entry.invalid", message: `symlink collision at ${symlinkPath}: existing target ${existing} differs from ${relativeTarget}`, }); @@ -391,7 +427,7 @@ interface RangeResolution { * `semver.maxSatisfying` against the current closure shape. * * Mirrors the resolver's first-arrival-per-`(name, range)` semantics - * on the loader side. Without this, two requirers with overlapping + * on the sidecar side. Without this, two requirers with overlapping * ranges of the same dep could each pick a different version of that * dep — `maxSatisfying` is deterministic given its candidate set, but * the candidate set is the full closure for the name and a transitive @@ -467,7 +503,7 @@ async function resolveRangesByFirstArrival( // `recorded.get(key)` returning `null` is the "we picked this // range against the closure and got nothing" cached answer. // Caching the null is safe only because the closure is static - // across this loader pass — `entriesByNameVersion` does not + // across this layout pass — `entriesByNameVersion` does not // grow underneath us. If a future change starts adding entries // mid-walk (e.g. lazy fetches during BFS), the cached null // would shadow the new candidates and produce a phantom miss; @@ -510,6 +546,7 @@ async function resolveRangesByFirstArrival( async function copyTree( srcDir: string, destDir: string, + packageLabel: string, extractionRoot: string = srcDir, ): Promise { await fs.mkdir(destDir, { recursive: true }); @@ -518,7 +555,7 @@ async function copyTree( const src = path.join(srcDir, entry.name); const dest = path.join(destDir, entry.name); if (entry.isDirectory()) { - await copyTree(src, dest, extractionRoot); + await copyTree(src, dest, packageLabel, extractionRoot); } else if (entry.isFile()) { try { await fs.copyFile(src, dest, fsConstants.COPYFILE_EXCL); @@ -571,9 +608,9 @@ async function copyTree( try { realExtractionRoot = await fs.realpath(extractionRoot); } catch (err) { - throw new ToolLoaderError({ + throw new ClosureMaterializationError({ category: "package.entry.invalid", - message: `tarball symlink ${src} → ${target}: extraction-root realpath failed: ${describeError(err)}`, + message: `tarball for ${packageLabel}: symlink ${src} → ${target}: extraction-root realpath failed: ${describeError(err)}`, }); } // `path.resolve` produces the absolute path the symlink would @@ -600,18 +637,18 @@ async function copyTree( targetAbs = await fs.realpath(path.resolve(path.dirname(src), target)); } catch (err) { if (!isENOENT(err)) { - throw new ToolLoaderError({ + throw new ClosureMaterializationError({ category: "package.entry.invalid", - message: `tarball contains symlink ${src} → ${target} whose target could not be resolved: ${describeError(err)}`, + message: `tarball for ${packageLabel} contains symlink ${src} → ${target} whose target could not be resolved: ${describeError(err)}`, }); } let srcDirReal: string; try { srcDirReal = await fs.realpath(path.dirname(src)); } catch (dirErr) { - throw new ToolLoaderError({ + throw new ClosureMaterializationError({ category: "package.entry.invalid", - message: `tarball symlink ${src} → ${target}: dirname realpath failed during dangling-link fallback: ${describeError(dirErr)}`, + message: `tarball for ${packageLabel}: symlink ${src} → ${target}: dirname realpath failed during dangling-link fallback: ${describeError(dirErr)}`, }); } targetAbs = path.resolve(srcDirReal, target); @@ -623,9 +660,9 @@ async function copyTree( targetAbs !== realExtractionRoot && !targetAbs.startsWith(realContainmentRoot) ) { - throw new ToolLoaderError({ + throw new ClosureMaterializationError({ category: "package.entry.invalid", - message: `tarball contains symlink ${src} → ${target} that escapes the package extraction directory`, + message: `tarball for ${packageLabel} contains symlink ${src} → ${target} that escapes the package extraction directory`, }); } try { @@ -666,10 +703,9 @@ async function readDirectDependencies( try { pkg = JSON.parse(pkgJsonRaw); } catch (err) { - throw new ToolLoaderError({ + throw new ClosureMaterializationError({ category: "package.entry.invalid", message: `malformed package.json in ${entry.name}@${entry.version}: ${describeError(err)}`, - package: { name: entry.name, version: entry.version }, }); } const byName = new Map(); @@ -694,10 +730,9 @@ async function readDirectDependencies( if (optionalDeps !== null && typeof optionalDeps === "object") { for (const [name, range] of Object.entries(optionalDeps)) { if (typeof range !== "string") { - throw new ToolLoaderError({ + throw new ClosureMaterializationError({ category: "package.entry.invalid", message: `package.json field optionalDependencies["${name}"] in ${entry.name}@${entry.version} is ${typeof range}, expected a string range`, - package: { name: entry.name, version: entry.version }, }); } byName.set(name, { @@ -714,10 +749,9 @@ async function readDirectDependencies( if (deps !== null && typeof deps === "object") { for (const [name, range] of Object.entries(deps)) { if (typeof range !== "string") { - throw new ToolLoaderError({ + throw new ClosureMaterializationError({ category: "package.entry.invalid", message: `package.json field dependencies["${name}"] in ${entry.name}@${entry.version} is ${typeof range}, expected a string range`, - package: { name: entry.name, version: entry.version }, }); } byName.set(name, { @@ -768,10 +802,9 @@ function assertDepMapShape( entry: { readonly name: string; readonly version: string }, ): void { if (Array.isArray(value)) { - throw new ToolLoaderError({ + throw new ClosureMaterializationError({ category: "package.entry.invalid", message: `package.json#${field} for ${entry.name}@${entry.version} must be an object map of name→range, not an array`, - package: { name: entry.name, version: entry.version }, }); } } diff --git a/packages/tool-packaging/src/tarball-fetcher.ts b/packages/tool-packaging/src/tarball-fetcher.ts new file mode 100644 index 000000000..0f61e1351 --- /dev/null +++ b/packages/tool-packaging/src/tarball-fetcher.ts @@ -0,0 +1,212 @@ +// eslint-disable-next-line @typescript-eslint/triple-slash-reference -- npm-team packages ship no types; declarations.d.ts must be visible to downstream typecheckers that import from this package's source. +/// +// The production tarball fetcher for closure materialization. Failures +// surface as `ClosureMaterializationError` with a `category` matching one of +// the `ClosureMaterializationErrorCategory` values. + +import { promises as fs } from "node:fs"; +import path from "node:path"; +import npmRegistryFetch from "npm-registry-fetch"; + +import type { ToolPackageManifestEntry } from "@intx/types/tool-packages"; + +import { + DEFAULT_MAX_REGISTRY_TARBALL_BYTES, + DEFAULT_REGISTRY_FETCH_TIMEOUT_MS, + ClosureMaterializationError, + describeError, +} from "./materialization-internal"; +import { + buildRegistryFetchOpts, + defaultTarballUrl, + readResponseWithLimit, +} from "./registry-fetch"; +import type { RegistryConfig } from "./resolver"; + +export interface TarballFetcherConfig { + /** + * Hard cap on the byte length of a tarball fetched from an HTTP + * registry. The default fetcher honors both the upstream + * `Content-Length` header (rejecting up front when the header value + * exceeds the cap) and the realized body byte count (aborting the + * read once the running total crosses the cap). Asset-sourced + * tarballs do not go through this path; their containment is the + * substrate's upload-time cap on the hub side. + * + * The default mirrors the hub's `HUB_MAX_TARBALL_BYTES` cap so a + * tarball legitimately accepted by the hub-side upload route is + * also legitimately fetchable from a registry mirror seeded from + * that hub. An operator pointing the sidecar at a third-party + * registry whose curated tarballs run larger should raise the cap + * explicitly rather than relying on the runtime to grow. + */ + readonly maxRegistryTarballBytes?: number; + /** + * Deadline in milliseconds for a single HTTP-registry tarball fetch, + * spanning the request and the streamed body read. A stalled registry + * cannot block the fetch -- and the closure materialization + * awaiting it -- past this bound. Defaults to + * `DEFAULT_REGISTRY_FETCH_TIMEOUT_MS`. Asset-sourced tarballs read from + * the local filesystem and are not subject to it. + */ + readonly registryFetchTimeoutMs?: number; +} + +export type TarballFetcher = ( + entry: ToolPackageManifestEntry, + ctx: { + registries: ReadonlyMap; + assetRoot: string; + assetMounts: ReadonlyMap; + }, +) => Promise; + +/** + * Build the production tarball fetcher: asset entries read from their mount + * under `assetRoot`, registry entries over HTTP through `npm-registry-fetch`. + */ +export function createTarballFetcher( + config: TarballFetcherConfig = {}, +): TarballFetcher { + const maxRegistryTarballBytes = + config.maxRegistryTarballBytes ?? DEFAULT_MAX_REGISTRY_TARBALL_BYTES; + if ( + !Number.isFinite(maxRegistryTarballBytes) || + maxRegistryTarballBytes <= 0 + ) { + throw new Error( + `createTarballFetcher: maxRegistryTarballBytes must be a positive finite number; got ${String(maxRegistryTarballBytes)}`, + ); + } + const registryFetchTimeoutMs = + config.registryFetchTimeoutMs ?? DEFAULT_REGISTRY_FETCH_TIMEOUT_MS; + if (!Number.isFinite(registryFetchTimeoutMs) || registryFetchTimeoutMs <= 0) { + throw new Error( + `createTarballFetcher: registryFetchTimeoutMs must be a positive finite number; got ${String(registryFetchTimeoutMs)}`, + ); + } + return async (entry, ctx) => { + if (entry.source.kind === "asset") { + // A source-format asset is a git subtree materialized by checkout + // in the store layout, never fetched as a tarball. Reaching the + // tarball fetcher with one is a bug; fail loud. + if (entry.source.package.format !== "tarball") { + throw new ClosureMaterializationError({ + category: "git.materialization.failed", + message: `source-format asset entry ${entry.name}@${entry.version} reached the tarball fetcher`, + }); + } + const tarballPath = entry.source.package.path; + // The mount lookup is guaranteed by `materializeClosure`'s + // pre-fetch gate, but reassert here so the narrowing is + // visible to readers — the caller of fetchTarball has no + // structural guarantee it ran through that gate. + const mount = ctx.assetMounts.get(entry.source.assetId); + if (mount === undefined) { + throw new ClosureMaterializationError({ + category: "asset.mount.missing", + message: `default fetcher reached for ${entry.name}@${entry.version} without a mount for assetId "${entry.source.assetId}"`, + }); + } + // Both `mount` and `entry.source.package.path` originate from the + // hub and cross the trust boundary into the sidecar process. A `..` + // segment in either would let a malicious manifest read any + // file the sidecar can open. Resolve the join and assert the + // result still sits under `assetRoot` so a traversal attempt + // surfaces as a structured manifest rejection rather than a + // silent arbitrary read. + // + // Reject absolute mount paths up front: `path.resolve` would + // discard the assetRoot prefix when handed an absolute segment, + // letting an absolute mount escape the containment check that + // follows. Defense-in-depth for the (today-trusted) hub-side + // mount producer. + if (path.isAbsolute(mount)) { + throw new ClosureMaterializationError({ + category: "package.entry.invalid", + message: `assetMounts entry for ${entry.name}@${entry.version} is absolute (${JSON.stringify(mount)}); mounts must be assetRoot-relative`, + }); + } + const mountAbs = path.resolve(ctx.assetRoot, mount); + const absPath = path.resolve(mountAbs, tarballPath); + const mountContainmentRoot = mountAbs.endsWith(path.sep) + ? mountAbs + : mountAbs + path.sep; + if (absPath !== mountAbs && !absPath.startsWith(mountContainmentRoot)) { + throw new ClosureMaterializationError({ + category: "package.entry.invalid", + message: `source.package.path for ${entry.name}@${entry.version} resolves to ${JSON.stringify(absPath)} which escapes the declared mount ${JSON.stringify(mountAbs)} (cross-mount traversal)`, + }); + } + try { + return await fs.readFile(absPath); + } catch (err) { + throw new ClosureMaterializationError({ + category: "tarball.missing", + message: `asset-stored tarball for ${entry.name}@${entry.version} not present at ${absPath}: ${describeError(err)}`, + }); + } + } + const registry = ctx.registries.get(entry.source.registry); + if (registry === undefined) { + throw new ClosureMaterializationError({ + category: "registry.unknown", + message: `manifest entry ${entry.name}@${entry.version} references registry "${entry.source.registry}" which is not in the sidecar config`, + }); + } + const tarballUrl = + entry.tarballUrl ?? + defaultTarballUrl(registry.url, entry.name, entry.version); + // Bound the whole fetch -- request and streamed body read -- so a + // stalled registry cannot block the awaiting deploy forever. + // npm-registry-fetch honors the signal for the request phase; + // readResponseWithLimit honors it for the manual body read. The + // timer spans both phases and is cleared only once the read settles. + const controller = new AbortController(); + const timer = setTimeout(() => { + controller.abort(); + }, registryFetchTimeoutMs); + try { + const res = await npmRegistryFetch(tarballUrl, { + ...buildRegistryFetchOpts(registry), + signal: controller.signal, + }); + if (res.status === 401 || res.status === 403) { + throw new ClosureMaterializationError({ + category: "registry.auth.failed", + message: `registry "${entry.source.registry}" rejected credentials for ${entry.name}@${entry.version} (HTTP ${String(res.status)})`, + }); + } + if (!res.ok) { + throw new ClosureMaterializationError({ + category: "registry.fetch.failed", + message: `registry "${entry.source.registry}" returned HTTP ${String(res.status)} fetching ${entry.name}@${entry.version}`, + }); + } + return await readResponseWithLimit( + res, + maxRegistryTarballBytes, + { + registry: entry.source.registry, + name: entry.name, + version: entry.version, + }, + controller.signal, + ); + } catch (err) { + if (err instanceof ClosureMaterializationError) throw err; + if (controller.signal.aborted) { + throw new ClosureMaterializationError({ + category: "registry.fetch.failed", + message: `registry "${entry.source.registry}" fetch for ${entry.name}@${entry.version} exceeded the ${String(registryFetchTimeoutMs)}ms timeout`, + }); + } + throw new ClosureMaterializationError({ + category: "registry.fetch.failed", + message: `registry "${entry.source.registry}" fetch failed for ${entry.name}@${entry.version}: ${describeError(err)}`, + }); + } finally { + clearTimeout(timer); + } + }; +} diff --git a/packages/tool-packaging/src/tool-materialization.test.ts b/packages/tool-packaging/src/tool-materialization.test.ts deleted file mode 100644 index b7fdb7ab5..000000000 --- a/packages/tool-packaging/src/tool-materialization.test.ts +++ /dev/null @@ -1,349 +0,0 @@ -import { describe, test, expect, afterEach } from "bun:test"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { - clearDirtyMarker, - materializeToolPackages, - parseActiveDeployId, - persistActiveDeployIdWithFallback, -} from "./tool-materialization"; - -const tempDirs: string[] = []; - -async function tempDir(): Promise { - const d = await fs.promises.mkdtemp( - path.join(os.tmpdir(), "default-harness-test-"), - ); - tempDirs.push(d); - return d; -} - -afterEach(async () => { - const dirs = tempDirs.splice(0); - await Promise.all( - dirs.map((d) => fs.promises.rm(d, { recursive: true, force: true })), - ); -}); - -describe("materializeToolPackages — manifest.invalid gate", () => { - test("returns empty factories when no manifest bytes supplied", async () => { - const storeDir = await tempDir(); - const result = await materializeToolPackages({ - assetMounts: new Map(), - rawManifestBytes: undefined, - storeDir, - agentAddress: "agent-no-manifest", - cacheRoot: "/unused-by-manifest-invalid-gate", - cacheMaxBytes: 1024 * 1024, - registryMaxTarballBytes: 10 * 1024 * 1024, - assetRoot: "/unused-by-manifest-invalid-gate", - registries: new Map([["npmjs", { url: "https://registry.npmjs.org" }]]), - host: { os: "linux", cpu: "x64" }, - }); - expect(result.factories).toEqual([]); - expect(result.pluginFactories).toEqual([]); - }); - - test("JSON.parse failure throws and persists the raw bytes verbatim", async () => { - const storeDir = await tempDir(); - const corrupt = "{this is not valid json"; - let caught: unknown; - try { - await materializeToolPackages({ - assetMounts: new Map(), - rawManifestBytes: corrupt, - storeDir, - agentAddress: "agent-corrupt", - cacheRoot: "/unused-by-manifest-invalid-gate", - cacheMaxBytes: 1024 * 1024, - registryMaxTarballBytes: 10 * 1024 * 1024, - assetRoot: "/unused-by-manifest-invalid-gate", - registries: new Map([["npmjs", { url: "https://registry.npmjs.org" }]]), - host: { os: "linux", cpu: "x64" }, - }); - } catch (err) { - caught = err; - } - expect(caught).toBeInstanceOf(Error); - expect(String(caught)).toMatch(/manifest\.invalid/); - expect(String(caught)).toMatch(/JSON\.parse failed/); - - const auditRoot = path.join(storeDir, "audit", "rejected-applies"); - const attemptDirs = await fs.promises.readdir(auditRoot); - expect(attemptDirs).toHaveLength(1); - const attemptDirName = attemptDirs[0]; - if (attemptDirName === undefined) { - throw new Error("no attempt dir was created"); - } - const attemptDir = path.join(auditRoot, attemptDirName); - - // The manifest file must contain the exact corrupt bytes, not a - // JSON-encoded string literal wrapping them — the audit-trail - // writer must not push the bytes through JSON.stringify, which - // would double-encode the string and erase the original-input - // evidence the audit file exists to preserve. - const persisted = await fs.promises.readFile( - path.join(attemptDir, "manifest.json"), - "utf-8", - ); - expect(persisted).toBe(corrupt); - - const errorJson = JSON.parse( - await fs.promises.readFile(path.join(attemptDir, "error.json"), "utf-8"), - ); - expect(errorJson.category).toBe("manifest.invalid"); - expect(errorJson.message).toMatch(/JSON\.parse failed/); - expect(errorJson.previousDeployId).toBe("none"); - expect(typeof errorJson.attemptId).toBe("string"); - expect(errorJson.attemptId.length).toBeGreaterThan(0); - }); - - test("schema failure throws and persists the raw bytes verbatim", async () => { - const storeDir = await tempDir(); - // Valid JSON but wrong shape — arktype rejects it. - const wrongShape = JSON.stringify({ schemaVersion: 99 }); - let caught: unknown; - try { - await materializeToolPackages({ - assetMounts: new Map(), - rawManifestBytes: wrongShape, - storeDir, - agentAddress: "agent-schema", - cacheRoot: "/unused-by-manifest-invalid-gate", - cacheMaxBytes: 1024 * 1024, - registryMaxTarballBytes: 10 * 1024 * 1024, - assetRoot: "/unused-by-manifest-invalid-gate", - registries: new Map([["npmjs", { url: "https://registry.npmjs.org" }]]), - host: { os: "linux", cpu: "x64" }, - }); - } catch (err) { - caught = err; - } - expect(caught).toBeInstanceOf(Error); - expect(String(caught)).toMatch(/manifest\.invalid/); - expect(String(caught)).toMatch(/schema validation failed/); - - const auditRoot = path.join(storeDir, "audit", "rejected-applies"); - const attemptDirs = await fs.promises.readdir(auditRoot); - expect(attemptDirs).toHaveLength(1); - const attemptDirName = attemptDirs[0]; - if (attemptDirName === undefined) { - throw new Error("no attempt dir was created"); - } - const attemptDir = path.join(auditRoot, attemptDirName); - - // The audit trail records the bytes-as-supplied for both - // failure modes, so a future investigator can replay the - // same input against a newer validator. - const persisted = await fs.promises.readFile( - path.join(attemptDir, "manifest.json"), - "utf-8", - ); - expect(persisted).toBe(wrongShape); - - const errorJson = JSON.parse( - await fs.promises.readFile(path.join(attemptDir, "error.json"), "utf-8"), - ); - expect(errorJson.category).toBe("manifest.invalid"); - }); - - describe("parseActiveDeployId", () => { - test("accepts a v1-prefixed id", () => { - expect(parseActiveDeployId("v1:abc-123", "/dummy")).toBe("abc-123"); - }); - - test("accepts a bare id for backward compatibility with pre-versioned files", () => { - expect(parseActiveDeployId("legacy-deploy-id", "/dummy")).toBe( - "legacy-deploy-id", - ); - }); - - test("rejects an unknown version prefix loudly", () => { - expect(() => parseActiveDeployId("v2:abc", "/dummy")).toThrow( - /unknown version prefix/, - ); - }); - - test("rejects a v1-prefixed but empty id", () => { - expect(() => parseActiveDeployId("v1:", "/dummy")).toThrow( - /carries the v1 prefix but no id/, - ); - }); - - test("rejects an empty file", () => { - expect(() => parseActiveDeployId("", "/dummy")).toThrow(/is empty/); - }); - }); - - describe("persistActiveDeployIdWithFallback", () => { - test("returns degraded=false and removes a stale dirty marker on full success", async () => { - const instanceDir = await tempDir(); - const activeIdFile = path.join(instanceDir, "active-deploy-id"); - const dirtyFile = `${activeIdFile}.dirty`; - await fs.promises.writeFile(dirtyFile, "v1:stale-id"); - - const outcome = await persistActiveDeployIdWithFallback( - instanceDir, - activeIdFile, - "fresh-id", - ); - expect(outcome.degraded).toBe(false); - const recorded = await fs.promises.readFile(activeIdFile, "utf-8"); - expect(recorded).toBe("v1:fresh-id"); - // The dirty marker must be cleared so the next boot reads the - // freshly-recorded id directly, not the stale marker that - // pre-dated the successful persist. - let dirtyExists = false; - try { - await fs.promises.access(dirtyFile); - dirtyExists = true; - } catch { - dirtyExists = false; - } - expect(dirtyExists).toBe(false); - }); - - test("the degraded-persist code path clears a stale dirty marker via the shared helper", async () => { - // The fsync'd primary persist and the no-fsync fallback persist - // both delegate to `clearDirtyMarker` after writing the recorded - // id. The fallback branch is hard to drive end-to-end without - // injecting an `fs` failure mode that distinguishes - // `fs.promises.open(path, "w")` from `fs.promises.writeFile(path)` - // — they share flags and permissions on every supported FS — so - // the contract is pinned at the helper level: when called against - // an instance dir that holds a stale marker, the marker is gone - // afterward. The wrapper's no-fsync branch routes through this - // helper, which gives the boot reader the freshly-recorded id on - // the next boot. - const instanceDir = await tempDir(); - const activeIdFile = path.join(instanceDir, "active-deploy-id"); - const dirtyFile = `${activeIdFile}.dirty`; - await fs.promises.writeFile(dirtyFile, "v1:stale-from-prior-apply"); - await clearDirtyMarker(activeIdFile, "test"); - let dirtyExists = false; - try { - await fs.promises.access(dirtyFile); - dirtyExists = true; - } catch { - dirtyExists = false; - } - expect(dirtyExists).toBe(false); - }); - - test("clearing a missing dirty marker is a no-op", async () => { - // ENOENT is the normal case when no prior apply wrote a marker. - // The helper must not throw or log when there is nothing to - // remove; the wrapper relies on the helper being safe to call - // unconditionally on every successful persist. - const instanceDir = await tempDir(); - const activeIdFile = path.join(instanceDir, "active-deploy-id"); - await clearDirtyMarker(activeIdFile, "test"); - }); - - test("writes the dirty marker when the primary persist cannot open the file", async () => { - // Simulate a primary persist failure by making activeIdFile itself - // a directory. Both the fsync'd open() and the no-fsync - // writeFile() will then fail with EISDIR, exercising the dirty- - // marker path. - const instanceDir = await tempDir(); - const activeIdFile = path.join(instanceDir, "active-deploy-id"); - await fs.promises.mkdir(activeIdFile); - - const outcome = await persistActiveDeployIdWithFallback( - instanceDir, - activeIdFile, - "new-id-42", - ); - expect(outcome.degraded).toBe(true); - expect(outcome.error).toBeInstanceOf(Error); - - const dirtyContents = await fs.promises.readFile( - `${activeIdFile}.dirty`, - "utf-8", - ); - expect(dirtyContents).toBe("v1:new-id-42"); - }); - }); - - test("boot reconciliation records the dirty marker as previousDeployId", async () => { - // Seed an instance where the prior apply could not durably record - // the committed deploy id and a dirty marker carries the truth. - // The next apply (any apply — here, a manifest-invalid one) must - // surface the marker's id as `previousDeployId` in the audit - // record, not the stale recorded id and not "none". - const storeDir = await tempDir(); - const instanceDir = path.join(storeDir, "tool-packages"); - await fs.promises.mkdir(instanceDir, { recursive: true }); - await fs.promises.writeFile( - path.join(instanceDir, "active-deploy-id"), - "v1:stale-recorded", - ); - await fs.promises.writeFile( - path.join(instanceDir, "active-deploy-id.dirty"), - "v1:truth-from-marker", - ); - - let caught: unknown; - try { - await materializeToolPackages({ - assetMounts: new Map(), - rawManifestBytes: "{ not json", - storeDir, - agentAddress: "agent-dirty-marker", - cacheRoot: "/unused-by-manifest-invalid-gate", - cacheMaxBytes: 1024 * 1024, - registryMaxTarballBytes: 10 * 1024 * 1024, - assetRoot: "/unused-by-manifest-invalid-gate", - registries: new Map([["npmjs", { url: "https://registry.npmjs.org" }]]), - host: { os: "linux", cpu: "x64" }, - }); - } catch (err) { - caught = err; - } - expect(caught).toBeInstanceOf(Error); - - const auditRoot = path.join(storeDir, "audit", "rejected-applies"); - const attemptDirs = await fs.promises.readdir(auditRoot); - expect(attemptDirs).toHaveLength(1); - const attemptDirName = attemptDirs[0]; - if (attemptDirName === undefined) { - throw new Error("no attempt dir was created"); - } - const errorJson = JSON.parse( - await fs.promises.readFile( - path.join(auditRoot, attemptDirName, "error.json"), - "utf-8", - ), - ); - expect(errorJson.previousDeployId).toBe("truth-from-marker"); - }); - - test("manifest.invalid throws and writes an audit entry", async () => { - const storeDir = await tempDir(); - let caught: unknown; - try { - await materializeToolPackages({ - assetMounts: new Map(), - rawManifestBytes: "{ not json", - storeDir, - agentAddress: "agent-throws", - cacheRoot: "/unused-by-manifest-invalid-gate", - cacheMaxBytes: 1024 * 1024, - registryMaxTarballBytes: 10 * 1024 * 1024, - assetRoot: "/unused-by-manifest-invalid-gate", - registries: new Map([["npmjs", { url: "https://registry.npmjs.org" }]]), - host: { os: "linux", cpu: "x64" }, - }); - } catch (err) { - caught = err; - } - expect(caught).toBeInstanceOf(Error); - expect(String(caught)).toMatch(/manifest\.invalid/); - - const auditRoot = path.join(storeDir, "audit", "rejected-applies"); - const attemptDirs = await fs.promises.readdir(auditRoot); - expect(attemptDirs).toHaveLength(1); - }); -}); diff --git a/packages/tool-packaging/src/tool-materialization.ts b/packages/tool-packaging/src/tool-materialization.ts deleted file mode 100644 index 032f0370c..000000000 --- a/packages/tool-packaging/src/tool-materialization.ts +++ /dev/null @@ -1,570 +0,0 @@ -// Per-step tool-package apply: validate the manifest, run `applyAtomic`, -// and persist the active deploy id. The registry map, host platform, and -// asset root come from the caller. - -import fs from "node:fs"; -import path from "node:path"; -import { type } from "arktype"; -import { type AnnotatedPluginFactory } from "@intx/agent"; -import { getLogger } from "@intx/log"; -import { hasCode } from "@intx/types/has-code"; -import type { ToolCredentialDeclaration } from "@intx/types/package-json"; -import { ToolPackageManifest } from "@intx/types/tool-packages"; - -import { applyAtomic } from "./atomic-apply"; -import { createTarballCache } from "./cache"; -import { - createToolLoader, - type HostPlatform, - type LoadedToolFactory, - type LoadedToolPackage, -} from "./loader"; -import type { RegistryConfig } from "./resolver"; - -const logger = getLogger(["sidecar", "harness-builder"]); - -// Sentinel `previousDeployId` for an instance that has never applied -// a deploy successfully. Encoded as a literal string so the value -// travels through `applyAtomic`'s `ApplyAtomicFailure.previousDeployId` -// channel unchanged; the hub treats it as "no prior deploy" rather -// than as a real id. -const NO_PRIOR_DEPLOY_ID = "none"; - -/** - * A loaded tool factory paired with the identity of the package it came - * from. `collectFactories` flattens every package's factories into one list - * but preserves this association per factory, because the per-bundle - * credential assembly downstream needs two things the bare factory (which - * carries only its bundle id) cannot supply: the package name, to derive the - * consumer identity (`toolConsumer(packageName)`) that Gate 2 checks, and the - * package's declared credential handles, to reconcile against the delivered - * bindings. `declaredCredentials` is the package-level set, repeated on each - * of that package's factories. - */ -export interface StepToolFactory { - readonly packageName: string; - readonly declaredCredentials: readonly ToolCredentialDeclaration[]; - readonly factory: LoadedToolFactory; -} - -interface MaterializedToolPackages { - readonly factories: readonly StepToolFactory[]; - readonly pluginFactories: readonly AnnotatedPluginFactory[]; -} - -// Exported for direct unit testing of the manifest-invalid gate -// (JSON.parse failure + arktype schema failure). Nothing in production -// calls it. -export async function materializeToolPackages(args: { - rawManifestBytes: string | undefined; - /** - * `assetId` → workspace-relative mount path, parsed from the deploy - * pack's `deploy/asset-mounts.json`. The loader resolves - * `kind: "asset"` manifest entries against this map. - */ - assetMounts: ReadonlyMap; - storeDir: string; - /** - * Workspace root the loader resolves `kind: "asset"` tarball mounts - * against (`//...`). - * - * The workflow-process child stages assets in the step's LEGACY agent - * dir workspace (where the hub's asset-pack push lands them) and roots - * the per-step apply-state + agent `env.workdir` under a distinct - * per-step store dir, so the loader's asset source and the apply-state - * root are two different directories. - */ - assetRoot: string; - /** - * Registry name → config. Parsed by the caller from - * `SIDECAR_TOOL_REGISTRIES` (or supplied directly by a test). - */ - registries: ReadonlyMap; - /** - * Host `os`/`cpu` pair the loader filters manifest entries against. - */ - host: HostPlatform; - agentAddress: string; - cacheRoot: string; - cacheMaxBytes: number; - registryMaxTarballBytes: number; -}): Promise { - if (args.rawManifestBytes === undefined) { - return { factories: [], pluginFactories: [] }; - } - // Capture into a local so closures below see a `string` (the property - // narrowing on args.rawManifestBytes does not survive the nested - // function boundary, even though the property is readonly). - const rawManifestBytes = args.rawManifestBytes; - - // The apply-state tree (`tool-packages/`) shares the agent's - // storeDir root with the workspace tree the tool factories run - // against. Factories execute with `cwd: env.workdir`; the loader - // and the agent runtime both rely on factories not writing outside - // their workspace. A factory that walks `..` out of the workspace - // could touch the apply-state tree. The substrate trusts factories - // to honor that boundary — a misbehaving factory is treated as a - // security regression against the package, not a sandboxing gap to - // close at this layer. - const instanceDir = path.join(args.storeDir, "tool-packages"); - await fs.promises.mkdir(instanceDir, { recursive: true }); - - const activeIdFile = path.join(instanceDir, "active-deploy-id"); - const activeIdDirtyFile = `${activeIdFile}.dirty`; - // The dirty marker is written by `persistActiveDeployId`'s catch - // path when the commit persist (the normal write + fsync + dir - // fsync) failed on the prior apply and even the no-fsync fallback - // failed. Its presence means the staged deploy was committed but the - // recorded `active-deploy-id` is stale, and the marker carries the id - // that belongs there. Read it first so a degraded prior boot doesn't - // surface as `previousDeployId="none"` and silently demote the - // committed deploy to "fresh instance". - let previousDeployId = NO_PRIOR_DEPLOY_ID; - try { - const dirtyRaw = ( - await fs.promises.readFile(activeIdDirtyFile, "utf-8") - ).trim(); - const dirtyId = parseActiveDeployId(dirtyRaw, activeIdDirtyFile); - logger.warn`active-deploy-id dirty marker present at ${activeIdDirtyFile}; the prior apply could not durably record the committed deploy id and the boot is reconciling from ${dirtyId}`; - previousDeployId = dirtyId; - } catch (err) { - if (!(hasCode(err) && err.code === "ENOENT")) { - throw err; - } - try { - const raw = (await fs.promises.readFile(activeIdFile, "utf-8")).trim(); - previousDeployId = parseActiveDeployId(raw, activeIdFile); - } catch (innerErr) { - if (!(hasCode(innerErr) && innerErr.code === "ENOENT")) { - throw innerErr; - } - } - } - - const attemptId = crypto.randomUUID(); - const newDeployId = crypto.randomUUID(); - - // Validate the manifest at the loader boundary. Both JSON parse - // failures and arktype schema failures route through the same - // `manifest.invalid` category so the hub sees a single failure - // shape for malformed manifests regardless of the specific defect. - // The helper persists the rejected bytes + the failure payload to - // the durable audit, then returns the Error for the caller to throw. - // Returning instead of throwing lets the caller use `throw await ...`, - // which TypeScript narrows control flow against without any - // dead-code suffix at the call site. - const rejectManifestInvalid = async (reason: string): Promise => { - const occurredAt = new Date().toISOString(); - logger.warn`tool-package apply rejected for ${args.agentAddress}: manifest.invalid — ${reason}`; - const message = `deploy/tool-packages-manifest.json could not be loaded: ${reason}`; - await writeRejectedApplyAudit({ - storeDir: args.storeDir, - attemptId, - manifestBytes: rawManifestBytes, - failure: { - attemptId, - previousDeployId, - category: "manifest.invalid", - message, - occurredAt, - }, - }); - return new Error( - `tool-package apply rejected (manifest.invalid): ${reason}`, - ); - }; - - let parsedManifest: unknown; - try { - parsedManifest = JSON.parse(rawManifestBytes); - } catch (err) { - throw await rejectManifestInvalid( - `JSON.parse failed: ${err instanceof Error ? err.message : String(err)}`, - ); - } - const validated = ToolPackageManifest(parsedManifest); - if (validated instanceof type.errors) { - throw await rejectManifestInvalid( - `schema validation failed: ${validated.summary}`, - ); - } - - const cache = createTarballCache({ - rootDir: args.cacheRoot, - maxBytes: args.cacheMaxBytes, - }); - const loader = createToolLoader({ - cache, - registries: args.registries, - host: args.host, - maxRegistryTarballBytes: args.registryMaxTarballBytes, - }); - const result = await applyAtomic({ - manifest: validated, - loader, - instanceDir, - assetRoot: args.assetRoot, - assetMounts: args.assetMounts, - // Step tool packages never source from git; only workflow-definition - // closures carry git entries. - gitDirs: new Map(), - attemptId, - previousDeployId, - newDeployId, - }); - if (result.status === "failed") { - logger.warn`tool-package apply rejected for ${args.agentAddress}: ${result.category} — ${result.message}`; - // A failed apply never wrote `active-deploy-id`: `applyAtomic` - // stages into a per-deploy-id directory and the commit is the - // persist below, which only runs on success. So the prior deploy is - // trivially still live and `result.previousDeployId` carries the - // unchanged prior id. There is no committed-but-failed swap to - // reconcile here — that case existed only under the old rename - // protocol. - await writeRejectedApplyAudit({ - storeDir: args.storeDir, - attemptId: result.attemptId, - // The raw bytes from disk, not the arktype-narrowed object. - // Re-serializing `validated` would drop unknown-tolerated - // fields, normalize key order, and lose whitespace — exactly - // the original-input evidence a future investigator needs to - // reproduce the failure against a newer validator. - manifestBytes: rawManifestBytes, - failure: result, - }); - throw new Error( - `tool-package apply rejected (${result.category}): ${result.message}`, - ); - } - - // Apply staged: the loader built the new deploy at - // `packages//`. Persisting the new active id is the - // commit — the single write that advances the live deploy from the - // prior id to this one. If the write or its fsync fails (disk full, - // EIO, EROFS), the on-disk state has diverged from the recorded id: - // `persistActiveDeployIdWithFallback` has already written the new id - // through its no-fsync / dirty-marker degradation ladder, so the new - // deploy is logically committed, but the id was not durably flushed. - // Record this as `apply.previous-rotation.failed` — for that category - // `previousDeployId` carries the **new** deploy id (the one now live) - // so the durable audit records the on-disk truth. Write the audit - // entry, then throw so the harness tears down: the durability gap - // means the next apply cannot trust `previousDeployId` until the next - // boot reconciles via the dirty marker. - const persistOutcome = await persistActiveDeployIdWithFallback( - instanceDir, - activeIdFile, - result.activeDeployId, - ); - if (persistOutcome.degraded) { - const err = persistOutcome.error; - const occurredAt = new Date().toISOString(); - const message = `active-deploy-id persist failed after staging deploy: ${err instanceof Error ? err.message : String(err)}`; - logger.error`tool-package apply: ${message}; active deploy ${result.activeDeployId} is live on disk but the recorded id was not durably written`; - await writeRejectedApplyAudit({ - storeDir: args.storeDir, - attemptId, - manifestBytes: rawManifestBytes, - failure: { - attemptId, - previousDeployId: result.activeDeployId, - category: "apply.previous-rotation.failed", - message, - occurredAt, - }, - }); - throw new Error( - `tool-package apply rejected (apply.previous-rotation.failed): ${message}`, - { cause: err }, - ); - } - return { - factories: collectFactories(result.loaded), - pluginFactories: collectPluginFactories(result.loaded), - }; -} - -/** - * Write the active deploy id to `activeIdFile` and fsync both the - * file's own data/metadata and the parent directory entry. POSIX does - * not guarantee a parent-directory entry is durably linked from a - * file's own fsync alone, so the dir handle is opened and synced - * separately. Without that, a crash between a deploy's commit and the - * next boot could leave the staged deploy directory present while - * active-deploy-id is not yet visible — the next apply would then read - * previousDeployId="none" and treat the committed deploy as belonging - * to a fresh, deploy-less instance. - * - * Dir-fsync is best-effort durability hardening, not part of the - * apply's structural success contract. Some filesystems (notably - * FAT/exFAT, certain network mounts) do not support fsync on a - * directory handle and will surface EINVAL / ENOTSUP. The deploy is - * already staged on disk and the deploy-id file's own fsync has - * landed; tearing the harness down at this point would force a - * restart for a degraded-durability condition the operator has no - * way to act on. Log it and continue. - */ -// Version prefix for the active-deploy-id file. A future format -// change (e.g. carrying additional fields alongside the id) bumps -// this and the reader rejects unknown prefixes loudly instead of -// silently treating an unrecognized payload as a deploy id. -const ACTIVE_DEPLOY_ID_VERSION = "v1"; -const ACTIVE_DEPLOY_ID_PREFIX = `${ACTIVE_DEPLOY_ID_VERSION}:`; - -// Exported for unit testing of the version-prefix gate. -export function parseActiveDeployId(raw: string, sourcePath: string): string { - if (raw.length === 0) { - throw new Error( - `active-deploy-id file ${sourcePath} is empty; expected ${ACTIVE_DEPLOY_ID_PREFIX}`, - ); - } - // Pre-versioning files carried just the raw deploy id. Accept those - // for backward compatibility so an upgrade does not require an - // operator-driven state rewrite, but require the prefix on any - // file the new code writes. - // - // Deprecation horizon: this branch exists to ease the v1: cutover - // from pre-versioning files. Once every deployed sidecar has been - // restarted at least once on a version that writes the prefix, the - // un-prefixed branch can be deleted and an un-prefixed file should - // be rejected as garbage. The horizon is "remove after a few - // sidecar minor versions have shipped that write the prefix"; the - // exact cutover is an operational decision, not a code-encoded - // one. - if (raw.startsWith(ACTIVE_DEPLOY_ID_PREFIX)) { - const id = raw.slice(ACTIVE_DEPLOY_ID_PREFIX.length); - if (id.length === 0) { - throw new Error( - `active-deploy-id file ${sourcePath} carries the ${ACTIVE_DEPLOY_ID_VERSION} prefix but no id`, - ); - } - return id; - } - if (raw.includes(":")) { - // A leading token shaped like a version prefix that we do not - // recognize. Surface loudly rather than treating the whole string - // as the deploy id. - const prefix = raw.slice(0, raw.indexOf(":") + 1); - throw new Error( - `active-deploy-id file ${sourcePath} carries unknown version prefix ${JSON.stringify(prefix)}; this sidecar understands ${JSON.stringify(ACTIVE_DEPLOY_ID_PREFIX)}`, - ); - } - return raw; -} - -function formatActiveDeployId(deployId: string): string { - return `${ACTIVE_DEPLOY_ID_PREFIX}${deployId}`; -} - -async function persistActiveDeployId( - instanceDir: string, - activeIdFile: string, - deployId: string, -): Promise { - const handle = await fs.promises.open(activeIdFile, "w"); - try { - await handle.writeFile(formatActiveDeployId(deployId)); - await handle.sync(); - } finally { - await handle.close(); - } - try { - const dirHandle = await fs.promises.open(instanceDir, "r"); - try { - await dirHandle.sync(); - } finally { - await dirHandle.close(); - } - } catch (err) { - logger.warn`parent-dir fsync failed for ${instanceDir} after deploy-id persist; deploy-id durability is degraded but the committed deploy is staged on disk — ${err instanceof Error ? err.message : String(err)}`; - } - await clearDirtyMarker(activeIdFile, "successful persist"); -} - -/** - * Remove a stale `.dirty` sibling of the active-deploy-id file. Called - * after either the fsync'd primary persist or the no-fsync fallback - * persist successfully writes a fresh active-deploy-id; in both cases a - * pre-existing marker carries a now-stale id that would otherwise - * shadow the recorded id on the next boot (the boot reader prefers the - * marker when present). - * - * Best-effort: a failure to remove the marker leaves the next boot - * reading the now-redundant marker (which carries a stale id), so the - * divergence is bounded to a noisier log path. ENOENT is the normal - * case when no prior failure had written a marker. - * - * Exported for direct unit testing of the cleanup contract. - */ -export async function clearDirtyMarker( - activeIdFile: string, - reason: string, -): Promise { - try { - await fs.promises.unlink(`${activeIdFile}.dirty`); - } catch (err) { - if (!(hasCode(err) && err.code === "ENOENT")) { - logger.warn`failed to clear active-deploy-id dirty marker after ${reason}: ${err instanceof Error ? err.message : String(err)}`; - } - } -} - -/** - * Wrap `persistActiveDeployId` with a degradation ladder: try the - * normal fsync'd write first, and on failure try a no-fsync write + - * write of a sibling `.dirty` marker so the next boot can reconcile. - * - * Returns `{ degraded: false }` on full success. Returns - * `{ degraded: true, error }` when the primary persist failed; the - * caller records an `apply.previous-rotation.failed` audit entry and - * throws on the back of it. - * - * If the fallback persist also fails, on-disk state will diverge from - * the recorded id for one boot cycle. Boot-time reconciliation reads - * the dirty marker and prefers it. - * - * Exported for direct unit testing of the dirty-marker contract. - */ -export async function persistActiveDeployIdWithFallback( - instanceDir: string, - activeIdFile: string, - deployId: string, -): Promise<{ readonly degraded: boolean; readonly error?: unknown }> { - try { - await persistActiveDeployId(instanceDir, activeIdFile, deployId); - return { degraded: false }; - } catch (primary) { - logger.warn`active-deploy-id primary persist failed for ${activeIdFile}: ${primary instanceof Error ? primary.message : String(primary)}; attempting degraded write`; - try { - await fs.promises.writeFile(activeIdFile, formatActiveDeployId(deployId)); - logger.warn`active-deploy-id degraded write succeeded (no fsync); next boot reads the recorded id from disk but a crash before flush may reveal the prior id`; - await clearDirtyMarker(activeIdFile, "degraded persist"); - return { degraded: true, error: primary }; - } catch (fallback) { - logger.error`active-deploy-id degraded write also failed for ${activeIdFile}: ${fallback instanceof Error ? fallback.message : String(fallback)}; writing dirty marker so the next boot can reconcile`; - try { - const dirtyPath = `${activeIdFile}.dirty`; - await fs.promises.writeFile(dirtyPath, formatActiveDeployId(deployId)); - logger.warn`active-deploy-id dirty marker written to ${dirtyPath}; next boot will prefer it over the stale recorded id`; - return { degraded: true, error: primary }; - } catch (marker) { - logger.error`active-deploy-id dirty marker write also failed for ${activeIdFile}.dirty: ${marker instanceof Error ? marker.message : String(marker)}; on-disk state will diverge from the recorded id for one boot cycle`; - return { degraded: true, error: primary }; - } - } - } -} - -/** - * Persist the rejected manifest and the failure payload to the agent's - * on-disk audit trail. The destination is - * `/audit/rejected-applies//`, two files: - * - * - `manifest.json` — the manifest bytes that were rejected, written - * verbatim — for every failure category, these - * are the original on-disk bytes the sidecar - * read from `deploy/tool-packages-manifest.json` - * (corrupt JSON, wrong-shape JSON, or a - * validator-accepted manifest the loader later - * rejected). Persisting the raw bytes lets a - * future investigator replay the same input - * against a newer validator without losing - * whitespace, key order, or tolerated fields - * the validator narrowed away. - * - `error.json` — `{ attemptId, previousDeployId, category, - * message, package?, occurredAt }` - * - * The files live in the agent's storeDir so a future - * git-commit-of-audit-entries pass can pick them up without rerouting - * the data. v1 writes them directly to disk; the formal git-commit - * step (turning these into a signed audit-trail commit) is a separate - * plumbing piece. - */ -async function writeRejectedApplyAudit(args: { - storeDir: string; - attemptId: string; - manifestBytes: string; - failure: { - attemptId: string; - previousDeployId: string; - category: string; - message: string; - package?: { name: string; version: string }; - occurredAt: string; - }; -}): Promise { - const dir = path.join( - args.storeDir, - "audit", - "rejected-applies", - args.attemptId, - ); - await fs.promises.mkdir(dir, { recursive: true }); - // fsync the files and their parent directory before returning so the - // rejection's evidence is durable before the caller throws and tears - // the apply down. Otherwise, a crash after the throw begins - // propagating but before the audit bytes hit disk would leave a - // rejected apply the sidecar cannot prove the manifest for on replay. - // fsync failures are downgraded to warnings: the bytes are written - // either way, and a refusing fsync (rare networked-FS failure mode) - // should not turn into a second cascading failure on an already- - // failing apply. - await fsyncWriteFile(path.join(dir, "manifest.json"), args.manifestBytes); - await fsyncWriteFile( - path.join(dir, "error.json"), - JSON.stringify(args.failure, null, 2), - ); - try { - const dirHandle = await fs.promises.open(dir, "r"); - try { - await dirHandle.sync(); - } finally { - await dirHandle.close(); - } - } catch (err) { - logger.warn`audit-dir fsync failed for ${dir}; rejected-apply durability is degraded but the files are written — ${err instanceof Error ? err.message : String(err)}`; - } -} - -async function fsyncWriteFile( - filePath: string, - contents: string, -): Promise { - const handle = await fs.promises.open(filePath, "w"); - try { - await handle.writeFile(contents); - try { - await handle.sync(); - } catch (err) { - logger.warn`fsync failed for ${filePath}; durability is degraded but the bytes are written — ${err instanceof Error ? err.message : String(err)}`; - } - } finally { - await handle.close(); - } -} - -function collectFactories( - loaded: readonly LoadedToolPackage[], -): readonly StepToolFactory[] { - const out: StepToolFactory[] = []; - for (const pkg of loaded) { - for (const f of pkg.factories) { - out.push({ - packageName: pkg.name, - declaredCredentials: pkg.credentials, - factory: f, - }); - } - } - return out; -} - -function collectPluginFactories( - loaded: readonly LoadedToolPackage[], -): readonly AnnotatedPluginFactory[] { - const out: AnnotatedPluginFactory[] = []; - for (const pkg of loaded) { - for (const p of pkg.plugins) out.push(p); - } - return out; -} diff --git a/packages/tool-packaging/src/workflow-closure-apply.test.ts b/packages/tool-packaging/src/workflow-closure-apply.test.ts index 4d25d1586..821c706d2 100644 --- a/packages/tool-packaging/src/workflow-closure-apply.test.ts +++ b/packages/tool-packaging/src/workflow-closure-apply.test.ts @@ -7,7 +7,7 @@ import path from "node:path"; import type { ToolPackageManifest } from "@intx/types/tool-packages"; import { createTarballCache, type TarballCache } from "./cache"; -import type { TarballFetcher } from "./loader"; +import type { TarballFetcher } from "./tarball-fetcher"; import { hostPlatform } from "./materialization-config"; import type { RegistryConfig } from "./resolver"; import { applyFrozenWorkflowClosure } from "./workflow-closure-apply"; @@ -55,7 +55,7 @@ afterEach(async () => { /** * Pack the fixture workflow package into an npm-style tarball (with the - * `package/` prefix the loader strips) and return its bytes plus the SRI + * `package/` prefix materialization strips) and return its bytes plus the SRI * integrity string the frozen manifest pins. */ async function packWorkflowFixture(): Promise<{ @@ -160,6 +160,48 @@ describe("applyFrozenWorkflowClosure", () => { ]); }); + test("replaces what an earlier apply left in the instance directory", async () => { + const fixture = await packWorkflowFixture(); + const apply = () => + applyFrozenWorkflowClosure({ + source: { kind: "registry", registry: REGISTRY_NAME }, + closure: { + schemaVersion: "1", + topLevel: [ + { name: WORKFLOW_PACKAGE_NAME, version: WORKFLOW_PACKAGE_VERSION }, + ], + entries: [ + { + name: WORKFLOW_PACKAGE_NAME, + version: WORKFLOW_PACKAGE_VERSION, + source: { + kind: "registry", + registry: REGISTRY_NAME, + integrity: fixture.integrity, + }, + }, + ], + }, + instanceDir, + cache, + registryMaxTarballBytes: 10_000_000, + registries: registries(), + host, + fetchTarball: async () => fixture.bytes, + }); + + const first = await apply(); + const leftover = path.join(first.packageDir, "left-by-earlier-attempt.js"); + await fs.writeFile(leftover, ""); + + const second = await apply(); + expect(second.packageDir).toBe(first.packageDir); + expect(await fs.readdir(second.packageDir)).not.toContain( + "left-by-earlier-attempt.js", + ); + expect(await fs.readdir(second.packageDir)).toContain("package.json"); + }); + test("rejects fetched bytes whose SRI does not match the frozen closure", async () => { const fixture = await packWorkflowFixture(); const manifest: ToolPackageManifest = { @@ -250,7 +292,7 @@ describe("applyFrozenWorkflowClosure", () => { // This is the durable-store read path: the deploy checked the source asset // out at //, and both deploy and restore materialize // from there with no HTTP fetch. Stage the tarball as a plain file where the - // loader resolves a kind:"asset" entry, then apply. + // materialization resolves a kind:"asset" entry, then apply. const fixture = await packWorkflowFixture(); const assetId = "asset_deploy"; const mountPath = "source-assets/asset_deploy/"; diff --git a/packages/tool-packaging/src/workflow-closure-apply.ts b/packages/tool-packaging/src/workflow-closure-apply.ts index 2d1a95eb9..fde5273e6 100644 --- a/packages/tool-packaging/src/workflow-closure-apply.ts +++ b/packages/tool-packaging/src/workflow-closure-apply.ts @@ -8,41 +8,36 @@ // pin against the registry at apply time. // // This is the durable deploy counterpart to the airlocked install-time -// probe: it reuses the same apply machinery (`createTarballCache` / -// `createToolLoader` / `applyAtomic`) that `tool-materialization.ts` uses -// for a step's tool closure, so the fetch + SRI-verify + extract + -// `node_modules` layout is not reimplemented here. +// probe: both lay the closure out with `materializeClosure`, so the fetch + +// SRI-verify + extract + `node_modules` layout is not reimplemented here. // -// A workflow-definition package declares `interchange.workflow` (the module -// whose evaluation produces the definition), NOT `interchange.tools`. -// `applyAtomic`'s load phase imports each TOP-LEVEL package's -// `interchange.tools` entry and rejects a package that has none -// (`package.entry.missing`), so the layout manifest handed to `applyAtomic` -// carries an EMPTY `topLevel`: every entry is still materialized and laid out -// (the dependency layout walks `entries`, and each dependency resolves against -// the frozen closure), but nothing is imported. The apply runs in the -// long-lived sidecar process, which keeps every module it imports, so the -// workflow child is the only process that evaluates the workflow entry. +// The apply imports nothing. It runs in the long-lived sidecar process, which +// keeps every module it imports, so the workflow child is the only process +// that evaluates the workflow entry. // // `host` comes from the caller. This package must not import // `@intx/hub-agent` or `@intx/workflow-host`, and it does not read the // process: the sidecar process edge resolves the platform. +import { promises as fs } from "node:fs"; import path from "node:path"; import { getLogger } from "@intx/log"; import type { ToolPackageManifest } from "@intx/types/tool-packages"; import type { WorkflowDefinitionSource } from "@intx/types/workflow-sources"; -import { applyAtomic } from "./atomic-apply"; import type { TarballCache } from "./cache"; import { - createToolLoader, - type HostPlatform, - type TarballFetcher, -} from "./loader"; + ClosureMaterializationError, + describeError, +} from "./materialization-internal"; import type { RegistryConfig } from "./resolver"; -import { storeEntryDir } from "./store-layout"; +import { + type HostPlatform, + materializeClosure, + storeEntryDir, +} from "./store-layout"; +import { createTarballFetcher, type TarballFetcher } from "./tarball-fetcher"; const logger = getLogger(["sidecar", "workflow-closure-apply"]); @@ -56,7 +51,8 @@ export interface ApplyFrozenWorkflowClosureArgs { readonly closure: ToolPackageManifest; /** * Durable per-deployment directory the closure is staged under - * (`/packages//store/...`). + * (`/packages/store/...`). An apply replaces whatever an earlier + * one left there. */ readonly instanceDir: string; /** @@ -67,9 +63,9 @@ export interface ApplyFrozenWorkflowClosureArgs { readonly cache: TarballCache; /** Byte cap for a single HTTP-registry tarball fetch. */ readonly registryMaxTarballBytes: number; - /** Registry identifier -> URL + credentials the loader resolves entries against. */ + /** Registry identifier -> URL + credentials materialization resolves entries against. */ readonly registries: ReadonlyMap; - /** npm `os`/`cpu` pair the loader selects optional dependencies with. */ + /** npm `os`/`cpu` pair materialization selects optional dependencies with. */ readonly host: HostPlatform; /** * Workspace root `kind: "asset"` closure entries mount against. A @@ -86,8 +82,8 @@ export interface ApplyFrozenWorkflowClosureArgs { */ readonly gitDirs?: ReadonlyMap; /** - * Test seam for tarball fetching, forwarded to `createToolLoader`. - * Production omits it and the loader fetches from the configured registry. + * Test seam for tarball fetching. Production omits it and gets + * `createTarballFetcher`. */ readonly fetchTarball?: TarballFetcher; } @@ -95,8 +91,6 @@ export interface ApplyFrozenWorkflowClosureArgs { export interface AppliedWorkflowClosure { /** Directory of the materialized workflow package within the closure. */ readonly packageDir: string; - /** The staged, never-renamed deploy directory the closure was laid out under. */ - readonly deployDir: string; } /** @@ -129,9 +123,9 @@ export async function applyFrozenWorkflowClosure( // Boundary check on the definition's source. The `registry` arm surfaces a // missing source registry loudly before any I/O (the per-entry registry - // gates fire again inside the loader). An `asset` closure materializes its + // gates fire again inside materialization). An `asset` closure materializes its // entries from the durable stores the caller populated: tarball entries from - // `assetMounts`, source entries from `gitDirs`; the loader fails loud + // `assetMounts`, source entries from `gitDirs`; materialization fails loud // (`asset.mount.missing` / `git.materialization.failed`) if either is absent. // The `never` default makes a future source kind a compile error rather than // a silent fallthrough. @@ -153,50 +147,47 @@ export async function applyFrozenWorkflowClosure( } } - const loader = createToolLoader({ - cache: args.cache, - registries: args.registries, - host: args.host, - maxRegistryTarballBytes: args.registryMaxTarballBytes, - ...(args.fetchTarball !== undefined - ? { fetchTarball: args.fetchTarball } - : {}), - }); - - // Apply EXACTLY the frozen entries. `topLevel` is emptied so `applyAtomic` - // imports no `interchange.tools` module (a workflow-definition package has - // none); the full `entries` set is still materialized and laid out. - const layoutManifest: ToolPackageManifest = { - schemaVersion: args.closure.schemaVersion, - topLevel: [], - entries: args.closure.entries, - }; + // The layout keeps files that already exist, so a tree left by an earlier + // attempt must not carry over into this one. + const packagesDir = path.join(args.instanceDir, "packages"); + await fs.rm(packagesDir, { recursive: true, force: true }); + await fs.mkdir(packagesDir, { recursive: true }); - const result = await applyAtomic({ - manifest: layoutManifest, - loader, - instanceDir: args.instanceDir, - assetRoot: args.assetRoot ?? path.join(args.instanceDir, "workspace"), - assetMounts: args.assetMounts ?? new Map(), - gitDirs: args.gitDirs ?? new Map(), - attemptId: crypto.randomUUID(), - // This apply stands alone per deployment: there is no prior deploy under - // `instanceDir` to retain, so the sentinel disables the retention window. - previousDeployId: "none", - newDeployId: crypto.randomUUID(), - }); - if (result.status === "failed") { + let storeDir: string; + try { + ({ storeDir } = await materializeClosure({ + manifest: args.closure, + instanceScratchDir: packagesDir, + assetRoot: args.assetRoot ?? path.join(args.instanceDir, "workspace"), + assetMounts: args.assetMounts ?? new Map(), + gitDirs: args.gitDirs ?? new Map(), + host: args.host, + cache: args.cache, + registries: args.registries, + fetchTarball: + args.fetchTarball ?? + createTarballFetcher({ + maxRegistryTarballBytes: args.registryMaxTarballBytes, + }), + })); + } catch (cause) { + await fs.rm(packagesDir, { recursive: true, force: true }); + const detail = + cause instanceof ClosureMaterializationError + ? `${cause.category}: ${cause.message}` + : describeError(cause); throw new Error( - `sidecar workflow-closure apply: materializing the frozen closure for ${workflowPin.name}@${workflowPin.version} failed (${result.category}): ${result.message}`, + `sidecar workflow-closure apply: materializing the frozen closure for ${workflowPin.name}@${workflowPin.version} failed (${detail})`, + { cause }, ); } const packageDir = storeEntryDir( - path.join(result.deployDir, "store"), + storeDir, workflowPin.name, workflowPin.version, ); logger.debug`applied frozen workflow closure ${workflowPin.name}@${workflowPin.version}`; - return { packageDir, deployDir: result.deployDir }; + return { packageDir }; } diff --git a/packages/tool-packaging/src/workflow-closure-materialization.test.ts b/packages/tool-packaging/src/workflow-closure-materialization.test.ts index 4d04046bb..031abed78 100644 --- a/packages/tool-packaging/src/workflow-closure-materialization.test.ts +++ b/packages/tool-packaging/src/workflow-closure-materialization.test.ts @@ -8,7 +8,7 @@ import type { WorkflowProbeRequestFrame } from "@intx/types/sidecar"; import type { ToolPackageManifest } from "@intx/types/tool-packages"; import { createTarballCache, type TarballCache } from "./cache"; -import type { TarballFetcher } from "./loader"; +import type { TarballFetcher } from "./tarball-fetcher"; import { hostPlatform } from "./materialization-config"; import type { RegistryConfig } from "./resolver"; import { @@ -69,7 +69,7 @@ afterEach(async () => { /** * Pack a package directory into an npm-style tarball (with the `package/` - * prefix the loader strips) and return its bytes plus the SRI integrity the + * prefix materialization strips) and return its bytes plus the SRI integrity the * frozen manifest pins. */ async function packFixture( diff --git a/packages/tool-packaging/src/workflow-closure-materialization.ts b/packages/tool-packaging/src/workflow-closure-materialization.ts index 045fced8b..2205af5ba 100644 --- a/packages/tool-packaging/src/workflow-closure-materialization.ts +++ b/packages/tool-packaging/src/workflow-closure-materialization.ts @@ -13,19 +13,10 @@ // the workflow-definition loader stay outside this package, and the probe // handler keeps the callback type the executor injects. // -// Phases 1-2 only, no `applyAtomic`: a probe is ephemeral and inert, so -// the durable-deploy lifecycle bookkeeping (`active-deploy-id`, the -// per-deploy-id retention ladder) is the wrong semantics. The closure is -// laid out under a per-probe scratch dir that `cleanup` removes once the -// child has been reaped. `createToolLoader(...).loadManifest(...)` is -// invoked with an EMPTIED `topLevel`: the loader's phase-3 import loop -// only imports packages named in `topLevel`, so an empty `topLevel` -// fetches + extracts + lays out every closure entry (the full `entries` -// set) while importing NONE of them. That runs exactly the eval-free -// `materializeClosure` phases the probe needs. The production registry -// fetcher is reached through `createToolLoader`, so the layout is driven -// through the loader rather than by calling `materializeClosure` with a -// fetcher of its own. +// A probe is ephemeral and inert, so the closure is laid out under a per-probe +// scratch dir that `cleanup` removes once the child has been reaped. +// `materializeClosure` imports no author code: the airlocked child owns the +// single import of the workflow entry. import { promises as fs } from "node:fs"; import path from "node:path"; @@ -40,13 +31,13 @@ import type { import type { ToolPackageManifest } from "@intx/types/tool-packages"; import type { TarballCache } from "./cache"; +import type { RegistryConfig } from "./resolver"; import { - createToolLoader, type HostPlatform, - type TarballFetcher, -} from "./loader"; -import type { RegistryConfig } from "./resolver"; -import { storeEntryDir } from "./store-layout"; + materializeClosure, + storeEntryDir, +} from "./store-layout"; +import { createTarballFetcher, type TarballFetcher } from "./tarball-fetcher"; const logger = getLogger(["sidecar", "workflow-closure-materialization"]); @@ -75,14 +66,14 @@ export interface WorkflowClosureMaterializerConfig { * before any pack is decoded. */ readonly maxAssetPayloadBytes: number; - /** Registry identifier -> URL + credentials the loader resolves entries against. */ + /** Registry identifier -> URL + credentials materialization resolves entries against. */ readonly registries: ReadonlyMap; /** * Root directory under which each probe's ephemeral closure scratch dir * is created (one per probe, removed by the returned `cleanup`). */ readonly scratchRoot: string; - /** npm `os`/`cpu` pair the loader selects optional dependencies with. */ + /** npm `os`/`cpu` pair materialization selects optional dependencies with. */ readonly host: HostPlatform; /** * Materialize the frame's inline-delivered assets into `assetRoot` and @@ -100,8 +91,8 @@ export interface WorkflowClosureMaterializerConfig { readonly gitDirs: ReadonlyMap; }>; /** - * Test seam for tarball fetching, forwarded to `createToolLoader`. - * Production omits it and the loader fetches from the configured registry. + * Test seam for tarball fetching. Production omits it and gets + * `createTarballFetcher`. */ readonly fetchTarball?: TarballFetcher; } @@ -144,7 +135,7 @@ export function createWorkflowClosureMaterializer( // Boundary check on the definition's source. The `registry` arm surfaces a // missing source registry loudly before any I/O (the per-entry registry - // gates fire again inside the loader). An `asset` closure is materialized + // gates fire again inside materialization). An `asset` closure is materialized // from the `assets` the frame delivers -- tarball entries from a plain-file // mount, source entries from an indexed gitDir -- checked below. The // `never` default makes a future source kind a compile error rather than a @@ -174,20 +165,10 @@ export function createWorkflowClosureMaterializer( }; try { - const loader = createToolLoader({ - cache: config.cache, - registries: config.registries, - host: config.host, - maxRegistryTarballBytes: config.registryMaxTarballBytes, - ...(config.fetchTarball !== undefined - ? { fetchTarball: config.fetchTarball } - : {}), - }); - // Materialize any inline-delivered assets under the probe scratch: // tarball entries as plain files under the workspace root, source entries - // as an indexed gitDir the loader checks subtrees out of. Registry-sourced - // closures deliver none; both maps stay empty and the loader fetches over + // as an indexed gitDir materialization checks subtrees out of. Registry-sourced + // closures deliver none; both maps stay empty and materialization fetches over // HTTP. const assetRoot = path.join(scratchDir, "workspace"); const gitDirRoot = path.join(scratchDir, "gitdirs"); @@ -217,25 +198,22 @@ export function createWorkflowClosureMaterializer( } } - // Lay out phases 1-2 only. `topLevel` is emptied so the loader's - // phase-3 loop imports nothing -- no author code is evaluated on the - // host; the airlocked child owns the single import of the workflow - // entry. The full `entries` set is still fetched, SRI-verified, - // extracted, and laid out with its `node_modules` graph. - const layoutManifest: ToolPackageManifest = { - schemaVersion: frame.closure.schemaVersion, - topLevel: [], - entries: frame.closure.entries, - }; - await loader.loadManifest({ - manifest: layoutManifest, + const { storeDir } = await materializeClosure({ + manifest: frame.closure, instanceScratchDir: scratchDir, assetRoot, assetMounts, gitDirs, + host: config.host, + cache: config.cache, + registries: config.registries, + fetchTarball: + config.fetchTarball ?? + createTarballFetcher({ + maxRegistryTarballBytes: config.registryMaxTarballBytes, + }), }); - const storeDir = path.join(scratchDir, "store"); const packageDir = storeEntryDir( storeDir, workflowPin.name, diff --git a/packages/tools-lsp/src/sidecar-bundle.ts b/packages/tools-lsp/src/sidecar-bundle.ts index bf466c0f0..bc2c0f9e9 100644 --- a/packages/tools-lsp/src/sidecar-bundle.ts +++ b/packages/tools-lsp/src/sidecar-bundle.ts @@ -1,6 +1,6 @@ // Sidecar-bundle entry for `@intx/tools-lsp` — a plugin-shaped factory -// the tool-package loader picks up and threads into `env.plugins` -// before tool factories run. +// the workflow child imports and threads into `env.plugins` before tool +// factories run. // // LSP is not a self-contained tool runner: it contributes one // standalone tool (`lsp_diagnostics` and friends) plus middleware @@ -15,12 +15,10 @@ import type { PosixToolEnv } from "@intx/tools-posix/sidecar-bundle"; import { createLSPPlugin, LSP_TOOL_DEFINITION } from "./index"; /** - * Named export the loader picks up. The factory returns a - * `ToolPlugin` (from `@intx/tools-posix`). Posix's sidecar-bundle - * filters `env.plugins` for entries that look like ToolPlugins and - * hands them to `createPosixTools`. The LSP tool surfaces to the - * model under posix's namespace because the plugin contributes it to - * posix's bundle. + * The factory returns a `ToolPlugin` (from `@intx/tools-posix`). Posix's + * sidecar-bundle filters `env.plugins` for entries that look like + * ToolPlugins and hands them to `createPosixTools`. The LSP tool reaches + * the model through posix's bundle because the plugin contributes it there. * * `createLSPPlugin` returns a ToolPlugin whose `dispose` chains * through to `lsp.dispose()` (see packages/tools-lsp/src/index.ts), diff --git a/packages/tools-mail/src/sidecar-bundle.ts b/packages/tools-mail/src/sidecar-bundle.ts index fb93ccf6f..b5ef103ae 100644 --- a/packages/tools-mail/src/sidecar-bundle.ts +++ b/packages/tools-mail/src/sidecar-bundle.ts @@ -1,5 +1,5 @@ // Sidecar-bundle entry for `@intx/tools-mail` — the convention-compliant -// factory the tool-package loader invokes. +// factory a workflow agent lists in its `toolFactories`. // // The bundle consumes the host-assembled runtime capabilities rather than // building its own. The host (the sidecar's step-env builder) owns the @@ -25,9 +25,7 @@ export interface MailToolEnv extends BaseEnv { } /** - * Named export the loader picks up. The id is package-namespaced per - * the convention; the model-facing tool names are synthesized by the - * loader as `@intx/tools-mail/sidecar-bundle:`. + * The id is package-namespaced per the convention. */ export const mail = defineTool({ id: "@intx/tools-mail/sidecar-bundle", diff --git a/packages/tools-posix/src/sidecar-bundle.ts b/packages/tools-posix/src/sidecar-bundle.ts index 87ba931b5..0b0987523 100644 --- a/packages/tools-posix/src/sidecar-bundle.ts +++ b/packages/tools-posix/src/sidecar-bundle.ts @@ -1,5 +1,5 @@ // Sidecar-bundle entry for `@intx/tools-posix` — the convention-compliant -// factory the tool-package loader invokes. +// factory a workflow agent lists in its `toolFactories`. // // The factory reads the working tree it operates on from `env.toolCwd` // and the blob store from the `BaseEnv` `storage` field, plus the @@ -47,8 +47,7 @@ function isToolPlugin(value: unknown): value is ToolPlugin { } /** - * Named export the loader picks up. The id is package-namespaced per - * the convention. + * The id is package-namespaced per the convention. */ export const posix = defineTool({ id: "@intx/tools-posix/sidecar-bundle", diff --git a/packages/types/README.md b/packages/types/README.md index a60fca385..60ccc2ed3 100644 --- a/packages/types/README.md +++ b/packages/types/README.md @@ -73,16 +73,14 @@ pull in the shapes they need: that round-trip through JSON. - `@intx/types/tool-packages` — schemas for the tool-package distribution path: pin shapes (`ToolPackagePin`, - `ToolPackagePinArray`, `ToolPackagePinName`), source variants + `ToolPackagePinName`), source variants (`ToolPackageRegistrySource`, `ToolPackageAssetSource`, and its `package` arms `ToolPackageAssetTarball` and - `ToolPackageAssetSourceTree`, unioned as `ToolPackageSource`), the - `getToolPackageSourceContentIdentity` helper that reads an entry's - content identity (tarball SRI or source tree oid), and the tool-package - manifest (`ToolPackageManifestEntry`, `ToolPackageManifest`). + `ToolPackageAssetSourceTree`, unioned as `ToolPackageSource`), and the + tool-package manifest (`ToolPackageManifestEntry`, `ToolPackageManifest`). - `@intx/types/package-json` — the `PackageJSON` validator for the - subset of `package.json` fields the asset substrate and tool-package - builders read, including the `interchange.tools` extension, plus + subset of `package.json` fields the asset substrate and closure tooling + read, including the `interchange.tools` extension, plus `isContainedEntryPath` for entry-path containment. - `@intx/types/wire-definition-hash` — `computeWireDefinitionHash` and the `canonicalJsonStringify` it builds on, the stable content hash a diff --git a/packages/types/package.json b/packages/types/package.json index 23a20cb5e..d781c9540 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -192,11 +192,7 @@ } }, "dependencies": { - "arktype": "catalog:", - "semver": "catalog:" - }, - "devDependencies": { - "@types/semver": "catalog:" + "arktype": "catalog:" }, "files": [ "dist", diff --git a/packages/types/src/credentials.ts b/packages/types/src/credentials.ts index be517c1bb..4341974d8 100644 --- a/packages/types/src/credentials.ts +++ b/packages/types/src/credentials.ts @@ -32,7 +32,7 @@ const CredentialSourceType = type.enumerated(...credentialRequirementSources); // A credential binding on a workflow definition maps one consumer's declared // credential handle to a concrete credential resolved fresh at launch. The -// consumer is a tool package, keyed `(package, handle)`, or an action handler, +// consumer is a tool factory, keyed `(package, handle)`, or an action handler, // keyed `(action, handle)`; a binding names exactly one of the two. `locator` // is which credential namespace the name is resolved in; today only `tenant` // exists (a tenant-owned credential, authorized by ownership). A second locator @@ -67,7 +67,7 @@ const credentialBindingFields = { // exported type. export const CredentialBinding = type({ package: type("string").describe( - "The tool package the declared handle belongs to: the tool factory's bundle id.", + "The tool factory the declared handle belongs to, given as its bundle id.", ), "action?": "never", ...credentialBindingFields, diff --git a/packages/types/src/grants.ts b/packages/types/src/grants.ts index 64d12fdc1..c79860aa3 100644 --- a/packages/types/src/grants.ts +++ b/packages/types/src/grants.ts @@ -80,7 +80,7 @@ export const EvaluateRequest = type({ resource: "string", action: "string", "consumer?": type("string > 0").describe( - "The workflow consumer to evaluate as, such as `action:` or `tool:`. A grant scoped to a consumer by a `{ tool }` condition matches only that consumer; without it, such a grant matches nothing.", + "The workflow consumer to evaluate as, such as `action:` or `tool:`. A grant scoped to a consumer by a `{ tool }` condition matches only that consumer; without it, such a grant matches nothing.", ), }); diff --git a/packages/types/src/package-json.test.ts b/packages/types/src/package-json.test.ts index edc457b8b..b7f6e8083 100644 --- a/packages/types/src/package-json.test.ts +++ b/packages/types/src/package-json.test.ts @@ -1,77 +1,42 @@ import { describe, test, expect } from "bun:test"; import { type } from "arktype"; -import { PackageJSON, isContainedEntryPath } from "./package-json"; +import { + PackageJSON, + ToolCredentialDeclarationArray, + isContainedEntryPath, +} from "./package-json"; -function accepts(data: unknown): boolean { - return !(PackageJSON(data) instanceof type.errors); -} +describe("ToolCredentialDeclarationArray", () => { + function acceptsDeclarations(data: unknown): boolean { + return !(ToolCredentialDeclarationArray(data) instanceof type.errors); + } -describe("PackageJSON interchange.credentials", () => { test("accepts a well-formed credential declaration", () => { expect( - accepts({ - name: "tools-x", - version: "1.0.0", - interchange: { - tools: "./dist/bundle.js", - credentials: [ - { handle: "gh", scopes: ["repo"] }, - { handle: "stripe" }, - ], - }, - }), - ).toBe(true); - }); - - test("accepts a package with no credential declaration", () => { - expect( - accepts({ - name: "tools-x", - version: "1.0.0", - interchange: { tools: "./dist/bundle.js" }, - }), + acceptsDeclarations([ + { handle: "gh", scopes: ["repo"] }, + { handle: "stripe" }, + ]), ).toBe(true); }); test("rejects a malformed handle", () => { - expect( - accepts({ - name: "tools-x", - version: "1.0.0", - interchange: { credentials: [{ handle: "Bad Handle!" }] }, - }), - ).toBe(false); + expect(acceptsDeclarations([{ handle: "Bad Handle!" }])).toBe(false); }); test("rejects non-array credentials", () => { - expect( - accepts({ - name: "tools-x", - version: "1.0.0", - interchange: { credentials: { handle: "gh" } }, - }), - ).toBe(false); + expect(acceptsDeclarations({ handle: "gh" })).toBe(false); }); test("rejects a non-string scope", () => { - expect( - accepts({ - name: "tools-x", - version: "1.0.0", - interchange: { credentials: [{ handle: "gh", scopes: [123] }] }, - }), - ).toBe(false); + expect(acceptsDeclarations([{ handle: "gh", scopes: [123] }])).toBe(false); }); - test("rejects a duplicate handle within one package", () => { - expect( - accepts({ - name: "tools-x", - version: "1.0.0", - interchange: { credentials: [{ handle: "gh" }, { handle: "gh" }] }, - }), - ).toBe(false); + test("rejects a duplicate handle", () => { + expect(acceptsDeclarations([{ handle: "gh" }, { handle: "gh" }])).toBe( + false, + ); }); }); diff --git a/packages/types/src/package-json.ts b/packages/types/src/package-json.ts index 53618b9cc..3278a1b3e 100644 --- a/packages/types/src/package-json.ts +++ b/packages/types/src/package-json.ts @@ -1,19 +1,17 @@ // Schema for the subset of `package.json` fields the asset substrate -// and tool-package builders read. +// and closure tooling read, plus the credential declaration `defineTool` +// takes, which is not a `package.json` field. // // Promoted here so the package-registry kind handler (in -// `@intx/hub-sessions`) and the workspace builtin-packing script -// (`bin/build-builtins.ts`) share one definition: the asset -// substrate's validation of an uploaded tarball must match the field -// set the build path emits, otherwise a freshly-packed builtin would -// be rejected for shape reasons the build did not anticipate. +// `@intx/hub-sessions`), closure materialization and the workflow child +// share one definition of what a package's manifest must hold. import { type } from "arktype"; import { normalize } from "./posix-path"; /** - * A tool package's static declaration of one provider-backed credential it + * A tool factory's static declaration of one provider-backed credential it * needs: an abstract handle plus optional scopes. Advisory only -- a request * the workflow definition later binds to a concrete credential and the launch-time * grant gate authorizes; a declaration consents to nothing on its own. The @@ -35,10 +33,10 @@ export type ReadonlyToolCredentialDeclaration = { }; /** - * The credential declarations for one package, with the unique-handle + * The credential declarations for one tool factory, with the unique-handle * invariant enforced at parse time: a handle is the binding/delivery key, so a - * duplicate within a single package is a defect the upload boundary must - * reject rather than let collapse silently downstream. + * duplicate is a defect `defineTool` and the inert projector must reject rather + * than let collapse silently downstream. */ export const ToolCredentialDeclarationArray = ToolCredentialDeclaration.array().narrow((decls, ctx) => { @@ -58,11 +56,10 @@ export type ToolCredentialDeclarationArray = /** * Required fields plus the `interchange` extensions used to identify - * interchange packages: `tools` names the sidecar-bundle entry, `credentials` - * statically declares the provider-backed credentials the package's tools may - * need, `workflow` names the module whose evaluation produces a workflow - * package's `WorkflowDefinition`, `directors` names the module whose exports - * are the package's custom `defineDirector` factories, `loops` names the module + * interchange packages: `tools` names the module plugin factories are imported + * from, `workflow` names the module whose evaluation produces a workflow + * package's `WorkflowDefinition`, `directors` names the module whose exports are + * the package's custom `defineDirector` factories, `loops` names the module * whose exports are the package's `loop` `while`/`carry` functions, and * `actions` names the module whose exports are the package's `action` handlers. * `loops` and `actions` refs are resolved by export name at establish. @@ -74,7 +71,6 @@ export const PackageJSON = type({ version: "string", "interchange?": type({ "tools?": "string", - "credentials?": ToolCredentialDeclarationArray, "workflow?": "string", "directors?": "string", "loops?": "string", @@ -88,8 +84,8 @@ export type PackageJSON = typeof PackageJSON.infer; * module path relative to its package -- stays inside the package directory. * An absolute path or a `..` traversal escapes and returns false. * - * This is the string-level half of the loader's containment rule. The - * load-time loader (`resolveContainedEntry`) pairs it with a realpath-based + * This is the string-level half of the containment rule. The workflow + * definition loader (`resolveContainedEntry`) pairs it with a realpath-based * symlink-escape check that only a materialized directory can run; the * push-time asset validator, which has no filesystem, relies on this string * half alone. Both boundaries call this one predicate so they cannot diverge diff --git a/packages/types/src/runtime-capabilities.ts b/packages/types/src/runtime-capabilities.ts index 374f9df7d..c5a36991c 100644 --- a/packages/types/src/runtime-capabilities.ts +++ b/packages/types/src/runtime-capabilities.ts @@ -99,11 +99,11 @@ export function createRuntimeCapabilities( * map and delegates every other key to `base`. * * The host uses this to add a per-bundle capability -- the consumer-scoped - * `credentials` handle, one instance per tool package -- onto a shared + * `credentials` handle, one instance per tool factory -- onto a shared * per-step base bag without re-plumbing the base's keys (`mail.transport` - * and any future shared key stay owned by the step bag). Each tool package's + * and any future shared key stay owned by the step bag). Each factory's * bundle receives the same base layered with ITS OWN credentials capability, - * so a package cannot resolve a handle scoped to a different package. + * so a factory cannot resolve a handle scoped to a different factory. * * `overrides` is snapshotted at construction, mirroring * `createRuntimeCapabilities`, so later mutation of the input is not diff --git a/packages/types/src/runtime-core.ts b/packages/types/src/runtime-core.ts index 9229079d0..9ac281444 100644 --- a/packages/types/src/runtime-core.ts +++ b/packages/types/src/runtime-core.ts @@ -673,10 +673,9 @@ export class MessageTransportError extends Error { } /** - * `instanceof` is not usable for this: a tool package is published as a bundle - * whose workspace imports are inlined (`bin/build-builtins.ts`), so a consumer - * loaded from a bundle holds its own copy of the class above and answers false - * against an error the host's copy constructed. + * `instanceof` is not usable for this: a workflow bundle inlines its workspace + * imports, so a consumer loaded from one holds its own copy of the class above + * and answers false against an error the host's copy constructed. */ export function isMessageTransportError( value: unknown, diff --git a/packages/types/src/runtime.test.ts b/packages/types/src/runtime.test.ts index 25bd2bdce..7eeb47972 100644 --- a/packages/types/src/runtime.test.ts +++ b/packages/types/src/runtime.test.ts @@ -1284,11 +1284,11 @@ describe("BoundedApprovalSnapshot", () => { describe("isMessageTransportError", () => { test("accepts a condition-bearing error the class did not construct", () => { - // The reason the guard is structural: a tool package is published as a - // bundle with its workspace imports inlined, so the consumer classifying a - // failure holds its own copy of MessageTransportError and an `instanceof` - // check against the host's copy answers false. This foreign class stands in - // for that second copy. + // The reason the guard is structural: a workflow bundle inlines its + // workspace imports, so the consumer classifying a failure holds its own + // copy of MessageTransportError and an `instanceof` check against the + // host's copy answers false. This foreign class stands in for that second + // copy. class ForeignTransportError extends Error { readonly condition = "NONEXISTENT"; } diff --git a/packages/types/src/sidecar.test.ts b/packages/types/src/sidecar.test.ts index 1b91b4299..246532e06 100644 --- a/packages/types/src/sidecar.test.ts +++ b/packages/types/src/sidecar.test.ts @@ -6,7 +6,6 @@ import { AgentDeployFrame, fitDeploymentError, CredentialsUpdateFrame, - DeployApplyErrorCategory, FrozenApprovalBundle, HubFrame, MAX_DEPLOYMENT_ADDRESS_LENGTH, @@ -82,36 +81,6 @@ describe("AgentDeployAckFrame public key", () => { }); }); -describe("DeployApplyErrorCategory", () => { - const allCategories = [ - "tarball.missing", - "integrity.mismatch", - "registry.fetch.failed", - "registry.unknown", - "registry.auth.failed", - "tarball.extract.failed", - "manifest.invalid", - "package.entry.missing", - "package.entry.invalid", - "factory.construct.failed", - "tool.name.duplicate", - "apply.swap.failed", - "apply.previous-rotation.failed", - ] as const; - - for (const category of allCategories) { - test(`accepts ${category}`, () => { - const result = DeployApplyErrorCategory(category); - expect(result instanceof type.errors).toBe(false); - }); - } - - test("rejects an unknown category", () => { - const result = DeployApplyErrorCategory("network.timeout"); - expect(result instanceof type.errors).toBe(true); - }); -}); - describe("PackRejectFrame reason forward-compat", () => { const base = { type: "repo.pack.reject" as const, @@ -190,7 +159,7 @@ describe("AgentDeployFrame", () => { // The source-ref pin every workflow frame carries: where the definition's // bytes come from plus the frozen dependency closure (empty here -- a - // workflow that pins no tool packages). + // workflow with no dependencies). const validSourceRef = { source: { kind: "registry", registry: "npmjs" }, closure: { schemaVersion: "1", topLevel: [], entries: [] }, diff --git a/packages/types/src/sidecar.ts b/packages/types/src/sidecar.ts index faa05398e..53019f556 100644 --- a/packages/types/src/sidecar.ts +++ b/packages/types/src/sidecar.ts @@ -729,7 +729,7 @@ export type FrozenApprovalBundle = typeof FrozenApprovalBundle.infer; * materialize a closure entry whose bytes live in that asset. `pack` is the * base64-encoded git packfile the hub produced for the asset (`createPack` * output); the sidecar checks out `commitSha` from it as plain files under - * `mountPath`, then the loader resolves each `kind:"asset"` closure entry + * `mountPath`, then materialization resolves each `kind:"asset"` closure entry * against that mount. `assetId` matches the `source.assetId` the closure * entries name. */ @@ -1069,114 +1069,6 @@ export const PackRejectFrame = type({ }); export type PackRejectFrame = typeof PackRejectFrame.infer; -/** - * Categories of deploy-apply failure surfaced by the sidecar's - * tool-package loader. Each value maps one-to-one to a distinct point in - * the apply pipeline; a single category fires per failed attempt. - * - * tarball.missing — a manifest entry's asset-sourced tarball - * is not present at the recorded path. - * asset.mount.missing — a `kind: "asset"` manifest entry names - * an `assetId` that the asset mount map - * does not cover. Indicates a mismatch between the - * resolver's view of attached assets and - * the materialization fan-out, not a - * missing file on disk. - * integrity.mismatch — fetched tarball bytes do not match the - * manifest's pinned SRI integrity. - * registry.fetch.failed — the configured registry refused or - * dropped the request for a tarball. - * registry.unknown — the manifest entry references a registry - * name not present in the sidecar's - * registry config. - * registry.auth.failed — the registry rejected the sidecar's - * credentials. - * tarball.extract.failed — tar extraction failed or the extracted - * tree was malformed. - * git.materialization.failed - * — a git-sourced entry could not be - * materialized from its checked-out - * subtree, or reached a loader that does - * not materialize git sources. - * manifest.invalid — the manifest itself did not validate - * at the loader boundary (JSON.parse - * failure or arktype schema failure). - * Peer-dependency violations are caught - * earlier by the hub's resolver and - * surface as a launch failure rather - * than this frame. - * package.entry.missing — a top-level package's package.json had - * no `interchange.tools` field. - * package.entry.invalid — the resolved `interchange.tools` module - * exported nothing that looked like an - * AnnotatedToolFactory. - * factory.construct.failed — a factory invocation threw, or required - * a capability key the env did not provide. - * tool.name.duplicate — a tool name is registered more than - * once in the apply's loaded set. The - * cross-bundle case (two pinned packages - * share a bundle id, producing colliding - * prefixed tool names) is rejected at - * apply time, before the caller commits. - * The intra-bundle case (one package - * exports two definitions sharing a raw - * name) surfaces at first agent - * construction with the same category - * instead of apply rejection: the loader - * cannot see `bundle.definitions` without - * invoking the factory, and the `BaseEnv` - * the factory needs is constructed by the - * workflow child's step build env AFTER - * the commit. Both paths carry the same - * category so the operator-facing failure - * shape is uniform regardless of which - * check fired; only the channel - * (apply.error frame vs runtime construct - * failure) differs. - * apply.swap.failed — DEPRECATED, no longer emitted. The apply - * protocol stages each deploy into a stable - * per-deploy-id directory and commits via a - * single `active-deploy-id` file write, so - * there is no filesystem rename that can - * fail. The value is retained in the enum - * for wire compatibility: during a rolling - * upgrade an older sidecar can still emit - * it, and dropping the member would make a - * newer hub's frame validator reject that - * frame. - * apply.previous-rotation.failed - * — every loaded factory validated and the - * new deploy was staged, but persisting the - * instance's `active-deploy-id` file (the - * commit) degraded: the id was written - * through the no-fsync / dirty-marker - * fallback ladder rather than durably - * flushed. The new deploy is logically - * live, so `previousDeployId` on this - * failure carries the NEW deploy id rather - * than the pre-apply one. The next boot - * reconciles the recorded id from the dirty - * marker. - */ -export const DeployApplyErrorCategory = type.enumerated( - "tarball.missing", - "asset.mount.missing", - "integrity.mismatch", - "registry.fetch.failed", - "registry.unknown", - "registry.auth.failed", - "tarball.extract.failed", - "git.materialization.failed", - "manifest.invalid", - "package.entry.missing", - "package.entry.invalid", - "factory.construct.failed", - "tool.name.duplicate", - "apply.swap.failed", - "apply.previous-rotation.failed", -); -export type DeployApplyErrorCategory = typeof DeployApplyErrorCategory.infer; - // --------------------------------------------------------------------------- // Workflow probe (bidirectional) // --------------------------------------------------------------------------- diff --git a/packages/types/src/tool-packages.test.ts b/packages/types/src/tool-packages.test.ts index 95292f5b9..562e7cf8e 100644 --- a/packages/types/src/tool-packages.test.ts +++ b/packages/types/src/tool-packages.test.ts @@ -2,11 +2,9 @@ import { describe, test, expect } from "bun:test"; import { type } from "arktype"; import { ToolPackagePin, - ToolPackagePinArray, ToolPackageSource, ToolPackageManifestEntry, ToolPackageManifest, - getToolPackageSourceContentIdentity, } from "./tool-packages"; describe("ToolPackagePin", () => { @@ -59,46 +57,6 @@ describe("ToolPackagePin", () => { }); }); -describe("ToolPackagePinArray", () => { - test("accepts an array with distinct names", () => { - const result = ToolPackagePinArray([ - { name: "@intx/tools-posix", version: "1.2.3" }, - { name: "@intx/tools-mail", version: "1.2.3" }, - ]); - expect(result instanceof type.errors).toBe(false); - }); - - test("accepts an empty array", () => { - const result = ToolPackagePinArray([]); - expect(result instanceof type.errors).toBe(false); - }); - - test("rejects an array with duplicate names", () => { - const result = ToolPackagePinArray([ - { name: "@intx/tools-posix", version: "1.2.3" }, - { name: "@intx/tools-posix", version: "1.3.0" }, - ]); - expect(result instanceof type.errors).toBe(true); - }); - - test("rejects an array containing an invalid pin name", () => { - const result = ToolPackagePinArray([{ name: "BadName", version: "1.2.3" }]); - expect(result instanceof type.errors).toBe(true); - }); - - test("rejects an array containing an unparseable semver range", () => { - const result = ToolPackagePinArray([ - { name: "left-pad", version: "not-a-range" }, - ]); - expect(result instanceof type.errors).toBe(true); - }); - - test("accepts a star range", () => { - const result = ToolPackagePinArray([{ name: "left-pad", version: "*" }]); - expect(result instanceof type.errors).toBe(false); - }); -}); - describe("ToolPackageSource", () => { test("accepts an asset tarball source", () => { const result = ToolPackageSource({ @@ -262,54 +220,13 @@ describe("ToolPackageSource", () => { }); }); -describe("getToolPackageSourceContentIdentity", () => { - test("returns the SRI for an asset tarball", () => { - expect( - getToolPackageSourceContentIdentity({ - kind: "asset", - assetId: "asset_abc", - package: { - format: "tarball", - path: "tarballs/x.tgz", - integrity: "sha512-AAAA", - }, - }), - ).toBe("sha512-AAAA"); - }); - - test("returns the SRI for a registry source", () => { - expect( - getToolPackageSourceContentIdentity({ - kind: "registry", - registry: "npmjs", - integrity: "sha512-BBBB", - }), - ).toBe("sha512-BBBB"); - }); - - test("returns the tree oid for an asset source-package", () => { - expect( - getToolPackageSourceContentIdentity({ - kind: "asset", - assetId: "asset_wf", - package: { - format: "source", - commitSha: "abc123", - packageDir: ".", - treeOid: "def456", - }, - }), - ).toBe("def456"); - }); -}); - describe("ToolPackageManifestEntry", () => { const validEntry = { name: "@intx/tools-posix", version: "1.2.3", source: { kind: "asset", - assetId: "asset_workspace_builtins", + assetId: "asset_tool_registry", package: { format: "tarball", path: "tarballs/intx-tools-posix-1.2.3.tgz", @@ -348,7 +265,7 @@ describe("ToolPackageManifestEntry", () => { version: validEntry.version, source: { kind: "asset", - assetId: "asset_workspace_builtins", + assetId: "asset_tool_registry", package: { format: "tarball", path: "tarballs/intx-tools-posix-1.2.3.tgz", @@ -377,7 +294,7 @@ describe("ToolPackageManifest", () => { version: "1.2.3", source: { kind: "asset", - assetId: "asset_workspace_builtins", + assetId: "asset_tool_registry", package: { format: "tarball", path: "tarballs/intx-tools-posix-1.2.3.tgz", @@ -412,7 +329,7 @@ describe("ToolPackageManifest", () => { version: "1.0.0", source: { kind: "asset", - assetId: "asset_workspace_builtins", + assetId: "asset_tool_registry", package: { format: "tarball", path: "tarballs/tools-with-deps-1.0.0.tgz", diff --git a/packages/types/src/tool-packages.ts b/packages/types/src/tool-packages.ts index 78cd0a792..5ef1f8d21 100644 --- a/packages/types/src/tool-packages.ts +++ b/packages/types/src/tool-packages.ts @@ -1,32 +1,24 @@ // Schemas for the tool-package distribution path. // -// An agent pins one or more tool packages via `ToolPackagePin[]`. At -// deploy-assembly time, the hub walks the pinned set, resolves the full -// dependency closure, and writes a `ToolPackageManifest` into the deploy -// pack. The sidecar reads the manifest at apply time and materializes -// every entry. +// At deploy-assembly time, the hub resolves a workflow's source to its +// full dependency closure and freezes it as a `ToolPackageManifest`. The +// sidecar materializes every entry of that manifest at apply time. // -// Only entries listed in `topLevel` contribute tools to the agent; -// transitive entries exist to satisfy `require()` / `import` resolution -// inside the top-level packages. +// `topLevel` names the packages the closure was resolved from; transitive +// entries exist to satisfy `require()` / `import` resolution inside them. import { type } from "arktype"; -import semver from "semver"; -import { - ToolCredentialDeclarationArray, - isContainedEntryPath, -} from "./package-json"; +import { isContainedEntryPath } from "./package-json"; /** * npm's documented package-name rules expressed as an arktype regex * literal: lowercase, may begin with a scope (`@scope/`), the rest of * each segment is URL-safe (letters, digits, `_`, `-`, `.`), no * leading dot or underscore, scoped names require a `/`. The npm - * registry rejects anything else; mirroring the rule at the REST - * boundary keeps mixed-case or malformed pins from threading past - * the API into the resolver, which would otherwise self-resolve - * them and then fail at the sidecar loader. + * registry rejects anything else; mirroring the rule here keeps a + * mixed-case or malformed name out of a manifest, where it would + * self-resolve and then fail at the sidecar. * * Using a regex literal (rather than a `narrow` predicate) lets the * JSON-Schema generator surface the rule as a `pattern` field in the @@ -37,27 +29,22 @@ export const ToolPackagePinName = type( ); /** - * A pin in an agent definition: name + version range. The hub resolves - * this against configured registries at deploy-assembly time. + * A package to resolve a closure from: name + version range. The hub + * resolves this against configured registries at deploy-assembly time. * * `version` is an npm-style spec ("^1.2.3", "~1.2", "1.2.3", "*"). * Resolution is performed by `npm-pick-manifest` against the registry - * packument. Semver-range validation lives on `ToolPackagePinArray` - * (below) so the JSON-Schema generator sees a plain string here; the - * array narrow is the actual REST boundary for pins and runs before - * any value reaches the resolver. + * packument. * * `name` must match npm's documented package-name rules — lowercase, * optional scope prefix, URL-safe characters only. npm itself rejects * uppercase names; packuments arrive lowercased, so a mixed-case pin - * would self-resolve and then silently fail the sidecar loader's + * would self-resolve and then silently fail the sidecar's * `${name}@${version}` lookup against the lowercase entry the * packument produced. * - * A `ToolPackagePin[]` must contain at most one entry per `name`. Use - * `ToolPackagePinArray` (below) at REST boundaries to enforce dedup - * before the resolver runs; the resolver still rejects duplicates at - * its own boundary as belt-and-suspenders. + * A `ToolPackagePin[]` must contain at most one entry per `name`; the + * resolver rejects duplicates. */ export const ToolPackagePin = type({ name: ToolPackagePinName, @@ -66,62 +53,17 @@ export const ToolPackagePin = type({ export type ToolPackagePin = typeof ToolPackagePin.infer; /** - * Array of pins with the no-duplicate-name and parseable-version - * invariants enforced at parse time. The downstream resolver keys - * its top-level resolution map by name; two pins of the same name - * would silently collapse to the first arrival's resolved version, - * and an unparseable semver range would fail mid-walk. Rejecting - * both at the REST boundary surfaces the bug to the caller instead - * of leaving it to misbehave at launch time. - * - * `*` is accepted as the documented any-version range; anything - * else must satisfy `semver.validRange`. - * - * NOTE: the same `*` special-case lives in `parsePin` inside the - * tool-packaging resolver. Any new magic-range additions need to be - * carved at both sites — the packages are separated by the wire-type - * vs. resolver boundary and cannot import each other. - */ -export const ToolPackagePinArray = ToolPackagePin.array().narrow( - (pins, ctx) => { - const seen = new Set(); - for (const pin of pins) { - if (seen.has(pin.name)) { - return ctx.mustBe( - `an array with no duplicate package names; "${pin.name}" appears more than once`, - ); - } - seen.add(pin.name); - if (pin.version !== "*" && semver.validRange(pin.version) === null) { - return ctx.mustBe( - `every pin to carry a parseable semver range; "${pin.name}" has version ${JSON.stringify(pin.version)}`, - ); - } - } - return true; - }, -); -export type ToolPackagePinArray = typeof ToolPackagePinArray.infer; - -/** - * A top-level manifest entry: a pinned package at its concrete resolved - * version, carrying the credential declarations harvested from the package's - * `interchange.credentials` (absent when it declares none). Only top-level - * pins contribute declarations; transitive dependencies never do, which is why - * this shape hangs off `topLevel` rather than `entries`. + * A top-level manifest entry: a root package at its concrete resolved version. */ export const ToolPackageTopLevelEntry = type({ name: ToolPackagePinName, version: "string", - "credentials?": ToolCredentialDeclarationArray, }); export type ToolPackageTopLevelEntry = typeof ToolPackageTopLevelEntry.infer; /** * The manifest's top-level entries with the no-duplicate-name invariant - * preserved -- the same guarantee `ToolPackagePinArray` gives agent-side pins. - * Versions here are concrete (already picked by the resolver), so the - * semver-range check that guards agent-side pins is unnecessary. + * preserved. Versions here are concrete, already picked by the resolver. */ export const ToolPackageTopLevelArray = ToolPackageTopLevelEntry.array().narrow( (entries, ctx) => { @@ -145,7 +87,7 @@ export type ToolPackageTopLevelArray = typeof ToolPackageTopLevelArray.infer; * credentials. * * `integrity` is the SRI string ("sha512-...") the registry served for - * the picked version. The loader verifies the fetched bytes against it + * the picked version. The sidecar verifies the fetched bytes against it * before unpacking and uses it as the content-addressed cache key. */ export const ToolPackageRegistrySource = type({ @@ -158,11 +100,11 @@ export type ToolPackageRegistrySource = typeof ToolPackageRegistrySource.infer; /** * The entry's bytes are a prepackaged npm tarball living at `path` inside * the asset's checkout (the package-registry kind stores them under - * `tarballs/.tgz`). The loader reads the blob and extracts it. + * `tarballs/.tgz`). The sidecar reads the blob and extracts it. * * `integrity` is the SRI string ("sha512-...") of the tarball bytes. A * reclassified npm tarball keeps its SRI: it is the same artifact an - * external registry would serve, so the loader verifies the read bytes + * external registry would serve, so the sidecar verifies the read bytes * against it and uses it as the content-addressed cache key, and a * byte-identical tarball has one identity regardless of transport. */ @@ -176,7 +118,7 @@ export type ToolPackageAssetTarball = typeof ToolPackageAssetTarball.infer; /** * The entry's bytes are a source package: the subtree at `packageDir` * of the asset's checkout at `commitSha`, used in place (not packed). - * The loader checks the tree out and copies the subtree into the store. + * The sidecar checks the tree out and copies the subtree into the store. * * `packageDir` is the resolved POSIX subtree path of this package within * the repo ("." for a single-package repo root, "packages/foo" for a @@ -186,7 +128,7 @@ export type ToolPackageAssetTarball = typeof ToolPackageAssetTarball.infer; * absolute paths and `..` traversal at the boundary. * * `treeOid` is the git tree object id of the subtree at `commitSha` -- - * the content identity the loader verifies the checked-out subtree + * the content identity the sidecar verifies the checked-out subtree * against. Unlike a tarball's `integrity`, it is a git tree oid, not an * SRI, because a source subtree has no tarball bytes to hash. */ @@ -206,7 +148,7 @@ export type ToolPackageAssetSourceTree = /** * A pinned entry's bytes come from a hub `asset` -- a checked-out git * repo attached to the agent at session time. `assetId` is the hub-side - * asset row id; the loader resolves it against its mount + * asset row id; the sidecar resolves it against its mount * map to reach the asset's checkout. The package lives at a location * within the checkout, either a prepackaged `tarball` or a `source` * subtree, discriminated by `package.format`. @@ -228,23 +170,6 @@ export const ToolPackageSource = ToolPackageRegistrySource.or( ); export type ToolPackageSource = typeof ToolPackageSource.infer; -/** - * A closure entry's content identity, whatever its source: the tarball - * SRI for a `registry` entry or an `asset` tarball, the subtree git tree - * oid for an `asset` source package. Cache-bust keys read this rather - * than reaching into a shape-specific field. - */ -export function getToolPackageSourceContentIdentity( - source: ToolPackageSource, -): string { - if (source.kind === "registry") { - return source.integrity; - } - return source.package.format === "tarball" - ? source.package.integrity - : source.package.treeOid; -} - /** * A single pinned package in the closure. * @@ -276,22 +201,17 @@ export type ToolPackageManifestEntry = typeof ToolPackageManifestEntry.infer; * The manifest of a resolved tool-package closure. * * `schemaVersion` is a literal "1" for now. Future schema changes bump - * this and the loader refuses unknown versions with `manifest.invalid`. + * this, and this validator refuses unknown versions. * - * `topLevel` enumerates the packages the agent definition explicitly - * pinned. The loader only scans these for `interchange.tools`; entries - * present in `entries` but absent from `topLevel` are transitive - * dependencies materialized for runtime `require()` / `import` - * resolution. + * `topLevel` enumerates the packages the closure was resolved from; + * entries present in `entries` but absent from `topLevel` are transitive + * dependencies materialized for runtime `require()` / `import` resolution. * - * `topLevel` extends the agent-side `ToolPackagePin` shape with the - * package's harvested `credentials` declarations. The `version` field - * here is always a concrete version (e.g. `"1.2.3"`), not a range. The resolver walks each - * agent-side pin's range through `npm-pick-manifest` and writes the - * picked version. The sidecar loader pairs `topLevel[i]` against - * `entries[j]` by `${name}@${version}` equality, so a range-form - * `version` here would never match any entry and the package would - * silently contribute no tool factories at apply time. + * The `version` field of a `topLevel` entry is always a concrete version + * (e.g. `"1.2.3"`), not a range. The resolver walks each pin's range through `npm-pick-manifest` and writes the + * picked version. The sidecar finds a top-level package in the store by + * this `${name}@${version}`, so a range-form `version` here would name a + * directory that does not exist. * * `entries` carries the full pinned closure: every top-level pin plus * every transitive dependency, deduped by `(name, version)`. The diff --git a/packages/workflow-deploy/README.md b/packages/workflow-deploy/README.md index 928e178a5..3f6d1a7d3 100644 --- a/packages/workflow-deploy/README.md +++ b/packages/workflow-deploy/README.md @@ -114,7 +114,7 @@ Public surface: reserved hub handle `interchange.hub`, and a duplicate binding. The hub's probe gate refuses a definition with any of these. - `collectHubConsumers(projection)` — the consumers that declare the reserved - hub handle anywhere in a frozen inert projection, as `tool:` and + hub handle anywhere in a frozen inert projection, as `tool:` and `action:`. Each gets a run token for the hub REST API when the run principal is created. - `inertNestedBodies(step, descent)` — the nested body projections one step of @@ -132,8 +132,6 @@ Public surface: referenced asset is deployed and walked on its own. - `deriveRunAddress` / `deriveWorkflowRunRepoId` — the pure address and id derivation helpers. -- `extractFoldedBody(definition)` — read the launch-relevant fields back - out of a folded single-step definition. The capability walk emits the v1 grant-shape vocabulary: `tool:`, `director:`, `capability:`, `inference.source:`, `mail.address:`, diff --git a/packages/workflow-deploy/package.json b/packages/workflow-deploy/package.json index 5b378f20c..451e75d08 100644 --- a/packages/workflow-deploy/package.json +++ b/packages/workflow-deploy/package.json @@ -10,11 +10,6 @@ "types": "./dist/index.d.ts", "default": "./dist/index.js" }, - "./testing": { - "intx-src": "./src/testing/index.ts", - "types": "./dist/testing/index.d.ts", - "default": "./dist/testing/index.js" - }, "./capabilities": { "intx-src": "./src/capability-walk.ts", "types": "./dist/capability-walk.d.ts", diff --git a/packages/workflow-deploy/src/capability-walk.ts b/packages/workflow-deploy/src/capability-walk.ts index d459179ce..0f8ef42d6 100644 --- a/packages/workflow-deploy/src/capability-walk.ts +++ b/packages/workflow-deploy/src/capability-walk.ts @@ -54,10 +54,10 @@ // // Director resolution: the walk calls `effectiveDirectorRef` so the // absent-director normalization stays identical between this walk and -// `getRequiredEnvKeys`. Director factories loaded from pinned tool -// packages via `interchange.directors` are folded into the registry by -// the caller; this module does not synthesize that registry itself -// because the loader is the layer that owns package materialization. +// `getRequiredEnvKeys`. Director factories from a workflow package's +// `interchange.directors` are folded into the registry by the caller; this +// module does not synthesize that registry itself because the definition +// loader owns importing them. // An unresolvable director surfaces on `unresolvedDirectors` rather // than raising -- the deploy flow translates that into a deploy-time // `"unresolvable director"` failure when it wires this output into @@ -291,7 +291,7 @@ export function credentialUseGrantRules( * credentials. This is the credential-cap input. It is not part of * `CapabilityWalkResult`: those strings are not operator-facing grants. * Plugin names are absent. A factory with an empty `definitions` array is - * still present, because the source arm keys Gate 2 on the factory id. + * still present, because Gate 2 is keyed on the factory id. */ export function collectCredentialConsumers( workflow: WorkflowDefinition, @@ -427,7 +427,7 @@ function collectOwnGrants( collected.grants.add(grant); } // Unlike a tool factory, which counts even with empty `definitions` - // because the source arm keys Gate 2 on its id, an action counts only when + // because Gate 2 is keyed on its id, an action counts only when // it carries a `credentials` field: that list is its declared credential // surface, so an action without the field consumes nothing. if (primitive.kind === "action" && primitive.credentials !== undefined) { diff --git a/packages/workflow-deploy/src/credential-binding-check.ts b/packages/workflow-deploy/src/credential-binding-check.ts index e3fa27204..4e3327735 100644 --- a/packages/workflow-deploy/src/credential-binding-check.ts +++ b/packages/workflow-deploy/src/credential-binding-check.ts @@ -216,7 +216,7 @@ export function checkCredentialBindings( /** * The consumers that declare the reserved hub handle anywhere in the workflow * -- the root and every nested body, which all run as the one run principal - * -- as `tool:` and `action:`, sorted and without repeats. Each + * -- as `tool:` and `action:`, sorted and without repeats. Each * gets a run token for the hub REST API when the run principal is created. */ export function collectHubConsumers( diff --git a/packages/workflow-deploy/src/fold-synthesis.test.ts b/packages/workflow-deploy/src/fold-synthesis.test.ts deleted file mode 100644 index eae70670d..000000000 --- a/packages/workflow-deploy/src/fold-synthesis.test.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { describe, test, expect } from "bun:test"; - -import type { AgentDefinition, BaseEnv } from "@intx/agent"; -import type { WorkflowDefinition } from "@intx/workflow/definition"; - -import { extractFoldedBody } from "./fold-synthesis"; -import { synthesizeFoldedWorkflow, type FoldedWorkflowInput } from "./testing"; - -const BASE: FoldedWorkflowInput = { - workflowId: "wf_agt_1", - mailAddress: "run_agt_1@tenant.interchange", - systemPrompt: "You are the folded agent.", - description: "A folded agent.", - inferencePreferences: [{ provider: "anthropic", model: "opus" }], - toolPackagePins: [{ name: "@intx/tools-mail", version: "0.1.2" }], -}; - -function stepAgent(wf: WorkflowDefinition): AgentDefinition { - const stepId = wf.stepOrder[0]; - if (stepId === undefined) throw new Error("expected one step"); - const primitive = wf.steps[stepId]; - if (primitive?.kind !== "step") throw new Error("expected a step primitive"); - return primitive.agent; -} - -describe("synthesizeFoldedWorkflow", () => { - test("produces a single-step workflow carrying the mail trigger", () => { - const wf = synthesizeFoldedWorkflow(BASE); - expect(wf.id).toBe("wf_agt_1"); - expect(wf.stepOrder).toHaveLength(1); - expect(wf.triggers).toEqual([ - { type: "mail", to: "run_agt_1@tenant.interchange" }, - ]); - }); - - test("maps the agent body onto the step agent", () => { - const agent = stepAgent(synthesizeFoldedWorkflow(BASE)); - expect(agent.systemPrompt).toBe("You are the folded agent."); - expect(agent.description).toBe("A folded agent."); - expect(agent.capabilities).toEqual([]); - expect(agent.toolPackagePins).toEqual([ - { name: "@intx/tools-mail", version: "0.1.2" }, - ]); - expect(agent.inference.sources).toEqual([ - { provider: "anthropic", model: "opus" }, - ]); - }); - - test("emits toolFactories as an explicit empty array", () => { - const agent = stepAgent(synthesizeFoldedWorkflow(BASE)); - expect(agent.toolFactories).toEqual([]); - // Must be present, not absent: a missing key hydrates to undefined and the - // capability walk's factory loop throws on a folded definition. - expect("toolFactories" in agent).toBe(true); - }); - - test("omits description when the agent has none", () => { - const agent = stepAgent( - synthesizeFoldedWorkflow({ ...BASE, description: null }), - ); - expect("description" in agent).toBe(false); - }); - - test("carries grantRequirements onto the workflow envelope", () => { - const grantRequirements = [ - { resource: "secret:vault", action: "use", source: "creator" as const }, - ]; - const wf = synthesizeFoldedWorkflow({ ...BASE, grantRequirements }); - expect(wf.grantRequirements).toEqual(grantRequirements); - }); - - test("raises when the agent has no system prompt", () => { - expect(() => - synthesizeFoldedWorkflow({ ...BASE, systemPrompt: null }), - ).toThrow(/no system prompt/); - }); -}); - -describe("extractFoldedBody", () => { - test("round-trips the synthesized body", () => { - const grantRequirements = [ - { resource: "secret:vault", action: "use", source: "creator" as const }, - ]; - const body = extractFoldedBody( - synthesizeFoldedWorkflow({ ...BASE, grantRequirements }), - ); - expect(body.systemPrompt).toBe("You are the folded agent."); - expect(body.toolPackagePins).toEqual(BASE.toolPackagePins); - expect(body.grantRequirements).toEqual(grantRequirements); - }); - - test("defaults grantRequirements to empty when the envelope has none", () => { - const body = extractFoldedBody(synthesizeFoldedWorkflow(BASE)); - expect(body.grantRequirements).toEqual([]); - }); - - test("surfaces the step agent's declared model", () => { - const body = extractFoldedBody(synthesizeFoldedWorkflow(BASE)); - expect(body.model).toBe("opus"); - }); - - test("surfaces a null model when the step declares no inference preference", () => { - const body = extractFoldedBody( - synthesizeFoldedWorkflow({ ...BASE, inferencePreferences: [] }), - ); - expect(body.model).toBeNull(); - }); - - test("raises when the definition is not single-step", () => { - const wf = synthesizeFoldedWorkflow(BASE); - const twoStep: WorkflowDefinition = { - ...wf, - stepOrder: [...wf.stepOrder, "extra"], - }; - expect(() => extractFoldedBody(twoStep)).toThrow(/single-step/); - }); -}); diff --git a/packages/workflow-deploy/src/fold-synthesis.ts b/packages/workflow-deploy/src/fold-synthesis.ts deleted file mode 100644 index 6bd3beb9f..000000000 --- a/packages/workflow-deploy/src/fold-synthesis.ts +++ /dev/null @@ -1,60 +0,0 @@ -// Agent-to-workflow fold: reading a folded definition's launch body. -// -// `extractFoldedBody` reads the launch-relevant fields back out of a folded -// single-step workflow definition. The inverse builder, `synthesizeFoldedWorkflow`, -// is test-only and lives in `@intx/workflow-deploy/testing`. - -import type { CredentialBinding, GrantRequirement } from "@intx/types"; -import type { ToolPackagePin } from "@intx/types/tool-packages"; -import type { WorkflowDefinition } from "@intx/workflow/definition"; - -/** The launch-relevant body a single-step definition carries. */ -export interface FoldedBody { - readonly systemPrompt: string; - readonly toolPackagePins: readonly ToolPackagePin[]; - readonly grantRequirements: readonly GrantRequirement[]; - /** - * The credential bindings the launch resolves into consumer-scoped - * `credential:{id}` / `use` grants. Empty when the definition declares - * none. - */ - readonly credentialBindings: readonly CredentialBinding[]; - /** - * The step agent's declared model, or null when it declares none. A - * definition with no `modelRequirements` manifest resolves its inference - * sources against the catalog from this model instead. - */ - readonly model: string | null; -} - -/** - * Read the launch body back out of a folded workflow definition -- the inverse - * of `synthesizeFoldedWorkflow`. A folded definition is a single `step`-kind - * primitive carrying the agent; the system prompt and tool-package pins live on - * that agent, the grant requirements on the envelope. Raises if the definition - * is not that single-step shape, so a malformed folded definition surfaces here - * rather than launching a broken instance. - */ -export function extractFoldedBody(definition: WorkflowDefinition): FoldedBody { - const [stepId, ...rest] = definition.stepOrder; - if (stepId === undefined || rest.length > 0) { - throw new Error( - `folded definition ${definition.id} is not single-step (${String( - definition.stepOrder.length, - )} steps)`, - ); - } - const primitive = definition.steps[stepId]; - if (primitive === undefined || primitive.kind !== "step") { - throw new Error( - `folded definition ${definition.id} step ${stepId} is not a step primitive`, - ); - } - return { - systemPrompt: primitive.agent.systemPrompt, - toolPackagePins: primitive.agent.toolPackagePins ?? [], - grantRequirements: definition.grantRequirements ?? [], - credentialBindings: definition.credentialBindings ?? [], - model: primitive.agent.inference.sources[0]?.model ?? null, - }; -} diff --git a/packages/workflow-deploy/src/index.ts b/packages/workflow-deploy/src/index.ts index 8373e878b..b92602c83 100644 --- a/packages/workflow-deploy/src/index.ts +++ b/packages/workflow-deploy/src/index.ts @@ -47,7 +47,6 @@ export { describeCredentialBindingViolation, type CredentialBindingViolation, } from "./credential-binding-check"; -export { extractFoldedBody, type FoldedBody } from "./fold-synthesis"; export { enumerateInertBodies, inertFlatNamespaceStepIds, @@ -62,7 +61,6 @@ export { collectAgentBearingStepIds, buildInertProjectionStepSources, buildInertBodyStepSources, - buildSingleStepAgentDefinition, WorkflowDefinitionInvalidError, } from "./orchestrator"; export { deriveRunAddress, deriveWorkflowRunRepoId } from "./addresses"; diff --git a/packages/workflow-deploy/src/orchestrator.ts b/packages/workflow-deploy/src/orchestrator.ts index c52ed0036..23cd3a325 100644 --- a/packages/workflow-deploy/src/orchestrator.ts +++ b/packages/workflow-deploy/src/orchestrator.ts @@ -9,14 +9,7 @@ // as an inert placeholder: the wire shape requires a source for every step, // but that step never issues a request through it. -import type { - AgentDefinition, - AnnotatedToolFactory, - BaseEnv, - InferencePreference, -} from "@intx/agent"; import type { HarnessConfig, InferenceSource } from "@intx/types/runtime"; -import type { ToolPackagePin } from "@intx/types/tool-packages"; import type { WorkflowProjectionDefinition } from "@intx/types/sidecar"; import { type ApprovalSet } from "./capability-approval"; @@ -342,35 +335,3 @@ export function buildInertBodyStepSources(args: { function sameInferenceSource(a: InferenceSource, b: InferenceSource): boolean { return a.id === b.id && a.provider === b.provider && a.model === b.model; } - -/** - * Assemble a single-step `AgentDefinition` from already-resolved fields. This - * is the single place the single-step agent shape is constructed, so the - * offline agent-to-workflow fold synthesis cannot drift on which fields a - * folded agent carries. Callers pass resolved inputs: the fold passes empty - * tool factories (its tools ride as `toolPackagePins`), the agent's own pins, - * and its catalog-resolved inference preferences. - */ -export function buildSingleStepAgentDefinition(args: { - id: string; - systemPrompt: string; - inferencePreferences: readonly InferencePreference[]; - toolFactories: readonly AnnotatedToolFactory[]; - capabilities?: readonly string[]; - description?: string; - toolPackagePins?: readonly ToolPackagePin[]; -}): AgentDefinition { - return { - id: args.id, - systemPrompt: args.systemPrompt, - toolFactories: args.toolFactories, - capabilities: args.capabilities ?? [], - inference: { sources: args.inferencePreferences }, - ...(args.description !== undefined - ? { description: args.description } - : {}), - ...(args.toolPackagePins !== undefined - ? { toolPackagePins: args.toolPackagePins } - : {}), - }; -} diff --git a/packages/workflow-deploy/src/testing/index.ts b/packages/workflow-deploy/src/testing/index.ts deleted file mode 100644 index 8f322d677..000000000 --- a/packages/workflow-deploy/src/testing/index.ts +++ /dev/null @@ -1,94 +0,0 @@ -// @intx/workflow-deploy/testing -- test-only fold synthesis. -// -// `synthesizeFoldedWorkflow` turns a legacy agent's resolved fields into a -// single-step `WorkflowDefinition` (the `workflow.json` a folded agent is -// stored as). Production no longer folds new agents: the one live fold path -// reads a folded definition back with `extractFoldedBody`, it never -// synthesizes one, so this builder exists only to construct folded definitions -// in tests. It is PURE: every impure input (the inference preferences resolved -// against the tenant catalog, the mail trigger address) is supplied by the -// caller, not computed here. -// -// Tools ride as `toolPackagePins` on the step agent (pins-only: the sidecar -// materializes them and its tool floor authorizes them at runtime, exactly as -// a deployed step agent does). The step agent's `toolFactories` is the empty array -// by construction -- a folded definition has no live factories, and the -// capability walk falls back to the (empty) factories to emit no tool grants. - -import type { CredentialBinding, GrantRequirement } from "@intx/types"; -import type { ToolPackagePin } from "@intx/types/tool-packages"; -import type { InferencePreference } from "@intx/agent"; -import { - defineWorkflow, - type WorkflowDefinition, -} from "@intx/workflow/definition"; - -import { buildSingleStepAgentDefinition } from "../orchestrator"; - -export interface FoldedWorkflowInput { - /** The workflow definition id (e.g. `wf_`). */ - readonly workflowId: string; - /** - * The mail trigger address. Supplied, never invented: routing keys on the - * deployment address, but this is the audit/grant record's mail identity and - * must match the deployment address the agent is folded onto. - */ - readonly mailAddress: string; - /** The agent's system prompt. Null makes the agent undeployable -- fail loud. */ - readonly systemPrompt: string | null; - /** Carried onto the step agent when present. */ - readonly description: string | null; - /** - * The credential-free `{ provider, model }` inference preferences resolved - * from the agent's model requirements against the tenant catalog. - */ - readonly inferencePreferences: readonly InferencePreference[]; - /** The agent's tool package pins, carried through as-is (pins-only). */ - readonly toolPackagePins: readonly ToolPackagePin[]; - /** The agent's grant requirements, carried onto the workflow envelope. */ - readonly grantRequirements?: readonly GrantRequirement[]; - /** The agent's credential bindings, carried onto the workflow envelope. */ - readonly credentialBindings?: readonly CredentialBinding[]; -} - -/** - * Synthesize a single-step `WorkflowDefinition` from a legacy agent's resolved - * fields. Raises if the agent has no system prompt (an empty-prompt agent is - * undeployable; synthesizing one would bury a broken definition rather than - * surface the source data problem). - */ -export function synthesizeFoldedWorkflow( - input: FoldedWorkflowInput, -): WorkflowDefinition { - if (input.systemPrompt === null) { - throw new Error( - `cannot synthesize a folded workflow for ${input.workflowId}: the agent has no system prompt`, - ); - } - - const agent = buildSingleStepAgentDefinition({ - id: input.workflowId, - systemPrompt: input.systemPrompt, - inferencePreferences: input.inferencePreferences, - // Pins-only: no live tool factories. The empty array is load-bearing -- a - // folded workflow.json must carry `toolFactories: []` explicitly (a missing - // key hydrates to undefined and the capability walk's factory loop throws). - toolFactories: [], - // Dormant on every deployed agent today; the instance wrap emits [] too. - capabilities: [], - ...(input.description !== null ? { description: input.description } : {}), - toolPackagePins: input.toolPackagePins, - }); - - return defineWorkflow({ - id: input.workflowId, - agent, - trigger: { type: "mail", to: input.mailAddress }, - ...(input.grantRequirements !== undefined - ? { grantRequirements: input.grantRequirements } - : {}), - ...(input.credentialBindings !== undefined - ? { credentialBindings: input.credentialBindings } - : {}), - }); -} diff --git a/packages/workflow-host/README.md b/packages/workflow-host/README.md index 58707f605..9765bb11c 100644 --- a/packages/workflow-host/README.md +++ b/packages/workflow-host/README.md @@ -411,8 +411,8 @@ await runWorkflowChildFromProcessEnv(createSubstrate, { The reference in-tree binary imports `createSubstrate` from `apps/sidecar/src/workflow-child-bindings.ts`, which closes this -package's `createSidecarSubstrateFactory` over the app's tool -materializer and the grant cap from `@intx/workflow-deploy`. The child +package's `createSidecarSubstrateFactory` over the grant cap from +`@intx/workflow-deploy`. The child binary path is resolved in `apps/sidecar/src/workflow-child-spawner.ts`. An alternative host follows the same pattern: write a substrate factory against its own diff --git a/packages/workflow-host/src/child/program.ts b/packages/workflow-host/src/child/program.ts index 32ca1cc00..be13b3461 100644 --- a/packages/workflow-host/src/child/program.ts +++ b/packages/workflow-host/src/child/program.ts @@ -1,13 +1,13 @@ // @intx/workflow-host/child -- the child process entry and substrate factory. // // Boots the workflow child from its process environment and builds its substrate. -// The host supplies tool materialization and the child-grant cap. +// The host supplies the child-grant cap. // The package barrel does not load this entry. export { SIDECAR_SUBSTRATE_CONFIG_KEYS, createSidecarSubstrateFactory, } from "./substrate-factory"; -export type { StepToolMaterialization } from "./step-tools"; +export type { StepToolRuntime } from "./step-tools"; export { runWorkflowChildFromProcessEnv } from "./from-process-env"; diff --git a/packages/workflow-host/src/child/step-tools.test.ts b/packages/workflow-host/src/child/step-tools.test.ts index bceb5a0d7..18dfa0dd8 100644 --- a/packages/workflow-host/src/child/step-tools.test.ts +++ b/packages/workflow-host/src/child/step-tools.test.ts @@ -50,7 +50,7 @@ import { createToolBearingAgentFactory, rewrapStepToolFactory, sourceStepToolFactories, - type StepToolMaterialization, + type StepToolRuntime, } from "./step-tools"; const tempDirs: string[] = []; @@ -156,10 +156,10 @@ describe("createToolBearingAgentFactory plugin/LSP lifecycle", () => { }), }); - const materialization: StepToolMaterialization = { + const stepTools: StepToolRuntime = { factories: [ { - packageName: "@intx/test-tool", + factoryId: "@intx/test-tool", declaredCredentials: [], factory: noopTool, }, @@ -168,7 +168,7 @@ describe("createToolBearingAgentFactory plugin/LSP lifecycle", () => { }; const env = await buildStepEnv(); - attachStepTools(env, materialization); + attachStepTools(env, stepTools); // The step-invoker adapter spreads the env (`{ ...envBase, authorize }`) // before handing it to the agent factory; replicate that spread so the // test exercises the symbol-slot-survives-spread path. @@ -245,13 +245,13 @@ describe("createToolBearingAgentFactory plugin/LSP lifecycle", () => { }, }); - const materialization: StepToolMaterialization = { + const stepTools: StepToolRuntime = { factories: [], pluginFactories: [firstPlugin, throwingPlugin], }; const env = await buildStepEnv(); - attachStepTools(env, materialization); + attachStepTools(env, stepTools); const agentFactory = createToolBearingAgentFactory(); const def = { @@ -308,13 +308,13 @@ describe("createToolBearingAgentFactory plugin/LSP lifecycle", () => { }), }); - const materialization: StepToolMaterialization = { + const stepTools: StepToolRuntime = { factories: [], pluginFactories: [throwingPlugin, survivorPlugin], }; const env = await buildStepEnv(); - attachStepTools(env, materialization); + attachStepTools(env, stepTools); const agentFactory = createToolBearingAgentFactory(); const def = { @@ -362,13 +362,13 @@ describe("createToolBearingAgentFactory plugin/LSP lifecycle", () => { }, }); - const materialization: StepToolMaterialization = { + const stepTools: StepToolRuntime = { factories: [], pluginFactories: [firstPlugin, throwingPlugin], }; const env = await buildStepEnv(); - attachStepTools(env, materialization); + attachStepTools(env, stepTools); const agentFactory = createToolBearingAgentFactory(); const def = { @@ -397,13 +397,13 @@ describe("createToolBearingAgentFactory plugin/LSP lifecycle", () => { }), }); - const materialization: StepToolMaterialization = { + const stepTools: StepToolRuntime = { factories: [], pluginFactories: [throwingPlugin], }; const env = await buildStepEnv(); - attachStepTools(env, materialization); + attachStepTools(env, stepTools); const agentFactory = createToolBearingAgentFactory(); const def = { @@ -510,8 +510,8 @@ describe("rewrapStepToolFactory", () => { if (seenEnv === undefined) { throw new Error("the bundle factory was not invoked"); } - // The bundle sees a layered bag resolving THIS package's capability -- the - // seam that must never hand a bundle another package's capability. + // The bundle sees a layered bag resolving THIS factory's capability -- the + // seam that must never hand a bundle another factory's capability. // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the slot is the RuntimeCapabilities resolver this test set const bag = Reflect.get(seenEnv, "capabilities") as RuntimeCapabilities; expect(bag.resolve("credentials")).toBe(capability); @@ -578,15 +578,15 @@ describe("sourceStepToolFactories", () => { expect( sourceStepToolFactories([declaring, legacy]).map((entry) => ({ - packageName: entry.packageName, + factoryId: entry.factoryId, declaredCredentials: entry.declaredCredentials, })), ).toEqual([ { - packageName: "@intx/test-tool/declaring", + factoryId: "@intx/test-tool/declaring", declaredCredentials: [{ handle: "api-token", scopes: ["read"] }], }, - { packageName: "@intx/test-tool/legacy", declaredCredentials: [] }, + { factoryId: "@intx/test-tool/legacy", declaredCredentials: [] }, ]); }); }); diff --git a/packages/workflow-host/src/child/step-tools.ts b/packages/workflow-host/src/child/step-tools.ts index 6b63ddb0e..e2cd47e98 100644 --- a/packages/workflow-host/src/child/step-tools.ts +++ b/packages/workflow-host/src/child/step-tools.ts @@ -30,15 +30,15 @@ import { const logger = getLogger(["sidecar", "workflow-child", "step-tools"]); /** - * Materialized tool runtime for one step's agent. Carried from + * Tool runtime for one step's agent. Carried from * `buildEnv` (which knows the step's identity) to the `agentFactory` * (which knows the agent definition + env) via a symbol-keyed slot on * the per-step env so the two callbacks of `createWorkflowStepInvoker` * can cooperate without widening the portable adapter's surface. */ -export interface StepToolMaterialization { +export interface StepToolRuntime { readonly factories: readonly { - readonly packageName: string; + readonly factoryId: string; readonly declaredCredentials: readonly ToolCredentialDeclaration[]; readonly factory: AnnotatedToolFactory; }[]; @@ -54,9 +54,9 @@ export interface StepToolMaterialization { */ export function sourceStepToolFactories( toolFactories: readonly AnnotatedToolFactory[], -): StepToolMaterialization["factories"] { +): StepToolRuntime["factories"] { return toolFactories.map((factory) => ({ - packageName: factory.id, + factoryId: factory.id, declaredCredentials: (factory.credentials ?? []).map((declaration) => ({ handle: declaration.handle, ...(declaration.scopes !== undefined @@ -81,27 +81,22 @@ const STEP_TOOLS = Symbol("intx.sidecar.step-tools"); // through `Reflect.get`/`Reflect.set` so neither site needs a type // assertion: `BaseEnv` is an interface without a symbol index // signature, so a structural cast would otherwise be required. -function setStepToolSlot( - env: object, - materialization: StepToolMaterialization, -): void { - Reflect.set(env, STEP_TOOLS, materialization); +function setStepToolSlot(env: object, stepTools: StepToolRuntime): void { + Reflect.set(env, STEP_TOOLS, stepTools); } -function getStepToolSlot(env: object): StepToolMaterialization | undefined { +function getStepToolSlot(env: object): StepToolRuntime | undefined { const value: unknown = Reflect.get(env, STEP_TOOLS); if (value === undefined) return undefined; - if (!isStepToolMaterialization(value)) { + if (!isStepToolRuntime(value)) { throw new Error( - "sidecar workflow-child step tools: the per-step env's tool slot is not a StepToolMaterialization; the slot is private to this module and must only be set by attachStepTools", + "sidecar workflow-child step tools: the per-step env's tool slot is not a StepToolRuntime; the slot is private to this module and must only be set by attachStepTools", ); } return value; } -function isStepToolMaterialization( - value: unknown, -): value is StepToolMaterialization { +function isStepToolRuntime(value: unknown): value is StepToolRuntime { return ( typeof value === "object" && value !== null && @@ -193,7 +188,7 @@ function requireCapabilitiesBag(env: BaseEnv): RuntimeCapabilities { } /** - * Attach a step's materialized tool runtime to the per-step env so the + * Attach a step's tool runtime to the per-step env so the * tool-bearing `agentFactory` can consume it. Called by `buildEnv` * once it has assembled the step's factories. * @@ -204,23 +199,23 @@ function requireCapabilitiesBag(env: BaseEnv): RuntimeCapabilities { */ export function attachStepTools( env: Omit, - materialization: StepToolMaterialization, + stepTools: StepToolRuntime, ): void { - setStepToolSlot(env, materialization); + setStepToolSlot(env, stepTools); } /** - * Re-wrap a loaded tool factory so its bundle's `dispose` (when present) - * is captured via `onDispose`, forwarding the loader's `id`, `requires`, - * and `definitions` so the result is a real `AnnotatedToolFactory`, not a + * Re-wrap a tool factory so its bundle's `dispose` (when present) is + * captured via `onDispose`, forwarding its `id`, `requires`, `definitions`, + * and `credentials` so the result is a real `AnnotatedToolFactory`, not a * hand-shaped lookalike. The static `definitions` declaration is * forwarded verbatim: this wrapper does not rename tools, so the names * the deploy-time walk enumerates must survive the re-wrap unchanged. * * When `credentials` is supplied, the bundle's factory sees a per-bundle - * capabilities bag: the step's base bag layered with THIS package's + * capabilities bag: the step's base bag layered with THIS factory's * consumer-scoped `credentials` capability, so the bundle resolves only the - * handles its own package is authorized for. When it is absent (the package + * handles its own factory is authorized for. When it is absent (the factory * declares and binds no credential), the base bag is passed through unchanged. * * Exported so the definitions-preservation contract is testable in @@ -273,16 +268,16 @@ function layerCredentialsOntoEnv( /** * Build the `agentFactory` the workflow step-invoker uses. The returned - * factory reads the materialized tool runtime off the env (set by + * factory reads the tool runtime off the env (set by * `buildEnv` via `attachStepTools`), augments the step's - * `AgentDefinition` with the loaded tool factories (wrapped to capture + * `AgentDefinition` with the tool factories (wrapped to capture * each bundle's disposer), constructs the plugin chain on `env.plugins`, * builds the agent, and wraps * `agent.close()` so every plugin instance and tool bundle is disposed * when the step's agent closes. * - * When the env carries no materialized tools (the `buildEnv` did not - * run materialization, e.g. a unit test using the bare factory), the + * When the env carries no tools (the `buildEnv` attached none, e.g. a + * unit test using the bare factory), the * factory falls back to `createAgent(def, env)` unchanged. */ export function createToolBearingAgentFactory(): ( @@ -293,15 +288,15 @@ export function createToolBearingAgentFactory(): ( def: AgentDefinition, env: EnvReq, ): Promise => { - const materialization = getStepToolSlot(env); - if (materialization === undefined) { + const stepTools = getStepToolSlot(env); + if (stepTools === undefined) { return createAgent(def, env); } - // Assemble each package's consumer-scoped `credentials` capability once, + // Assemble each factory's consumer-scoped `credentials` capability once, // when the step carries credential wiring (a bare test build carries none, - // yielding an empty map). Every factory in a package shares the one - // capability; a package that declares a handle no binding resolves fails + // yielding an empty map). A factory that declares a handle no binding + // resolves fails // the launch here, loudly, rather than at the tool's first resolve. This // can throw (reconcile fail-closed, malformed delivery) before any handle // is shaped, so nothing is orphaned on that path. @@ -309,27 +304,24 @@ export function createToolBearingAgentFactory(): ( const credentialCapabilities = credentialWiring === undefined ? new Map() - : buildCredentialCapabilities( - materialization.factories, - credentialWiring, - ); + : buildCredentialCapabilities(stepTools.factories, credentialWiring); - // Wrap each loaded tool factory so its bundle's `dispose` (when + // Wrap each tool factory so its bundle's `dispose` (when // present) is captured. Dedupe by closure identity: a factory whose // bundle returns the same `dispose` on every invocation must not be - // torn down once per push. Each package's credentials capability joins + // torn down once per push. Each factory's credentials capability joins // the same teardown set so its shaped handles are released with the agent. const capturedDisposers = new Set<() => unknown>(); for (const capability of credentialCapabilities.values()) { capturedDisposers.add(() => capability.dispose()); } - const factoriesWithCapture = materialization.factories.map((stf) => + const factoriesWithCapture = stepTools.factories.map((stf) => rewrapStepToolFactory( stf.factory, (dispose) => { capturedDisposers.add(dispose); }, - credentialCapabilities.get(stf.packageName), + credentialCapabilities.get(stf.factoryId), ), ); @@ -354,12 +346,9 @@ export function createToolBearingAgentFactory(): ( return failures; }; - // Rebuild the def with the materialized tool factories. The - // serialized `def.toolFactories` carry only `{ id, requires }` - // metadata (the workflow projection strips closures on the wire), - // so the runnable factories come from materialization, not the - // incoming def. `defineAgent` owns the contravariance escape for - // the `BaseEnv`-typed loader factories (see its `EnvRequiredByAll` + // Rebuild the def with the wrapped factories, which capture disposers + // and layer credentials. `defineAgent` owns the contravariance escape + // for the `BaseEnv`-typed factories (see its `EnvRequiredByAll` // machinery). const toolDef = defineAgent({ id: def.id, @@ -387,7 +376,7 @@ export function createToolBearingAgentFactory(): ( const pluginInstances: unknown[] = []; let chainEnv: BaseEnv = env; try { - for (const factory of materialization.pluginFactories) { + for (const factory of stepTools.pluginFactories) { const instance = factory(chainEnv); pluginInstances.push(instance); chainEnv = { diff --git a/packages/workflow-host/src/child/substrate-factory.ts b/packages/workflow-host/src/child/substrate-factory.ts index d5f259e9b..83d73a8b7 100644 --- a/packages/workflow-host/src/child/substrate-factory.ts +++ b/packages/workflow-host/src/child/substrate-factory.ts @@ -168,7 +168,7 @@ import { attachStepTools, createToolBearingAgentFactory, sourceStepToolFactories, - type StepToolMaterialization, + type StepToolRuntime, } from "./step-tools"; import { createSupervisorBackedTransport, @@ -1057,14 +1057,14 @@ export function createSidecarStepBuildEnv( // factories. The child holds the live re-verified `WorkflowDefinition`, and // the runtime threads each step's live agent in as `req.agent`, so // `req.agent.toolFactories` are the real callable `AnnotatedToolFactory`s, - // bare-named because no namespacing loader runs. `packageName` is the - // factory's bundle `id` (the credential Gate-2 key), and its `defineTool` - // declaration feeds the declared-versus-bound check; a mail/plain tool - // declares no credentials, so it stays inert. Plugin factories -- which have - // no agent slot -- are materialized separately from the frozen closure the - // child already holds (`materializeSourcePluginFactories`), and the per-step - // plugin chain wires them onto `env.plugins`. - const materialization: StepToolMaterialization = { + // bare-named. `factoryId` is the factory's bundle `id` (the credential + // Gate-2 key), and its `defineTool` declaration feeds the declared-versus- + // bound check; a mail/plain tool declares no credentials, so it stays inert. + // Plugin factories -- which have no agent slot -- are materialized separately + // from the frozen closure the child already holds + // (`materializeSourcePluginFactories`), and the per-step plugin chain wires + // them onto `env.plugins`. + const stepTools: StepToolRuntime = { factories: sourceStepToolFactories(req.agent.toolFactories), pluginFactories: await materializeSourcePluginFactories(deps, req), }; @@ -1132,16 +1132,16 @@ export function createSidecarStepBuildEnv( address: deps.mailboxAddress, capabilities, }; - // Carry the materialized tool runtime to the tool-bearing + // Carry the tool runtime to the tool-bearing // `agentFactory` via the env's symbol-keyed slot. The step-invoker // adapter spreads this env (`{ ...envBase, authorize }`) before // handing it to `agentFactory`; object spread preserves own // symbol-keyed properties, so the slot survives the spread. - attachStepTools(env, materialization); + attachStepTools(env, stepTools); // Attach the credential wiring so the `agentFactory` can assemble each // bundle's consumer-scoped `credentials` capability. Grants are wired as a - // THUNK, resolved only when a package actually needs a capability -- a - // step with no credential-consuming package never reads them, so a + // THUNK, resolved only when a factory actually needs a capability -- a + // step with no credential-consuming factory never reads them, so a // self-discovery resume that precedes the grants barrier does not fault on // a missing snapshot. Omitted when no credential context was // threaded, in which case a credential-consuming tool fails closed and loud @@ -2080,8 +2080,7 @@ async function buildChildRunEnv(args: { // `sourcesRef`, the event funnel, and the credential context (so the step's // inference reads the run's live material). One invoker serves both the // childWorkflow path and the onTrigger body path: each runs a real - // tool-bearing agent through the source-tools arm against the same - // `sourcesRef`. + // tool-bearing agent against the same `sourcesRef`. const invokeStep: WorkflowRuntimeEnv["invokeStep"] = (req) => deps.invokeStep( req, @@ -2463,9 +2462,9 @@ export function createSidecarSubstrateFactory( ...(durableConversation !== undefined ? { durableConversation } : {}), }); - // The tool-bearing agent factory reads the materialized tool + // The tool-bearing agent factory reads the tool // runtime off the per-step env (set by `buildStepEnv` via - // `attachStepTools`), attaches the loaded tool factories to the + // `attachStepTools`), attaches the tool factories to the // step's `AgentDefinition`, builds the plugin chain on // `env.plugins`, and wraps `agent.close()` so every plugin (the LSP // subprocess included) and tool bundle is torn down with the agent @@ -2494,10 +2493,10 @@ export function createSidecarSubstrateFactory( // Spawned-child step build env (INTR-310). Every spawned child's steps -- // a childWorkflow child's and an onTrigger section body's alike -- run real, - // TOOL-BEARING agents through the same source-tools arm the top level uses: + // TOOL-BEARING agents through the same build the top level uses: // the child holds the live re-verified definition, so each step agent // carries live `toolFactories` fed from `req.agent.toolFactories`, and the - // source arm materializes declared plugins from the shared closure. Tools + // build materializes declared plugins from the shared closure. Tools // are available wherever inference runs, so a body agent gets the same // build a top-level step agent gets (see the tool-availability invariant in // `packages/workflow/README.md`). diff --git a/packages/workflow-host/src/child/warm-agent-cache.ts b/packages/workflow-host/src/child/warm-agent-cache.ts index 442397557..0c72d786d 100644 --- a/packages/workflow-host/src/child/warm-agent-cache.ts +++ b/packages/workflow-host/src/child/warm-agent-cache.ts @@ -1,8 +1,8 @@ // Warm-agent cache for the workflow-process child (design §3b). // -// A long-lived single-step agent is built once -- tools materialized, +// A long-lived single-step agent is built once -- tool bundles built, // plugins instantiated, the LSP subprocess spawned -- and reused across -// every inbound message. Re-materializing tools and re-spawning the LSP +// every inbound message. Rebuilding tool bundles and re-spawning the LSP // per message is the instantiate-send-teardown cost the warm cache // removes; keeping the agent alive across messages is also what // preserves in-memory conversation continuity (durability across child diff --git a/packages/workflow-host/src/deploy/agent-key-registration-lifecycle.test.ts b/packages/workflow-host/src/deploy/agent-key-registration-lifecycle.test.ts index 5df9d8fec..74b790314 100644 --- a/packages/workflow-host/src/deploy/agent-key-registration-lifecycle.test.ts +++ b/packages/workflow-host/src/deploy/agent-key-registration-lifecycle.test.ts @@ -253,7 +253,6 @@ describe("agent signing-key registration lifecycle on the host transport", () => applyFrozenWorkflowClosure: (applyArgs) => Promise.resolve({ packageDir: path.join(applyArgs.instanceDir, "package"), - deployDir: path.join(applyArgs.instanceDir, "deploy"), }), multistepMailRouter: createMultistepMailRouter(), multistepSignalRouter: createMultistepSignalRouter(), diff --git a/packages/workflow-host/src/deploy/workflow-host-wiring-deploy-failure.test.ts b/packages/workflow-host/src/deploy/workflow-host-wiring-deploy-failure.test.ts index ae4df39d5..9ccd67cd3 100644 --- a/packages/workflow-host/src/deploy/workflow-host-wiring-deploy-failure.test.ts +++ b/packages/workflow-host/src/deploy/workflow-host-wiring-deploy-failure.test.ts @@ -116,7 +116,6 @@ function stubApplyFrozenWorkflowClosure(): NonNullable< return (applyArgs) => Promise.resolve({ packageDir: pathJoin(applyArgs.instanceDir, "package"), - deployDir: pathJoin(applyArgs.instanceDir, "deploy"), }); } diff --git a/packages/workflow-host/src/deploy/workflow-host-wiring-undeploy-supervisor.test.ts b/packages/workflow-host/src/deploy/workflow-host-wiring-undeploy-supervisor.test.ts index 3124bf213..ab0801c33 100644 --- a/packages/workflow-host/src/deploy/workflow-host-wiring-undeploy-supervisor.test.ts +++ b/packages/workflow-host/src/deploy/workflow-host-wiring-undeploy-supervisor.test.ts @@ -271,7 +271,6 @@ describe("createSidecarDeployRouter multi-step undeploy shuts the supervisor dow applyFrozenWorkflowClosure: (applyArgs) => Promise.resolve({ packageDir: path.join(applyArgs.instanceDir, "package"), - deployDir: path.join(applyArgs.instanceDir, "deploy"), }), multistepMailRouter: mailRouter, multistepSignalRouter: signalRouter, diff --git a/packages/workflow-host/src/deploy/workflow-host-wiring.test.ts b/packages/workflow-host/src/deploy/workflow-host-wiring.test.ts index 32a65013a..8cb4bf330 100644 --- a/packages/workflow-host/src/deploy/workflow-host-wiring.test.ts +++ b/packages/workflow-host/src/deploy/workflow-host-wiring.test.ts @@ -846,7 +846,6 @@ describe("createSidecarDeployRouter multi-step branch", () => { const deploymentId = path.basename(applyArgs.instanceDir); return Promise.resolve({ packageDir: path.join(tempBase, "closure-package", deploymentId), - deployDir: path.join(tempBase, "closure-deploy", deploymentId), }); }; const router = createSidecarDeployRouter({ @@ -2631,7 +2630,6 @@ describe("createSidecarDeployRouter multi-step branch", () => { } return Promise.resolve({ packageDir: path.join(dataDir, "closure-package", deploymentId), - deployDir: path.join(dataDir, "closure-deploy", deploymentId), }); }, }); @@ -3137,7 +3135,6 @@ describe("createSidecarDeployRouter multi-step branch", () => { }); return Promise.resolve({ packageDir: fakePackageDir, - deployDir: path.join(dataDir, "fake-deploy-dir"), }); }; diff --git a/packages/workflow-host/src/deploy/workflow-host-wiring.ts b/packages/workflow-host/src/deploy/workflow-host-wiring.ts index 0637dcbbc..ca4d8255c 100644 --- a/packages/workflow-host/src/deploy/workflow-host-wiring.ts +++ b/packages/workflow-host/src/deploy/workflow-host-wiring.ts @@ -221,8 +221,8 @@ async function makeDirectoriesWritable(dir: string): Promise { * Resolve the durable source-asset store root and `assetId -> mountPath` map a * pinned deployment materializes its `kind:"asset"` closure entries from, * asserting every referenced asset's mount directory is present on disk. This - * is a cheap early gate for a missing checkout; the loader still SRI-verifies - * each tarball's bytes at materialization. Derived purely from the pin, so the + * is a cheap early gate for a missing checkout; closure materialization still + * SRI-verifies each tarball's bytes. Derived purely from the pin, so the * deploy path and the boot-time restore path (which has only the pin, no * re-delivery) resolve the identical mounts. A missing mount is a broken * deployment that cannot run without it, so it fails loud rather than @@ -1020,7 +1020,6 @@ export function createSidecarDeployRouter< readonly gitDirs: ReadonlyMap; }) => Promise<{ readonly packageDir: string; - readonly deployDir: string; }>; /** On-disk size cap of the tarball cache closure applies share. */ closureCacheMaxBytes: number; @@ -1589,13 +1588,12 @@ export function createSidecarDeployRouter< * registry table. Both callers take `packageDir`, which the workflow child * evaluates; restore also re-carries the pin on its spec. * - * The instance dir is force-reclaimed before the apply. `deploymentId` is + * The apply clears the closure's directory before writing. `deploymentId` is * deterministic per agent address, so a redeploy or a boot-restore reuses the * same dir; a prior soft-failed deploy or a dead prior process can leave it - * half-materialized. The rm makes the apply write into a clean dir either - * way (a no-op on a never-deployed address). It is safe to reclaim only - * because no live reader holds the dir when this runs -- a precondition each - * caller establishes and notes at its call site. + * half-materialized. Clearing it is safe only because no live reader holds + * the dir when this runs -- a precondition each caller establishes and notes + * at its call site. */ async function materializeDeploymentClosure( dataDir: string, @@ -1607,7 +1605,6 @@ export function createSidecarDeployRouter< "workflow-definition-closures", deploymentId, ); - await rm(instanceDir, { recursive: true, force: true }); // Tarball `kind:"asset"` entries read from the durable plain-file store; a // source-format entry checks its subtree out of the durable indexed git @@ -2255,7 +2252,7 @@ export function createSidecarDeployRouter< maxAssetPayloadBytes: deps.maxInlineAssetPayloadBytes, }); } - // Safe to reclaim the instance dir inside the helper: the `deploying` + // Safe for the apply to clear the closure's directory: the `deploying` // entry set above refuses a concurrent deploy of the address, and the // child is not yet spawned, so no live reader holds it. const applied = await materializeDeploymentClosure( @@ -2685,11 +2682,11 @@ export function createSidecarDeployRouter< try { // Re-materialize the pinned closure for the restored child to // evaluate and verify, as on deploy; the closure IS the source of - // truth, and this process never evaluates it. The helper reclaims - // the instance dir first, which is safe here because the prior - // process (the only reader) is dead, each deployment has its own - // dir, and restore runs before `hubLink.connect()`, so no deploy - // reads it. Registry-sourced entries come from the + // truth, and this process never evaluates it. The apply clears + // the closure's directory first, which is safe here because the + // prior process (the only reader) is dead, each deployment has + // its own dir, and restore runs before `hubLink.connect()`, so no + // deploy reads it. Registry-sourced entries come from the // content-addressed closure cache, or from their registry when the // cache, which every deployment on this sidecar shares, has // evicted them; asset-sourced entries read from the durable source diff --git a/packages/workflow-host/src/local.ts b/packages/workflow-host/src/local.ts index 4a62818e7..bdd99109e 100644 --- a/packages/workflow-host/src/local.ts +++ b/packages/workflow-host/src/local.ts @@ -26,7 +26,7 @@ export { attachStepCredentialWiring, attachStepTools, createToolBearingAgentFactory, - type StepToolMaterialization, + type StepToolRuntime, } from "./child/step-tools"; export { createCredentialsBackedAuthorize, diff --git a/packages/workflow-host/src/workflow-definition-loader.ts b/packages/workflow-host/src/workflow-definition-loader.ts index 4102fda6b..75340fb25 100644 --- a/packages/workflow-host/src/workflow-definition-loader.ts +++ b/packages/workflow-host/src/workflow-definition-loader.ts @@ -11,16 +11,14 @@ // validated at this boundary before being returned. // // Materialization is deliberately NOT done here. `@intx/workflow-host` -// stays free of a `@intx/tool-packaging` dependency (that layer lives -// in `packages/tool-packaging/src/tool-materialization.ts`), so the -// caller runs the closure machinery and hands the resulting package -// directory in. This module only performs the import + evaluate + -// validate step, which is the part that must run inside the child's -// address space because it evaluates author code. The loader resolves -// whatever the materializer laid out: the closure lays direct deps -// (workspace members or pinned registry deps) into the workflow -// package's `node_modules/`, and a published tarball that bundles its -// own `node_modules` can also place them there. +// stays free of a `@intx/tool-packaging` dependency, so the caller runs +// the closure machinery and hands the resulting package directory in. This +// module only performs the import + evaluate + validate step, which is the +// part that must run inside the child's address space because it evaluates +// author code. The loader resolves whatever the materializer laid out: the +// closure lays direct deps (workspace members or pinned registry deps) into +// the workflow package's `node_modules/`, and a published tarball that bundles +// its own `node_modules` can also place them there. import { promises as fs } from "node:fs"; import path from "node:path"; @@ -614,8 +612,7 @@ export interface LoadWorkflowPluginsFromClosureArgs { /** * Import each declared plugin package's `interchange.tools` module from the * materialized workflow closure and collect the `AnnotatedPluginFactory` - * values it exports. This is the run-child counterpart to the tool-package - * loader's plugin channel: a source-ref workflow contributes no plugin factory + * values it exports. A source-ref workflow contributes no plugin factory * through its agent definition (a plugin has no agent slot), so the child * materializes the declared plugins straight from the already-laid-out closure * -- no re-download, no manifest -- and feeds them into the existing per-step diff --git a/packages/workflow/src/definition/workflow.ts b/packages/workflow/src/definition/workflow.ts index 038bb07ad..3efbc2d01 100644 --- a/packages/workflow/src/definition/workflow.ts +++ b/packages/workflow/src/definition/workflow.ts @@ -60,7 +60,7 @@ export interface WorkflowDefinition { grantRequirements?: readonly GrantRequirement[]; /** * The credential bindings a launch resolves against tenant-owned - * credentials. Each maps a tool package's or an action handler's + * credentials. Each maps a tool factory's or an action handler's * declared handle to a provider. Ownership authorizes the use, and the * launch stamps a consumer-scoped `credential:{id}` / `use` grant with * `origin = 'system'`. diff --git a/packages/workflow/src/live-inert-projector.ts b/packages/workflow/src/live-inert-projector.ts index 0f690a16e..8983d890c 100644 --- a/packages/workflow/src/live-inert-projector.ts +++ b/packages/workflow/src/live-inert-projector.ts @@ -38,7 +38,6 @@ import type { InferencePreference, ToolDeclaration, } from "@intx/agent"; -import type { ToolPackagePin } from "@intx/types/tool-packages"; import type { CredentialBinding, SidecarCapabilityPolicy } from "@intx/types"; import type { InboundMailPolicy } from "@intx/types/inbound-mail-policy"; import type { MailAccept } from "@intx/types/mail-accept"; @@ -129,7 +128,6 @@ export interface InertAgent { readonly plugins?: readonly InertPlugin[]; readonly modelSources: readonly InertModelSource[]; readonly tags?: Readonly>; - readonly toolPackagePins?: readonly ToolPackagePin[]; } export interface InertStepStep { @@ -228,7 +226,7 @@ export interface InertWorkflowDefinition { readonly stepOrder: readonly string[]; readonly steps: Record; readonly state?: { readonly schema?: unknown }; - // The definition's credential bindings -- each maps a tool package's or an + // The definition's credential bindings -- each maps a tool factory's or an // action handler's declared handle to a concrete provider/locator. These are // pure plain data (no secret material; the secret is resolved fresh at // deploy), so they project verbatim. They are part of the hashed surface on @@ -622,9 +620,6 @@ function projectAgent( : {}), modelSources: agent.inference.sources.map(projectModelSource), ...(agent.tags !== undefined ? { tags: { ...agent.tags } } : {}), - ...(agent.toolPackagePins !== undefined - ? { toolPackagePins: [...agent.toolPackagePins] } - : {}), }; } diff --git a/tests/db/asset-resolution.test.ts b/tests/db/asset-resolution.test.ts index 621986d7a..3eba466a6 100644 --- a/tests/db/asset-resolution.test.ts +++ b/tests/db/asset-resolution.test.ts @@ -256,13 +256,13 @@ describe.skipIf(!harnessDbEnvAvailable())("asset-resolution (real DB)", () => { id: "ast_pkg_outer", tenantId: "tnt_root", kind: "package-registry", - name: "workspace-builtins", + name: "tool-registry", }); await seedAsset(h.db, { id: "ast_pkg_inner", tenantId: "tnt_leaf", kind: "package-registry", - name: "workspace-builtins", + name: "tool-registry", }); const list = await listAssetsForTenant( h.db, diff --git a/tests/db/persisted-approval-compat.test.ts b/tests/db/persisted-approval-compat.test.ts index 338c44fc8..24636039f 100644 --- a/tests/db/persisted-approval-compat.test.ts +++ b/tests/db/persisted-approval-compat.test.ts @@ -53,7 +53,6 @@ function legacyRow(approvedGrants: unknown): LaunchSpecRow { }, sourceOfferingIds: ["off_1"], defaultSourceOfferingId: "off_1", - toolPackagePins: null, createdAt: new Date("2026-01-01T00:00:00Z"), }; } diff --git a/tests/db/workflow-run-launch-spec-store.test.ts b/tests/db/workflow-run-launch-spec-store.test.ts index 11fe6833e..97b27ac0c 100644 --- a/tests/db/workflow-run-launch-spec-store.test.ts +++ b/tests/db/workflow-run-launch-spec-store.test.ts @@ -92,7 +92,6 @@ describe.skipIf(!harnessDbEnvAvailable())( frozenApprovalBundle: FROZEN_APPROVAL_BUNDLE, sourceOfferingIds: ["offering-primary", "offering-fallback"], defaultSourceOfferingId: "offering-primary", - toolPackagePins: [{ name: "@intx/search", version: "^1.2.0" }], }); const stored = await store.get(ANCHOR_RUN_ID); diff --git a/tests/tool-packaging/build-determinism.test.ts b/tests/tool-packaging/build-determinism.test.ts deleted file mode 100644 index efb78248a..000000000 --- a/tests/tool-packaging/build-determinism.test.ts +++ /dev/null @@ -1,213 +0,0 @@ -// Determinism gate for `bin/build-builtins.ts`. -// -// The packer's contract is bit-identical output across runs and -// machines: same workspace inputs produce the same tarball bytes (and -// therefore the same SRI integrity) so the seed's pin set can refer to -// an exact version without coordinating with the build step. The -// gate boils down to three invariants: -// -// - `listFilesSorted` walks the staging tree in lexical order so -// `tar.create` writes a stable entry sequence regardless of -// `fs.readdir` ordering. -// - `normalizeStagingModes` rewrites mode bits to canonical values -// before tar reads the entry stats, so two builders with different -// umasks emit identical headers. -// - `tar.create` is called with `{ mtime: epoch, portable: true, -// noDirRecurse: true }`, zeroing per-entry mtimes and stripping -// uid/gid/uname/gname. -// -// `packStaging` and `normalizeStagingModes` mirror the production -// helpers in `bin/build-builtins.ts` step-for-step. Keeping the test -// self-contained avoids dragging `bin/`'s tsconfig project into the -// test project graph, at the cost of having to keep the two copies in -// lockstep — if a future change rewrites the production packer, this -// test must follow. -// -// The umask test is the load-bearing piece: without -// `normalizeStagingModes`, two builders with different umasks produce -// distinct headers and the SRI diverges, but the same-tree tests -// above would still pass. - -import { describe, test, expect, beforeEach, afterEach } from "bun:test"; -import { promises as fs } from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import ssri from "ssri"; -import * as tar from "tar"; - -let scratch: string; - -beforeEach(async () => { - scratch = await fs.mkdtemp(path.join(os.tmpdir(), "build-det-")); -}); - -afterEach(async () => { - await fs.rm(scratch, { recursive: true, force: true }); -}); - -async function listFilesSorted(cwd: string, root: string): Promise { - const acc: string[] = []; - async function walk(rel: string): Promise { - const abs = path.join(cwd, rel); - const entries = await fs.readdir(abs, { withFileTypes: true }); - entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); - for (const entry of entries) { - const childRel = path.join(rel, entry.name); - acc.push(childRel); - if (entry.isDirectory()) { - await walk(childRel); - } - } - } - acc.push(root); - await walk(root); - return acc; -} - -async function normalizeStagingModes( - cwd: string, - entries: string[], -): Promise { - for (const rel of entries) { - const abs = path.join(cwd, rel); - const stat = await fs.lstat(abs); - if (stat.isSymbolicLink()) continue; - if (stat.isDirectory()) { - await fs.chmod(abs, 0o755); - } else if (stat.isFile()) { - await fs.chmod(abs, 0o644); - } - } -} - -async function packStaging( - stagingDir: string, - outPath: string, -): Promise { - const entries = await listFilesSorted(stagingDir, "package"); - await normalizeStagingModes(stagingDir, entries); - const opts: tar.TarOptionsWithAliasesAsyncFile = { - cwd: stagingDir, - gzip: true, - file: outPath, - portable: true, - mtime: new Date(0), - noDirRecurse: true, - }; - await tar.create(opts, entries); - const bytes = await fs.readFile(outPath); - return ssri.fromData(bytes, { algorithms: ["sha512"] }).toString(); -} - -async function writeMinimalPackage(stagingDir: string): Promise { - const pkgDir = path.join(stagingDir, "package"); - const subDir = path.join(pkgDir, "dist"); - await fs.mkdir(subDir, { recursive: true }); - await fs.writeFile( - path.join(pkgDir, "package.json"), - JSON.stringify( - { - name: "tools-determinism", - version: "1.0.0", - interchange: { tools: "./dist/main.js" }, - }, - null, - 2, - ), - ); - await fs.writeFile( - path.join(subDir, "main.js"), - "export const main = () => ({});\n", - ); -} - -describe("build-builtins determinism", () => { - test("packing the same staging tree twice yields identical SRI", async () => { - const stagingDir = path.join(scratch, "pkg"); - await writeMinimalPackage(stagingDir); - - const outA = path.join(scratch, "first.tgz"); - const outB = path.join(scratch, "second.tgz"); - const integrityA = await packStaging(stagingDir, outA); - const integrityB = await packStaging(stagingDir, outB); - - expect(integrityA).toBe(integrityB); - - const bytesA = await fs.readFile(outA); - const bytesB = await fs.readFile(outB); - expect(bytesA).toEqual(bytesB); - }); - - test("packing two staging trees with different mtimes still yields identical SRI", async () => { - // The packer zeroes mtimes via `{ mtime: epoch }`; the source - // files' mtimes must not leak into the tarball. Create the same - // tree twice with deliberately different filesystem mtimes and - // assert the integrity is unchanged. - async function buildTree(dir: string, mtimeMs: number): Promise { - const pkgDir = path.join(dir, "package"); - await fs.mkdir(pkgDir, { recursive: true }); - const pkgJsonPath = path.join(pkgDir, "package.json"); - await fs.writeFile( - pkgJsonPath, - JSON.stringify( - { - name: "tools-determinism", - version: "1.0.0", - interchange: { tools: "./main.js" }, - }, - null, - 2, - ), - ); - const mainPath = path.join(pkgDir, "main.js"); - await fs.writeFile(mainPath, "export const main = () => ({});\n"); - const mtime = new Date(mtimeMs); - await fs.utimes(pkgJsonPath, mtime, mtime); - await fs.utimes(mainPath, mtime, mtime); - } - - const stagingA = path.join(scratch, "a"); - const stagingB = path.join(scratch, "b"); - await buildTree(stagingA, 1_000_000_000_000); - await buildTree(stagingB, 1_700_000_000_000); - - const outA = path.join(scratch, "a.tgz"); - const outB = path.join(scratch, "b.tgz"); - const integrityA = await packStaging(stagingA, outA); - const integrityB = await packStaging(stagingB, outB); - - expect(integrityA).toBe(integrityB); - }); - - test("packing two staging trees with different mode bits still yields identical SRI", async () => { - // `normalizeStagingModes` runs before `tar.create` reads each - // entry's stat, so two builders whose umasks landed different mode - // bits on disk emit identical headers. Without that step (or if a - // future refactor reordered it after the tar walk), this test - // would fail because tar would record the as-on-disk modes. - const stagingA = path.join(scratch, "modes-a"); - const stagingB = path.join(scratch, "modes-b"); - await writeMinimalPackage(stagingA); - await writeMinimalPackage(stagingB); - - // Force divergent mode bits on the two staging trees, both for a - // directory and a regular file, so the normalization step has - // work to do on both shapes. - await fs.chmod(path.join(stagingA, "package"), 0o700); - await fs.chmod(path.join(stagingA, "package", "package.json"), 0o600); - await fs.chmod(path.join(stagingA, "package", "dist"), 0o711); - await fs.chmod(path.join(stagingA, "package", "dist", "main.js"), 0o755); - - await fs.chmod(path.join(stagingB, "package"), 0o777); - await fs.chmod(path.join(stagingB, "package", "package.json"), 0o664); - await fs.chmod(path.join(stagingB, "package", "dist"), 0o775); - await fs.chmod(path.join(stagingB, "package", "dist", "main.js"), 0o600); - - const outA = path.join(scratch, "modes-a.tgz"); - const outB = path.join(scratch, "modes-b.tgz"); - const integrityA = await packStaging(stagingA, outA); - const integrityB = await packStaging(stagingB, outB); - - expect(integrityA).toBe(integrityB); - }); -}); diff --git a/tests/tool-packaging/end-to-end.test.ts b/tests/tool-packaging/end-to-end.test.ts index e217be0d2..a34b5ce30 100644 --- a/tests/tool-packaging/end-to-end.test.ts +++ b/tests/tool-packaging/end-to-end.test.ts @@ -1,22 +1,19 @@ // End-to-end integration of the tool-package pipeline. // // Exercises the gluing between the hub-side resolver and the sidecar-side -// loader+atomic-apply. The test does -// not spin up a subprocess sidecar — the same harness builder code -// runs in-process here against synthetic fixtures, which is what the -// production path runs against real ones. +// closure materialization, in-process against synthetic fixtures. // // What the test demonstrates per scenario: // -// - happy path: resolver walks pins → loader materializes → -// applyAtomic swaps → loaded factories registered. +// - happy path: resolver walks pins → manifest round-trips through disk → +// materializeClosure lays the closure out. // // - integrity.mismatch: bytes served by the mock registry do not -// match the resolver's pinned integrity → applyAtomic returns -// failed with the right category and previousDeployId. +// match the resolver's pinned integrity → materializeClosure rejects +// with the right category. // // - registry.unknown: manifest references a registry the sidecar -// was not told about → loader rejects with registry.unknown. +// was not told about → materializeClosure rejects with registry.unknown. import { describe, test, expect, beforeEach, afterEach } from "bun:test"; import { promises as fs } from "node:fs"; @@ -29,19 +26,18 @@ import { type Packument, type PackumentFetcher, type RegistryConfig, - type RegistrySource, AssetRegistrySource, HttpRegistrySource, - applyAtomic, createClosureResolver, createTarballCache, - createToolLoader, + createTarballFetcher, + materializeClosure, + storeEntryDir, } from "@intx/tool-packaging"; import type { ToolPackageManifest, ToolPackagePin, } from "@intx/types/tool-packages"; -import { getToolPackageSourceContentIdentity } from "@intx/types/tool-packages"; let scratch: string; let fixtureRoot: string; @@ -93,10 +89,7 @@ async function buildFixture(spec: { interchange: { tools: "./tools.js" }, }), ); - await fs.writeFile( - path.join(packageDir, "tools.js"), - "// stub; loader replaces import via importModule\n", - ); + await fs.writeFile(path.join(packageDir, "tools.js"), "export {};\n"); const tarballPath = path.join(stagingDir, "out.tgz"); await tar.create({ cwd: stagingDir, gzip: true, file: tarballPath }, [ @@ -132,20 +125,23 @@ function makePackumentFromFixtures(fixtures: readonly Fixture[]): Packument[] { })); } -function fakeFactory(id: string) { - const fn = () => ({ - definitions: [], - run: async () => ({ callId: "stub", content: "ok" }), - }); - return Object.assign(fn, { - id, - requires: [] as readonly string[], - definitions: [], - }); +async function storedPackageJSON( + storeDir: string, + fixture: { name: string; version: string }, +): Promise { + return JSON.parse( + await fs.readFile( + path.join( + storeEntryDir(storeDir, fixture.name, fixture.version), + "package.json", + ), + "utf8", + ), + ); } describe("hub→sidecar pipeline (happy path)", () => { - test("resolved manifest round-trips through disk and loads cleanly", async () => { + test("resolved manifest round-trips through disk and lays out cleanly", async () => { const fixture = await buildFixture({ name: "tools-fixture", version: "1.0.0", @@ -172,8 +168,7 @@ describe("hub→sidecar pipeline (happy path)", () => { fetchPackument, }); const resolver = createClosureResolver({ - registries: new Map([[source.name, source]]), - defaultRegistry: source.name, + source, }); const pins: ToolPackagePin[] = [ { name: fixture.name, version: fixture.version }, @@ -182,9 +177,10 @@ describe("hub→sidecar pipeline (happy path)", () => { expect(manifest.entries).toHaveLength(1); const materializedEntry = manifest.entries[0]; if (materializedEntry === undefined) throw new Error("missing entry"); - expect(getToolPackageSourceContentIdentity(materializedEntry.source)).toBe( - fixture.integrity, - ); + expect(materializedEntry.source).toMatchObject({ + kind: "registry", + integrity: fixture.integrity, + }); // Persist the manifest; round-trip it. const deployDir = path.join(instanceDir, "deploy"); @@ -201,36 +197,27 @@ describe("hub→sidecar pipeline (happy path)", () => { ); expect(readBack).toEqual(manifest); - // Sidecar side: instantiate cache + loader against the same - // registries, fetch via a stub that hands back the fixture bytes. + // Sidecar side: materialize against the same registries, fetch via a + // stub that hands back the fixture bytes. const cache = createTarballCache({ rootDir: cacheDir, maxBytes: 100_000_000, }); - const loader = createToolLoader({ - cache, - registries, - host: { os: "linux", cpu: "x64" }, - fetchTarball: async () => fixture.bytes, - importModule: async () => ({ main: fakeFactory("@vendor/fixture/main") }), - }); - - const result = await applyAtomic({ + const { storeDir } = await materializeClosure({ manifest: readBack, - loader, - instanceDir, + instanceScratchDir: instanceDir, assetRoot, assetMounts: new Map(), gitDirs: new Map(), - attemptId: "atp_happy", - previousDeployId: "none", - newDeployId: "dpl_1", + host: { os: "linux", cpu: "x64" }, + cache, + registries, + fetchTarball: async () => fixture.bytes, + }); + expect(await storedPackageJSON(storeDir, fixture)).toMatchObject({ + name: fixture.name, + version: fixture.version, }); - expect(result.status).toBe("ok"); - if (result.status !== "ok") return; - expect(result.activeDeployId).toBe("dpl_1"); - expect(result.loaded).toHaveLength(1); - expect(result.loaded[0]?.factories[0]?.id).toBe("@vendor/fixture/main"); }); }); @@ -264,30 +251,21 @@ describe("hub→sidecar pipeline (failure paths)", () => { rootDir: cacheDir, maxBytes: 100_000_000, }); - const loader = createToolLoader({ - cache, - registries, - host: { os: "linux", cpu: "x64" }, - // Hand back bytes that do not match the pinned integrity. - fetchTarball: async () => - new TextEncoder().encode("definitely the wrong bytes"), - }); - - const result = await applyAtomic({ - manifest, - loader, - instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - attemptId: "atp_mismatch", - previousDeployId: "dpl_prior", - newDeployId: "dpl_attempted", - }); - expect(result.status).toBe("failed"); - if (result.status !== "failed") return; - expect(result.category).toBe("integrity.mismatch"); - expect(result.previousDeployId).toBe("dpl_prior"); + await expect( + materializeClosure({ + manifest, + instanceScratchDir: instanceDir, + assetRoot, + assetMounts: new Map(), + gitDirs: new Map(), + host: { os: "linux", cpu: "x64" }, + cache, + registries, + // Hand back bytes that do not match the pinned integrity. + fetchTarball: async () => + new TextEncoder().encode("definitely the wrong bytes"), + }), + ).rejects.toMatchObject({ category: "integrity.mismatch" }); }); test("registry.unknown when manifest names a registry not in the sidecar config", async () => { @@ -316,34 +294,25 @@ describe("hub→sidecar pipeline (failure paths)", () => { rootDir: cacheDir, maxBytes: 100_000_000, }); - const loader = createToolLoader({ - cache, - // Sidecar only knows about a different registry. - registries: new Map([ - ["fixture-registry", { url: "https://registry.test" }], - ]), - host: { os: "linux", cpu: "x64" }, - fetchTarball: async () => fixture.bytes, - }); - - const result = await applyAtomic({ - manifest, - loader, - instanceDir, - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - attemptId: "atp_unknown", - previousDeployId: "dpl_prior", - newDeployId: "dpl_attempted", - }); - expect(result.status).toBe("failed"); - if (result.status !== "failed") return; - expect(result.category).toBe("registry.unknown"); - expect(result.previousDeployId).toBe("dpl_prior"); + await expect( + materializeClosure({ + manifest, + instanceScratchDir: instanceDir, + assetRoot, + assetMounts: new Map(), + gitDirs: new Map(), + host: { os: "linux", cpu: "x64" }, + cache, + // Sidecar only knows about a different registry. + registries: new Map([ + ["fixture-registry", { url: "https://registry.test" }], + ]), + fetchTarball: async () => fixture.bytes, + }), + ).rejects.toMatchObject({ category: "registry.unknown" }); }); - test("manifest.invalid when resolver detects unsatisfied peer deps", async () => { + test("ManifestInvalidError when resolver detects unsatisfied peer deps", async () => { const fetchPackument: PackumentFetcher = async (name) => { if (name === "needs-react") { return { @@ -369,8 +338,7 @@ describe("hub→sidecar pipeline (failure paths)", () => { fetchPackument, }); const resolver = createClosureResolver({ - registries: new Map([[source.name, source]]), - defaultRegistry: source.name, + source, }); let caught: unknown; try { @@ -416,45 +384,28 @@ describe("hub→sidecar pipeline (cache behavior)", () => { maxBytes: 100_000_000, }); let fetchCount = 0; - const loader = createToolLoader({ - cache, - registries, - host: { os: "linux", cpu: "x64" }, - fetchTarball: async () => { - fetchCount += 1; - return fixture.bytes; - }, - importModule: async () => ({ main: fakeFactory("@vendor/cached/main") }), - }); + const materialize = (instanceScratchDir: string) => + materializeClosure({ + manifest, + instanceScratchDir, + assetRoot, + assetMounts: new Map(), + gitDirs: new Map(), + host: { os: "linux", cpu: "x64" }, + cache, + registries, + fetchTarball: async () => { + fetchCount += 1; + return fixture.bytes; + }, + }); // First apply: fetch happens. - const first = await applyAtomic({ - manifest, - loader, - instanceDir: path.join(instanceDir, "first"), - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - attemptId: "atp_first", - previousDeployId: "none", - newDeployId: "dpl_1", - }); - expect(first.status).toBe("ok"); + await materialize(path.join(instanceDir, "first")); expect(fetchCount).toBe(1); // Second apply against a fresh instance dir but the SAME cache. - const second = await applyAtomic({ - manifest, - loader, - instanceDir: path.join(instanceDir, "second"), - assetRoot, - assetMounts: new Map(), - gitDirs: new Map(), - attemptId: "atp_second", - previousDeployId: "dpl_1", - newDeployId: "dpl_2", - }); - expect(second.status).toBe("ok"); + await materialize(path.join(instanceDir, "second")); // Cache hit; fetcher was not called again. expect(fetchCount).toBe(1); }); @@ -469,8 +420,8 @@ describe("hub→sidecar pipeline (asset-backed registry)", () => { * The fixture returns the bound `readBlob`/`listBlobs` pair the * resolver consumes, plus a writeMount helper that lays the same * tarball bytes onto disk at the mount path the mount map will - * point the loader at. End-to-end: the resolver picks an entry, - * the mount map records `(assetId, mount)`, the loader joins + * point materialization at. End-to-end: the resolver picks an entry, + * the mount map records `(assetId, mount)`, materialization joins * `assetRoot + mount + path` and reads the same bytes back. */ async function buildAssetRegistryFixture( @@ -544,25 +495,19 @@ describe("hub→sidecar pipeline (asset-backed registry)", () => { }; } - test("resolver picks asset entries, loader reads them through the mount", async () => { + test("resolver picks asset entries, materialization reads them through the mount", async () => { const fixture = await buildAssetRegistryFixture([ { name: "asset-tool", version: "1.2.3", exportName: "main" }, ]); - const assetId = "asset_workspace_builtins"; + const assetId = "asset_tool_registry"; const assetSource = new AssetRegistrySource({ - name: "workspace-builtins", + name: "tool-registry", assetId, readBlob: fixture.readBlob, listBlobs: fixture.listBlobs, }); - const registries = new Map([ - [assetSource.name, assetSource], - ]); - const resolver = createClosureResolver({ - registries, - defaultRegistry: assetSource.name, - }); + const resolver = createClosureResolver({ source: assetSource }); const pins: ToolPackagePin[] = [{ name: "asset-tool", version: "1.2.3" }]; const manifest = await resolver.resolveClosure(pins); expect(manifest.entries).toHaveLength(1); @@ -585,7 +530,7 @@ describe("hub→sidecar pipeline (asset-backed registry)", () => { const mountPath = `package-registries/${assetSource.name}/`; - // Materialize the asset tarballs onto disk at the mount. The loader's default tarball fetcher + // Materialize the asset tarballs onto disk at the mount. The default tarball fetcher // resolves `kind: "asset"` entries against `assetRoot + mount + path`. await fixture.writeMount(path.join(assetRoot, mountPath)); @@ -593,28 +538,22 @@ describe("hub→sidecar pipeline (asset-backed registry)", () => { rootDir: cacheDir, maxBytes: 100_000_000, }); - const loader = createToolLoader({ - cache, - registries: new Map([["npmjs", { url: "https://r.test" }]]), - host: { os: "linux", cpu: "x64" }, - importModule: async () => ({ - main: fakeFactory("@asset-vendor/tool/main"), - }), - }); - const result = await applyAtomic({ + const { storeDir } = await materializeClosure({ manifest, - loader, - instanceDir, + instanceScratchDir: instanceDir, assetRoot, assetMounts: new Map([[assetId, mountPath]]), gitDirs: new Map(), - attemptId: "atp_asset", - previousDeployId: "none", - newDeployId: "dpl_asset", + host: { os: "linux", cpu: "x64" }, + cache, + registries: new Map([["npmjs", { url: "https://r.test" }]]), + fetchTarball: createTarballFetcher(), }); - expect(result.status).toBe("ok"); - if (result.status !== "ok") return; - expect(result.loaded).toHaveLength(1); - expect(result.loaded[0]?.factories[0]?.id).toBe("@asset-vendor/tool/main"); + expect( + await storedPackageJSON(storeDir, { + name: "asset-tool", + version: "1.2.3", + }), + ).toMatchObject({ name: "asset-tool", version: "1.2.3" }); }); }); diff --git a/tests/workflow-deploy/README.md b/tests/workflow-deploy/README.md index 59149840d..e507d3797 100644 --- a/tests/workflow-deploy/README.md +++ b/tests/workflow-deploy/README.md @@ -6,7 +6,7 @@ latency of the workflow-run substrate. They are `*.bench.ts`, not them via this directory's tsconfig. Both drive one warm agent back-to-back with inference mocked -- message N+1 fires only after reply N (no pipelining) -- and discard the first (cold) message so agent -build, tool materialization, and LSP spawn cost are excluded. The +build, tool construction, and LSP spawn cost are excluded. The steady-state per-message round-trip is what they measure. ## Running diff --git a/tests/workflow-deploy/boot-restore.bench.ts b/tests/workflow-deploy/boot-restore.bench.ts index 277019648..62fa06b0b 100644 --- a/tests/workflow-deploy/boot-restore.bench.ts +++ b/tests/workflow-deploy/boot-restore.bench.ts @@ -270,7 +270,6 @@ const stubApplyFrozenWorkflowClosure: NonNullable< > = (applyArgs) => Promise.resolve({ packageDir: path.join(applyArgs.instanceDir, "package"), - deployDir: path.join(applyArgs.instanceDir, "deploy"), }); const resolvedInboundMailPolicy = { diff --git a/tests/workflow-deploy/child-workflow-roundtrip.test.ts b/tests/workflow-deploy/child-workflow-roundtrip.test.ts index fc56e53a6..7ee23e0aa 100644 --- a/tests/workflow-deploy/child-workflow-roundtrip.test.ts +++ b/tests/workflow-deploy/child-workflow-roundtrip.test.ts @@ -560,11 +560,10 @@ describe.skipIf(!harnessDbEnvAvailable())( }, 180_000); // A tool-bearing child. The child step's agent carries the inline - // `mail_send` tool, so the child runs a REAL tool-bearing agent through the - // source-tools arm -- not a toolless one. The mock inference echoes the - // exposed tool names into the reply, so the child's `StepCompleted` reply - // listing the tool name is the proof the child materialized its tool from - // the shared closure. A regression that ran the child toolless (or failed + // `mail_send` tool, so the child runs a REAL tool-bearing agent -- not a + // toolless one. The mock inference echoes the exposed tool names into the + // reply, so the child's `StepCompleted` reply listing the tool name is the + // proof the child built its tool from the shared closure. A regression that ran the child toolless (or failed // to materialize the source tool) would omit the name from the reply. test("a childWorkflow child runs a real tool-bearing agent", async () => { const parentMailAddress = deriveRunAddress({ @@ -686,7 +685,7 @@ describe.skipIf(!harnessDbEnvAvailable())( const childStepCompleted = childEvents.find( (e) => e.type === "StepCompleted" && e.body["stepId"] === "childStep", ); - // The reply lists the materialized tool name: the child ran a real + // The reply lists the tool name: the child ran a real // tool-bearing agent. A toolless run would carry the bare // "I see these tools:" prefix with no tool name. const childReply = readStepReply(childStepCompleted); diff --git a/tests/workflow-deploy/fixtures/child-workflow.ts b/tests/workflow-deploy/fixtures/child-workflow.ts index 465ffea82..d69c54899 100644 --- a/tests/workflow-deploy/fixtures/child-workflow.ts +++ b/tests/workflow-deploy/fixtures/child-workflow.ts @@ -34,8 +34,7 @@ export type AgentStepSpec = { /** * When set, the step agent carries the inline `mail_send` tool of this * variant, so the mock inference reply lists the tool name -- the proof that - * a childWorkflow step resolves its source-tools arm and runs a real - * tool-bearing agent, not a toolless one. + * a childWorkflow step runs a real tool-bearing agent, not a toolless one. */ tool?: MailToolVariant; }; diff --git a/tests/workflow-deploy/fixtures/credential-tool-bundle.ts b/tests/workflow-deploy/fixtures/credential-tool-bundle.ts index 1e85b658d..bae245a8f 100644 --- a/tests/workflow-deploy/fixtures/credential-tool-bundle.ts +++ b/tests/workflow-deploy/fixtures/credential-tool-bundle.ts @@ -1,8 +1,8 @@ // A real, type-checked credential-consuming tool bundle for the single-step // credential-tool e2e. `credential-tool-workflow.ts` inlines THIS module into // the workflow's source closure via `bundleWorkflowEntry`, so the e2e drives -// the production tool loader against a genuine bundle rather than a -// hand-written string blob. +// the production closure evaluation and `sourceStepToolFactories` against a +// genuine bundle rather than a hand-written string blob. // // The tool resolves one credential handle from the host-assembled // `credentials` capability into an http mediated credential, @@ -33,7 +33,7 @@ export const CREDENTIAL_HANDLE = "api-token"; */ export const BUNDLE_ID = "@intx/tools-credential-probe/sidecar-bundle"; -/** The probe definition's name; a source-ref tool keeps its bare name. */ +/** The name the probe tool is called by. */ export const PROBE_DEFINITION_NAME = "probe"; /** diff --git a/tests/workflow-deploy/fixtures/credential-tool-workflow.ts b/tests/workflow-deploy/fixtures/credential-tool-workflow.ts index b2606156b..e95bc237d 100644 --- a/tests/workflow-deploy/fixtures/credential-tool-workflow.ts +++ b/tests/workflow-deploy/fixtures/credential-tool-workflow.ts @@ -5,13 +5,12 @@ // tenant-owned credential. The returned string is a `@intx/*`-importing entry // module that also imports the tool by absolute path; `bundleWorkflowEntry` // inlines both into a self-contained `.mjs` the sidecar evaluates in-child, so -// the probe runs from the workflow's own source closure rather than a pinned -// tarball. +// the probe runs from the workflow's own source closure. // -// The binding's `package` is the probe factory's `id` (`BUNDLE_ID`). The -// source-ref arm keys a source tool's synthetic `StepToolFactory.packageName` -// to the factory `id`, and a credential is keyed to its consuming tool by -// `toolConsumer(packageName)` on both the delivery descriptor and the Gate-2 +// The binding's `package` is the probe factory's `id` (`BUNDLE_ID`). +// `sourceStepToolFactories` keys a source tool's `factoryId` to the factory +// `id`, and a credential is keyed to its consuming tool by +// `toolConsumer(factoryId)` on both the delivery descriptor and the Gate-2 // grant. So the author names `credentialBindings[].package` = the factory id, // and the delivered credential's consumer matches the source tool at run time. @@ -31,8 +30,8 @@ const CREDENTIAL_TOOL_MODULE = path.join( /** * The credential binding's `package`, which MUST equal the probe factory's - * `id`: the source-ref arm sets the synthetic tool's `packageName` to the - * factory id, and a credential's consumer is `toolConsumer(packageName)`. A + * `id`: `sourceStepToolFactories` sets each factory's `factoryId` to the + * factory id, and a credential's consumer is `toolConsumer(factoryId)`. A * mismatch would deliver the credential to a consumer the source tool does not * key on, so the probe's `resolve` would find no binding. */ diff --git a/tests/workflow-deploy/folded-tools-failover-real-agent.test.ts b/tests/workflow-deploy/folded-tools-failover-real-agent.test.ts index 6f759ede0..526e47b9a 100644 --- a/tests/workflow-deploy/folded-tools-failover-real-agent.test.ts +++ b/tests/workflow-deploy/folded-tools-failover-real-agent.test.ts @@ -9,7 +9,7 @@ // head (HTTP 500) followed by the healthy mock, both operator-approved. The // child's reactor fails over forward off the dead head to the healthy tail -- // which drives the tool call -- so a single run proves (1) the agent's own -// source-closure tool materialized and ran in-child, and (2) whole-workflow +// source-closure tool was built and ran in-child, and (2) whole-workflow // failover is preserved for a single-step agent. import fs from "node:fs"; @@ -266,7 +266,7 @@ describe.skipIf(!harnessDbEnvAvailable())( expect(headRequests).toBeGreaterThanOrEqual(1); expect(env.inference.requests.length).toBeGreaterThanOrEqual(2); - // The materialized tool reached inference -- proof the source closure + // The tool reached inference -- proof the source closure // evaluated in-child and the factory's definition reached inference. const firstServed = env.inference.requests[0]; if (firstServed === undefined) { @@ -332,7 +332,7 @@ describe.skipIf(!harnessDbEnvAvailable())( ]); // A tool-less single-step agent: the negative control is about the - // inference-failure contract, not tool materialization. + // inference-failure contract, not tools. const entryModule = singleStepAgentEntry({ stepId: STEP_ID, systemPrompt: "You are the folded failover negative-control agent.", diff --git a/tests/workflow-deploy/latency-gate.bench.ts b/tests/workflow-deploy/latency-gate.bench.ts index be736588a..980d2221d 100644 --- a/tests/workflow-deploy/latency-gate.bench.ts +++ b/tests/workflow-deploy/latency-gate.bench.ts @@ -38,7 +38,7 @@ // Both paths measure the SUSTAINED interactive case: back-to-back // messages on ONE warm agent, message N+1 fired only after reply N (no // pipelining), and the FIRST (cold) message is discarded so the agent -// build / tool materialization / LSP spawn cost is excluded -- the +// build / tool construction / LSP spawn cost is excluded -- the // steady-state per-message round-trip is what the gate is about. // // Run: diff --git a/tests/workflow-deploy/on-trigger-tool-invoke-roundtrip.test.ts b/tests/workflow-deploy/on-trigger-tool-invoke-roundtrip.test.ts index f0fa5ae18..4d9768794 100644 --- a/tests/workflow-deploy/on-trigger-tool-invoke-roundtrip.test.ts +++ b/tests/workflow-deploy/on-trigger-tool-invoke-roundtrip.test.ts @@ -12,7 +12,7 @@ // deploy path: the capability walk collects its `tool:` grant, the gate // rejects the deploy with `grants_not_approved` if the operator withholds it, // and the frozen projection carries it. At run time the body step invoker -// builds its env with tool materialization skipped, so the provider is handed +// builds its env without the step's tools, so the provider is handed // an EMPTY tool list. No step fails, no error is raised, and no log line is // emitted at any level -- the operator's approval decision simply has no // effect. A test asserting "no error occurred" would pass today and prove diff --git a/tests/workflow-deploy/single-step-credential-tool.test.ts b/tests/workflow-deploy/single-step-credential-tool.test.ts index 18a6f20fe..3fa7e4d3a 100644 --- a/tests/workflow-deploy/single-step-credential-tool.test.ts +++ b/tests/workflow-deploy/single-step-credential-tool.test.ts @@ -15,9 +15,9 @@ // pinned origin -- an in-process mock server that records the Authorization // header it sees. // -// The consumer both ends key on is the probe factory's `id`: the source-ref arm -// sets the synthetic `StepToolFactory.packageName` to the factory id, and the -// author names `credentialBindings[].package` = that same id, so the delivered +// The consumer both ends key on is the probe factory's `id`: +// `sourceStepToolFactories` sets each step factory's `factoryId` to its id, +// and the author names `credentialBindings[].package` = that same id, so the delivered // descriptor's `consumer` and the source tool's runtime consumer match. // // The proofs: @@ -89,7 +89,7 @@ const STEP_ID = "step1"; const AGENT_ID = "single-step-credential-tool-agent"; // The consumer identity Gate 2 checks. On the source path the tool's -// `packageName` is the probe factory's `id`; the binding names that same id in +// `factoryId` is the probe factory's `id`; the binding names that same id in // `package`, so the delivered descriptor's `consumer` and the credential-use // grant's `{ tool }` condition both equal this. const CONSUMER = toolConsumer(CREDENTIAL_BINDING_PACKAGE); diff --git a/tests/workflow-deploy/single-step-posix-tool.test.ts b/tests/workflow-deploy/single-step-posix-tool.test.ts index 678ba139e..41ca9a3c5 100644 --- a/tests/workflow-deploy/single-step-posix-tool.test.ts +++ b/tests/workflow-deploy/single-step-posix-tool.test.ts @@ -1,4 +1,4 @@ -// Phase 2 proof: a step's agent runs a REAL tool materialized in the +// Phase 2 proof: a step's agent runs a REAL tool built in the // spawned workflow-process child. // // Deploys a one-step workflow BY SOURCE-REF whose agent carries the inline diff --git a/tests/workflow-deploy/source-workflow.e2e.test.ts b/tests/workflow-deploy/source-workflow.e2e.test.ts index 62bf25380..897578196 100644 --- a/tests/workflow-deploy/source-workflow.e2e.test.ts +++ b/tests/workflow-deploy/source-workflow.e2e.test.ts @@ -425,8 +425,8 @@ describe.skipIf(!harnessDbEnvAvailable())("source-sourced workflow e2e", () => { // frame -- so the deployment becoming routable again is the load-bearing // restore assertion. Restore runs the full re-materialization // (`resolveDeploymentAssetMounts` asserts the durable git store is present, - // then the loader checks the pinned subtree out of it and re-verifies its - // tree oid) BEFORE it re-registers the address, so an absent or corrupt git + // then `materializeGitEntry` checks the pinned subtree out of it and + // re-verifies its tree oid) BEFORE it re-registers the address, so an absent or corrupt git // store fails the restore closed and the address never re-routes. // // Re-triggering the run is deliberately NOT asserted here: a mail-triggered diff --git a/tests/workflow-deploy/tarball-route-deploy.test.ts b/tests/workflow-deploy/tarball-route-deploy.test.ts index 8c4ee1f39..08daf57eb 100644 --- a/tests/workflow-deploy/tarball-route-deploy.test.ts +++ b/tests/workflow-deploy/tarball-route-deploy.test.ts @@ -1,15 +1,14 @@ // Route-to-run coverage for a workflow deploy sourced from a packed tarball. // // The definition is published as an npm tarball into a `package-registry` -// asset, the same substrate tool packages use, and deployed through the -// PRODUCTION `POST /workflows/deployments` route with `format: "tarball"` and a -// `name@range` pin. The tarball is written through the repo store, so the -// package-registry push validator admits it; the route admits the -// package-registry asset; the real session service resolves the pin from the -// asset's blobs and packs the asset for the probe; the deploy delivers it to a -// live sidecar process. The run is then fired by mail and asserted complete, -// so a 201 alone cannot pass this test: the route, the resolvers and the -// sidecar all have to read the same repository. +// asset and deployed through the PRODUCTION `POST /workflows/deployments` +// route with `format: "tarball"` and a `name@range` pin. The tarball is written +// through the repo store, so the package-registry push validator admits it; the +// route admits the package-registry asset; the real session service resolves +// the pin from the asset's blobs and packs the asset for the probe; the deploy +// delivers it to a live sidecar process. The run is then fired by mail and +// asserted complete, so a 201 alone cannot pass this test: the route, the +// resolvers and the sidecar all have to read the same repository. import { promises as fs } from "node:fs"; import os from "node:os"; diff --git a/tests/workflow/run-local-real-step-invoker.test.ts b/tests/workflow/run-local-real-step-invoker.test.ts index bbf399619..60bb62597 100644 --- a/tests/workflow/run-local-real-step-invoker.test.ts +++ b/tests/workflow/run-local-real-step-invoker.test.ts @@ -73,7 +73,7 @@ interface LocalToolEnv extends BaseEnv { const noArgs = { type: "object", properties: {} } as const; -// One tool package with a plain tool and a credential tool. Each run records +// One tool factory with a plain tool and a credential tool. Each run records // its calls so a test can tell an executed tool from a refused one. function localTools(calls: string[]) { return defineTool({ @@ -210,7 +210,7 @@ describe("runLocal with the real step invoker", () => { }; attachStepTools(env, { factories: req.agent.toolFactories.map((factory) => ({ - packageName: factory.id, + factoryId: factory.id, declaredCredentials: [], factory, })),