Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 26 additions & 12 deletions packages/longevity-sim/src/stack.ts
Original file line number Diff line number Diff line change
Expand Up @@ -437,7 +437,9 @@ export async function bootLongevityStack(
// credential row still requires some string, so each target's own
// `apiKey` (expected to be the same placeholder) is threaded through
// unchanged rather than this package inventing its own convention.
async function seedCatalogChain(target: InferenceTarget): Promise<void> {
async function seedCatalogChain(
target: InferenceTarget,
): Promise<{ offeringId: string }> {
const model = await api(
hub.baseUrl,
"POST",
Expand Down Expand Up @@ -509,10 +511,15 @@ export async function bootLongevityStack(
owner.cookies,
);
expectStatus(`create catalog offering ${target.model}`, offering, 201);
return {
offeringId: stringField(offering.data, "id", "create catalog offering"),
};
}

const offeringIdByTargetLabel = new Map<string, string>();
for (const target of options.realTargets) {
await seedCatalogChain(target);
const { offeringId } = await seedCatalogChain(target);
offeringIdByTargetLabel.set(target.label, offeringId);
}

for (const skill of options.skills ?? []) {
Expand Down Expand Up @@ -668,6 +675,13 @@ export async function bootLongevityStack(
workflowJson,
});

const offeringId = offeringIdByTargetLabel.get(target.label);
if (offeringId === undefined) {
throw new Error(
`agent ${spec.key}: no catalog offering seeded for realTarget "${target.label}"`,
);
}

const deployDeadline = Date.now() + 90_000;
for (;;) {
if (sidecar.exited()) {
Expand All @@ -682,11 +696,8 @@ export async function bootLongevityStack(
workflowDeployBody({
assetId,
commitSha: pushed.commitSha,
sourceId: `src-agent-${spec.handle}`,
provider: target.provider,
baseURL: target.baseURL,
apiKey: target.apiKey,
model: target.model,
sourceOfferingIds: [offeringId],
defaultSourceOfferingId: offeringId,
}),
owner.cookies,
);
Expand Down Expand Up @@ -847,6 +858,12 @@ export async function bootLongevityStack(
if (routineTarget === undefined) {
throw new Error("unreachable: realTargets checked non-empty above");
}
const routineOfferingId = offeringIdByTargetLabel.get(routineTarget.label);
if (routineOfferingId === undefined) {
throw new Error(
`no catalog offering seeded for realTarget "${routineTarget.label}"`,
);
}
const assetName = "longevity-heartbeat";
const heartbeatAssetCreated = await api(
hub.baseUrl,
Expand Down Expand Up @@ -912,11 +929,8 @@ export async function bootLongevityStack(
workflowDeployBody({
assetId: heartbeatAssetId,
commitSha: heartbeatPushed.commitSha,
sourceId: "src-longevity-heartbeat",
provider: "openai-compatible",
baseURL: routineTarget.baseURL,
apiKey: routineTarget.apiKey,
model: routineTarget.model,
sourceOfferingIds: [routineOfferingId],
defaultSourceOfferingId: routineOfferingId,
}),
owner.cookies,
);
Expand Down
48 changes: 43 additions & 5 deletions scripts/e2e/chat.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,47 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => {
log: () => undefined,
});

// The echo deploy resolves against the catalog offering `seedCatalog`
// just planted for the same provider/model this workflow declares,
// rather than seeding a second inference chain.
const echoModelsListed = await api(
"GET",
`/api/tenants/${tenantId}/catalog/models`,
undefined,
user1.cookies,
);
expectStatus("list catalog models", echoModelsListed, 200);
const echoModelRows = (
echoModelsListed.data as { data: { id: string; canonicalName: string }[] }
).data;
const echoModelId = echoModelRows.find(
(row) => row.canonicalName === "claude-sonnet-5",
)?.id;
if (echoModelId === undefined) {
throw new Error(
`no catalog model named "claude-sonnet-5": ${JSON.stringify(echoModelsListed.data)}`,
);
}

const echoOfferingsListed = await api(
"GET",
`/api/tenants/${tenantId}/catalog/offerings`,
undefined,
user1.cookies,
);
expectStatus("list catalog offerings", echoOfferingsListed, 200);
const echoOfferingRows = (
echoOfferingsListed.data as { data: { id: string; modelId: string }[] }
).data;
const echoOfferingId = echoOfferingRows.find(
(row) => row.modelId === echoModelId,
)?.id;
if (echoOfferingId === undefined) {
throw new Error(
`no catalog offering for model "${echoModelId}": ${JSON.stringify(echoOfferingsListed.data)}`,
);
}

// Seed the echo workflow as a deployed, invitable definition: the
// same asset-publish → git-token → smart-HTTP push → native deploy
// path `scripts/e2e/walking-skeleton.test.ts` proves end to end.
Expand Down Expand Up @@ -334,11 +375,8 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => {
workflowDeployBody({
assetId: echoAssetId,
commitSha: echoPushed.commitSha,
sourceId: "src-echo-e2e",
provider: "anthropic",
baseURL: "https://inference.invalid",
apiKey: "e2e-placeholder",
model: "claude-sonnet-5",
sourceOfferingIds: [echoOfferingId],
defaultSourceOfferingId: echoOfferingId,
}),
user1.cookies,
);
Expand Down
128 changes: 67 additions & 61 deletions scripts/e2e/cl-6324-launch-proof.ts
Original file line number Diff line number Diff line change
Expand Up @@ -369,62 +369,77 @@ async function main(): Promise<void> {
// on purpose, so it narrows the bench's own catalog to that model
// through the catalog API rather than leaving the turn's model to a
// coin flip the proof is not about.
await hop("narrow the bench catalog to the pinned model", async () => {
await plantGrant("model-offering:*", "read");
await plantGrant("model-offering:*", "manage");
await plantGrant("model:*", "read");
const pinnedOfferingId = await hop(
"narrow the bench catalog to the pinned model",
async () => {
await plantGrant("model-offering:*", "read");
await plantGrant("model-offering:*", "manage");
await plantGrant("model:*", "read");

const models = await api(
hub.baseUrl,
"GET",
`/api/tenants/${tenant.tenantId}/catalog/models?limit=200`,
undefined,
user.cookies,
);
expectStatus("list the bench catalog models", models, 200);
const modelRows = arrayField(models.data, "data", "catalog models") as {
id: string;
canonicalName: string;
}[];
const pinned = modelRows.find(
(row) => row.canonicalName === proofModelSource.model,
);
if (pinned === undefined) {
throw new Error(
`the bench catalog carries no model named ${proofModelSource.model}; ` +
`it has ${JSON.stringify(modelRows.map((m) => m.canonicalName))}`,
const models = await api(
hub.baseUrl,
"GET",
`/api/tenants/${tenant.tenantId}/catalog/models?limit=200`,
undefined,
user.cookies,
);
}
expectStatus("list the bench catalog models", models, 200);
const modelRows = arrayField(models.data, "data", "catalog models") as {
id: string;
canonicalName: string;
}[];
const pinned = modelRows.find(
(row) => row.canonicalName === proofModelSource.model,
);
if (pinned === undefined) {
throw new Error(
`the bench catalog carries no model named ${proofModelSource.model}; ` +
`it has ${JSON.stringify(modelRows.map((m) => m.canonicalName))}`,
);
}

const offerings = await api(
hub.baseUrl,
"GET",
`/api/tenants/${tenant.tenantId}/catalog/offerings?limit=200`,
undefined,
user.cookies,
);
expectStatus("list the bench model offerings", offerings, 200);
const offeringRows = arrayField(
offerings.data,
"data",
"model offerings",
) as { id: string; modelId: string; disabled: boolean }[];
for (const offering of offeringRows) {
if (offering.modelId === pinned.id || offering.disabled) continue;
const patched = await api(
const offerings = await api(
hub.baseUrl,
"PATCH",
`/api/tenants/${tenant.tenantId}/catalog/offerings/${offering.id}`,
{ disabled: true },
"GET",
`/api/tenants/${tenant.tenantId}/catalog/offerings?limit=200`,
undefined,
user.cookies,
);
expectStatus(`disable offering ${offering.id}`, patched, 200);
}
console.log(
` TRANSCRIPT — bench catalog narrowed to ${proofModelSource.model} ` +
`(${String(offeringRows.length - 1)} other offerings disabled)`,
);
});
expectStatus("list the bench model offerings", offerings, 200);
const offeringRows = arrayField(
offerings.data,
"data",
"model offerings",
) as { id: string; modelId: string; disabled: boolean }[];
let pinnedOffering: { id: string; modelId: string } | undefined;
for (const offering of offeringRows) {
if (offering.modelId === pinned.id) {
pinnedOffering = offering;
continue;
}
if (offering.disabled) continue;
const patched = await api(
hub.baseUrl,
"PATCH",
`/api/tenants/${tenant.tenantId}/catalog/offerings/${offering.id}`,
{ disabled: true },
user.cookies,
);
expectStatus(`disable offering ${offering.id}`, patched, 200);
}
if (pinnedOffering === undefined) {
throw new Error(
`no catalog offering resolves the pinned model ${proofModelSource.model} ` +
`(id ${pinned.id})`,
);
}
console.log(
` TRANSCRIPT — bench catalog narrowed to ${proofModelSource.model} ` +
`(${String(offeringRows.length - 1)} other offerings disabled)`,
);
return pinnedOffering.id;
},
);

const assistantDefinitionId = await hop(
"PROOF 1 — 'assistant' is invitable tenant-wide",
Expand Down Expand Up @@ -694,7 +709,6 @@ async function main(): Promise<void> {
// own durable event log, which is the artefact the milestone's
// `RunStarted` assertion was always about.
const SECTION_ASSET_NAME = "cl6324-section";
const SECTION_SOURCE_ID = "cl6324-section-source";
const SECTION_TURN_TIMEOUT_MS = 180_000;
const sectionAddress = `${SECTION_ASSET_NAME}@${tenant.tenantDomain}`;

Expand Down Expand Up @@ -780,16 +794,8 @@ async function main(): Promise<void> {
package: { format: "source", commitSha: pushed.commitSha },
},
entry: WORKFLOW_SOURCE_ENTRY,
sources: [
{
id: SECTION_SOURCE_ID,
provider: model.provider,
baseURL: model.baseURL,
apiKey: model.apiKey,
model: model.model,
},
],
defaultSource: SECTION_SOURCE_ID,
sourceOfferingIds: [pinnedOfferingId],
defaultSourceOfferingId: pinnedOfferingId,
},
user.cookies,
);
Expand Down
24 changes: 24 additions & 0 deletions scripts/e2e/harness.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
createCleanupHarness,
parseEnvFileDatabaseUrl,
runCleanups,
workflowDeployBody,
type SpawnedApp,
} from "./harness.ts";

Expand Down Expand Up @@ -67,6 +68,29 @@ describe("createCleanupHarness", () => {
});
});

describe("workflowDeployBody", () => {
test("carries catalog offering ids, never a raw provider/apiKey source", () => {
const body = workflowDeployBody({
assetId: "asset-1",
commitSha: "deadbeef",
sourceOfferingIds: ["offering-1", "offering-2"],
defaultSourceOfferingId: "offering-1",
});
expect(body).toEqual({
source: {
kind: "asset",
assetId: "asset-1",
package: { format: "source", commitSha: "deadbeef" },
},
entry: (body as { entry: string }).entry,
sourceOfferingIds: ["offering-1", "offering-2"],
defaultSourceOfferingId: "offering-1",
});
expect(body).not.toHaveProperty("sources");
expect(body).not.toHaveProperty("defaultSource");
});
});

describe("parseEnvFileDatabaseUrl", () => {
test("reads DATABASE_URL and ignores comments and blanks", () => {
expect(
Expand Down
26 changes: 10 additions & 16 deletions scripts/e2e/harness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@
try {
return await run();
} catch (error) {
throw new Error(

Check failure on line 119 in scripts/e2e/harness.ts

View workflow job for this annotation

GitHub Actions / e2e

error: walking-skeleton hop "workflow deploy" failed: deploy workbench-digest workflow: expected HTTP 201

at hop (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:119:15) at async <anonymous> (/home/runner/work/workbench/workbench/scripts/e2e/workbench-digest.test.ts:222:32)

Check failure on line 119 in scripts/e2e/harness.ts

View workflow job for this annotation

GitHub Actions / e2e

error: walking-skeleton hop "the real

at hop (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:119:15)

Check failure on line 119 in scripts/e2e/harness.ts

View workflow job for this annotation

GitHub Actions / e2e

error: walking-skeleton hop "workflow deploy" failed: deploy heartbeat workflow: expected HTTP 201

at hop (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:119:15) at async <anonymous> (/home/runner/work/workbench/workbench/scripts/e2e/smoke-webhook.test.ts:198:34)

Check failure on line 119 in scripts/e2e/harness.ts

View workflow job for this annotation

GitHub Actions / e2e

error: walking-skeleton hop "the real

at hop (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:119:15)

Check failure on line 119 in scripts/e2e/harness.ts

View workflow job for this annotation

GitHub Actions / e2e

error: walking-skeleton hop "workflow deploy" failed: deploy heartbeat workflow: expected HTTP 201

at hop (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:119:15) at async <anonymous> (/home/runner/work/workbench/workbench/scripts/e2e/heartbeat.test.ts:218:32)

Check failure on line 119 in scripts/e2e/harness.ts

View workflow job for this annotation

GitHub Actions / e2e

error: walking-skeleton hop "workflow deploy" failed: deploy workbench-digest workflow: expected HTTP 201

at hop (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:119:15) at async <anonymous> (/home/runner/work/workbench/workbench/scripts/e2e/workbench-digest.test.ts:222:32)

Check failure on line 119 in scripts/e2e/harness.ts

View workflow job for this annotation

GitHub Actions / e2e

error: walking-skeleton hop "the real

at hop (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:119:15)

Check failure on line 119 in scripts/e2e/harness.ts

View workflow job for this annotation

GitHub Actions / e2e

error: walking-skeleton hop "workflow deploy" failed: deploy heartbeat workflow: expected HTTP 201

at hop (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:119:15) at async <anonymous> (/home/runner/work/workbench/workbench/scripts/e2e/smoke-webhook.test.ts:198:34)

Check failure on line 119 in scripts/e2e/harness.ts

View workflow job for this annotation

GitHub Actions / e2e

error: walking-skeleton hop "the real

at hop (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:119:15)

Check failure on line 119 in scripts/e2e/harness.ts

View workflow job for this annotation

GitHub Actions / e2e

error: walking-skeleton hop "workflow deploy" failed: deploy heartbeat workflow: expected HTTP 201

at hop (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:119:15) at async <anonymous> (/home/runner/work/workbench/workbench/scripts/e2e/heartbeat.test.ts:218:32)
`walking-skeleton hop "${name}" failed: ${
error instanceof Error ? error.message : String(error)
}`,
Expand Down Expand Up @@ -485,7 +485,7 @@
expected: number,
): void {
if (result.status !== expected) {
throw new Error(

Check failure on line 488 in scripts/e2e/harness.ts

View workflow job for this annotation

GitHub Actions / e2e

error: deploy workbench-digest workflow: expected HTTP 201

at expectStatus (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:488:15) at <anonymous> (/home/runner/work/workbench/workbench/scripts/e2e/workbench-digest.test.ts:253:7) at async hop (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:117:18) at async <anonymous> (/home/runner/work/workbench/workbench/scripts/e2e/workbench-digest.test.ts:222:32)

Check failure on line 488 in scripts/e2e/harness.ts

View workflow job for this annotation

GitHub Actions / e2e

error: deploy heartbeat workflow: expected HTTP 201

at expectStatus (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:488:15) at <anonymous> (/home/runner/work/workbench/workbench/scripts/e2e/smoke-webhook.test.ts:228:11) at async hop (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:117:18) at async <anonymous> (/home/runner/work/workbench/workbench/scripts/e2e/smoke-webhook.test.ts:198:34)

Check failure on line 488 in scripts/e2e/harness.ts

View workflow job for this annotation

GitHub Actions / e2e

error: deploy heartbeat workflow: expected HTTP 201

at expectStatus (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:488:15) at <anonymous> (/home/runner/work/workbench/workbench/scripts/e2e/heartbeat.test.ts:249:7) at async hop (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:117:18) at async <anonymous> (/home/runner/work/workbench/workbench/scripts/e2e/heartbeat.test.ts:218:32)

Check failure on line 488 in scripts/e2e/harness.ts

View workflow job for this annotation

GitHub Actions / e2e

error: deploy workbench-digest workflow: expected HTTP 201

at expectStatus (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:488:15) at <anonymous> (/home/runner/work/workbench/workbench/scripts/e2e/workbench-digest.test.ts:253:7) at async hop (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:117:18) at async <anonymous> (/home/runner/work/workbench/workbench/scripts/e2e/workbench-digest.test.ts:222:32)

Check failure on line 488 in scripts/e2e/harness.ts

View workflow job for this annotation

GitHub Actions / e2e

error: deploy heartbeat workflow: expected HTTP 201

at expectStatus (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:488:15) at <anonymous> (/home/runner/work/workbench/workbench/scripts/e2e/smoke-webhook.test.ts:228:11) at async hop (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:117:18) at async <anonymous> (/home/runner/work/workbench/workbench/scripts/e2e/smoke-webhook.test.ts:198:34)

Check failure on line 488 in scripts/e2e/harness.ts

View workflow job for this annotation

GitHub Actions / e2e

error: deploy heartbeat workflow: expected HTTP 201

at expectStatus (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:488:15) at <anonymous> (/home/runner/work/workbench/workbench/scripts/e2e/heartbeat.test.ts:249:7) at async hop (/home/runner/work/workbench/workbench/scripts/e2e/harness.ts:117:18) at async <anonymous> (/home/runner/work/workbench/workbench/scripts/e2e/heartbeat.test.ts:218:32)
`${what}: expected HTTP ${expected}, got ${result.status}: ` +
JSON.stringify(result.data),
);
Expand Down Expand Up @@ -653,16 +653,18 @@
/**
* The deploy body a code-sourced asset deployment takes: the pushed
* commit is the definition's pin, and the entry names the
* `interchange.workflow` module the sidecar evaluates.
* `interchange.workflow` module the sidecar evaluates. The native route
* resolves inference against the tenant's own catalog, so the caller
* supplies the ordered catalog offering ids to deploy against rather
* than a raw provider/baseURL/apiKey triple — see
* `@corbits/seeding`'s `ensureNoopCatalogOffering` for the zero-cost way
* to obtain one.
*/
export function workflowDeployBody(options: {
assetId: string;
commitSha: string;
sourceId: string;
provider: string;
baseURL: string;
apiKey: string;
model: string;
sourceOfferingIds: string[];
defaultSourceOfferingId: string;
}): Record<string, unknown> {
return {
source: {
Expand All @@ -671,15 +673,7 @@
package: { format: "source", commitSha: options.commitSha },
},
entry: WORKFLOW_SOURCE_ENTRY,
sources: [
{
id: options.sourceId,
provider: options.provider,
baseURL: options.baseURL,
apiKey: options.apiKey,
model: options.model,
},
],
defaultSource: options.sourceId,
sourceOfferingIds: options.sourceOfferingIds,
defaultSourceOfferingId: options.defaultSourceOfferingId,
};
}
Loading
Loading