From d434f5fc1574dd2c3729f1e5d6aed62b3db01a1a Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 17:37:47 -0700 Subject: [PATCH] chore(packages)!: delete @corbits/evals (CL-8335) --- bun.lock | 65 -- package.json | 1 - packages/evals/LICENSE | 176 ------ packages/evals/README.md | 69 -- packages/evals/eval-report.md | 5 - packages/evals/package.json | 43 -- packages/evals/src/cases/ai-daily-research.ts | 64 -- .../evals/src/cases/docs-on-sdk-change.ts | 64 -- packages/evals/src/cases/exa-topic-digest.ts | 66 -- packages/evals/src/cases/index.ts | 14 - packages/evals/src/define-eval.test.ts | 39 -- packages/evals/src/define-eval.ts | 44 -- packages/evals/src/fakes/github-rest-fake.ts | 66 -- .../evals/src/fakes/mcp-fake-server.test.ts | 66 -- packages/evals/src/fakes/mcp-fake-server.ts | 117 ---- packages/evals/src/fakes/recording.test.ts | 38 -- packages/evals/src/fakes/recording.ts | 61 -- packages/evals/src/fakes/recordings.ts | 8 - .../evals/src/fakes/recordings/github.json | 32 - packages/evals/src/index.ts | 70 -- packages/evals/src/model-call.test.ts | 42 -- packages/evals/src/model-call.ts | 38 -- packages/evals/src/persona-runner.test.ts | 135 ---- packages/evals/src/persona-runner.ts | 42 -- packages/evals/src/persona.test.ts | 53 -- packages/evals/src/persona.ts | 71 --- packages/evals/src/report.test.ts | 66 -- packages/evals/src/report.ts | 39 -- packages/evals/src/runner.test.ts | 316 --------- packages/evals/src/runner.ts | 131 ---- packages/evals/src/scorers/scorers.test.ts | 598 ------------------ packages/evals/src/scorers/scorers.ts | 552 ---------------- packages/evals/src/scorers/tool-names.test.ts | 34 - packages/evals/src/scorers/tool-names.ts | 31 - .../evals/src/scorers/world-scorers.test.ts | 112 ---- packages/evals/src/scorers/world-scorers.ts | 70 -- packages/evals/src/store/migrations.ts | 53 -- packages/evals/src/store/pg-store.ts | 100 --- packages/evals/src/store/schema.ts | 28 - packages/evals/src/store/store.test.ts | 123 ---- packages/evals/src/store/store.ts | 18 - .../evals/src/targets/git-workflow-push.ts | 215 ------- packages/evals/src/targets/json-fields.ts | 18 - packages/evals/src/targets/trace.test.ts | 84 --- packages/evals/src/targets/trace.ts | 121 ---- .../evals/src/targets/world-snapshot.test.ts | 261 -------- packages/evals/src/targets/world-snapshot.ts | 144 ----- packages/evals/src/types.ts | 217 ------- packages/evals/tsconfig.json | 8 - packages/migration-runner/README.md | 6 +- 50 files changed, 3 insertions(+), 4831 deletions(-) delete mode 100644 packages/evals/LICENSE delete mode 100644 packages/evals/README.md delete mode 100644 packages/evals/eval-report.md delete mode 100644 packages/evals/package.json delete mode 100644 packages/evals/src/cases/ai-daily-research.ts delete mode 100644 packages/evals/src/cases/docs-on-sdk-change.ts delete mode 100644 packages/evals/src/cases/exa-topic-digest.ts delete mode 100644 packages/evals/src/cases/index.ts delete mode 100644 packages/evals/src/define-eval.test.ts delete mode 100644 packages/evals/src/define-eval.ts delete mode 100644 packages/evals/src/fakes/github-rest-fake.ts delete mode 100644 packages/evals/src/fakes/mcp-fake-server.test.ts delete mode 100644 packages/evals/src/fakes/mcp-fake-server.ts delete mode 100644 packages/evals/src/fakes/recording.test.ts delete mode 100644 packages/evals/src/fakes/recording.ts delete mode 100644 packages/evals/src/fakes/recordings.ts delete mode 100644 packages/evals/src/fakes/recordings/github.json delete mode 100644 packages/evals/src/index.ts delete mode 100644 packages/evals/src/model-call.test.ts delete mode 100644 packages/evals/src/model-call.ts delete mode 100644 packages/evals/src/persona-runner.test.ts delete mode 100644 packages/evals/src/persona-runner.ts delete mode 100644 packages/evals/src/persona.test.ts delete mode 100644 packages/evals/src/persona.ts delete mode 100644 packages/evals/src/report.test.ts delete mode 100644 packages/evals/src/report.ts delete mode 100644 packages/evals/src/runner.test.ts delete mode 100644 packages/evals/src/runner.ts delete mode 100644 packages/evals/src/scorers/scorers.test.ts delete mode 100644 packages/evals/src/scorers/scorers.ts delete mode 100644 packages/evals/src/scorers/tool-names.test.ts delete mode 100644 packages/evals/src/scorers/tool-names.ts delete mode 100644 packages/evals/src/scorers/world-scorers.test.ts delete mode 100644 packages/evals/src/scorers/world-scorers.ts delete mode 100644 packages/evals/src/store/migrations.ts delete mode 100644 packages/evals/src/store/pg-store.ts delete mode 100644 packages/evals/src/store/schema.ts delete mode 100644 packages/evals/src/store/store.test.ts delete mode 100644 packages/evals/src/store/store.ts delete mode 100644 packages/evals/src/targets/git-workflow-push.ts delete mode 100644 packages/evals/src/targets/json-fields.ts delete mode 100644 packages/evals/src/targets/trace.test.ts delete mode 100644 packages/evals/src/targets/trace.ts delete mode 100644 packages/evals/src/targets/world-snapshot.test.ts delete mode 100644 packages/evals/src/targets/world-snapshot.ts delete mode 100644 packages/evals/src/types.ts delete mode 100644 packages/evals/tsconfig.json diff --git a/bun.lock b/bun.lock index 7fdfa565f0..7aee71829a 100644 --- a/bun.lock +++ b/bun.lock @@ -11,7 +11,6 @@ "@corbits/artifacts": "github:corbitsdev/corbits-artifacts#5309a15d4aa504f132c54b4da08bb672b4903873", "@corbits/codex-provider": "github:corbitsdev/corbits-codex-provider#a1f475f341261c05fa8dade0c82f1246c175b5b3", "@corbits/connections": "workspace:*", - "@corbits/evals": "workspace:*", "@corbits/mailbox": "github:corbitsdev/corbits-mailbox#e85b81e4a2e0cc318dd33a4f0f92337b6fac7f38", "@corbits/ollama-adapter": "workspace:*", "@corbits/webhook-triggers": "workspace:*", @@ -338,38 +337,6 @@ "@intx/log": "0.3.0", }, }, - "packages/evals": { - "name": "@corbits/evals", - "version": "0.0.1", - "dependencies": { - "@corbits/agent-directory": "workspace:*", - "@corbits/connections": "workspace:*", - "@corbits/error-sink": "workspace:*", - "@corbits/hub-api-client": "workspace:*", - "@corbits/migration-runner": "workspace:*", - "@corbits/myra": "workspace:*", - "@corbits/webhook-triggers": "workspace:*", - "@corbits/workflows": "workspace:*", - "@intx/agent": "workspace:*", - "@intx/db": "workspace:*", - "@intx/hub-sessions": "workspace:*", - "@intx/log": "0.3.0", - "@intx/types": "workspace:*", - "@intx/workflow": "workspace:*", - "@modelcontextprotocol/sdk": "catalog:", - "arktype": "catalog:", - "drizzle-orm": "catalog:", - "postgres": "catalog:", - }, - "devDependencies": { - "@corbits/agent-directory-tools": "workspace:*", - "@corbits/connections-tools": "workspace:*", - "@corbits/github-tools": "workspace:*", - "@corbits/mcp-tools": "workspace:*", - "@types/bun": "catalog:", - "typescript": "catalog:", - }, - }, "packages/hub-api-client": { "name": "@corbits/hub-api-client", "version": "0.0.1", @@ -1185,8 +1152,6 @@ "@corbits/error-sink": ["@corbits/error-sink@workspace:packages/error-sink"], - "@corbits/evals": ["@corbits/evals@workspace:packages/evals"], - "@corbits/github-tools": ["@corbits/github-tools@workspace:tools/github"], "@corbits/gotenberg": ["@corbits/gotenberg@workspace:tools/gotenberg"], @@ -2403,10 +2368,6 @@ "@corbits/codex-provider/@corbits/oauth-core": ["@corbits/oauth-core@github:corbitsdev/corbits-oauth-core#1b7f9fb", { "dependencies": { "arktype": "2.2.3" } }, "corbitsdev-corbits-oauth-core-1b7f9fb", "sha512-PscyiDAl46umTM44NQPN1q5+PsGUV53Rxr4Wz/Ofz+k9LOOz6FBKamRFtWR8YWdKJt9I8sPOLOyEcMNNN2i+YQ=="], - "@corbits/connections/@corbits/codex-provider": ["@corbits/codex-provider@github:corbitsdev/corbits-codex-provider#a1f475f", { "dependencies": { "@corbits/oauth-core": "github:corbitsdev/corbits-oauth-core", "@corbits/openai-responses": "github:corbitsdev/corbits-openai-responses", "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-codex-provider-a1f475f", "sha512-D6d0lblWx/4KudfLwQoQGrLB8GKRLP/HVp1cx08L2/hOGZAgH6txLPpeEtBqjl+R520qXSSPfM0kDC+3XNaAfw=="], - - "@corbits/connections/@corbits/xai-provider": ["@corbits/xai-provider@github:corbitsdev/corbits-xai-provider#9f2d4ba", { "dependencies": { "@corbits/oauth-core": "github:corbitsdev/corbits-oauth-core", "@corbits/openai-responses": "github:corbitsdev/corbits-openai-responses", "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-xai-provider-9f2d4ba", "sha512-ZjvKxGcp+QzCh8zeXHg7kNbtkpR/itZkmQAcdG1aOAOjHaNwfGVpkPc/c/XtwPNsQyaDjF8W9CHb4rH1xI2uCA=="], - "@corbits/mailbox/hono-openapi": ["hono-openapi@1.3.1", "", { "peerDependencies": { "@hono/standard-validator": "^0.2.0", "@standard-community/standard-json": "^0.3.5", "@standard-community/standard-openapi": "^0.2.9", "@types/json-schema": "^7.0.15", "hono": "^4.11.2", "openapi-types": "^12.1.3" }, "optionalPeers": ["@hono/standard-validator", "hono"] }, "sha512-NLVeVkhKZ3drmQNEIPac8HX8Y54uf1hJAgIM/7MfDsaeVVmB+QILWQxx5x3R3NvRHgedcbEbOCGY2uR7WQYyMw=="], "@corbits/xai-provider/@corbits/oauth-core": ["@corbits/oauth-core@github:corbitsdev/corbits-oauth-core#1b7f9fb", { "dependencies": { "arktype": "2.2.3" } }, "corbitsdev-corbits-oauth-core-1b7f9fb", "sha512-PscyiDAl46umTM44NQPN1q5+PsGUV53Rxr4Wz/Ofz+k9LOOz6FBKamRFtWR8YWdKJt9I8sPOLOyEcMNNN2i+YQ=="], @@ -2427,16 +2388,8 @@ "@tailwindcss/oxide-wasm32-wasi/tslib": ["tslib@2.8.1", "", { "bundled": true }, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], - "@workbench/hub/@corbits/artifacts": ["@corbits/artifacts@github:corbitsdev/corbits-artifacts#5309a15", { "dependencies": { "@hono/standard-validator": "^0.2.3" }, "peerDependencies": { "@intx/hub-api": "^0.3.0", "@intx/types": "^0.3.0", "arktype": "^2.1.29", "drizzle-orm": "^0.45.2", "hono": "^4.12.32", "hono-openapi": "^1.2.0", "postgres": "^3.4.9" } }, "corbitsdev-corbits-artifacts-5309a15", "sha512-zGHkq++juclUrOd0O8/Z5ziTldmJZ6gqyH/c9cs5S10UPU4gqn8CHwq174SiJXY0ujJ3cNpJXnrr+0fcqlfQ1w=="], - - "@workbench/hub/@corbits/codex-provider": ["@corbits/codex-provider@github:corbitsdev/corbits-codex-provider#a1f475f", { "dependencies": { "@corbits/oauth-core": "github:corbitsdev/corbits-oauth-core", "@corbits/openai-responses": "github:corbitsdev/corbits-openai-responses", "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-codex-provider-a1f475f", "sha512-D6d0lblWx/4KudfLwQoQGrLB8GKRLP/HVp1cx08L2/hOGZAgH6txLPpeEtBqjl+R520qXSSPfM0kDC+3XNaAfw=="], - - "@workbench/hub/@corbits/mailbox": ["@corbits/mailbox@github:corbitsdev/corbits-mailbox#e85b81e", { "dependencies": { "@hono/standard-validator": "0.2.3", "@intx/crypto": "0.3.0", "@standard-community/standard-json": "0.3.5", "@standard-community/standard-openapi": "0.2.9", "arktype": "2.1.29", "hono-openapi": "1.3.1" }, "peerDependencies": { "@intx/log": "^0.2.2", "drizzle-orm": "^0.45.2", "hono": "^4.12.0", "postgres": "^3.4.0" } }, "corbitsdev-corbits-mailbox-e85b81e", "sha512-ATPP/pll1sYG1DLKV1HvtboK8q1tFkqayQqRXdUmRUIsFJx5wqzd8YA1XELjXzLu1P2f2uCMe2CjsJHGv6E90Q=="], - "@workbench/hub/@corbits/memory": ["@corbits/memory@github:corbitsdev/corbits-memory#e74da20", { "dependencies": { "@intx/agent": "0.2.2", "@intx/authz": "0.2.2", "@intx/hub-api": "0.2.2", "@intx/log": "0.2.2", "@intx/workflow": "0.2.2", "arktype": "^2.1.29", "drizzle-orm": "^0.45.1", "hono": "^4.9.0", "hono-openapi": "^1.3.1", "postgres": "^3.4.7" } }, "corbitsdev-corbits-memory-e74da20", "sha512-/uA9xTcF7XwMXgnR8OZamierzkLG4g4mhw3u9Fl3Uj6ait8pH3wcoQ6V32on1nMdaqtPRdYVFnxof5O+4ZacGA=="], - "@workbench/hub/@corbits/xai-provider": ["@corbits/xai-provider@github:corbitsdev/corbits-xai-provider#9f2d4ba", { "dependencies": { "@corbits/oauth-core": "github:corbitsdev/corbits-oauth-core", "@corbits/openai-responses": "github:corbitsdev/corbits-openai-responses", "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-xai-provider-9f2d4ba", "sha512-ZjvKxGcp+QzCh8zeXHg7kNbtkpR/itZkmQAcdG1aOAOjHaNwfGVpkPc/c/XtwPNsQyaDjF8W9CHb4rH1xI2uCA=="], - "better-call/@better-auth/utils": ["@better-auth/utils@0.5.0", "", { "dependencies": { "@noble/hashes": "^2.0.1" } }, "sha512-BL8W4EfIZFwlu0r54m3v1ztjDhu6dDe/amLTm0xybmbZaNgYUqhD3SjpAsnq0q8YD6/ki4iwIgxJNLP/N3TxiA=="], "body-parser/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], @@ -2483,14 +2436,6 @@ "@babel/helper-compilation-targets/lru-cache/yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="], - "@corbits/connections/@corbits/codex-provider/@corbits/oauth-core": ["@corbits/oauth-core@github:corbitsdev/corbits-oauth-core#1b7f9fb", { "dependencies": { "arktype": "2.2.3" } }, "corbitsdev-corbits-oauth-core-1b7f9fb", "sha512-PscyiDAl46umTM44NQPN1q5+PsGUV53Rxr4Wz/Ofz+k9LOOz6FBKamRFtWR8YWdKJt9I8sPOLOyEcMNNN2i+YQ=="], - - "@corbits/connections/@corbits/codex-provider/@corbits/openai-responses": ["@corbits/openai-responses@github:corbitsdev/corbits-openai-responses#7efe000", { "dependencies": { "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-openai-responses-7efe000", "sha512-pH6C8T1WDJjqM+YbnQpofd7HgP5krEj48JCCQK16KI98a6kaz+gffpgdOdQpU3aNVZnFlyADXod1Z4tqUG/3ww=="], - - "@corbits/connections/@corbits/xai-provider/@corbits/oauth-core": ["@corbits/oauth-core@github:corbitsdev/corbits-oauth-core#1b7f9fb", { "dependencies": { "arktype": "2.2.3" } }, "corbitsdev-corbits-oauth-core-1b7f9fb", "sha512-PscyiDAl46umTM44NQPN1q5+PsGUV53Rxr4Wz/Ofz+k9LOOz6FBKamRFtWR8YWdKJt9I8sPOLOyEcMNNN2i+YQ=="], - - "@corbits/connections/@corbits/xai-provider/@corbits/openai-responses": ["@corbits/openai-responses@github:corbitsdev/corbits-openai-responses#7efe000", { "dependencies": { "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-openai-responses-7efe000", "sha512-pH6C8T1WDJjqM+YbnQpofd7HgP5krEj48JCCQK16KI98a6kaz+gffpgdOdQpU3aNVZnFlyADXod1Z4tqUG/3ww=="], - "@esbuild-kit/core-utils/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.18.20", "", { "os": "android", "cpu": "arm" }, "sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw=="], "@esbuild-kit/core-utils/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.18.20", "", { "os": "android", "cpu": "arm64" }, "sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ=="], @@ -2535,16 +2480,6 @@ "@esbuild-kit/core-utils/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.18.20", "", { "os": "win32", "cpu": "x64" }, "sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ=="], - "@workbench/hub/@corbits/codex-provider/@corbits/oauth-core": ["@corbits/oauth-core@github:corbitsdev/corbits-oauth-core#1b7f9fb", { "dependencies": { "arktype": "2.2.3" } }, "corbitsdev-corbits-oauth-core-1b7f9fb", "sha512-PscyiDAl46umTM44NQPN1q5+PsGUV53Rxr4Wz/Ofz+k9LOOz6FBKamRFtWR8YWdKJt9I8sPOLOyEcMNNN2i+YQ=="], - - "@workbench/hub/@corbits/codex-provider/@corbits/openai-responses": ["@corbits/openai-responses@github:corbitsdev/corbits-openai-responses#7efe000", { "dependencies": { "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-openai-responses-7efe000", "sha512-pH6C8T1WDJjqM+YbnQpofd7HgP5krEj48JCCQK16KI98a6kaz+gffpgdOdQpU3aNVZnFlyADXod1Z4tqUG/3ww=="], - - "@workbench/hub/@corbits/mailbox/hono-openapi": ["hono-openapi@1.3.1", "", { "peerDependencies": { "@hono/standard-validator": "^0.2.0", "@standard-community/standard-json": "^0.3.5", "@standard-community/standard-openapi": "^0.2.9", "@types/json-schema": "^7.0.15", "hono": "^4.11.2", "openapi-types": "^12.1.3" }, "optionalPeers": ["@hono/standard-validator", "hono"] }, "sha512-NLVeVkhKZ3drmQNEIPac8HX8Y54uf1hJAgIM/7MfDsaeVVmB+QILWQxx5x3R3NvRHgedcbEbOCGY2uR7WQYyMw=="], - - "@workbench/hub/@corbits/xai-provider/@corbits/oauth-core": ["@corbits/oauth-core@github:corbitsdev/corbits-oauth-core#1b7f9fb", { "dependencies": { "arktype": "2.2.3" } }, "corbitsdev-corbits-oauth-core-1b7f9fb", "sha512-PscyiDAl46umTM44NQPN1q5+PsGUV53Rxr4Wz/Ofz+k9LOOz6FBKamRFtWR8YWdKJt9I8sPOLOyEcMNNN2i+YQ=="], - - "@workbench/hub/@corbits/xai-provider/@corbits/openai-responses": ["@corbits/openai-responses@github:corbitsdev/corbits-openai-responses#7efe000", { "dependencies": { "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-openai-responses-7efe000", "sha512-pH6C8T1WDJjqM+YbnQpofd7HgP5krEj48JCCQK16KI98a6kaz+gffpgdOdQpU3aNVZnFlyADXod1Z4tqUG/3ww=="], - "cross-spawn/which/isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], "minipass-flush/minipass/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="], diff --git a/package.json b/package.json index 308d7e6292..863accf74b 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,6 @@ "devDependencies": { "@corbits/artifacts": "github:corbitsdev/corbits-artifacts#5309a15d4aa504f132c54b4da08bb672b4903873", "@corbits/codex-provider": "github:corbitsdev/corbits-codex-provider#a1f475f341261c05fa8dade0c82f1246c175b5b3", - "@corbits/evals": "workspace:*", "@corbits/ollama-adapter": "workspace:*", "@corbits/xai-provider": "github:corbitsdev/corbits-xai-provider#9f2d4bac40ea075df092fef807404a13c638cf14", "@intx/db": "workspace:*", diff --git a/packages/evals/LICENSE b/packages/evals/LICENSE deleted file mode 100644 index 173c3a21d0..0000000000 --- a/packages/evals/LICENSE +++ /dev/null @@ -1,176 +0,0 @@ -GNU LESSER GENERAL PUBLIC LICENSE - -Version 2.1, February 1999 - -Copyright (C) 1991, 1999 Free Software Foundation, Inc. -51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. - -[This is the first released version of the Lesser GPL. It also counts as the successor of the GNU Library Public License, version 2, hence the version number 2.1.] - -Preamble - -The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public Licenses are intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. - -This license, the Lesser General Public License, applies to some specially designated software packages--typically libraries--of the Free Software Foundation and other authors who decide to use it. You can use it too, but we suggest you first think carefully about whether this license or the ordinary General Public License is the better strategy to use in any particular case, based on the explanations below. - -When we speak of free software, we are referring to freedom of use, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish); that you receive source code or can get it if you want it; that you can change the software and use pieces of it in new free programs; and that you are informed that you can do these things. - -To protect your rights, we need to make restrictions that forbid distributors to deny you these rights or to ask you to surrender these rights. These restrictions translate to certain responsibilities for you if you distribute copies of the library or if you modify it. - -For example, if you distribute copies of the library, whether gratis or for a fee, you must give the recipients all the rights that we gave you. You must make sure that they, too, receive or can get the source code. If you link other code with the library, you must provide complete object files to the recipients, so that they can relink them with the library after making changes to the library and recompiling it. And you must show them these terms so they know their rights. - -We protect your rights with a two-step method: (1) we copyright the library, and (2) we offer you this license, which gives you legal permission to copy, distribute and/or modify the library. - -To protect each distributor, we want to make it very clear that there is no warranty for the free library. Also, if the library is modified by someone else and passed on, the recipients should know that what they have is not the original version, so that the original author's reputation will not be affected by problems that might be introduced by others. - -Finally, software patents pose a constant threat to the existence of any free program. We wish to make sure that a company cannot effectively restrict the users of a free program by obtaining a restrictive license from a patent holder. Therefore, we insist that any patent license obtained for a version of the library must be consistent with the full freedom of use specified in this license. - -Most GNU software, including some libraries, is covered by the ordinary GNU General Public License. This license, the GNU Lesser General Public License, applies to certain designated libraries, and is quite different from the ordinary General Public License. We use this license for certain libraries in order to permit linking those libraries into non-free programs. - -When a program is linked with a library, whether statically or using a shared library, the combination of the two is legally speaking a combined work, a derivative of the original library. The ordinary General Public License therefore permits such linking only if the entire combination fits its criteria of freedom. The Lesser General Public License permits more lax criteria for linking other code with the library. - -We call this license the "Lesser" General Public License because it does Less to protect the user's freedom than the ordinary General Public License. It also provides other free software developers Less of an advantage over competing non-free programs. These disadvantages are the reason we use the ordinary General Public License for many libraries. However, the Lesser license provides advantages in certain special circumstances. - -For example, on rare occasions, there may be a special need to encourage the widest possible use of a certain library, so that it becomes a de-facto standard. To achieve this, non-free programs must be allowed to use the library. A more frequent case is that a free library does the same job as widely used non-free libraries. In this case, there is little to gain by limiting the free library to free software only, so we use the Lesser General Public License. - -In other cases, permission to use a particular library in non-free programs enables a greater number of people to use a large body of free software. For example, permission to use the GNU C Library in non-free programs enables many more people to use the whole GNU operating system, as well as its variant, the GNU/Linux operating system. - -Although the Lesser General Public License is Less protective of the users' freedom, it does ensure that the user of a program that is linked with the Library has the freedom and the wherewithal to run that program using a modified version of the Library. - -The precise terms and conditions for copying, distribution and modification follow. Pay close attention to the difference between a "work based on the library" and a "work that uses the library". The former contains code derived from the library, whereas the latter must be combined with the library in order to run. - -GNU LESSER GENERAL PUBLIC LICENSE -TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - -0. This License Agreement applies to any software library or other program which contains a notice placed by the copyright holder or other authorized party saying it may be distributed under the terms of this Lesser General Public License (also called "this License"). Each licensee is addressed as "you". - -A "library" means a collection of software functions and/or data prepared so as to be conveniently linked with application programs (which use some of those functions and data) to form executables. - -The "Library", below, refers to any such software library or work which has been distributed under these terms. A "work based on the Library" means either the Library or any derivative work under copyright law: that is to say, a work containing the Library or a portion of it, either verbatim or with modifications and/or translated straightforwardly into another language. (Hereinafter, translation is included without limitation in the term "modification".) - -"Source code" for a work means the preferred form of the work for making modifications to it. For a library, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the library. - -Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running a program using the Library is not restricted, and output from such a program is covered only if its contents constitute a work based on the Library (independent of the use of the Library in a tool for writing it). Whether that is true depends on what the Library does and what the program that uses the Library does. - -1. You may copy and distribute verbatim copies of the Library's complete source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and distribute a copy of this License along with the Library. - -You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. - -2. You may modify your copy or copies of the Library or any portion of it, thus forming a work based on the Library, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: - - a) The modified work must itself be a software library. - - b) You must cause the files modified to carry prominent notices stating that you changed the files and the date of any change. - - c) You must cause the whole of the work to be licensed at no charge to all third parties under the terms of this License. - - d) If a facility in the modified Library refers to a function or a table of data to be supplied by an application program that uses the facility, other than as an argument passed when the facility is invoked, then you must make a good faith effort to ensure that, in the event an application does not supply such function or table, the facility still operates, and performs whatever part of its purpose remains meaningful. - -(For example, a function in a library to compute square roots has a purpose that is entirely well-defined independent of the application. Therefore, Subsection 2d requires that any application-supplied function or table used by this function must be optional: if the application does not supply it, the square root function must still compute square roots.) - -These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Library, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Library, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Library. - -In addition, mere aggregation of another work not based on the Library with the Library (or with a work based on the Library) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. - -3. You may opt to apply the terms of the ordinary GNU General Public License instead of this License to a given copy of the Library. To do this, you must alter all the notices that refer to this License, so that they refer to the ordinary GNU General Public License, version 2, instead of to this License. (If a newer version than version 2 of the ordinary GNU General Public License has appeared, then you can specify that version instead if you wish.) Do not make any other change in these notices. - -Once this change is made in a given copy, it is irreversible for that copy, so the ordinary GNU General Public License applies to all subsequent copies and derivative works made from that copy. - -This option is useful when you wish to copy part of the code of the Library into a program that is not a library. - -4. You may copy and distribute the Library (or a portion or derivative of it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange. - -If distribution of object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place satisfies the requirement to distribute the source code, even though third parties are not compelled to copy the source along with the object code. - -5. A program that contains no derivative of any portion of the Library, but is designed to work with the Library by being compiled or linked with it, is called a "work that uses the Library". Such a work, in isolation, is not a derivative work of the Library, and therefore falls outside the scope of this License. - -However, linking a "work that uses the Library" with the Library creates an executable that is a derivative of the Library (because it contains portions of the Library), rather than a "work that uses the library". The executable is therefore covered by this License. Section 6 states terms for distribution of such executables. - -When a "work that uses the Library" uses material from a header file that is part of the Library, the object code for the work may be a derivative work of the Library even though the source code is not. Whether this is true is especially significant if the work can be linked without the Library, or if the work is itself a library. The threshold for this to be true is not precisely defined by law. - -If such an object file uses only numerical parameters, data structure layouts and accessors, and small macros and small inline functions (ten lines or less in length), then the use of the object file is unrestricted, regardless of whether it is legally a derivative work. (Executables containing this object code plus portions of the Library will still fall under Section 6.) - -Otherwise, if the work is a derivative of the Library, you may distribute the object code for the work under the terms of Section 6. Any executables containing that work also fall under Section 6, whether or not they are linked directly with the Library itself. - -6. As an exception to the Sections above, you may also combine or link a "work that uses the Library" with the Library to produce a work containing portions of the Library, and distribute that work under terms of your choice, provided that the terms permit modification of the work for the customer's own use and reverse engineering for debugging such modifications. - -You must give prominent notice with each copy of the work that the Library is used in it and that the Library and its use are covered by this License. You must supply a copy of this License. If the work during execution displays copyright notices, you must include the copyright notice for the Library among them, as well as a reference directing the user to the copy of this License. Also, you must do one of these things: - - a) Accompany the work with the complete corresponding machine-readable source code for the Library including whatever changes were used in the work (which must be distributed under Sections 1 and 2 above); and, if the work is an executable linked with the Library, with the complete machine-readable "work that uses the Library", as object code and/or source code, so that the user can modify the Library and then relink to produce a modified executable containing the modified Library. (It is understood that the user who changes the contents of definitions files in the Library will not necessarily be able to recompile the application to use the modified definitions.) - - b) Use a suitable shared library mechanism for linking with the Library. A suitable mechanism is one that (1) uses at run time a copy of the library already present on the user's computer system, rather than copying library functions into the executable, and (2) will operate properly with a modified version of the library, if the user installs one, as long as the modified version is interface-compatible with the version that the work was made with. - - c) Accompany the work with a written offer, valid for at least three years, to give the same user the materials specified in Subsection 6a, above, for a charge no more than the cost of performing this distribution. - - d) If distribution of the work is made by offering access to copy from a designated place, offer equivalent access to copy the above specified materials from the same place. - - e) Verify that the user has already received a copy of these materials or that you have already sent this user a copy. - -For an executable, the required form of the "work that uses the Library" must include any data and utility programs needed for reproducing the executable from it. However, as a special exception, the materials to be distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. - -It may happen that this requirement contradicts the license restrictions of other proprietary libraries that do not normally accompany the operating system. Such a contradiction means you cannot use both them and the Library together in an executable that you distribute. - -7. You may place library facilities that are a work based on the Library side-by-side in a single library together with other library facilities not covered by this License, and distribute such a combined library, provided that the separate distribution of the work based on the Library and of the other library facilities is otherwise permitted, and provided that you do these two things: - - a) Accompany the combined library with a copy of the same work based on the Library, uncombined with any other library facilities. This must be distributed under the terms of the Sections above. - - b) Give prominent notice with the combined library of the fact that part of it is a work based on the Library, and explaining where to find the accompanying uncombined form of the same work. - -8. You may not copy, modify, sublicense, link with, or distribute the Library except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense, link with, or distribute the Library is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. - -9. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Library or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Library (or any work based on the Library), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Library or works based on it. - -10. Each time you redistribute the Library (or any work based on the Library), the recipient automatically receives a license from the original licensor to copy, distribute, link with or modify the Library subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties with this License. - -11. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Library at all. For example, if a patent license would not permit royalty-free redistribution of the Library by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Library. - -If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply, and the section as a whole is intended to apply in other circumstances. - -It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. - -This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. - -12. If the distribution and/or use of the Library is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Library under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. - -13. The Free Software Foundation may publish revised and/or new versions of the Lesser General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. - -Each version is given a distinguishing version number. If the Library specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Library does not specify a license version number, you may choose any version ever published by the Free Software Foundation. - -14. If you wish to incorporate parts of the Library into other free programs whose distribution conditions are incompatible with these, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. - -NO WARRANTY - -15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - -16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -END OF TERMS AND CONDITIONS - -How to Apply These Terms to Your New Libraries - -If you develop a new library, and you want it to be of the greatest possible use to the public, we recommend making it free software that everyone can redistribute and change. You can do so by permitting redistribution under these terms (or, alternatively, under the terms of the ordinary General Public License). - -To apply these terms, attach the following notices to the library. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. - - one line to give the library's name and an idea of what it does. - Copyright (C) year name of author - - This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; version 2.1 of the License. - - This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Also add information on how to contact you by electronic and paper mail. - -You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the library, if necessary. Here is a sample; alter the names: - -Yoyodyne, Inc., hereby disclaims all copyright interest in -the library `Frob' (a library for tweaking knobs) written -by James Random Hacker. - -signature of Ty Coon, 1 April 1990 -Ty Coon, President of Vice -That's all there is to it! diff --git a/packages/evals/README.md b/packages/evals/README.md deleted file mode 100644 index abb41fc875..0000000000 --- a/packages/evals/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# @corbits/evals - -Myra evals: `defineEval`/`runEval`, composable scorers, and a -persisted `evals.run` store that plays a hardcoded scripted scenario -against a real Myra deployment and grades what she actually did — never -what a prompt merely claims. - -## Factory eval: `github-pr-review-factory` - -`src/cases/github-pr-review-factory.ts` encodes the plan's §8.2 case — -"connect this GitHub organization and put every PR through an automated -review with greybeard, cto, and critique, plus suggested fixes" — in the -same `defineEval` idiom as the other two cases. It was written to be RED -by design — the case is the work list, not a bug report against itself -— and `bun run eval` (plumbing-only, no `EVAL_PROVIDER_API_KEY`) proves -end to end against a real hub + sidecar that every step still produces -a turn even with every scorer red. - -Two rulings from the owner are baked into the case's step 4: - -- Posting a review comment is **free** under a valid per-repo GitHub - grant — it must never wait on a human approval phrase, but every post - must still be audit-attributable to the posting agent. -- A merge-class action (landing a merge, not reviewing) **does park** - behind an explicit human approval, the same way `routine_create` - already does. - -`outwardGitHubActionsRespectGrantBoundary` grades both halves of that -ruling in one scorer, so a case that accidentally gates reviews or -accidentally frees merges fails loudly either way. - -### Scoreboard (install deploys the block workflow, GitHub REST rides the seam) - -The install path now deploys the template's referenced `code-review` -block workflow per tenant (`instantiateWorkbenchTemplate`'s -`deployBlockWorkflow` port -> `POST /template-blocks/:assetName/deploy`), -the eval boot stands up a fake GitHub REST origin and threads it through -the hub's `GITHUB_API_BASE_URL`, connects the PAT -through the real `/connections/github/complete` route, and drives the -connect card's start-reviewing step after install — so the per-repo -grant and `webhook_trigger` row mint for real and the fire-webhook step -fires an actual trigger. - -| # | Scorer | Result on a scratch-hub run | Why | -| --- | ------------------------------------------ | --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | `githubConnectedViaConnectionsLayer` | **PASS** | Both halves now: the MCP fake connects through the real `POST /mcp-servers` route, and the Plugins PAT proves against the fake REST origin through the real `/:connectorId/complete` (`probeBaseUrls`). | -| 2 | `agentDefinitionsHaveToolGrants` | **PASS** | The three reviewer definitions materialize via the real install, and the install now also deploys the `code-review` block workflow carrying the `@corbits/github-tools` pin (product fix). The snapshot's `name` is the stable definition handle (`displayName` carries the label), so handle matching is exact. | -| 3 | `triggerIsWebhookPerPr` | **PASS** | Install drives start-reviewing against the fake REST origin's repo list; one enabled `webhook_trigger` row mints per repo, bound to the deployed `code-review` definition. | -| 4 | `reviewCommentsAttributable` | **SKIP** (product gap) | `WorldSnapshot` has no `reviewComments` field — blocked on `onTrigger` adoption giving each fired occurrence its own child run id. | -| 5 | `suggestedFixesStructurallyValid` | **FAIL** (gap 1 below) | The launch itself succeeds: the template-block deploy freezes its definition through the native `workflowDeployer.deploy` path, so the fired trigger answers 202 with a real run instance instead of `DefinitionProjectionMissingError`. What remains is that a posted review needs genuine model tool calls, i.e. a live `EVAL_PROVIDER_API_KEY` run — plumbing mode's stub credential can never call `github_post_pr_review`. | -| 6 | `outwardGitHubActionsRespectGrantBoundary` | **FAIL** (gap 1 below) | Same blocker as #5. | -| 7 | `wholeRunInspectable` | **SKIP** (product gap) | `WorldSnapshot` has no `runs` field. Blocked on the same `onTrigger` adoption as #4. | - -### Remaining gaps, precisely - -1. **Scorers #5/#6 need a live run.** They grade genuine - `github_post_pr_review` tool calls off the trace; a plumbing-mode - stub credential produces a credential-error turn with no tool calls - by design. Re-run with `EVAL_PROVIDER_API_KEY`. -2. **`onTrigger` adoption** — unblocks #4/#7 - (per-comment and per-run ids in the snapshot). Unchanged. - -Run `bun test packages/evals` to see every scorer's current -`skipped`/`fail` reason first-hand — each one names its own blocker in -`reason`, not just in this table. Run `DATABASE_URL=... bun run eval` -for the full plumbing-mode harness proof (it now also prints every -failing scorer's reason and each harness step's outcome line); pass a -real `EVAL_PROVIDER_API_KEY` for genuine scorer grading against live -model output. diff --git a/packages/evals/eval-report.md b/packages/evals/eval-report.md deleted file mode 100644 index 58e8b67981..0000000000 --- a/packages/evals/eval-report.md +++ /dev/null @@ -1,5 +0,0 @@ -| Eval | plumbing-only | -| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| ai-daily-research | asksQuestions: FAIL
noToolCalls: PASS
noBuildBeforeAnswers: PASS
noBuildBeforeAnswers: PASS
memoryWritten: FAIL
namesRequiredTools: FAIL
agentCreatedInWorkbench: FAIL
routineCreatedOnlyAfterOk: PASS
approvalGated: PASS
namesRequiredTools: FAIL
routineCreated: FAIL
judge: skip | -| docs-on-sdk-change | asksQuestions: FAIL
noToolCalls: PASS
noBuildBeforeAnswers: PASS
noBuildBeforeAnswers: PASS
memoryWritten: FAIL
namesRequiredTools: FAIL
agentCreatedInWorkbench: FAIL
routineCreatedOnlyAfterOk: PASS
approvalGated: PASS
namesRequiredTools: FAIL
routineCreated: FAIL
judge: skip | -| github-pr-review-factory | noBuildBeforeAnswers: PASS
judge: skip
githubConnectedViaConnectionsLayer: PASS
agentDefinitionsHaveToolGrants: PASS
routineCreatedOnlyAfterOk: PASS
approvalGated: PASS
triggerIsWebhookPerPr: PASS
judge: skip
reviewCommentsAttributable: skip
suggestedFixesStructurallyValid: FAIL
outwardGitHubActionsRespectGrantBoundary: FAIL
wholeRunInspectable: skip | diff --git a/packages/evals/package.json b/packages/evals/package.json deleted file mode 100644 index 09cd8e84ab..0000000000 --- a/packages/evals/package.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "name": "@corbits/evals", - "version": "0.0.1", - "private": true, - "description": "Myra evals: a first-class eval library — defineEval/runEval, composable scorers, and a persisted evals.run store — that plays a hardcoded scripted scenario against a real Myra deployment (real hub + sidecar) and grades what she actually did, not what a prompt merely claims.", - "license": "LGPL-2.1-only", - "type": "module", - "exports": { - ".": "./src/index.ts" - }, - "scripts": { - "typecheck": "tsc --noEmit", - "test": "bun test" - }, - "dependencies": { - "@corbits/agent-directory": "workspace:*", - "@corbits/connections": "workspace:*", - "@corbits/error-sink": "workspace:*", - "@corbits/hub-api-client": "workspace:*", - "@corbits/migration-runner": "workspace:*", - "@corbits/myra": "workspace:*", - "@corbits/webhook-triggers": "workspace:*", - "@corbits/workflows": "workspace:*", - "@intx/agent": "workspace:*", - "@intx/db": "workspace:*", - "@intx/hub-sessions": "workspace:*", - "@intx/log": "0.3.0", - "@intx/types": "workspace:*", - "@intx/workflow": "workspace:*", - "@modelcontextprotocol/sdk": "catalog:", - "arktype": "catalog:", - "drizzle-orm": "catalog:", - "postgres": "catalog:" - }, - "devDependencies": { - "@corbits/agent-directory-tools": "workspace:*", - "@corbits/connections-tools": "workspace:*", - "@corbits/github-tools": "workspace:*", - "@corbits/mcp-tools": "workspace:*", - "@types/bun": "catalog:", - "typescript": "catalog:" - } -} diff --git a/packages/evals/src/cases/ai-daily-research.ts b/packages/evals/src/cases/ai-daily-research.ts deleted file mode 100644 index 784258fc2f..0000000000 --- a/packages/evals/src/cases/ai-daily-research.ts +++ /dev/null @@ -1,64 +0,0 @@ -// Case 1: "do research on AI daily for me" — the owner's -// canonical interview -> tool-map -> memory -> specialist -> routine -// flow. Expects Myra to: (1) ask a short interview about -// topics/sources/cadence/delivery rather than building anything first, -// (2) create a web-search-tools researcher agent with its own chat, -// (3) write the choices to firm memory, and (4) create the -// daily delivery routine only after an explicit go-ahead. -import { defineEval } from "../define-eval.ts"; -import { - agentCreatedInWorkbench, - approvalGated, - asksQuestions, - judge, - memoryWritten, - namesRequiredTools, - noBuildBeforeAnswers, - noToolCalls, - routineCreated, - routineCreatedOnlyAfterOk, -} from "../scorers/scorers.ts"; -import { CREATE_AGENT_TOOL, ROUTINE_CREATE_TOOL } from "../scorers/tool-names.ts"; - -export const aiDailyResearchEval = defineEval({ - name: "ai-daily-research", - description: - "'do research on AI daily for me' -> interview (topics/sources/" + - "cadence/delivery) -> web-search-tools researcher agent -> daily " + - "routine delivering here", - steps: [ - { - human: "do research on AI daily for me", - expect: [ - asksQuestions({ max: 4 }), - noToolCalls(["create_agent", "routine_create", "routine_run_now"]), - noBuildBeforeAnswers(1), - ], - }, - { - human: - "topics: new model releases and agent tooling; sources: X and " + - "Hacker News; cadence: daily every morning; deliver it here in " + - "this chat", - expect: [ - noBuildBeforeAnswers(1), - memoryWritten(["daily"]), - namesRequiredTools([CREATE_AGENT_TOOL]), - agentCreatedInWorkbench(), - ], - }, - { - human: "yes, go ahead and set up the daily routine", - expect: [ - routineCreatedOnlyAfterOk(2), - approvalGated([ROUTINE_CREATE_TOOL]), - namesRequiredTools([ROUTINE_CREATE_TOOL]), - routineCreated({ trigger: "daily" }), - judge( - "The reply confirms the routine is set up in a warm, direct " + - "teammate tone — not a wizard checklist, not pushy.", - ), - ], - }, - ], -}); diff --git a/packages/evals/src/cases/docs-on-sdk-change.ts b/packages/evals/src/cases/docs-on-sdk-change.ts deleted file mode 100644 index ff75e4e55f..0000000000 --- a/packages/evals/src/cases/docs-on-sdk-change.ts +++ /dev/null @@ -1,64 +0,0 @@ -// Case 2: "automatically update my docs when my SDK changes" — -// the same interview -> tool-map -> memory -> specialist -> routine -// flow, but the routine is webhook-triggered (see -// packages/webhook-triggers) rather than a schedule, and the specialist -// must never commit a doc change without an explicit review/approval -// step, per the owner's script. -import { defineEval } from "../define-eval.ts"; -import { - agentCreatedInWorkbench, - approvalGated, - asksQuestions, - judge, - memoryWritten, - namesRequiredTools, - noBuildBeforeAnswers, - noToolCalls, - routineCreated, - routineCreatedOnlyAfterOk, -} from "../scorers/scorers.ts"; -import { CREATE_AGENT_TOOL, ROUTINE_CREATE_TOOL } from "../scorers/tool-names.ts"; - -export const docsOnSdkChangeEval = defineEval({ - name: "docs-on-sdk-change", - description: - "'automatically update my docs when my SDK changes' -> interview " + - "(repo, docs location, review-before-commit?) -> github-tools " + - "agent -> webhook-triggered routine with approval before any write", - steps: [ - { - human: "automatically update my docs when my SDK changes", - expect: [ - asksQuestions({ max: 4 }), - noToolCalls(["create_agent", "routine_create", "routine_run_now"]), - noBuildBeforeAnswers(1), - ], - }, - { - human: - "repo: github.com/example/sdk; docs live in the docs/ folder of " + - "the same repo; yes, always review the diff before committing " + - "anything", - expect: [ - noBuildBeforeAnswers(1), - memoryWritten(["docs/"]), - namesRequiredTools([CREATE_AGENT_TOOL]), - agentCreatedInWorkbench(), - ], - }, - { - human: "looks good, wire up the webhook routine", - expect: [ - routineCreatedOnlyAfterOk(2), - approvalGated([ROUTINE_CREATE_TOOL]), - namesRequiredTools([ROUTINE_CREATE_TOOL]), - routineCreated({ trigger: "webhook" }), - judge( - "The reply confirms the webhook routine is wired up AND makes " + - "clear no doc change ever commits without a human reviewing " + - "it first.", - ), - ], - }, - ], -}); diff --git a/packages/evals/src/cases/exa-topic-digest.ts b/packages/evals/src/cases/exa-topic-digest.ts deleted file mode 100644 index 224cc9d061..0000000000 --- a/packages/evals/src/cases/exa-topic-digest.ts +++ /dev/null @@ -1,66 +0,0 @@ -// "we need someone to watch our Exa topics and post a digest to -// this bench" — Myra coordinates rather than declining or answering -// inline: the same interview -> tool-map -> memory -> specialist -> -// routine flow as ai-daily-research, with the plan itself naming the -// teammate (agent/workflow) it will create. -import { defineEval } from "../define-eval.ts"; -import { - agentCreatedInWorkbench, - approvalGated, - asksQuestions, - judge, - memoryWritten, - namesRequiredTools, - noBuildBeforeAnswers, - noToolCalls, - routineCreated, - routineCreatedOnlyAfterOk, -} from "../scorers/scorers.ts"; -import { CREATE_AGENT_TOOL, ROUTINE_CREATE_TOOL } from "../scorers/tool-names.ts"; - -export const exaTopicDigestEval = defineEval({ - name: "exa-topic-digest", - description: - "'we need someone to watch our Exa topics and post a digest to this " + - "bench' -> interview (topics/cadence) -> specialist teammate that " + - "watches Exa -> routine posting the digest here", - steps: [ - { - human: "we need someone to watch our Exa topics and post a digest to " + "this bench", - expect: [ - asksQuestions({ max: 4 }), - noToolCalls(["create_agent", "routine_create", "routine_run_now"]), - noBuildBeforeAnswers(1), - judge( - "The reply proposes creating a teammate (an agent or workflow) " + - "to own this job rather than declining the request or " + - "trying to answer it inline as a one-off.", - ), - ], - }, - { - human: - "topics: competitor launches and pricing changes; cadence: " + - "daily every morning; post the digest here in this chat", - expect: [ - noBuildBeforeAnswers(1), - memoryWritten(["daily"]), - namesRequiredTools([CREATE_AGENT_TOOL]), - agentCreatedInWorkbench(), - ], - }, - { - human: "yes, set up the daily digest routine", - expect: [ - routineCreatedOnlyAfterOk(2), - approvalGated([ROUTINE_CREATE_TOOL]), - namesRequiredTools([ROUTINE_CREATE_TOOL]), - routineCreated({ trigger: "daily" }), - judge( - "The reply confirms the digest routine is set up in a warm, " + - "direct teammate tone — not a wizard checklist, not pushy.", - ), - ], - }, - ], -}); diff --git a/packages/evals/src/cases/index.ts b/packages/evals/src/cases/index.ts deleted file mode 100644 index 945bb4ee71..0000000000 --- a/packages/evals/src/cases/index.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { aiDailyResearchEval } from "./ai-daily-research.ts"; -import { docsOnSdkChangeEval } from "./docs-on-sdk-change.ts"; -import { exaTopicDigestEval } from "./exa-topic-digest.ts"; -import type { EvalDefinition } from "../types.ts"; - -export { aiDailyResearchEval } from "./ai-daily-research.ts"; -export { docsOnSdkChangeEval } from "./docs-on-sdk-change.ts"; -export { exaTopicDigestEval } from "./exa-topic-digest.ts"; - -export const ALL_EVALS: readonly EvalDefinition[] = [ - aiDailyResearchEval, - docsOnSdkChangeEval, - exaTopicDigestEval, -]; diff --git a/packages/evals/src/define-eval.test.ts b/packages/evals/src/define-eval.test.ts deleted file mode 100644 index 08498e77ff..0000000000 --- a/packages/evals/src/define-eval.test.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { expect, test } from "bun:test"; - -import { defineEval } from "./define-eval.ts"; - -test("defineEval rejects a persona step with maxTurns below 1", () => { - expect(() => - defineEval({ - name: "zero-turns", - description: "test", - steps: [ - { - kind: "persona", - opening: "set up my digest", - persona: { name: "Dana", goal: "get a digest", knownFacts: {} }, - maxTurns: 0, - expect: [], - }, - ], - }), - ).toThrow('defineEval("zero-turns") step 0: maxTurns must be at least 1'); -}); - -test("defineEval accepts a persona step with maxTurns of 1", () => { - expect(() => - defineEval({ - name: "one-turn", - description: "test", - steps: [ - { - kind: "persona", - opening: "set up my digest", - persona: { name: "Dana", goal: "get a digest", knownFacts: {} }, - maxTurns: 1, - expect: [], - }, - ], - }), - ).not.toThrow(); -}); diff --git a/packages/evals/src/define-eval.ts b/packages/evals/src/define-eval.ts deleted file mode 100644 index 1c898cb673..0000000000 --- a/packages/evals/src/define-eval.ts +++ /dev/null @@ -1,44 +0,0 @@ -import type { EvalDefinition, EvalStep } from "./types.ts"; - -export interface DefineEvalConfig { - readonly name: string; - readonly description: string; - readonly steps: readonly EvalStep[]; - readonly memorySeed?: readonly string[]; -} - -/** Validates and returns an `EvalDefinition` — a hardcoded expected - * scenario, never generated or inferred. Fails loudly on an empty name - * or an eval with no steps, rather than silently accepting a - * definition that could never run or never grade anything. */ -export function defineEval(config: DefineEvalConfig): EvalDefinition { - if (config.name === "") { - throw new Error("defineEval requires a non-empty name"); - } - if (config.steps.length === 0) { - throw new Error(`defineEval("${config.name}") requires at least one step`); - } - config.steps.forEach((step, index) => { - if (step.kind === "persona" && step.maxTurns < 1) { - throw new Error( - `defineEval("${config.name}") step ${String(index)}: maxTurns must be at least 1`, - ); - } - }); - const steps = config.steps.map((step) => - step.kind === undefined ? { ...step, kind: "scripted" as const } : step, - ); - if (config.memorySeed !== undefined) { - return { - name: config.name, - description: config.description, - steps, - memorySeed: config.memorySeed, - }; - } - return { - name: config.name, - description: config.description, - steps, - }; -} diff --git a/packages/evals/src/fakes/github-rest-fake.ts b/packages/evals/src/fakes/github-rest-fake.ts deleted file mode 100644 index 713b1c484e..0000000000 --- a/packages/evals/src/fakes/github-rest-fake.ts +++ /dev/null @@ -1,66 +0,0 @@ -// A fake GitHub REST origin for the `GITHUB_API_BASE_URL` seam: the hub, booted with -// `GITHUB_API_BASE_URL` pointing here, runs its `github` connector PAT -// probe (`GET /user`), the connect card's authenticated-login + -// repo-listing reads (`GET /user`, `GET /user/repos`, -// `GET /search/issues` for the per-repo open-PR count), and — through -// the stored provider origin — any run-time `@corbits/github-tools` -// call, against this process instead of `https://api.github.com`. -// Serves fixed fixture data over `Bun.serve`, mirroring -// `./mcp-fake-server.ts`'s shape: `receipts()` records every request -// so a scorer (or a debugging run) can see exactly what the fake was -// asked. -import type { FakeReceipt } from "../types.ts"; - -export interface GithubRestFakeRepo { - readonly id: number; - readonly full_name: string; -} - -export interface GithubRestFakeHandle { - readonly url: string; - receipts(): readonly FakeReceipt[]; - stop(): void; -} - -export function startGithubRestFake( - port: number, - fixture: { - readonly login: string; - readonly repos: readonly GithubRestFakeRepo[]; - }, -): GithubRestFakeHandle { - const receipts: FakeReceipt[] = []; - - const server = Bun.serve({ - port, - fetch(request) { - const url = new URL(request.url); - receipts.push({ - server: "github-rest", - toolName: `${request.method} ${url.pathname}`, - arguments: Object.fromEntries(url.searchParams.entries()), - }); - if (request.method === "GET" && url.pathname === "/user") { - return Response.json({ login: fixture.login }); - } - if (request.method === "GET" && url.pathname === "/user/repos") { - return Response.json(fixture.repos); - } - if (request.method === "GET" && url.pathname === "/search/issues") { - return Response.json({ total_count: 1, items: [] }); - } - return Response.json( - { message: `github-rest fake has no route for ${url.pathname}` }, - { status: 404 }, - ); - }, - }); - - return { - url: `http://localhost:${String(port)}`, - receipts: () => receipts, - stop: () => { - void server.stop(true); - }, - }; -} diff --git a/packages/evals/src/fakes/mcp-fake-server.test.ts b/packages/evals/src/fakes/mcp-fake-server.test.ts deleted file mode 100644 index 5357c8953a..0000000000 --- a/packages/evals/src/fakes/mcp-fake-server.test.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { afterEach, expect, test } from "bun:test"; -import { callMcpTool, listMcpTools, withMcpConnection } from "@corbits/mcp-tools"; - -import githubFixture from "./recordings/github.json"; -import { parseMcpFakeRecording } from "./recording.ts"; -import { startMcpFake, type McpFakeHandle } from "./mcp-fake-server.ts"; - -const recording = parseMcpFakeRecording(githubFixture); - -let handle: McpFakeHandle | undefined; - -afterEach(() => { - handle?.stop(); - handle = undefined; -}); - -test("tools/list surfaces exactly the fixture's tools", async () => { - handle = startMcpFake(recording); - const tools = await withMcpConnection({ url: handle.url, fetchImpl: fetch }, (client) => - listMcpTools(client), - ); - expect(tools.map((tool) => tool.name)).toEqual(["list_pull_requests"]); -}); - -test("tools/call replays the recorded response and logs a receipt", async () => { - handle = startMcpFake(recording); - const result = await withMcpConnection({ url: handle.url, fetchImpl: fetch }, (client) => - callMcpTool(client, { - name: "list_pull_requests", - arguments: { owner: "acme-corp", repo: "widget-service" }, - }), - ); - expect(result.isError).toBe(false); - expect(JSON.stringify(result.content)).toContain("Fix flaky checkout test"); - expect(handle.receipts()).toEqual([ - { - server: "github", - toolName: "list_pull_requests", - arguments: { owner: "acme-corp", repo: "widget-service" }, - }, - ]); -}); - -test("an unrecorded call fails loudly instead of silently defaulting", async () => { - handle = startMcpFake(recording); - await expect( - withMcpConnection({ url: handle.url, fetchImpl: fetch }, (client) => - callMcpTool(client, { - name: "list_pull_requests", - arguments: { owner: "someone-else", repo: "unknown-repo" }, - }), - ), - ).rejects.toThrow(/No recorded call for github\.list_pull_requests/); - expect(handle.receipts()).toEqual([]); -}); - -test("argument key order does not affect matching", async () => { - handle = startMcpFake(recording); - const result = await withMcpConnection({ url: handle.url, fetchImpl: fetch }, (client) => - callMcpTool(client, { - name: "list_pull_requests", - arguments: { repo: "widget-service", owner: "acme-corp" }, - }), - ); - expect(result.isError).toBe(false); -}); diff --git a/packages/evals/src/fakes/mcp-fake-server.ts b/packages/evals/src/fakes/mcp-fake-server.ts deleted file mode 100644 index ce612e6af3..0000000000 --- a/packages/evals/src/fakes/mcp-fake-server.ts +++ /dev/null @@ -1,117 +0,0 @@ -// Stands up a real MCP server — same SDK pieces -// `@corbits/mcp-tools`' own test double already proves against -// (`tools/mcp/test/stub-mcp-server.ts`) — that replays a -// checked-in `McpFakeRecording` (./recording.ts) instead of a -// hand-written tool set. Connected into a tenant through the exact -// same `POST /mcp-servers` route real users use -// (`packages/connections/src/mcp-server-routes.ts`) — this file only -// serves the wire protocol, it never fakes the connect mechanism. -import { Server } from "@modelcontextprotocol/sdk/server/index.js"; -import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js"; -import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js"; - -import type { FakeReceipt } from "../types.ts"; -import type { McpFakeRecording, RecordedCall } from "./recording.ts"; - -export interface McpFakeHandle { - readonly url: string; - /** Every call the fake actually received so far, in arrival order — - * gap 1's `WorldSnapshot.fakeReceipts` reads this through the - * `fakeReceipts` hook `real-target.ts` folds it into. */ - receipts(): readonly FakeReceipt[]; - stop(): void; -} - -function canonical(value: unknown): string { - if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; - if (typeof value === "object" && value !== null) { - const entries = Object.entries(value as Record).sort(([a], [b]) => - a < b ? -1 : a > b ? 1 : 0, - ); - return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonical(v)}`).join(",")}}`; - } - return JSON.stringify(value); -} - -function findRecordedCall( - recording: McpFakeRecording, - tool: string, - args: Record, -): RecordedCall | undefined { - const wanted = canonical(args); - return recording.calls.find((call) => call.tool === tool && canonical(call.arguments) === wanted); -} - -function buildServer(recording: McpFakeRecording, receipts: FakeReceipt[]): Server { - const server = new Server( - { name: `${recording.server}-fake`, version: "0.0.1" }, - { capabilities: { tools: {} } }, - ); - - server.setRequestHandler(ListToolsRequestSchema, () => ({ - tools: recording.tools.map((tool) => ({ - name: tool.name, - ...(tool.description !== undefined ? { description: tool.description } : {}), - inputSchema: tool.inputSchema, - })), - })); - - server.setRequestHandler(CallToolRequestSchema, (request) => { - const args = (request.params.arguments ?? {}) as Record; - const match = findRecordedCall(recording, request.params.name, args); - if (match === undefined) { - throw new Error( - `No recorded call for ${recording.server}.${request.params.name}(${JSON.stringify(args)})`, - ); - } - receipts.push({ - server: recording.server, - toolName: request.params.name, - arguments: args, - }); - return match.response; - }); - - return server; -} - -/** - * Starts a fake MCP server over Streamable HTTP on `port` (0 lets the - * OS pick a free one). A fresh `Server`/transport pair is minted per - * MCP session, keyed by the `Mcp-Session-Id` header — the same - * per-session-map pattern `stub-mcp-server.ts` establishes, needed - * because this SDK's `Server` cannot outlive one `initialize` handshake. - */ -export function startMcpFake(recording: McpFakeRecording, port = 0): McpFakeHandle { - const receipts: FakeReceipt[] = []; - const sessions = new Map(); - - const httpServer = Bun.serve({ - port, - fetch: async (req) => { - const sessionId = req.headers.get("mcp-session-id"); - const existing = sessionId !== null ? sessions.get(sessionId) : undefined; - if (existing !== undefined) { - return existing.handleRequest(req); - } - const transport: WebStandardStreamableHTTPServerTransport = - new WebStandardStreamableHTTPServerTransport({ - sessionIdGenerator: () => crypto.randomUUID(), - onsessioninitialized: (id: string): void => { - sessions.set(id, transport); - }, - onsessionclosed: (id: string): void => { - sessions.delete(id); - }, - }); - await buildServer(recording, receipts).connect(transport); - return transport.handleRequest(req); - }, - }); - - return { - url: `http://127.0.0.1:${String(httpServer.port)}/mcp`, - receipts: () => receipts, - stop: () => httpServer.stop(true), - }; -} diff --git a/packages/evals/src/fakes/recording.test.ts b/packages/evals/src/fakes/recording.test.ts deleted file mode 100644 index 582d73bfd3..0000000000 --- a/packages/evals/src/fakes/recording.test.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { expect, test } from "bun:test"; - -import { parseMcpFakeRecording } from "./recording.ts"; - -const valid = { - server: "github", - tools: [ - { - name: "list_pull_requests", - inputSchema: { type: "object", properties: {} }, - }, - ], - calls: [ - { - tool: "list_pull_requests", - arguments: { owner: "acme-corp" }, - response: { isError: false, content: "ok" }, - }, - ], -}; - -test("parses a well-formed recording", () => { - const parsed = parseMcpFakeRecording(valid); - expect(parsed.server).toBe("github"); - expect(parsed.calls).toHaveLength(1); -}); - -test("rejects a recording missing required fields", () => { - expect(() => parseMcpFakeRecording({ server: "github" })).toThrow(/invalid MCP fake recording/); -}); - -test("rejects a call whose response is missing isError", () => { - const broken = { - ...valid, - calls: [{ tool: "list_pull_requests", arguments: {}, response: { content: "x" } }], - }; - expect(() => parseMcpFakeRecording(broken)).toThrow(/invalid MCP fake recording/); -}); diff --git a/packages/evals/src/fakes/recording.ts b/packages/evals/src/fakes/recording.ts deleted file mode 100644 index ab1ebe1d19..0000000000 --- a/packages/evals/src/fakes/recording.ts +++ /dev/null @@ -1,61 +0,0 @@ -// A recorded MCP fake's fixture shape: what tools an -// external MCP server (GitHub, Attio, Sumble, ...) advertises, and the -// canned `(tool, arguments) -> response` pairs a fake stood up from -// `startMcpFake` (./mcp-fake-server.ts) replays. Checked into the repo -// as plain JSON under `recordings/` — deterministic and reviewable in -// a PR diff, the same "record-as-truth JSON, no migration" convention -// other product JSON columns already use. -import { type } from "arktype"; - -export interface RecordedCall { - readonly tool: string; - readonly arguments: Record; - readonly response: { readonly isError: boolean; readonly content: unknown }; -} - -export interface McpFakeToolDefinition { - readonly name: string; - readonly description?: string; - readonly inputSchema: Record; -} - -/** Matched by (tool, arguments) equality; first match wins, exact-match - * only — an unrecorded call is a hard fixture gap the fake fails loudly - * on, never silently defaulted (see mcp-fake-server.ts). */ -export interface McpFakeRecording { - readonly server: string; - readonly tools: readonly McpFakeToolDefinition[]; - readonly calls: readonly RecordedCall[]; -} - -const RecordedCallSchema = type({ - tool: "string", - arguments: "Record", - response: { - isError: "boolean", - content: "unknown", - }, -}); - -const McpFakeToolDefinitionSchema = type({ - name: "string", - "description?": "string", - inputSchema: "Record", -}); - -const McpFakeRecordingSchema = type({ - server: "string", - tools: McpFakeToolDefinitionSchema.array(), - calls: RecordedCallSchema.array(), -}); - -/** Parses untrusted JSON (a fixture file read off disk) into a - * `McpFakeRecording` — the trust-boundary check every external-data - * read in this repo goes through, never a bare `as McpFakeRecording`. */ -export function parseMcpFakeRecording(data: unknown): McpFakeRecording { - const parsed = McpFakeRecordingSchema(data); - if (parsed instanceof type.errors) { - throw new Error(`invalid MCP fake recording: ${parsed.summary}`); - } - return parsed; -} diff --git a/packages/evals/src/fakes/recordings.ts b/packages/evals/src/fakes/recordings.ts deleted file mode 100644 index 654bd132fb..0000000000 --- a/packages/evals/src/fakes/recordings.ts +++ /dev/null @@ -1,8 +0,0 @@ -// The checked-in recordings, parsed once through the fixture schema so -// a caller (scripts/evals-run.ts wiring a fake into `bootMyraTarget`) -// never re-reads the JSON path itself. -import githubJson from "./recordings/github.json"; - -import { parseMcpFakeRecording, type McpFakeRecording } from "./recording.ts"; - -export const GITHUB_MCP_FAKE_RECORDING: McpFakeRecording = parseMcpFakeRecording(githubJson); diff --git a/packages/evals/src/fakes/recordings/github.json b/packages/evals/src/fakes/recordings/github.json deleted file mode 100644 index 4b0cfddbcc..0000000000 --- a/packages/evals/src/fakes/recordings/github.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "server": "github", - "tools": [ - { - "name": "list_pull_requests", - "description": "Lists open pull requests for a repository.", - "inputSchema": { - "type": "object", - "properties": { - "owner": { "type": "string" }, - "repo": { "type": "string" } - }, - "required": ["owner", "repo"] - } - } - ], - "calls": [ - { - "tool": "list_pull_requests", - "arguments": { "owner": "acme-corp", "repo": "widget-service" }, - "response": { - "isError": false, - "content": [ - { - "type": "text", - "text": "[{\"number\":42,\"title\":\"Fix flaky checkout test\",\"state\":\"open\"}]" - } - ] - } - } - ] -} diff --git a/packages/evals/src/index.ts b/packages/evals/src/index.ts deleted file mode 100644 index 1c9582d3ec..0000000000 --- a/packages/evals/src/index.ts +++ /dev/null @@ -1,70 +0,0 @@ -export type { - EvalDefinition, - EvalRunResult, - EvalStep, - EvalStepRecord, - FakeReceipt, - PersonaBrief, - PersonaEvalStep, - RunConfig, - Scorer, - ScorerContext, - ScorerReport, - ScorerResult, - ScriptedEvalStep, - Target, - ToolCall, - Turn, - WorldAgentDefinition, - WorldConnection, - WorldSnapshot, - WorldWebhookTrigger, -} from "./types.ts"; -export { defineEval } from "./define-eval.ts"; -export { runEval, runMatrix } from "./runner.ts"; -export { callEvalModel } from "./model-call.ts"; -export type { ModelCallResult } from "./model-call.ts"; -export { personaAnswer } from "./persona.ts"; -export type { PersonaReply } from "./persona.ts"; -export { runPersonaStep } from "./persona-runner.ts"; -export { renderResultsMarkdown } from "./report.ts"; -export { - agentCreatedInWorkbench, - agentDefinitionsHaveToolGrants, - approvalGated, - asksQuestions, - githubConnectedViaConnectionsLayer, - judge, - memoryWritten, - namesRequiredTools, - noBuildBeforeAnswers, - noToolCalls, - outwardGitHubActionsRespectGrantBoundary, - reviewCommentsAttributable, - routineCreated, - routineCreatedOnlyAfterOk, - suggestedFixesStructurallyValid, - triggerIsWebhookPerPr, - wholeRunInspectable, -} from "./scorers/scorers.ts"; -export { agentHasTools, connectionIsLive, fakeReceived } from "./scorers/world-scorers.ts"; -export { - parseMcpFakeRecording, - type McpFakeRecording, - type McpFakeToolDefinition, - type RecordedCall, -} from "./fakes/recording.ts"; -export { GITHUB_MCP_FAKE_RECORDING } from "./fakes/recordings.ts"; -export { - ALL_EVALS, - aiDailyResearchEval, - docsOnSdkChangeEval, - exaTopicDigestEval, -} from "./cases/index.ts"; -export type { EvalRunRecord, EvalRunStore } from "./store/store.ts"; -export { createPostgresEvalRunStore } from "./store/pg-store.ts"; -export { applyEvalsMigrations } from "./store/migrations.ts"; -export { newToolCallsSince, readAllToolCalls } from "./targets/trace.ts"; -export type { SqlClientLike } from "./targets/trace.ts"; -export { captureWorldSnapshot } from "./targets/world-snapshot.ts"; -export type { WorldSnapshotInfra } from "./targets/world-snapshot.ts"; diff --git a/packages/evals/src/model-call.test.ts b/packages/evals/src/model-call.test.ts deleted file mode 100644 index f4851a5ae5..0000000000 --- a/packages/evals/src/model-call.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { expect, test } from "bun:test"; - -import { callEvalModel } from "./model-call.ts"; -import type { FetchLike } from "./model-call.ts"; - -test("callEvalModel posts to the Anthropic Messages API and extracts the text block", async () => { - const calls: { url: string; init: RequestInit }[] = []; - const fetchImpl: FetchLike = async (url, init) => { - calls.push({ url, init }); - return new Response(JSON.stringify({ content: [{ type: "text", text: "hello there" }] }), { - status: 200, - }); - }; - - const result = await callEvalModel("say hi", "test-key", undefined, fetchImpl); - - expect(result.text).toBe("hello there"); - expect(calls).toHaveLength(1); - expect(calls[0]?.url).toBe("https://api.anthropic.com/v1/messages"); - const body = JSON.parse((calls[0]?.init.body as string) ?? "{}") as { - model: string; - messages: { role: string; content: string }[]; - }; - expect(body.model).toBe("claude-3-5-haiku-20241022"); - expect(body.messages).toEqual([{ role: "user", content: "say hi" }]); -}); - -test("callEvalModel honors an explicit model override", async () => { - const fetchImpl: FetchLike = async () => - new Response(JSON.stringify({ content: [{ type: "text", text: "ok" }] }), { - status: 200, - }); - const result = await callEvalModel("prompt", "test-key", "claude-3-opus-20240229", fetchImpl); - expect(result.text).toBe("ok"); -}); - -test("callEvalModel returns empty text when no text block is present", async () => { - const fetchImpl: FetchLike = async () => - new Response(JSON.stringify({ content: [] }), { status: 200 }); - const result = await callEvalModel("prompt", "test-key", undefined, fetchImpl); - expect(result.text).toBe(""); -}); diff --git a/packages/evals/src/model-call.ts b/packages/evals/src/model-call.ts deleted file mode 100644 index 859814202e..0000000000 --- a/packages/evals/src/model-call.ts +++ /dev/null @@ -1,38 +0,0 @@ -// The one shared seam every eval-side model call goes through — a small -// POST to the Anthropic Messages API. `judge()` (scorers/scorers.ts) and -// `personaAnswer()` (persona.ts) both call this instead of inlining -// `fetch`, so there is exactly one place that knows the wire format. - -export interface ModelCallResult { - readonly text: string; -} - -const DEFAULT_MODEL = "claude-3-5-haiku-20241022"; - -export type FetchLike = (url: string, init: RequestInit) => Promise; - -export async function callEvalModel( - prompt: string, - apiKey: string, - model = DEFAULT_MODEL, - fetchImpl: FetchLike = fetch, -): Promise { - const res = await fetchImpl("https://api.anthropic.com/v1/messages", { - method: "POST", - headers: { - "content-type": "application/json", - "x-api-key": apiKey, - "anthropic-version": "2023-06-01", - }, - body: JSON.stringify({ - model, - max_tokens: 200, - messages: [{ role: "user", content: prompt }], - }), - }); - const data = (await res.json()) as { - content?: { type: string; text?: string }[]; - }; - const text = data.content?.find((block) => block.type === "text")?.text ?? ""; - return { text }; -} diff --git a/packages/evals/src/persona-runner.test.ts b/packages/evals/src/persona-runner.test.ts deleted file mode 100644 index 81c853adc7..0000000000 --- a/packages/evals/src/persona-runner.test.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { expect, test } from "bun:test"; - -import { runPersonaStep } from "./persona-runner.ts"; -import type { PersonaBrief, PersonaEvalStep, Target, Turn } from "./types.ts"; - -const brief: PersonaBrief = { - name: "Dana", - goal: "get a daily digest set up", - knownFacts: { cadence: "every weekday at 8am" }, -}; - -function scriptedTarget(replies: Record): Target { - return { - configName: "test", - async sendTurn(human) { - const turn = replies[human]; - if (turn === undefined) { - throw new Error(`scriptedTarget: no reply scripted for "${human}"`); - } - return turn; - }, - async close() {}, - }; -} - -test("stops when the agent's reply carries no question", async () => { - const target = scriptedTarget({ - "set up my digest": { - human: "set up my digest", - replyText: "Done — your digest is set up.", - toolCalls: [], - }, - }); - const step: PersonaEvalStep = { - kind: "persona", - opening: "set up my digest", - persona: brief, - maxTurns: 5, - expect: [], - }; - let callCount = 0; - const call = async () => { - callCount += 1; - return { text: "should not be reached" }; - }; - - const turns = await runPersonaStep(step, target, call); - - expect(turns).toHaveLength(1); - expect(turns[0]?.replyText).toBe("Done — your digest is set up."); - expect(callCount).toBe(0); -}); - -test("stops when the persona replies DONE", async () => { - const target = scriptedTarget({ - "set up my digest": { - human: "set up my digest", - replyText: "What cadence works for you?", - toolCalls: [], - }, - }); - const step: PersonaEvalStep = { - kind: "persona", - opening: "set up my digest", - persona: brief, - maxTurns: 5, - expect: [], - }; - const call = async () => ({ text: "DONE" }); - - const turns = await runPersonaStep(step, target, call); - - expect(turns).toHaveLength(1); -}); - -test("stops at maxTurns even if the agent keeps asking questions", async () => { - let questionCount = 0; - const target: Target = { - configName: "test", - async sendTurn() { - questionCount += 1; - return { - human: `turn ${String(questionCount)}`, - replyText: "What else can you tell me?", - toolCalls: [], - }; - }, - async close() {}, - }; - const step: PersonaEvalStep = { - kind: "persona", - opening: "set up my digest", - persona: brief, - maxTurns: 3, - expect: [], - }; - const call = async () => ({ text: "here's more info" }); - - const turns = await runPersonaStep(step, target, call); - - expect(turns).toHaveLength(3); -}); - -test("feeds the persona's answer back into the next sendTurn call", async () => { - const sentHumans: string[] = []; - let turnCount = 0; - const target: Target = { - configName: "test", - async sendTurn(human) { - sentHumans.push(human); - turnCount += 1; - if (turnCount === 1) { - return { - human, - replyText: "What cadence works for you?", - toolCalls: [], - }; - } - return { human, replyText: "Great, all set.", toolCalls: [] }; - }, - async close() {}, - }; - const step: PersonaEvalStep = { - kind: "persona", - opening: "set up my digest", - persona: brief, - maxTurns: 5, - expect: [], - }; - const call = async () => ({ text: "every weekday at 8am" }); - - await runPersonaStep(step, target, call); - - expect(sentHumans).toEqual(["set up my digest", "every weekday at 8am"]); -}); diff --git a/packages/evals/src/persona-runner.ts b/packages/evals/src/persona-runner.ts deleted file mode 100644 index 90dec0683b..0000000000 --- a/packages/evals/src/persona-runner.ts +++ /dev/null @@ -1,42 +0,0 @@ -// Plays a `PersonaEvalStep`'s human side turn-by-turn against a -// `Target`, generating each follow-up message from `personaAnswer` -// instead of a scripted string. Termination is triple-guarded so the -// loop never runs unbounded: (a) `maxTurns` is a hard cap; (b) the -// agent's reply carrying no "?" is a structural "stopped asking" -// signal that needs no model call — the same heuristic -// `asksQuestions` (scorers/scorers.ts) already codifies; (c) the -// persona itself may decide it has nothing left to volunteer. - -import { personaAnswer } from "./persona.ts"; -import type { PersonaEvalStep, Target, Turn } from "./types.ts"; - -function stillAsking(replyText: string): boolean { - return (replyText.match(/\?/g) ?? []).length > 0; -} - -export async function runPersonaStep( - step: PersonaEvalStep, - target: Target, - call?: (prompt: string) => Promise<{ text: string }>, -): Promise { - const turns: Turn[] = []; - const history: { human: string; agentReply: string }[] = []; - let human = step.opening; - - for (let i = 0; i < step.maxTurns; i += 1) { - const turn = await target.sendTurn(human); - turns.push(turn); - history.push({ human, agentReply: turn.replyText }); - - if (!stillAsking(turn.replyText)) { - break; - } - const reply = await personaAnswer(step.persona, turn.replyText, history, call); - if (reply.kind === "done") { - break; - } - human = reply.text; - } - - return turns; -} diff --git a/packages/evals/src/persona.test.ts b/packages/evals/src/persona.test.ts deleted file mode 100644 index aa23065e6d..0000000000 --- a/packages/evals/src/persona.test.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { expect, test } from "bun:test"; - -import { personaAnswer } from "./persona.ts"; -import type { PersonaBrief } from "./types.ts"; - -const brief: PersonaBrief = { - name: "Dana", - goal: "get a daily AI news digest set up", - knownFacts: { - cadence: "every weekday morning at 8am", - sources: "The Verge, TechCrunch, and Ars Technica", - }, - tone: "terse", -}; - -test("personaAnswer builds a prompt from only the brief and the last agent reply, not the whole transcript", async () => { - let seenPrompt = ""; - const call = async (prompt: string) => { - seenPrompt = prompt; - return { text: "every weekday morning at 8am" }; - }; - const history = [{ human: "set up my digest", agentReply: "Happy to help — what topic?" }]; - - await personaAnswer(brief, "What cadence works for you?", history, call); - - expect(seenPrompt).toContain("What cadence works for you?"); - expect(seenPrompt).toContain(brief.goal); - expect(seenPrompt).toContain("every weekday morning at 8am"); - // The prior turn's agent reply text is not repeated verbatim in the - // prompt — only the brief and the current question drive the answer. - expect(seenPrompt).not.toContain("Happy to help"); -}); - -test("personaAnswer returns the persona's reply text as a message", async () => { - const call = async () => ({ text: "every weekday morning at 8am" }); - const reply = await personaAnswer(brief, "What cadence works for you?", [], call); - expect(reply).toEqual({ - kind: "message", - text: "every weekday morning at 8am", - }); -}); - -test("personaAnswer maps an exact DONE response to a done reply", async () => { - const call = async () => ({ text: "DONE" }); - const reply = await personaAnswer(brief, "Great, I'll set that up now.", [], call); - expect(reply).toEqual({ kind: "done" }); -}); - -test("personaAnswer maps a DONE response with surrounding whitespace to a done reply", async () => { - const call = async () => ({ text: " DONE\n" }); - const reply = await personaAnswer(brief, "All set, running it now.", [], call); - expect(reply).toEqual({ kind: "done" }); -}); diff --git a/packages/evals/src/persona.ts b/packages/evals/src/persona.ts deleted file mode 100644 index 0b19667c2c..0000000000 --- a/packages/evals/src/persona.ts +++ /dev/null @@ -1,71 +0,0 @@ -// A simulated human that answers what's asked and volunteers nothing -// else. `personaAnswer` turns the agent's latest reply into the -// persona's next message (or a signal that the persona has nothing -// left to say) by asking a model to play the persona against its brief -// — the same injectable-call seam `judge()` uses (scorers/scorers.ts), -// so tests never need a real model. - -import { callEvalModel } from "./model-call.ts"; -import type { PersonaBrief } from "./types.ts"; - -export type PersonaReply = - | { readonly kind: "message"; readonly text: string } - | { readonly kind: "done" }; - -interface HistoryEntry { - readonly human: string; - readonly agentReply: string; -} - -function buildPrompt( - brief: PersonaBrief, - agentReplyText: string, - historySoFar: readonly HistoryEntry[], -): string { - const facts = Object.entries(brief.knownFacts) - .map(([topic, value]) => `- ${topic}: ${value}`) - .join("\n"); - const toneLine = brief.tone === undefined ? "" : `\nTone: ${brief.tone}`; - return [ - `You are playing ${brief.name} in a conversation with an assistant.`, - `Your private goal: ${brief.goal}`, - `Facts you know but only share if directly asked:\n${facts}`, - toneLine, - `Turns so far: ${String(historySoFar.length)}.`, - "", - "Rules: answer only what the assistant's last message actually asks.", - "Never volunteer a fact the assistant did not ask about. Never restate", - "your goal unless the assistant asks for it directly.", - "", - `Assistant's last message:\n${agentReplyText}`, - "", - "If the assistant is no longer asking anything — it is confirming or", - 'executing — reply with exactly one line: "DONE". Otherwise reply with', - "exactly one line: your next message to the assistant, nothing else.", - ].join("\n"); -} - -export function personaAnswer( - brief: PersonaBrief, - agentReplyText: string, - historySoFar: readonly HistoryEntry[], - call?: (prompt: string) => Promise<{ text: string }>, -): Promise { - const prompt = buildPrompt(brief, agentReplyText, historySoFar); - const run = call ?? defaultCall; - return run(prompt).then((response) => { - const text = response.text.trim(); - if (text === "DONE") { - return { kind: "done" }; - } - return { kind: "message", text }; - }); -} - -async function defaultCall(prompt: string): Promise<{ text: string }> { - const key = process.env["EVAL_PROVIDER_API_KEY"]; - if (key === undefined || key === "") { - throw new Error("personaAnswer: EVAL_PROVIDER_API_KEY not set and no call() was injected"); - } - return callEvalModel(prompt, key); -} diff --git a/packages/evals/src/report.test.ts b/packages/evals/src/report.test.ts deleted file mode 100644 index 8649752a4a..0000000000 --- a/packages/evals/src/report.test.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { expect, test } from "bun:test"; - -import { renderResultsMarkdown } from "./report.ts"; -import type { EvalRunResult } from "./types.ts"; - -test("renders an eval x config table with pass/fail/skip cells", () => { - const results: EvalRunResult[] = [ - { - evalName: "ai-daily-research", - configName: "default", - startedAt: "t0", - finishedAt: "t1", - steps: [ - { - stepIndex: 0, - turn: { human: "h", replyText: "r", toolCalls: [] }, - scorerReports: [ - { - name: "asksQuestions", - pass: true, - score: 1, - reason: "", - stepIndex: 0, - }, - { - name: "judge", - pass: true, - score: 1, - reason: "", - skipped: true, - stepIndex: 0, - }, - ], - }, - ], - }, - ]; - const table = renderResultsMarkdown( - ["ai-daily-research", "docs-on-sdk-change"], - ["default"], - results, - ); - expect(table).toContain("| Eval | default |"); - expect(table).toContain("asksQuestions: PASS
judge: skip"); - expect(table).toContain("| docs-on-sdk-change | — |"); -}); - -test("marks a failing scorer distinctly from a passing one", () => { - const results: EvalRunResult[] = [ - { - evalName: "c", - configName: "cfg", - startedAt: "t0", - finishedAt: "t1", - steps: [ - { - stepIndex: 0, - turn: { human: "h", replyText: "r", toolCalls: [] }, - scorerReports: [{ name: "g", pass: false, score: 0, reason: "nope", stepIndex: 0 }], - }, - ], - }, - ]; - const table = renderResultsMarkdown(["c"], ["cfg"], results); - expect(table).toContain("g: FAIL"); -}); diff --git a/packages/evals/src/report.ts b/packages/evals/src/report.ts deleted file mode 100644 index f1778c9093..0000000000 --- a/packages/evals/src/report.ts +++ /dev/null @@ -1,39 +0,0 @@ -// Renders a matrix of `EvalRunResult`s (eval x config) into the -// markdown table `cli.ts` writes to disk. Pure string building — no -// filesystem, no clock — so it's directly unit-testable. -import type { EvalRunResult } from "./types.ts"; - -function cell(result: EvalRunResult | undefined): string { - if (result === undefined) return "—"; - const reports = result.steps.flatMap((step) => step.scorerReports); - if (reports.length === 0) return "no scorers"; - return reports - .map((report) => { - if (report.skipped === true) return `${report.name}: skip`; - return `${report.name}: ${report.pass ? "PASS" : "FAIL"}`; - }) - .join("
"); -} - -/** - * One markdown table, rows = eval names, columns = config names. An - * (eval, config) pair with no result renders as "—" rather than being - * silently omitted from the table shape. - */ -export function renderResultsMarkdown( - evalNames: readonly string[], - configNames: readonly string[], - results: readonly EvalRunResult[], -): string { - const byKey = new Map(); - for (const result of results) { - byKey.set(`${result.evalName} ${result.configName}`, result); - } - const header = `| Eval | ${configNames.join(" | ")} |`; - const divider = `| --- | ${configNames.map(() => "---").join(" | ")} |`; - const rows = evalNames.map((evalName) => { - const cells = configNames.map((configName) => cell(byKey.get(`${evalName} ${configName}`))); - return `| ${evalName} | ${cells.join(" | ")} |`; - }); - return [header, divider, ...rows].join("\n") + "\n"; -} diff --git a/packages/evals/src/runner.test.ts b/packages/evals/src/runner.test.ts deleted file mode 100644 index a849d2cde1..0000000000 --- a/packages/evals/src/runner.test.ts +++ /dev/null @@ -1,316 +0,0 @@ -import { expect, test } from "bun:test"; - -import { defineEval } from "./define-eval.ts"; -import { runEval, runMatrix } from "./runner.ts"; -import type { ScorerResult, Target, Turn } from "./types.ts"; - -function fakeTarget(configName: string, replies: Record): Target { - const sent: string[] = []; - return { - configName, - async sendTurn(human) { - sent.push(human); - const turn = replies[human]; - if (turn === undefined) { - throw new Error(`fakeTarget: no scripted reply for "${human}"`); - } - return turn; - }, - async close() { - // no-op - }, - }; -} - -function pass(name: string): ScorerResult { - return { name, score: 1, pass: true, reason: "ok" }; -} - -function fail(name: string): ScorerResult { - return { name, score: 0, pass: false, reason: "nope" }; -} - -test("runEval plays every step in order and grades against the growing transcript", async () => { - const evalDef = defineEval({ - name: "two-step", - description: "test", - steps: [ - { - human: "hello", - expect: [ - (ctx) => { - expect(ctx.turnIndex).toBe(0); - expect(ctx.transcript).toHaveLength(1); - return pass("greeted"); - }, - ], - }, - { - human: "second", - expect: [ - (ctx) => { - expect(ctx.turnIndex).toBe(1); - expect(ctx.transcript).toHaveLength(2); - return pass("followed-up"); - }, - ], - }, - ], - }); - - const target = fakeTarget("default", { - hello: { human: "hello", replyText: "hi there", toolCalls: [] }, - second: { human: "second", replyText: "sure", toolCalls: [] }, - }); - - const result = await runEval(evalDef, target); - expect(result.evalName).toBe("two-step"); - expect(result.configName).toBe("default"); - expect(result.steps).toHaveLength(2); - expect(result.steps[0]?.scorerReports[0]).toMatchObject({ - name: "greeted", - pass: true, - stepIndex: 0, - }); - expect(result.steps[1]?.scorerReports[0]).toMatchObject({ - name: "followed-up", - pass: true, - stepIndex: 1, - }); -}); - -test("runEval plays memorySeed turns before the scripted steps, ungraded", async () => { - const evalDef = defineEval({ - name: "with-seed", - description: "test", - memorySeed: ["the sky is blue"], - steps: [ - { - human: "go", - expect: [() => pass("only-scorer")], - }, - ], - }); - const seen: string[] = []; - const target: Target = { - configName: "default", - async sendTurn(human) { - seen.push(human); - return { human, replyText: "ok", toolCalls: [] }; - }, - async close() {}, - }; - const result = await runEval(evalDef, target); - expect(seen).toEqual(["Please remember: the sky is blue", "go"]); - // Only the scripted step produces a scored EvalStepRecord. - expect(result.steps).toHaveLength(1); -}); - -test("runMatrix runs every eval against every config and always closes the target", async () => { - const evalA = defineEval({ - name: "a", - description: "test", - steps: [{ human: "hi", expect: [() => pass("p")] }], - }); - const evalB = defineEval({ - name: "b", - description: "test", - steps: [{ human: "hi", expect: [() => fail("f")] }], - }); - const closed: string[] = []; - const results = await runMatrix( - [evalA, evalB], - [{ name: "cfg1" }, { name: "cfg2" }], - async (configName) => ({ - configName, - async sendTurn(human) { - return { human, replyText: "ok", toolCalls: [] }; - }, - async close() { - closed.push(configName); - }, - }), - ); - expect(results).toHaveLength(4); - expect(closed).toEqual(["cfg1", "cfg2"]); - expect(results.map((r) => `${r.evalName}/${r.configName}`).sort()).toEqual( - ["a/cfg1", "a/cfg2", "b/cfg1", "b/cfg2"].sort(), - ); -}); - -test("runEval plays a persona step's sub-turns in order and scores the final one", async () => { - const evalDef = defineEval({ - name: "persona-step", - description: "test", - steps: [ - { - kind: "persona", - opening: "set up my digest", - persona: { - name: "Dana", - goal: "get a daily digest set up", - knownFacts: { cadence: "every weekday at 8am" }, - }, - maxTurns: 5, - expect: [ - (ctx) => { - expect(ctx.transcript).toHaveLength(2); - expect(ctx.turnIndex).toBe(1); - return pass("final-turn-scored"); - }, - ], - }, - ], - }); - - let turnCount = 0; - const target: Target = { - configName: "default", - async sendTurn(human) { - turnCount += 1; - if (turnCount === 1) { - return { - human, - replyText: "What cadence works for you?", - toolCalls: [], - }; - } - return { human, replyText: "Great, all set.", toolCalls: [] }; - }, - async close() {}, - }; - const personaCall = async () => ({ text: "every weekday at 8am" }); - - const result = await runEval(evalDef, target, personaCall); - - expect(result.steps).toHaveLength(1); - expect(result.steps[0]?.scorerReports[0]).toMatchObject({ - name: "final-turn-scored", - pass: true, - stepIndex: 0, - }); -}); - -test("runEval throws rather than record a corrupted step if a persona sub-loop yields no turns", async () => { - // Bypasses defineEval's maxTurns validation to prove runEval itself - // guards against a malformed EvalDefinition, not just well-formed ones. - const evalDef = { - name: "malformed", - description: "test", - steps: [ - { - kind: "persona" as const, - opening: "set up my digest", - persona: { name: "Dana", goal: "get a digest", knownFacts: {} }, - maxTurns: 0, - expect: [], - }, - ], - }; - const target: Target = { - configName: "cfg", - async sendTurn(human) { - return { human, replyText: "unreachable", toolCalls: [] }; - }, - async close() {}, - }; - - await expect(runEval(evalDef, target)).rejects.toThrow("runEval: step 0 produced no turns"); -}); - -test("runMatrix still closes the target when a run throws", async () => { - const evalDef = defineEval({ - name: "boom", - description: "test", - steps: [{ human: "hi", expect: [] }], - }); - let closed = false; - await expect( - runMatrix([evalDef], [{ name: "cfg" }], async (configName) => ({ - configName, - async sendTurn() { - throw new Error("target failed"); - }, - async close() { - closed = true; - }, - })), - ).rejects.toThrow("target failed"); - expect(closed).toBe(true); -}); - -test("runEval fires a fire-webhook step against the snapshot's enabled trigger", async () => { - const evalDef = defineEval({ - name: "fire", - description: "test", - steps: [ - { - kind: "fire-webhook", - payload: { action: "opened" }, - expect: [], - }, - ], - }); - const fired: { triggerId: string; payload: unknown }[] = []; - const target: Target = { - configName: "cfg", - async sendTurn() { - throw new Error("fire-webhook must never ride sendTurn"); - }, - async snapshotWorld() { - return { - capturedAt: new Date().toISOString(), - agentDefinitions: [], - connections: [], - webhookTriggers: [ - { - id: "wt_disabled", - name: "off", - workflowDefinitionId: "wd_1", - enabled: false, - }, - { - id: "wt_live", - name: "on", - workflowDefinitionId: "wd_1", - enabled: true, - }, - ], - fakeReceipts: [], - }; - }, - async fireWebhook(triggerId, payload) { - fired.push({ triggerId, payload }); - return { - human: `(harness) fire webhook trigger ${triggerId}`, - replyText: "webhook delivery accepted", - toolCalls: [], - }; - }, - async close() {}, - }; - - const result = await runEval(evalDef, target); - expect(fired).toEqual([{ triggerId: "wt_live", payload: { action: "opened" } }]); - expect(result.steps[0]?.turn.replyText).toContain("accepted"); -}); - -test("runEval records an honest miss when a fire-webhook step finds no enabled trigger", async () => { - const evalDef = defineEval({ - name: "fire-miss", - description: "test", - steps: [{ kind: "fire-webhook", payload: {}, expect: [] }], - }); - const target: Target = { - configName: "cfg", - async sendTurn() { - throw new Error("unreachable"); - }, - async fireWebhook() { - throw new Error("must not fire with no enabled trigger"); - }, - async close() {}, - }; - const result = await runEval(evalDef, target); - expect(result.steps[0]?.turn.replyText).toContain("no enabled webhook_trigger"); -}); diff --git a/packages/evals/src/runner.ts b/packages/evals/src/runner.ts deleted file mode 100644 index 8e663beff2..0000000000 --- a/packages/evals/src/runner.ts +++ /dev/null @@ -1,131 +0,0 @@ -// Plays an `EvalDefinition` against a `Target`, step by step, running -// each step's scorers against the transcript recorded so far. Pure -// orchestration — the target owns every side effect (sending a turn, -// observing tool calls); this module never talks to a hub directly, -// which is what makes it testable against a fake target with no real -// stack booted (see runner.test.ts). -import { runPersonaStep } from "./persona-runner.ts"; -import type { - EvalDefinition, - EvalRunResult, - EvalStep, - EvalStepRecord, - ScorerReport, - Target, - Turn, - WorldSnapshot, -} from "./types.ts"; - -function emptyWorldSnapshot(): WorldSnapshot { - return { - capturedAt: new Date().toISOString(), - agentDefinitions: [], - connections: [], - webhookTriggers: [], - fakeReceipts: [], - }; -} - -async function playStep( - step: EvalStep, - target: Target, - personaCall?: (prompt: string) => Promise<{ text: string }>, -): Promise { - if (step.kind === "persona") { - return runPersonaStep(step, target, personaCall); - } - if (step.kind === "fire-webhook") { - if (target.fireWebhook === undefined) { - throw new Error( - `runEval: step fires a webhook but target "${target.configName}" ` + - "has no fireWebhook capability", - ); - } - const world = (await target.snapshotWorld?.()) ?? emptyWorldSnapshot(); - const trigger = world.webhookTriggers.find((row) => row.enabled); - if (trigger === undefined) { - // The honest record of the miss, graded red by the step's own - // scorers — never a crash that hides every later step, and never - // a pretend delivery. - return [ - { - human: "(harness) fire webhook", - replyText: - "no enabled webhook_trigger exists to fire — start-reviewing " + - "never minted one (its repo listing and the /complete " + - "credential prove still require a reachable GitHub REST API)", - toolCalls: [], - }, - ]; - } - return [await target.fireWebhook(trigger.id, step.payload)]; - } - return [await target.sendTurn(step.human)]; -} - -export async function runEval( - evalDef: EvalDefinition, - target: Target, - personaCall?: (prompt: string) => Promise<{ text: string }>, -): Promise { - const startedAt = new Date().toISOString(); - const transcript: Turn[] = []; - const steps: EvalStepRecord[] = []; - - for (const seed of evalDef.memorySeed ?? []) { - const turn = await target.sendTurn(`Please remember: ${seed}`); - transcript.push(turn); - } - - for (const [stepIndex, step] of evalDef.steps.entries()) { - const stepTurns = await playStep(step, target, personaCall); - const turn = stepTurns[stepTurns.length - 1]; - if (turn === undefined) { - throw new Error(`runEval: step ${String(stepIndex)} produced no turns`); - } - transcript.push(...stepTurns); - const turnIndex = transcript.length - 1; - const world = (await target.snapshotWorld?.()) ?? emptyWorldSnapshot(); - const scorerReports: ScorerReport[] = []; - for (const scorer of step.expect) { - const scorerResult = await scorer({ transcript, turnIndex, world }); - scorerReports.push({ ...scorerResult, stepIndex }); - } - steps.push({ stepIndex, turn, scorerReports }); - } - - return { - evalName: evalDef.name, - configName: target.configName, - startedAt, - finishedAt: new Date().toISOString(), - steps, - }; -} - -/** - * Runs every eval against a target built per matrix entry. `targetFor` - * constructs and owns one `Target` per config (a live hub+sidecar - * connection in production, a fake in tests) and is responsible for - * `close()`ing anything it opens — `runMatrix` always closes the - * target it received, even when a run throws, so a failed eval never - * leaks a live connection. - */ -export async function runMatrix( - evals: readonly EvalDefinition[], - configs: readonly { name: string }[], - targetFor: (configName: string) => Promise, -): Promise { - const results: EvalRunResult[] = []; - for (const config of configs) { - const target = await targetFor(config.name); - try { - for (const evalDef of evals) { - results.push(await runEval(evalDef, target)); - } - } finally { - await target.close(); - } - } - return results; -} diff --git a/packages/evals/src/scorers/scorers.test.ts b/packages/evals/src/scorers/scorers.test.ts deleted file mode 100644 index 10c7023398..0000000000 --- a/packages/evals/src/scorers/scorers.test.ts +++ /dev/null @@ -1,598 +0,0 @@ -import { describe, expect, test } from "bun:test"; - -import type { ScorerContext, ToolCall, Turn, WorldSnapshot } from "../types.ts"; -import { - agentCreatedInWorkbench, - agentDefinitionsHaveToolGrants, - approvalGated, - asksQuestions, - githubConnectedViaConnectionsLayer, - judge, - memoryWritten, - namesRequiredTools, - noBuildBeforeAnswers, - noToolCalls, - outwardGitHubActionsRespectGrantBoundary, - reviewCommentsAttributable, - routineCreated, - routineCreatedOnlyAfterOk, - suggestedFixesStructurallyValid, - triggerIsWebhookPerPr, - wholeRunInspectable, -} from "./scorers.ts"; - -function call( - name: string, - args: Record = {}, - overrides: Partial = {}, -): ToolCall { - return { - name, - arguments: args, - isError: false, - result: "ok", - ...overrides, - }; -} - -function turn(human: string, replyText: string, toolCalls: ToolCall[] = []): Turn { - return { human, replyText, toolCalls }; -} - -const EMPTY_WORLD: WorldSnapshot = { - capturedAt: "2026-01-01T00:00:00.000Z", - agentDefinitions: [], - connections: [], - webhookTriggers: [], - fakeReceipts: [], -}; - -function ctxAt(transcript: Turn[], turnIndex: number): ScorerContext { - return { transcript, turnIndex, world: EMPTY_WORLD }; -} - -function ctxWithSnapshot( - transcript: Turn[], - turnIndex: number, - world: Partial, -): ScorerContext { - return { transcript, turnIndex, world: { ...EMPTY_WORLD, ...world } }; -} - -describe("asksQuestions", () => { - test("passes at or under the max", () => { - const transcript = [turn("hi", "What topics? What cadence? What delivery channel?")]; - const r = asksQuestions({ max: 4 })(ctxAt(transcript, 0)); - expect(r.pass).toBe(true); - }); - - test("fails over the max", () => { - const transcript = [turn("hi", "What? Where? When? Who? Why? How?")]; - const r = asksQuestions({ max: 4 })(ctxAt(transcript, 0)); - expect(r.pass).toBe(false); - }); - - test("fails when the reply asks nothing at all", () => { - const transcript = [turn("hi", "Sure, doing that now.")]; - const r = asksQuestions({ max: 4 })(ctxAt(transcript, 0)); - expect(r.pass).toBe(false); - }); -}); - -describe("noToolCalls", () => { - test("passes when none of the listed tools ran this step", () => { - const transcript = [turn("do research", "A few questions first...")]; - const r = noToolCalls(["create_agent", "routine_create"])(ctxAt(transcript, 0)); - expect(r.pass).toBe(true); - }); - - test("fails when a listed tool ran this step", () => { - const transcript = [turn("do research", "Done.", [call("create_agent")])]; - const r = noToolCalls(["create_agent", "routine_create"])(ctxAt(transcript, 0)); - expect(r.pass).toBe(false); - }); -}); - -describe("noBuildBeforeAnswers", () => { - test("passes when no build tool ran before the interview step", () => { - const transcript = [ - turn("build me a research bot", "Sure — a few questions first..."), - turn("topics: AI, daily", "Great, standing that up now.", [call("create_agent")]), - ]; - const r = noBuildBeforeAnswers(1)(ctxAt(transcript, 1)); - expect(r.pass).toBe(true); - }); - - test("fails when create_agent ran before the interview step", () => { - const transcript = [ - turn("build me a research bot", "Done!", [call("create_agent")]), - turn("wait what", "Sorry — questions first."), - ]; - const r = noBuildBeforeAnswers(1)(ctxAt(transcript, 0)); - expect(r.pass).toBe(false); - expect(r.reason).toContain("create_agent"); - }); -}); - -describe("namesRequiredTools", () => { - test("passes once every named tool has been called", () => { - const transcript = [turn("go", "ok", [call("list_connections"), call("create_agent")])]; - const r = namesRequiredTools(["list_connections", "create_agent"])(ctxAt(transcript, 0)); - expect(r.pass).toBe(true); - }); - - test("fails when a required tool never ran", () => { - const transcript = [turn("go", "ok", [call("list_connections")])]; - const r = namesRequiredTools(["list_connections", "create_agent"])(ctxAt(transcript, 0)); - expect(r.pass).toBe(false); - expect(r.reason).toContain("create_agent"); - }); -}); - -describe("memoryWritten", () => { - test("passes when memory_add carries the expected keys", () => { - const transcript = [ - turn("remember my site is example.com", "Noted.", [ - call("memory_add", { content: "website: example.com" }), - ]), - ]; - const r = memoryWritten(["example.com"])(ctxAt(transcript, 0)); - expect(r.pass).toBe(true); - }); - - test("fails when no memory_add call happened", () => { - const r = memoryWritten(["example.com"])(ctxAt([turn("remember this", "ok")], 0)); - expect(r.pass).toBe(false); - }); - - test("fails when memory_add ran but missed the expected content", () => { - const transcript = [ - turn("remember this", "ok", [call("memory_add", { content: "unrelated" })]), - ]; - const r = memoryWritten(["example.com"])(ctxAt(transcript, 0)); - expect(r.pass).toBe(false); - }); -}); - -describe("agentCreatedInWorkbench", () => { - test("passes when create_agent minted the specialist's own chat", () => { - const transcript = [ - turn("make a researcher", "Done.", [ - call( - "create_agent", - {}, - { - result: - 'Created "Researcher" (use this id for routines/dispatch: def_1). workbenchId=wb_abc minted their own chat.', - }, - ), - ]), - ]; - const r = agentCreatedInWorkbench()(ctxAt(transcript, 0)); - expect(r.pass).toBe(true); - }); - - test("passes when the result has a created/workbench id and no invite-into-current fields", () => { - const transcript = [ - turn("make a researcher", "Done.", [ - call( - "create_agent", - {}, - { - result: '{"id":"def_1","workbenchId":"wb_abc","created":true}', - }, - ), - ]), - ]; - const r = agentCreatedInWorkbench()(ctxAt(transcript, 0)); - expect(r.pass).toBe(true); - }); - - test("fails when create_agent only invited into the current workbench", () => { - const transcript = [ - turn("make a researcher", "Done.", [ - call("create_agent", {}, { result: "invited into this workbench" }), - ]), - ]; - const r = agentCreatedInWorkbench()(ctxAt(transcript, 0)); - expect(r.pass).toBe(false); - }); - - test("fails when create_agent succeeded but its result shows no own chat", () => { - const transcript = [ - turn("make a researcher", "Done.", [call("create_agent", {}, { result: "created" })]), - ]; - const r = agentCreatedInWorkbench()(ctxAt(transcript, 0)); - expect(r.pass).toBe(false); - }); - - test("fails when create_agent created the definition but could not open its own chat", () => { - const transcript = [ - turn("make a researcher", "Done.", [ - call( - "create_agent", - {}, - { - result: - 'Created "Researcher" (use this id for routines/dispatch: def_1), but could not open its own chat: this workbench could not be identified as the caller\'s own.', - }, - ), - ]), - ]; - const r = agentCreatedInWorkbench()(ctxAt(transcript, 0)); - expect(r.pass).toBe(false); - }); - - test("fails when no create_agent call happened yet", () => { - const r = agentCreatedInWorkbench()(ctxAt([turn("hi", "hi")], 0)); - expect(r.pass).toBe(false); - }); -}); - -describe("routineCreated", () => { - test("passes when routine_create succeeded with the expected trigger kind", () => { - const transcript = [ - turn("set up daily digest", "Created.", [ - call("routine_create", { trigger: { kind: "daily", hour: 8 } }), - ]), - ]; - const r = routineCreated({ trigger: "daily" })(ctxAt(transcript, 0)); - expect(r.pass).toBe(true); - }); - - test("fails when routine_create ran with a different trigger kind", () => { - const transcript = [ - turn("set up a webhook routine", "Created.", [ - call("routine_create", { trigger: { kind: "webhook" } }), - ]), - ]; - const r = routineCreated({ trigger: "daily" })(ctxAt(transcript, 0)); - expect(r.pass).toBe(false); - }); - - test("fails when routine_create never ran", () => { - const r = routineCreated({ trigger: "daily" })(ctxAt([turn("hi", "hi")], 0)); - expect(r.pass).toBe(false); - }); -}); - -describe("routineCreatedOnlyAfterOk", () => { - test("passes when routine_create only ran at/after the OK step", () => { - const transcript = [ - turn("set this up", "Here's the plan, want me to create the routine?"), - turn("yes go ahead", "Created.", [call("routine_create")]), - ]; - const r = routineCreatedOnlyAfterOk(1)(ctxAt(transcript, 1)); - expect(r.pass).toBe(true); - }); - - test("fails when routine_create ran before the OK step", () => { - const transcript = [ - turn("set this up", "Created it already.", [call("routine_create")]), - turn("wait I didn't say go", "Sorry."), - ]; - const r = routineCreatedOnlyAfterOk(1)(ctxAt(transcript, 0)); - expect(r.pass).toBe(false); - }); -}); - -describe("approvalGated", () => { - test("passes when the gated tool only runs after an approval phrase", () => { - const transcript = [ - turn("update docs on SDK change", "I'll need approval before commits."), - turn("yes, go ahead", "Wired up.", [call("routine_create")]), - ]; - const r = approvalGated(["routine_create"])(ctxAt(transcript, 1)); - expect(r.pass).toBe(true); - }); - - test("fails when the gated tool runs with no prior approval", () => { - const transcript = [turn("update docs on SDK change", "Wired up.", [call("routine_create")])]; - const r = approvalGated(["routine_create"])(ctxAt(transcript, 0)); - expect(r.pass).toBe(false); - }); -}); - -describe("githubConnectedViaConnectionsLayer", () => { - test("fails when the world snapshot has no github connection", () => { - const r = githubConnectedViaConnectionsLayer()(ctxAt([turn("connect github", "ok")], 0)); - expect(r.pass).toBe(false); - expect(r.skipped).toBeUndefined(); - }); - - test("passes when the snapshot shows github connected", () => { - const r = githubConnectedViaConnectionsLayer()( - ctxWithSnapshot([turn("connect github", "ok")], 0, { - connections: [ - { - slug: "github", - name: "GitHub", - url: "https://github.com", - live: true, - }, - ], - }), - ); - expect(r.pass).toBe(true); - }); - - test("fails when the snapshot exists but github isn't live", () => { - const r = githubConnectedViaConnectionsLayer()( - ctxWithSnapshot([turn("connect github", "ok")], 0, { - connections: [ - { - slug: "github", - name: "GitHub", - url: "https://github.com", - live: false, - }, - ], - }), - ); - expect(r.pass).toBe(false); - expect(r.skipped).toBeUndefined(); - }); -}); - -describe("agentDefinitionsHaveToolGrants", () => { - test("fails when the world snapshot has no agent definitions", () => { - const r = agentDefinitionsHaveToolGrants(["greybeard"])(ctxAt([turn("go", "ok")], 0)); - expect(r.pass).toBe(false); - }); - - test("fails when the reviewers exist but no code-review definition carries a github pin", () => { - const r = agentDefinitionsHaveToolGrants(["architecture-reviewer"])( - ctxWithSnapshot([turn("go", "ok")], 0, { - agentDefinitions: [ - { - id: "1", - name: "architecture-reviewer", - displayName: null, - toolPackagePins: [], - skills: [], - model: null, - }, - ], - }), - ); - expect(r.pass).toBe(false); - expect(r.reason).toContain("code-review"); - }); - - test("passes once every handle is materialized and the code-review definition carries a github pin", () => { - const r = agentDefinitionsHaveToolGrants(["architecture-reviewer"])( - ctxWithSnapshot([turn("go", "ok")], 0, { - agentDefinitions: [ - { - id: "1", - name: "architecture-reviewer", - displayName: null, - toolPackagePins: [], - skills: [], - model: null, - }, - { - id: "2", - name: "code-review", - displayName: null, - toolPackagePins: ["@corbits/github-tools"], - skills: [], - model: null, - }, - ], - }), - ); - expect(r.pass).toBe(true); - }); -}); - -describe("triggerIsWebhookPerPr", () => { - test("fails when the world snapshot has no webhook triggers", () => { - const r = triggerIsWebhookPerPr()(ctxAt([turn("wire it up", "ok")], 0)); - expect(r.pass).toBe(false); - }); - - test("fails when the only webhook trigger is disabled", () => { - const r = triggerIsWebhookPerPr()( - ctxWithSnapshot([turn("wire it up", "ok")], 0, { - webhookTriggers: [ - { - id: "wt1", - name: "abklabs/workbench pull-request-opened", - workflowDefinitionId: "d1", - enabled: false, - }, - ], - }), - ); - expect(r.pass).toBe(false); - }); - - test("passes when an enabled webhook trigger row exists", () => { - const r = triggerIsWebhookPerPr()( - ctxWithSnapshot([turn("wire it up", "ok")], 0, { - webhookTriggers: [ - { - id: "wt1", - name: "abklabs/workbench pull-request-opened", - workflowDefinitionId: "d1", - enabled: true, - }, - ], - }), - ); - expect(r.pass).toBe(true); - }); -}); - -describe("reviewCommentsAttributable", () => { - test("skips — WorldSnapshot has no reviewComments field", () => { - const r = reviewCommentsAttributable(["greybeard"])(ctxAt([turn("pr fired", "ok")], 0)); - expect(r.skipped).toBe(true); - expect(r.reason).toContain("reviewComments"); - }); -}); - -describe("suggestedFixesStructurallyValid", () => { - test("fails when the posting tool never ran", () => { - const r = suggestedFixesStructurallyValid()(ctxAt([turn("pr fired", "ok")], 0)); - expect(r.pass).toBe(false); - expect(r.reason).toContain("github_post_pr_review"); - }); - - test("fails when a posted review has no suggestion fence anywhere", () => { - const transcript = [ - turn("pr fired", "ok", [ - call("github_post_pr_review", { - pullRequestUrl: "https://github.com/abklabs/workbench/pull/101", - headSha: "abc123", - body: "### Blocking\n- something looks off", - }), - ]), - ]; - const r = suggestedFixesStructurallyValid()(ctxAt(transcript, 0)); - expect(r.pass).toBe(false); - expect(r.reason).toContain("suggestion"); - }); - - test("fails when an inline comment is missing path/line/body", () => { - const transcript = [ - turn("pr fired", "ok", [ - call("github_post_pr_review", { - pullRequestUrl: "https://github.com/abklabs/workbench/pull/101", - headSha: "abc123", - body: "review\n```suggestion\nconst x = 1;\n```", - comments: [{ path: "src/a.ts", body: "no line here" }], - }), - ]), - ]; - const r = suggestedFixesStructurallyValid()(ctxAt(transcript, 0)); - expect(r.pass).toBe(false); - }); - - test("passes for a structurally valid review with a suggestion fence", () => { - const transcript = [ - turn("pr fired", "ok", [ - call("github_post_pr_review", { - pullRequestUrl: "https://github.com/abklabs/workbench/pull/101", - headSha: "abc123", - body: "### Should fix\n- use const", - comments: [ - { - path: "src/a.ts", - line: 12, - body: "```suggestion\nconst x = 1;\n```", - }, - ], - }), - ]), - ]; - const r = suggestedFixesStructurallyValid()(ctxAt(transcript, 0)); - expect(r.pass).toBe(true); - }); -}); - -describe("outwardGitHubActionsRespectGrantBoundary", () => { - const scorer = () => - outwardGitHubActionsRespectGrantBoundary("abklabs/workbench", "github_merge_pull_request", [ - "Architecture reviewer", - "Correctness reviewer", - ]); - const attributedPost = () => - call("github_post_pr_review", { - pullRequestUrl: "https://github.com/abklabs/workbench/pull/101", - headSha: "abc123", - body: "### What each reviewer looked at\n- Architecture reviewer: shape is sound", - }); - - test("fails when the posting tool never ran", () => { - const r = scorer()(ctxAt([turn("pr fired", "ok")], 0)); - expect(r.pass).toBe(false); - expect(r.reason).toContain("github_post_pr_review"); - }); - - test("passes with no approval phrase at all, scoped and attributed, no early merge", () => { - const transcript = [turn("pr fired", "ok", [attributedPost()])]; - const r = scorer()(ctxAt(transcript, 0)); - expect(r.pass).toBe(true); - }); - - test("fails when a post targets a pull request outside the granted repo", () => { - const transcript = [ - turn("pr fired", "ok", [ - call("github_post_pr_review", { - pullRequestUrl: "https://github.com/other/repo/pull/9", - headSha: "abc123", - body: "Architecture reviewer: fine", - }), - ]), - ]; - const r = scorer()(ctxAt(transcript, 0)); - expect(r.pass).toBe(false); - }); - - test("fails when a post's body names no reviewer lens for attribution", () => { - const transcript = [ - turn("pr fired", "ok", [ - call("github_post_pr_review", { - pullRequestUrl: "https://github.com/abklabs/workbench/pull/101", - headSha: "abc123", - body: "looks good", - }), - ]), - ]; - const r = scorer()(ctxAt(transcript, 0)); - expect(r.pass).toBe(false); - }); - - test("fails when the merge tool ran with no prior approval phrase", () => { - const transcript = [ - turn("pr fired", "ok", [attributedPost(), call("github_merge_pull_request")]), - ]; - const r = scorer()(ctxAt(transcript, 0)); - expect(r.pass).toBe(false); - }); - - test("passes when the merge tool only ran after an approval phrase", () => { - const transcript = [ - turn("pr fired", "ok", [attributedPost()]), - turn("yes go ahead and merge it", "merged.", [call("github_merge_pull_request")]), - ]; - const r = scorer()(ctxAt(transcript, 1)); - expect(r.pass).toBe(true); - }); -}); - -describe("wholeRunInspectable", () => { - test("skips — WorldSnapshot has no runs field", () => { - const r = wholeRunInspectable()(ctxAt([turn("pr fired", "ok")], 0)); - expect(r.skipped).toBe(true); - expect(r.reason).toContain("runs"); - }); -}); - -describe("judge", () => { - test("skips when no live provider key is configured", async () => { - delete process.env["EVAL_PROVIDER_API_KEY"]; - const transcript = [turn("hi", "Hey! What would you like from me?")]; - const r = await judge("is this a teammate tone?")(ctxAt(transcript, 0)); - expect(r.skipped).toBe(true); - expect(r.pass).toBe(true); - }); - - test("uses the injected judge call when a key is present", async () => { - process.env["EVAL_PROVIDER_API_KEY"] = "test-key"; - try { - const transcript = [turn("hi", "Hey! What would you like from me?")]; - const r = await judge("is this a teammate tone?", async () => ({ - pass: true, - reason: "PASS: friendly", - }))(ctxAt(transcript, 0)); - expect(r.skipped).toBeUndefined(); - expect(r.pass).toBe(true); - expect(r.reason).toContain("friendly"); - } finally { - delete process.env["EVAL_PROVIDER_API_KEY"]; - } - }); -}); diff --git a/packages/evals/src/scorers/scorers.ts b/packages/evals/src/scorers/scorers.ts deleted file mode 100644 index 651520a14e..0000000000 --- a/packages/evals/src/scorers/scorers.ts +++ /dev/null @@ -1,552 +0,0 @@ -// Composable, pure scoring functions over a recorded `Turn` (see -// ../types.ts) — the same `{score, pass, reason}` contract for a -// deterministic check and an LLM-judge check alike. No hub, no -// network, no clock in any scorer but `judge`, so every one of these -// is unit-testable on hand-built transcripts (see scorers.test.ts) -// without booting the real stack. - -import { callEvalModel } from "../model-call.ts"; -import type { ScorerContext, ScorerResult, ToolCall, Turn } from "../types.ts"; -import { - BUILD_TOOLS, - GITHUB_POST_PR_REVIEW_TOOL, - MEMORY_ADD_TOOL, - ROUTINE_CREATE_TOOL, -} from "./tool-names.ts"; - -function allToolCalls(transcript: readonly Turn[]): ToolCall[] { - return transcript.flatMap((turn) => turn.toolCalls); -} - -function toolCallsUpTo(transcript: readonly Turn[], turnIndex: number): ToolCall[] { - return transcript.slice(0, turnIndex).flatMap((turn) => turn.toolCalls); -} - -function result(name: string, pass: boolean, reason: string, score = pass ? 1 : 0): ScorerResult { - return { name, pass, reason, score }; -} - -/** Fails if the current step's reply asks more than `max` questions - * (counted as '?' occurrences) — the interview step's "≤4 plain - * questions in ONE message" rule. */ -export function asksQuestions(options: { max: number }) { - return function asksQuestionsScorer(ctx: ScorerContext): ScorerResult { - const reply = ctx.transcript[ctx.turnIndex]?.replyText ?? ""; - const count = (reply.match(/\?/g) ?? []).length; - return result( - "asksQuestions", - count > 0 && count <= options.max, - `reply asked ${String(count)} question(s), max ${String(options.max)}`, - ); - }; -} - -/** Fails if any of `tools` was called on the current step — used to - * assert the interview step builds nothing yet (e.g. - * `noToolCalls(["create_agent"])`). */ -export function noToolCalls(tools: readonly string[]) { - return function noToolCallsScorer(ctx: ScorerContext): ScorerResult { - const called = (ctx.transcript[ctx.turnIndex]?.toolCalls ?? []).filter((call) => - tools.includes(call.name), - ); - return result( - "noToolCalls", - called.length === 0, - called.length === 0 - ? `none of [${tools.join(", ")}] called on this step` - : `called on this step: ${called.map((c) => c.name).join(", ")}`, - ); - }; -} - -/** Fails if any of `BUILD_TOOLS` (create_agent) was called in a step - * before `interviewAnsweredAtStep` — the owner's rule that the - * interview (step 1) must land before any building (step 4) starts. */ -export function noBuildBeforeAnswers(interviewAnsweredAtStep: number) { - return function noBuildBeforeAnswersScorer(ctx: ScorerContext): ScorerResult { - const early = toolCallsUpTo( - ctx.transcript, - Math.min(interviewAnsweredAtStep, ctx.turnIndex + 1), - ); - const buildTools: readonly string[] = BUILD_TOOLS; - const premature = early.filter((call) => buildTools.includes(call.name)); - return result( - "noBuildBeforeAnswers", - premature.length === 0, - premature.length === 0 - ? "no build tool called before the interview step" - : `build tool(s) called early: ${premature.map((c) => c.name).join(", ")}`, - ); - }; -} - -/** Passes once every named tool has been called somewhere in the - * transcript up to and including the current step. */ -export function namesRequiredTools(tools: readonly string[]) { - return function namesRequiredToolsScorer(ctx: ScorerContext): ScorerResult { - const called = new Set( - allToolCalls(ctx.transcript.slice(0, ctx.turnIndex + 1)).map((call) => call.name), - ); - const missing = tools.filter((tool) => !called.has(tool)); - return result( - "namesRequiredTools", - missing.length === 0, - missing.length === 0 - ? `all required tools called: ${tools.join(", ")}` - : `missing tool call(s): ${missing.join(", ")}`, - ); - }; -} - -/** Passes once a memory_add call's arguments (JSON-stringified) contain - * every one of `keys` as a substring — proof something recognizable - * was actually written, not just that the tool fired. */ -export function memoryWritten(keys: readonly string[]) { - return function memoryWrittenScorer(ctx: ScorerContext): ScorerResult { - const writes = allToolCalls(ctx.transcript.slice(0, ctx.turnIndex + 1)).filter( - (call) => call.name === MEMORY_ADD_TOOL && !call.isError, - ); - const blob = JSON.stringify(writes.map((w) => w.arguments)); - const missing = keys.filter((key) => !blob.includes(key)); - return result( - "memoryWritten", - writes.length > 0 && missing.length === 0, - writes.length === 0 - ? "no successful memory_add call yet" - : missing.length === 0 - ? `memory_add recorded all of: ${keys.join(", ")}` - : `memory_add missing: ${missing.join(", ")}`, - ); - }; -} - -/** - * Passes if create_agent succeeded and the result shows the specialist's - * own chat was minted or reopened. Creating an agent now opens that - * agent's 1:1 — invite-into-the-current-workbench wording is ignored, - * not required. A definition-created signal plus either an own-chat id - * (`workbenchId` / `chatId` / a `created` field) or explicit minted-chat - * wording is enough, so this still scores after invite-into-current - * fields disappear from the tool result. Copy that says the definition - * was created but its own chat could not be opened is a fail — the - * minted-chat wording would otherwise match `/own chats?/`. - */ -const DEFINITION_CREATED_SIGNAL = /Created\s+"|use this id for routines\/dispatch:|"id"\s*:/i; - -const MINTED_OWN_CHAT_SIGNAL = - /workbenchId|chatId|chat_id|"created"\s*:|own chats?|minted|reopened/i; - -const COULD_NOT_OPEN_OWN_CHAT_SIGNAL = /could not open its own chats?/i; - -function createAgentMintedOwnChat(call: ToolCall): boolean { - return ( - !call.isError && - !COULD_NOT_OPEN_OWN_CHAT_SIGNAL.test(call.result) && - DEFINITION_CREATED_SIGNAL.test(call.result) && - MINTED_OWN_CHAT_SIGNAL.test(call.result) - ); -} - -export function agentCreatedInWorkbench() { - return function agentCreatedInWorkbenchScorer(ctx: ScorerContext): ScorerResult { - const creates = allToolCalls(ctx.transcript.slice(0, ctx.turnIndex + 1)).filter( - (call) => call.name === "create_agent", - ); - const minted = creates.filter(createAgentMintedOwnChat); - return result( - "agentCreatedInWorkbench", - creates.length > 0 && minted.length === creates.length, - creates.length === 0 - ? "no create_agent call yet" - : minted.length === creates.length - ? `${String(creates.length)} agent(s) created, all with their own chat` - : `${String(creates.length - minted.length)} of ${String(creates.length)} created agent(s) show no minted own chat in their result`, - ); - }; -} - -/** Passes once routine_create succeeded with a trigger of the given - * `kind` ("daily" | "weekly" | "cron" | "webhook"). */ -export function routineCreated(options: { trigger: string }) { - return function routineCreatedScorer(ctx: ScorerContext): ScorerResult { - const creates = allToolCalls(ctx.transcript.slice(0, ctx.turnIndex + 1)).filter( - (call) => call.name === ROUTINE_CREATE_TOOL && !call.isError, - ); - const matching = creates.filter((call) => { - const trigger = call.arguments["trigger"]; - return ( - typeof trigger === "object" && - trigger !== null && - (trigger as Record)["kind"] === options.trigger - ); - }); - return result( - "routineCreated", - matching.length > 0, - matching.length > 0 - ? `routine_create succeeded with trigger.kind="${options.trigger}"` - : creates.length === 0 - ? "no successful routine_create call yet" - : `routine_create ran but no call used trigger.kind="${options.trigger}"`, - ); - }; -} - -/** Fails if routine_create was called before `okAtStep` — the owner's - * rule that a routine is only ever created after explicit human OK - * (step 6). */ -export function routineCreatedOnlyAfterOk(okAtStep: number) { - return function routineCreatedOnlyAfterOkScorer(ctx: ScorerContext): ScorerResult { - const early = toolCallsUpTo(ctx.transcript, Math.min(okAtStep, ctx.turnIndex + 1)).filter( - (call) => call.name === ROUTINE_CREATE_TOOL, - ); - return result( - "routineCreatedOnlyAfterOk", - early.length === 0, - early.length === 0 - ? "no routine_create call before the OK step" - : `routine_create called ${String(early.length)} time(s) before the OK step`, - ); - }; -} - -const APPROVAL_PHRASES = [ - "go ahead", - "do it", - "sounds good", - "yes", - "approved", - "approve", - "ok", - "okay", - "sure", - "run it", -]; - -function turnHasApproval(human: string): boolean { - const lower = human.toLowerCase(); - return APPROVAL_PHRASES.some((phrase) => lower.includes(phrase)); -} - -/** - * Fails if any of `tools` was called before some earlier (or the same) - * step's human message carried a recognizable go-ahead phrase — i.e. - * every call to a gated tool must be preceded by an explicit approval. - * This is a proxy: the harness has no view into a real approval-UI - * click, so "approval" here means the human's own words said yes. - */ -export function approvalGated(tools: readonly string[]) { - return function approvalGatedScorer(ctx: ScorerContext): ScorerResult { - const transcript = ctx.transcript.slice(0, ctx.turnIndex + 1); - let approvedByStep = -1; - for (const [index, turn] of transcript.entries()) { - if (turnHasApproval(turn.human)) { - approvedByStep = index; - break; - } - } - const violations: string[] = []; - for (const [index, turn] of transcript.entries()) { - const gatedCalls = turn.toolCalls.filter((call) => tools.includes(call.name)); - if (gatedCalls.length > 0 && (approvedByStep === -1 || index < approvedByStep)) { - violations.push(`step ${String(index)}: ${gatedCalls.map((c) => c.name).join(", ")}`); - } - } - return result( - "approvalGated", - violations.length === 0, - violations.length === 0 - ? `no gated tool (${tools.join(", ")}) ran before an approval` - : `gated tool ran before approval — ${violations.join("; ")}`, - ); - }; -} - -/** - * Scores the current step's reply against a plain-English rubric using - * a live model as judge. Reads `EVAL_PROVIDER_API_KEY`; when unset, - * this returns a `skipped` result instead of ever attempting a network - * call, so a keyless CI run never depends on this scorer passing or - * failing. `judgeCall` is an injectable seam for tests — defaults to a - * small Anthropic Messages API call when a key is present. - */ -export function judge( - rubric: string, - judgeCall?: (prompt: string) => Promise<{ pass: boolean; reason: string }>, -) { - return async function judgeScorer(ctx: ScorerContext): Promise { - const key = process.env["EVAL_PROVIDER_API_KEY"]; - if (key === undefined || key === "") { - return { - name: "judge", - score: 1, - pass: true, - skipped: true, - reason: "skipped: EVAL_PROVIDER_API_KEY not set", - }; - } - const reply = ctx.transcript[ctx.turnIndex]?.replyText ?? ""; - const prompt = - `Rubric: ${rubric}\n\nReply to judge:\n${reply}\n\n` + - 'Answer with exactly one line: "PASS: " or "FAIL: ".'; - const run = - judgeCall ?? - (async (p: string) => { - const { text } = await callEvalModel(p, key); - return { pass: text.trim().startsWith("PASS"), reason: text.trim() }; - }); - const { pass, reason } = await run(prompt); - return { name: "judge", score: pass ? 1 : 0, pass, reason }; - }; -} - -// --- §8.2 scorers ------------------------------------------- -// -// Every scorer below grades "what Myra actually built" (plan.md §8.1 -// item 1), not what a tool call merely asked for, against the product -// that actually shipped: the code-review template instantiated from -// the seeded library, per-repo grants + webhook triggers minted at -// repo selection (start-reviewing), and one aggregated comment-only -// review per PR posted free under the grant by -// `github_post_pr_review` — never N per-reviewer `create_agent`-shaped -// posts. `WorldSnapshot` still has no `reviewComments` or `runs` -// field, so the two scorers needing those skip, naming that gap. - -function worldSnapshotFieldMissing(name: string, needs: string): ScorerResult { - return { - name, - score: 1, - pass: true, - skipped: true, - reason: - `skipped: WorldSnapshot has no ${needs} field — no Target populates it; ` + - "blocked on extending WorldSnapshot/snapshotWorld (see types.ts, targets/world-snapshot.ts)", - }; -} - -/** Passes once the world snapshot's `github` connection is live — i.e. - * the connection went through `@corbits/connections`, not a - * hand-rolled token stashed some other way. */ -export function githubConnectedViaConnectionsLayer() { - return function githubConnectedViaConnectionsLayerScorer(ctx: ScorerContext): ScorerResult { - const github = ctx.world.connections.find((connection) => connection.slug === "github"); - const connected = github?.live === true; - return result( - "githubConnectedViaConnectionsLayer", - connected, - connected - ? "github connection reads live=true from the world snapshot" - : `github connection not live in the world snapshot (found: ${JSON.stringify(github)})`, - ); - }; -} - -/** Passes once every named reviewer handle has a materialized agent - * definition AND the tenant's deployed code-review workflow definition - * carries a GitHub-shaped tool-package pin. This is the shipped - * install shape: `instantiateWorkbenchTemplate` creates the - * reviewer roster as prompt-only agent definitions, while the GitHub - * reach lives on the one `code-review` workflow the template's blocks - * install — the reviewers themselves never carry a github pin. */ -export function agentDefinitionsHaveToolGrants(handles: readonly string[]) { - return function agentDefinitionsHaveToolGrantsScorer(ctx: ScorerContext): ScorerResult { - const definitions = ctx.world.agentDefinitions; - const missing = handles.filter( - (handle) => !definitions.some((definition) => definition.name === handle), - ); - const codeReviewPinned = definitions.some( - (definition) => - /code.?review/i.test(definition.name) && - definition.toolPackagePins.some((pin) => /github/i.test(pin)), - ); - const pass = missing.length === 0 && codeReviewPinned; - return result( - "agentDefinitionsHaveToolGrants", - pass, - pass - ? `all of [${handles.join(", ")}] are materialized and the code-review definition carries a github-shaped tool pin` - : missing.length > 0 - ? `missing a materialized definition for: ${missing.join(", ")}` - : "no deployed code-review definition carries a github-shaped tool pin", - ); - }; -} - -/** Passes once the snapshot carries an enabled `webhook_trigger` row — - * the shipped per-repo trigger start-reviewing mints (one - * per selected repo, bound to the deployed code-review definition), - * not a chat-driven `routine_create` with a webhook trigger kind. */ -export function triggerIsWebhookPerPr() { - return function triggerIsWebhookPerPrScorer(ctx: ScorerContext): ScorerResult { - const live = ctx.world.webhookTriggers.find((trigger) => trigger.enabled); - return result( - "triggerIsWebhookPerPr", - live !== undefined, - live !== undefined - ? `webhook trigger ${live.id} ("${live.name}") is enabled against definition ${live.workflowDefinitionId}` - : "no enabled webhook_trigger row in the snapshot — start-reviewing never ran", - ); - }; -} - -/** Passes once every named reviewer handle posted at least one review - * comment, and every posted comment carries its own child run id — the - * per-turn/per-reviewer run tracing `onTrigger` adoption is meant to - * produce. `WorldSnapshot` has no - * `reviewComments` field today, so this always skips naming that gap. */ -export function reviewCommentsAttributable(handles: readonly string[]) { - return function reviewCommentsAttributableScorer(ctx: ScorerContext): ScorerResult { - void ctx; - void handles; - return worldSnapshotFieldMissing("reviewCommentsAttributable", "reviewComments"); - }; -} - -/** Passes once every successful `github_post_pr_review` call is - * structurally sound in the shipped aggregated-review shape: a - * non-empty markdown `body`, a `headSha` anchoring the review, - * and every inline comment carrying `path`/`line`/`body` — with at - * least one GitHub `suggestion` fence somewhere in the review, the - * form `aggregateReview` renders a reviewer's `suggestedFix` into. */ -export function suggestedFixesStructurallyValid() { - return function suggestedFixesStructurallyValidScorer(ctx: ScorerContext): ScorerResult { - const calls = allToolCalls(ctx.transcript.slice(0, ctx.turnIndex + 1)).filter( - (call) => call.name === GITHUB_POST_PR_REVIEW_TOOL && !call.isError, - ); - if (calls.length === 0) { - return result( - "suggestedFixesStructurallyValid", - false, - `no successful ${GITHUB_POST_PR_REVIEW_TOOL} call yet — the fired ` + - "PR event never produced a posted review", - ); - } - const problems: string[] = []; - let sawSuggestionFence = false; - for (const [index, call] of calls.entries()) { - const body = call.arguments["body"]; - const headSha = call.arguments["headSha"]; - if (typeof body !== "string" || body.trim() === "") { - problems.push(`call ${String(index)}: empty body`); - } - if (typeof headSha !== "string" || headSha.trim() === "") { - problems.push(`call ${String(index)}: missing headSha`); - } - const comments = call.arguments["comments"]; - const commentList = Array.isArray(comments) ? comments : []; - for (const [commentIndex, comment] of commentList.entries()) { - const entry = comment as Record; - if ( - typeof entry["path"] !== "string" || - typeof entry["line"] !== "number" || - typeof entry["body"] !== "string" - ) { - problems.push( - `call ${String(index)} comment ${String(commentIndex)}: missing path/line/body`, - ); - } - } - const allText = [ - typeof body === "string" ? body : "", - ...commentList.map((comment) => String((comment as Record)["body"] ?? "")), - ].join("\n"); - if (allText.includes("```suggestion")) sawSuggestionFence = true; - } - if (!sawSuggestionFence) { - problems.push("no ```suggestion fence in any posted review"); - } - return result( - "suggestedFixesStructurallyValid", - problems.length === 0, - problems.length === 0 - ? `all ${String(calls.length)} posted review(s) are structurally valid with at least one suggestion fence` - : problems.join("; "), - ); - }; -} - -/** - * Encodes the owner's ruling on where the approval boundary sits for - * an outward GitHub action: posting a review comment is FREE under a - * valid per-repo grant (must NOT wait on a human approval phrase), but - * a merge-class action (opening/landing a merge, not reviewing) DOES - * park behind one — the same contract `approvalGated` already proves - * for `routine_create`, applied here to `mergeTool`. Two assertions, - * one scorer, because they're the same ruling read two ways: passes - * once (a) every successful `github_post_pr_review` call targets a - * pull request under the granted `repo` and its aggregated body names - * at least one of `attributionMarkers` (the reviewer lens names - * `aggregateReview` renders into the "What each reviewer looked at" - * section — the audit-attribution the shipped one-review shape - * carries), with no approval-phrase requirement, and (b) any - * `mergeTool` call found only ever follows an approval phrase. No - * merge-class tool exists in `@corbits/github-tools` today (posting is - * comment-only by design), so (b) is vacuously satisfied until one - * ships. - */ -export function outwardGitHubActionsRespectGrantBoundary( - repo: string, - mergeTool: string, - attributionMarkers: readonly string[], -) { - return function outwardGitHubActionsRespectGrantBoundaryScorer(ctx: ScorerContext): ScorerResult { - const transcript = ctx.transcript.slice(0, ctx.turnIndex + 1); - const postCalls = allToolCalls(transcript).filter( - (call) => call.name === GITHUB_POST_PR_REVIEW_TOOL && !call.isError, - ); - if (postCalls.length === 0) { - return result( - "outwardGitHubActionsRespectGrantBoundary", - false, - `no successful ${GITHUB_POST_PR_REVIEW_TOOL} call yet — the fired ` + - "PR event never produced a posted review", - ); - } - const repoPrefix = `https://github.com/${repo}/pull/`; - const offRepo = postCalls.filter((call) => { - const url = call.arguments["pullRequestUrl"]; - return typeof url !== "string" || !url.startsWith(repoPrefix); - }); - const unattributed = postCalls.filter((call) => { - const body = call.arguments["body"]; - if (typeof body !== "string") return true; - return !attributionMarkers.some((marker) => body.includes(marker)); - }); - - let approvedByStep = -1; - for (const [index, turn] of transcript.entries()) { - if (turnHasApproval(turn.human)) { - approvedByStep = index; - break; - } - } - const mergedBeforeApproval: string[] = []; - for (const [index, turn] of transcript.entries()) { - const gated = turn.toolCalls.filter((call) => call.name === mergeTool); - if (gated.length > 0 && (approvedByStep === -1 || index < approvedByStep)) { - mergedBeforeApproval.push(`step ${String(index)}`); - } - } - - const pass = - offRepo.length === 0 && unattributed.length === 0 && mergedBeforeApproval.length === 0; - return result( - "outwardGitHubActionsRespectGrantBoundary", - pass, - pass - ? `all ${String(postCalls.length)} review(s) posted free under the ${repo} grant with reviewer attribution in the body, and no ${mergeTool} call ran before approval` - : `${String(offRepo.length)} post(s) outside the ${repo} grant, ${String(unattributed.length)} with no reviewer attribution in the body, ${mergeTool} ran before approval at: ${mergedBeforeApproval.join(", ") || "none"}`, - ); - }; -} - -/** Passes once every run is inspectable after the fact — the whole run, - * including each reviewer's own per-PR pass (plan.md §8.2 item 7). - * `WorldSnapshot` has no `runs` field today, so this always skips - * naming that gap. */ -export function wholeRunInspectable() { - return function wholeRunInspectableScorer(ctx: ScorerContext): ScorerResult { - void ctx; - return worldSnapshotFieldMissing("wholeRunInspectable", "runs"); - }; -} diff --git a/packages/evals/src/scorers/tool-names.test.ts b/packages/evals/src/scorers/tool-names.test.ts deleted file mode 100644 index 28453acd6c..0000000000 --- a/packages/evals/src/scorers/tool-names.test.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { expect, test } from "bun:test"; - -import { CREATE_AGENT_TOOL as REAL_CREATE_AGENT_TOOL } from "@corbits/agent-directory-tools"; -import { LIST_CONNECTIONS_TOOL as REAL_LIST_CONNECTIONS_TOOL } from "@corbits/connections-tools"; -import { - GITHUB_PULL_REQUEST_DIFF_TOOL as REAL_GITHUB_PULL_REQUEST_DIFF_TOOL, - GITHUB_POST_PULL_REQUEST_REVIEW_TOOL as REAL_GITHUB_POST_PR_REVIEW_TOOL, -} from "@corbits/github-tools"; - -import { - CREATE_AGENT_TOOL, - MEMORY_ADD_TOOL, - LIST_CONNECTIONS_TOOL, - GITHUB_PULL_REQUEST_DIFF_TOOL, - GITHUB_POST_PR_REVIEW_TOOL, - BUILD_TOOLS, -} from "./tool-names.ts"; - -test("eval tool-name constants match the real manager-tools bundles", () => { - expect(CREATE_AGENT_TOOL).toBe(REAL_CREATE_AGENT_TOOL); - // `@corbits/memory` exports no name constant — its - // `memoryAdd` tool factory names itself "memory_add" inline. - expect(MEMORY_ADD_TOOL).toBe("memory_add"); - expect(LIST_CONNECTIONS_TOOL).toBe(REAL_LIST_CONNECTIONS_TOOL); -}); - -test("GitHub tool-name constants match the real @corbits/github-tools bundle", () => { - expect(GITHUB_PULL_REQUEST_DIFF_TOOL).toBe(REAL_GITHUB_PULL_REQUEST_DIFF_TOOL); - expect(GITHUB_POST_PR_REVIEW_TOOL).toBe(REAL_GITHUB_POST_PR_REVIEW_TOOL); -}); - -test("BUILD_TOOLS names the state-changing manager tools", () => { - expect(BUILD_TOOLS).toEqual([CREATE_AGENT_TOOL]); -}); diff --git a/packages/evals/src/scorers/tool-names.ts b/packages/evals/src/scorers/tool-names.ts deleted file mode 100644 index 43a33661ad..0000000000 --- a/packages/evals/src/scorers/tool-names.ts +++ /dev/null @@ -1,31 +0,0 @@ -// Tool-name constants a scorer checks for. Kept in lockstep with each -// manager-tools bundle's own export (see each package's src/tool.ts) — -// tool-names.test.ts pins these against the real bundles so a rename -// there fails this package's tests instead of a scorer silently never -// matching. -export const CREATE_AGENT_TOOL = "create_agent"; -export const ROUTINE_CREATE_TOOL = "routine_create"; -export const ROUTINE_RUN_NOW_TOOL = "routine_run_now"; -export const MEMORY_ADD_TOOL = "memory_add"; -export const MEMORY_SEARCH_TOOL = "memory_search"; -export const MEMORY_LIST_TOOL = "memory_list"; -export const LIST_CONNECTIONS_TOOL = "list_connections"; -export const REQUEST_CONNECTION_TOOL = "request_connection"; - -/** The tools a "build" step uses to stand up real, lasting workbench - * state — the ones an interview must precede. */ -export const BUILD_TOOLS = [CREATE_AGENT_TOOL] as const; - -// The shipped GitHub write path: one diff read and one aggregated -// comment-only review per PR, posted -// by the code-review workflow run itself. Pinned against -// `@corbits/github-tools`' own exports in tool-names.test.ts. -export const GITHUB_PULL_REQUEST_DIFF_TOOL = "github_pull_request_diff"; -export const GITHUB_POST_PR_REVIEW_TOOL = "github_post_pr_review"; - -// Merge-class action name the owner ruling parks behind explicit human -// approval. No such tool exists in `@corbits/github-tools` today — -// posting is comment-only by design — so the parked-merge half of -// `outwardGitHubActionsRespectGrantBoundary` is vacuously satisfiable -// until one ships; the constant names what the scorer watches for. -export const GITHUB_MERGE_PULL_REQUEST_TOOL = "github_merge_pull_request"; diff --git a/packages/evals/src/scorers/world-scorers.test.ts b/packages/evals/src/scorers/world-scorers.test.ts deleted file mode 100644 index 795625fff5..0000000000 --- a/packages/evals/src/scorers/world-scorers.test.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { describe, expect, test } from "bun:test"; - -import type { ScorerContext, WorldSnapshot } from "../types.ts"; -import { agentHasTools, connectionIsLive, fakeReceived } from "./world-scorers.ts"; - -function worldCtx(world: Partial): ScorerContext { - return { - transcript: [], - turnIndex: 0, - world: { - capturedAt: "2026-01-01T00:00:00.000Z", - agentDefinitions: [], - connections: [], - webhookTriggers: [], - fakeReceipts: [], - ...world, - }, - }; -} - -describe("agentHasTools", () => { - test("passes when the named agent has every tool pinned", () => { - const ctx = worldCtx({ - agentDefinitions: [ - { - id: "def-1", - name: "AI Daily researcher", - displayName: null, - toolPackagePins: ["@corbits/web-search-tools", "@corbits/memory"], - skills: [], - model: null, - }, - ], - }); - const r = agentHasTools("AI Daily researcher", ["@corbits/web-search-tools"])(ctx); - expect(r.pass).toBe(true); - }); - - test("fails when the agent is missing a required tool", () => { - const ctx = worldCtx({ - agentDefinitions: [ - { - id: "def-1", - name: "AI Daily researcher", - displayName: null, - toolPackagePins: ["@corbits/memory"], - skills: [], - model: null, - }, - ], - }); - const r = agentHasTools("AI Daily researcher", ["@corbits/web-search-tools"])(ctx); - expect(r.pass).toBe(false); - expect(r.reason).toContain("@corbits/web-search-tools"); - }); - - test("fails when the agent doesn't exist yet", () => { - const r = agentHasTools("Nobody", ["x"])(worldCtx({})); - expect(r.pass).toBe(false); - }); -}); - -describe("connectionIsLive", () => { - test("passes for a live connection, fails for none found", () => { - const ctx = worldCtx({ - connections: [{ slug: "github", name: "GitHub", url: "https://x", live: true }], - }); - expect(connectionIsLive("github")(ctx).pass).toBe(true); - expect(connectionIsLive("attio")(ctx).pass).toBe(false); - }); - - test("fails for a connection that exists but isn't live", () => { - const ctx = worldCtx({ - connections: [{ slug: "github", name: "GitHub", url: "https://x", live: false }], - }); - expect(connectionIsLive("github")(ctx).pass).toBe(false); - }); -}); - -describe("fakeReceived", () => { - test("passes once a matching call was received", () => { - const ctx = worldCtx({ - fakeReceipts: [ - { - server: "github", - toolName: "list_pull_requests", - arguments: { repo: "corbitsdev/workbench" }, - }, - ], - }); - expect(fakeReceived("github", "list_pull_requests")(ctx).pass).toBe(true); - expect(fakeReceived("github", "create_issue")(ctx).pass).toBe(false); - }); - - test("applies the optional argument matcher", () => { - const ctx = worldCtx({ - fakeReceipts: [ - { - server: "github", - toolName: "create_issue", - arguments: { repo: "a" }, - }, - ], - }); - expect(fakeReceived("github", "create_issue", (args) => args["repo"] === "b")(ctx).pass).toBe( - false, - ); - expect(fakeReceived("github", "create_issue", (args) => args["repo"] === "a")(ctx).pass).toBe( - true, - ); - }); -}); diff --git a/packages/evals/src/scorers/world-scorers.ts b/packages/evals/src/scorers/world-scorers.ts deleted file mode 100644 index 7edeba9d5b..0000000000 --- a/packages/evals/src/scorers/world-scorers.ts +++ /dev/null @@ -1,70 +0,0 @@ -// Pure `Scorer` factories over `ctx.world` (see ../types.ts) — the -// same `{name, score, pass, reason}` contract `scorers.ts` uses for -// transcript-based checks, applied instead to what the tenant's own -// tables actually hold. No network, no clock; unit-testable against a -// hand-built `WorldSnapshot` (see world-scorers.test.ts). -import type { FakeReceipt, ScorerContext, ScorerResult } from "../types.ts"; - -function result(name: string, pass: boolean, reason: string, score = pass ? 1 : 0): ScorerResult { - return { name, pass, reason, score }; -} - -/** Passes once an agent definition named `agentName` exists and has - * every one of `tools` among its pinned tool packages. */ -export function agentHasTools(agentName: string, tools: readonly string[]) { - return function agentHasToolsScorer(ctx: ScorerContext): ScorerResult { - const agent = ctx.world.agentDefinitions.find((definition) => definition.name === agentName); - if (agent === undefined) { - return result("agentHasTools", false, `no agent definition named "${agentName}" exists yet`); - } - const missing = tools.filter((tool) => !agent.toolPackagePins.includes(tool)); - return result( - "agentHasTools", - missing.length === 0, - missing.length === 0 - ? `"${agentName}" has all of: ${tools.join(", ")}` - : `"${agentName}" is missing: ${missing.join(", ")}`, - ); - }; -} - -/** Passes once a connection with the given `slug` is live (its - * credential is active). */ -export function connectionIsLive(slug: string) { - return function connectionIsLiveScorer(ctx: ScorerContext): ScorerResult { - const connection = ctx.world.connections.find((c) => c.slug === slug); - if (connection === undefined) { - return result("connectionIsLive", false, `no connection "${slug}" found`); - } - return result( - "connectionIsLive", - connection.live, - connection.live ? `"${slug}" is live` : `"${slug}" is connected but not live`, - ); - }; -} - -/** Passes once the recording fake for `server` received a call to - * `toolName` whose arguments satisfy `matcher` (default: any - * arguments). */ -export function fakeReceived( - server: string, - toolName: string, - matcher?: (args: Record) => boolean, -) { - return function fakeReceivedScorer(ctx: ScorerContext): ScorerResult { - const matches = ctx.world.fakeReceipts.filter( - (receipt: FakeReceipt) => - receipt.server === server && - receipt.toolName === toolName && - (matcher === undefined || matcher(receipt.arguments)), - ); - return result( - "fakeReceived", - matches.length > 0, - matches.length > 0 - ? `${server}.${toolName} received ${String(matches.length)} matching call(s)` - : `${server}.${toolName} received no matching call`, - ); - }; -} diff --git a/packages/evals/src/store/migrations.ts b/packages/evals/src/store/migrations.ts deleted file mode 100644 index 89abed7d81..0000000000 --- a/packages/evals/src/store/migrations.ts +++ /dev/null @@ -1,53 +0,0 @@ -// Package-owned migrations for @corbits/evals. Bookkeeping uses its -// own ledger table so the package can be extracted without -// disentangling history from the platform drizzle journal. Every -// table this package owns — including its ledger — lives in its own -// `evals` Postgres schema, never `public`; see -// docs/package-migrations.md. Mechanics (schema/ledger bootstrap, -// transactional apply, the advisory lock across concurrent hub -// replicas) live in @corbits/migration-runner — this file owns only -// the domain SQL. -import { - applyPackageMigrations, - type ApplyPackageMigrationsReport, - type PackageMigration, -} from "@corbits/migration-runner"; - -export type EvalsMigration = PackageMigration; - -const SCHEMA = "evals"; - -export const evalsMigrations: readonly EvalsMigration[] = [ - { - name: "0001_run", - sql: ` - CREATE TABLE IF NOT EXISTS "evals"."run" ( - "id" text PRIMARY KEY, - "eval_name" text NOT NULL, - "config_name" text NOT NULL, - "started_at" timestamptz NOT NULL, - "finished_at" timestamptz NOT NULL, - "steps" jsonb NOT NULL, - "recorded_at" timestamptz NOT NULL DEFAULT now() - ); - CREATE INDEX IF NOT EXISTS "run_eval_name_recorded_idx" - ON "evals"."run" ("eval_name", "recorded_at"); - `, - }, -]; - -const LEDGER_TABLE = "evals_migrations"; - -export type ApplyEvalsMigrationsReport = ApplyPackageMigrationsReport; - -export async function applyEvalsMigrations( - databaseUrl: string, -): Promise { - return applyPackageMigrations({ - databaseUrl, - schema: SCHEMA, - ledgerTable: LEDGER_TABLE, - migrations: evalsMigrations, - packageLabel: "evals", - }); -} diff --git a/packages/evals/src/store/pg-store.ts b/packages/evals/src/store/pg-store.ts deleted file mode 100644 index e7db5a0702..0000000000 --- a/packages/evals/src/store/pg-store.ts +++ /dev/null @@ -1,100 +0,0 @@ -// Postgres-backed EvalRunStore against the package-owned evals.run -// table. Created after applyEvalsMigrations on the same URL. -import postgres from "postgres"; - -import type { EvalRunResult } from "../types.ts"; -import type { EvalRunRecord, EvalRunStore } from "./store.ts"; - -type Sql = ReturnType; - -function parseJsonbSteps(value: unknown): EvalRunResult["steps"] { - if (typeof value === "string") { - return JSON.parse(value) as EvalRunResult["steps"]; - } - return (value ?? []) as EvalRunResult["steps"]; -} - -interface RunRow { - id: string; - eval_name: string; - config_name: string; - started_at: Date; - finished_at: Date; - steps: unknown; -} - -function toRecord(row: RunRow): EvalRunRecord { - return { - id: row.id, - evalName: row.eval_name, - configName: row.config_name, - startedAt: row.started_at.toISOString(), - finishedAt: row.finished_at.toISOString(), - steps: parseJsonbSteps(row.steps), - }; -} - -export function createPostgresEvalRunStore(databaseUrl: string): { - store: EvalRunStore; - close: () => Promise; -} { - const sql: Sql = postgres(databaseUrl, { max: 4, onnotice: () => undefined }); - - const store: EvalRunStore = { - async save(result) { - const id = `evalrun_${crypto.randomUUID()}`; - await sql` - INSERT INTO evals.run - (id, eval_name, config_name, started_at, finished_at, steps) - VALUES ( - ${id}, - ${result.evalName}, - ${result.configName}, - ${result.startedAt}, - ${result.finishedAt}, - ${sql.json(result.steps as never)} - ) - `; - return id; - }, - - async recent(evalName, limit) { - const rows = await sql` - SELECT id, eval_name, config_name, started_at, finished_at, steps - FROM evals.run - WHERE eval_name = ${evalName} - ORDER BY recorded_at DESC - LIMIT ${limit} - `; - return rows.map(toRecord); - }, - - async recentAcrossEvals(limit) { - const rows = await sql` - SELECT id, eval_name, config_name, started_at, finished_at, steps - FROM evals.run - ORDER BY recorded_at DESC - LIMIT ${limit} - `; - return rows.map(toRecord); - }, - - async get(id) { - const rows = await sql` - SELECT id, eval_name, config_name, started_at, finished_at, steps - FROM evals.run - WHERE id = ${id} - LIMIT 1 - `; - const row = rows[0]; - return row === undefined ? null : toRecord(row); - }, - }; - - return { - store, - close: async () => { - await sql.end({ timeout: 5 }); - }, - }; -} diff --git a/packages/evals/src/store/schema.ts b/packages/evals/src/store/schema.ts deleted file mode 100644 index e5e3840ac4..0000000000 --- a/packages/evals/src/store/schema.ts +++ /dev/null @@ -1,28 +0,0 @@ -// Workbench-owned eval-run history: one row per (eval, config) run, -// carrying the whole step/scorer transcript as JSONB rather than a -// fully normalized shape — a run result is written once and read back -// whole (by the CLI reporter, and eventually a read route), never -// queried per-scorer, so normalizing it into more tables would only -// add join cost with no query this package needs. -// -// This table lives in its own `evals` Postgres schema, fully siloed -// from the platform's `public` schema — see docs/package-migrations.md. -// `id` is this package's own id, not a workflow_run id: one eval run -// can (and for a live config, does) drive many workflow runs across -// its steps. -import { jsonb, pgSchema, text, timestamp } from "drizzle-orm/pg-core"; - -export const evalsSchema = pgSchema("evals"); - -export const evalRun = evalsSchema.table("run", { - id: text("id").primaryKey(), - evalName: text("eval_name").notNull(), - configName: text("config_name").notNull(), - startedAt: timestamp("started_at", { withTimezone: true }).notNull(), - finishedAt: timestamp("finished_at", { withTimezone: true }).notNull(), - /** The run's `EvalStepRecord[]` — see ../types.ts — verbatim. */ - steps: jsonb("steps").notNull(), - recordedAt: timestamp("recorded_at", { withTimezone: true }).notNull().defaultNow(), -}); - -export type EvalRunRow = typeof evalRun.$inferSelect; diff --git a/packages/evals/src/store/store.test.ts b/packages/evals/src/store/store.test.ts deleted file mode 100644 index 2122a08543..0000000000 --- a/packages/evals/src/store/store.test.ts +++ /dev/null @@ -1,123 +0,0 @@ -// Round-trips EvalRunResult through a real, uniquely-named scratch -// Postgres database — never the developer's own DATABASE_URL. Skips -// without DATABASE_URL (like every other real-Postgres suite in this -// repo); CI=true turns that skip into a loud CI failure. -import { afterAll, describe, expect, test } from "bun:test"; -import postgres from "postgres"; - -import { applyEvalsMigrations } from "./migrations.ts"; -import { createPostgresEvalRunStore } from "./pg-store.ts"; -import type { EvalRunResult } from "../types.ts"; -import { assertDatabaseConfigured } from "../../../../test/db-gate.ts"; - -function scratchDatabaseUrl(): string | undefined { - const base = process.env["DATABASE_URL"]; - if (base === undefined || base === "") { - assertDatabaseConfigured(undefined, "evals store suite"); - return undefined; - } - const url = new URL(base); - const database = url.pathname.replace(/^\//, ""); - if (database === "") { - throw new Error(`DATABASE_URL names no database (empty path): ${base}`); - } - const suffix = crypto.randomUUID().slice(0, 8); - url.pathname = `/${database}_evals_store_${suffix}`; - return url.toString(); -} - -const databaseUrl = scratchDatabaseUrl(); - -function fixtureResult(overrides: Partial = {}): EvalRunResult { - return { - evalName: "ai-daily-research", - configName: "default", - startedAt: "2026-08-16T00:00:00.000Z", - finishedAt: "2026-08-16T00:01:00.000Z", - steps: [ - { - stepIndex: 0, - turn: { human: "hi", replyText: "hello", toolCalls: [] }, - scorerReports: [ - { - name: "asksQuestions", - score: 1, - pass: true, - reason: "ok", - stepIndex: 0, - }, - ], - }, - ], - ...overrides, - }; -} - -describe.skipIf(databaseUrl === undefined)( - "createPostgresEvalRunStore against a real scratch database", - () => { - const url = databaseUrl as string; - let admin: ReturnType; - - afterAll(async () => { - const database = new URL(url).pathname.replace(/^\//, ""); - const adminUrl = new URL(url); - adminUrl.pathname = "/postgres"; - admin = postgres(adminUrl.toString(), { - max: 1, - onnotice: () => undefined, - }); - await admin.unsafe(`DROP DATABASE IF EXISTS "${database}"`); - await admin.end({ timeout: 5 }); - }); - - test("migrates, saves a run, and reads it back by eval name", async () => { - const database = new URL(url).pathname.replace(/^\//, ""); - const adminUrl = new URL(url); - adminUrl.pathname = "/postgres"; - const bootstrap = postgres(adminUrl.toString(), { - max: 1, - onnotice: () => undefined, - }); - await bootstrap.unsafe(`CREATE DATABASE "${database}"`); - await bootstrap.end({ timeout: 5 }); - - const report = await applyEvalsMigrations(url); - expect(report.applied).toContain("0001_run"); - - const { store, close } = createPostgresEvalRunStore(url); - try { - const id = await store.save(fixtureResult()); - expect(id).toMatch(/^evalrun_/); - - const recent = await store.recent("ai-daily-research", 5); - expect(recent).toHaveLength(1); - expect(recent[0]?.id).toBe(id); - expect(recent[0]?.evalName).toBe("ai-daily-research"); - expect(recent[0]?.configName).toBe("default"); - expect(recent[0]?.steps).toEqual(fixtureResult().steps); - - const other = await store.recent("docs-on-sdk-change", 5); - expect(other).toHaveLength(0); - - const second = await store.save(fixtureResult({ evalName: "docs-on-sdk-change" })); - const across = await store.recentAcrossEvals(10); - expect(across.map((r) => r.id).sort()).toEqual([id, second].sort()); - - const fetched = await store.get(id); - expect(fetched?.id).toBe(id); - expect(fetched?.evalName).toBe("ai-daily-research"); - - expect(await store.get("evalrun_does-not-exist")).toBeNull(); - } finally { - await close(); - } - }, 30_000); - - test("re-running migrations is idempotent", async () => { - const first = await applyEvalsMigrations(url); - expect(first.applied).toEqual([]); - expect(first.alreadyApplied).toContain("0001_run"); - }); - }, -); diff --git a/packages/evals/src/store/store.ts b/packages/evals/src/store/store.ts deleted file mode 100644 index 16d0c6a677..0000000000 --- a/packages/evals/src/store/store.ts +++ /dev/null @@ -1,18 +0,0 @@ -import type { EvalRunResult } from "../types.ts"; - -/** An `EvalRunResult` as read back from storage, carrying the store's own - * `id` (`evalrun_`) — never part of `EvalRunResult` itself, since - * that type is also what a fresh run produces before it has one. */ -export interface EvalRunRecord extends EvalRunResult { - readonly id: string; -} - -export interface EvalRunStore { - save(result: EvalRunResult): Promise; - recent(evalName: string, limit: number): Promise; - /** Most recent runs across every eval, newest first — the read - * route's list view. */ - recentAcrossEvals(limit: number): Promise; - /** One run by its store id, or `null` if no such run exists. */ - get(id: string): Promise; -} diff --git a/packages/evals/src/targets/git-workflow-push.ts b/packages/evals/src/targets/git-workflow-push.ts deleted file mode 100644 index e76bb9ff17..0000000000 --- a/packages/evals/src/targets/git-workflow-push.ts +++ /dev/null @@ -1,215 +0,0 @@ -// Eval-harness fixture: pushes a workflow definition into its -// asset repo over the hub's smart-HTTP git route, using the system git -// binary with a bearer token as the basic-auth password and a -// GIT_ASKPASS shim as the non-interactive fallback — the platform's -// established asset-push convention. Content-aware: an identical tree -// is a reported skip, not a duplicate commit, which is what makes -// re-running an eval target's setup convergence safe. -// -// A local copy of the hub's own push transport on purpose: the shared -// workflow-push helper this duplicates now lives at -// `@corbits/connections/workflow-push`, and the eval -// harness needs the dumb transport (caller-supplied bytes in, a commit -// sha out — no catalog, no seed order) to plant the eval tenant's -// workflows against its spawned hub. The pushed tree is the source -// codebase `@corbits/workflows`'s `./source` renders — the one shape a -// workflow-kind asset accepts (see -// `vendor/intx/hub-sessions/src/workflow-kind.ts`). - -import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { renderWorkflowSourceTree } from "@corbits/workflows"; -import { HubApiError } from "@corbits/hub-api-client"; - -export type PushOutcome = "pushed" | "unchanged"; - -/** - * What the push left on the asset's `main`: whether it wrote a commit, - * and the sha that commit (or the already-current one) sits at. The sha - * IS the deploy pin — a code-sourced deploy sources - * `package: { format: "source", commitSha }`. - */ -export type PushResult = { outcome: PushOutcome; commitSha: string }; - -export type WorkflowPusher = (args: { - remoteUrl: string; - tokenSecret: string; - workflowJson: string; - /** Name the rendered source package declares; never leaves the asset. */ - packageName: string; -}) => Promise; - -function requireGit(): void { - if (Bun.which("git") === null) { - throw new HubApiError( - "git is not installed or not on PATH; the workflow push uses the system git binary", - "install git (macOS: `xcode-select --install`), then re-run the eval", - ); - } -} - -const GIT_DIR_VARS = new Set([ - "GIT_DIR", - "GIT_WORK_TREE", - "GIT_INDEX_FILE", - "GIT_OBJECT_DIRECTORY", - "GIT_ALTERNATE_OBJECT_DIRECTORIES", -]); - -function gitProcessEnv(extra: Record): Record { - const env: Record = {}; - for (const [key, value] of Object.entries(process.env)) { - if (value === undefined) continue; - if (GIT_DIR_VARS.has(key)) continue; - env[key] = value; - } - return { ...env, ...extra }; -} - -async function runGit( - args: string[], - cwd: string, - env: Record, -): Promise<{ code: number; output: string }> { - const child = Bun.spawn(["git", "-c", "core.hooksPath=", ...args], { - cwd, - env: gitProcessEnv(env), - stdout: "pipe", - stderr: "pipe", - }); - const [stdout, stderr, code] = await Promise.all([ - new Response(child.stdout).text(), - new Response(child.stderr).text(), - child.exited, - ]); - return { code, output: `${stdout}${stderr}`.trim() }; -} - -/** - * The pushed commit is the definition's pin: a code-sourced deploy names - * `package.format: "source"` plus this sha, so the pusher is the only - * place that can report it. - */ -async function headSha(repoDir: string): Promise { - const result = await runGit(["rev-parse", "HEAD"], repoDir, {}); - if (result.code !== 0) { - throw new HubApiError( - `reading the pushed workflow commit failed: ${result.output}`, - "confirm the eval hub is running and check the eval logs for the underlying git failure", - ); - } - return result.output.trim(); -} - -function withToken(remoteUrl: string, tokenSecret: string): string { - const url = new URL(remoteUrl); - url.username = "x-access-token"; - url.password = tokenSecret; - return url.toString(); -} - -export function createGitWorkflowPusher(): WorkflowPusher { - return async ({ remoteUrl, tokenSecret, workflowJson, packageName }) => { - requireGit(); - const work = await mkdtemp(join(tmpdir(), "workbench-eval-push-")); - try { - const askpass = join(work, "askpass.sh"); - await writeFile( - askpass, - `#!/bin/sh\nprintf '%s\\n' '${tokenSecret.replace(/'/g, "'\\''")}'\n`, - "utf-8", - ); - await chmod(askpass, 0o755); - const gitEnv = { - GIT_ASKPASS: askpass, - GIT_TERMINAL_PROMPT: "0", - GIT_AUTHOR_NAME: "Workbench", - GIT_AUTHOR_EMAIL: "hub@workbench.localhost", - GIT_COMMITTER_NAME: "Workbench", - GIT_COMMITTER_EMAIL: "hub@workbench.localhost", - }; - const authRemote = withToken(remoteUrl, tokenSecret); - const repoDir = join(work, "repo"); - - const clone = await runGit( - ["-c", "credential.helper=", "clone", authRemote, repoDir], - work, - gitEnv, - ); - if (clone.code !== 0) { - throw new HubApiError( - `cloning the workflow asset repo failed: ${clone.output}`, - "confirm the eval hub is running and check the eval logs for the underlying git failure", - ); - } - - const tree = renderWorkflowSourceTree({ - packageName, - workflowJson, - }); - let unchanged = true; - for (const [file, contents] of Object.entries(tree)) { - const target = join(repoDir, file); - let existing: string | null = null; - try { - existing = await readFile(target, "utf-8"); - } catch (cause) { - // A missing file is the normal case (a new tree entry); any - // other read failure is rethrown — failing the push loudly - // rather than silently pushing a tree never fully read. - if ((cause as { code?: string })?.code !== "ENOENT") throw cause; - existing = null; - } - if (existing === contents) continue; - unchanged = false; - await writeFile(target, contents, "utf-8"); - } - if (unchanged) { - return { outcome: "unchanged", commitSha: await headSha(repoDir) }; - } - - const steps: { label: string; args: string[] }[] = [ - { label: "stage", args: ["add", ...Object.keys(tree)] }, - { - label: "commit", - args: [ - // Pushed commits are authored as hub@workbench.localhost and - // must never inherit the operator's signing identity - //: a global `commit.gpgsign = true` would fail the - // commit — or worse, sign a bot commit with a personal key. - "-c", - "commit.gpgsign=false", - "commit", - "--no-verify", - "-m", - "Deploy the default workflow definition", - ], - }, - { - // Forced deliberately: this asset repo is harness-owned (this - // pusher is its only writer), so `main` always carries - // exactly the canonical tree this run computed. A plain push - // 409s as "non-fast-forward" the moment the remote's `main` - // shares no ancestry with this run's fresh clone, which would - // otherwise fail the entire setup on a re-run rather than - // repointing the ref it owns. - label: "push", - args: ["-c", "credential.helper=", "push", "--force", authRemote, "HEAD:main"], - }, - ]; - for (const step of steps) { - const result = await runGit(step.args, repoDir, gitEnv); - if (result.code !== 0) { - throw new HubApiError( - `the workflow source ${step.label} failed: ${result.output}`, - "confirm the eval hub is running and check the eval logs for the underlying git failure", - ); - } - } - return { outcome: "pushed", commitSha: await headSha(repoDir) }; - } finally { - await rm(work, { recursive: true, force: true }); - } - }; -} diff --git a/packages/evals/src/targets/json-fields.ts b/packages/evals/src/targets/json-fields.ts deleted file mode 100644 index 73d901d4f3..0000000000 --- a/packages/evals/src/targets/json-fields.ts +++ /dev/null @@ -1,18 +0,0 @@ -// Shared field readers for parsing hub API JSON responses — both -// real-target.ts and fire-routine.ts read fields off `unknown` response -// bodies and need the same loud-on-missing behavior. -export function stringField(data: unknown, field: string, what: string): string { - if (typeof data === "object" && data !== null && field in data) { - const value = (data as Record)[field]; - if (typeof value === "string" && value !== "") return value; - } - throw new Error(`${what}: missing string field "${field}": ${JSON.stringify(data)}`); -} - -export function arrayField(data: unknown, field: string, what: string): unknown[] { - if (typeof data === "object" && data !== null && field in data) { - const value = (data as Record)[field]; - if (Array.isArray(value)) return value; - } - throw new Error(`${what}: missing array field "${field}": ${JSON.stringify(data)}`); -} diff --git a/packages/evals/src/targets/trace.test.ts b/packages/evals/src/targets/trace.test.ts deleted file mode 100644 index 5dfb43319b..0000000000 --- a/packages/evals/src/targets/trace.test.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { expect, test } from "bun:test"; - -import { newToolCallsSince, readAllToolCalls, type SqlClientLike } from "./trace.ts"; - -function fakeSql(rows: Record[]): SqlClientLike { - return { unsafe: async () => rows }; -} - -function callRow(ordinal: number, callId: string, name: string, args: unknown) { - return { - metadata: { kind: "call", callId, name, arguments: args }, - ordinal, - started_at: new Date(0), - }; -} - -function resultRow(ordinal: number, callId: string, content: unknown, isError: boolean) { - return { - metadata: { kind: "result", callId, content, isError }, - ordinal, - started_at: new Date(0), - }; -} - -test("pairs a call with its result into one ToolCall", async () => { - const sql = fakeSql([ - callRow(0, "c1", "memory_add", { content: "website: example.com" }), - resultRow(1, "c1", "added", false), - ]); - const calls = await readAllToolCalls(sql, "tenant-1", "run-1"); - expect(calls).toEqual([ - { - name: "memory_add", - arguments: { content: "website: example.com" }, - isError: false, - result: "added", - }, - ]); -}); - -test("omits a call with no result yet (turn still in flight)", async () => { - const sql = fakeSql([callRow(0, "c1", "create_agent", { name: "Researcher" })]); - const calls = await readAllToolCalls(sql, "tenant-1", "run-1"); - expect(calls).toEqual([]); -}); - -test("preserves call order and marks a failed result", async () => { - const sql = fakeSql([ - callRow(0, "c1", "request_connection", { connector: "attio" }), - resultRow(1, "c1", "unknown connector: attio", true), - callRow(2, "c2", "list_connections", {}), - resultRow(3, "c2", "Connected: none.", false), - ]); - const calls = await readAllToolCalls(sql, "tenant-1", "run-1"); - expect(calls.map((c) => c.name)).toEqual(["request_connection", "list_connections"]); - expect(calls[0]?.isError).toBe(true); - expect(calls[1]?.isError).toBe(false); -}); - -test("ignores non-tool turn_part rows (e.g. step-start, error parts)", async () => { - const sql = fakeSql([ - { metadata: { model: "claude" }, ordinal: 0, started_at: new Date(0) }, - callRow(1, "c1", "memory_list", {}), - resultRow(2, "c1", "[]", false), - ]); - const calls = await readAllToolCalls(sql, "tenant-1", "run-1"); - expect(calls).toHaveLength(1); - expect(calls[0]?.name).toBe("memory_list"); -}); - -test("newToolCallsSince returns only the calls beyond what was already consumed", () => { - const all = [ - { name: "a", arguments: {}, isError: false, result: "" }, - { name: "b", arguments: {}, isError: false, result: "" }, - { name: "c", arguments: {}, isError: false, result: "" }, - ]; - const first = newToolCallsSince(all.slice(0, 1), 0); - expect(first.newCalls.map((c) => c.name)).toEqual(["a"]); - expect(first.consumed).toBe(1); - - const second = newToolCallsSince(all, first.consumed); - expect(second.newCalls.map((c) => c.name)).toEqual(["b", "c"]); - expect(second.consumed).toBe(3); -}); diff --git a/packages/evals/src/targets/trace.ts b/packages/evals/src/targets/trace.ts deleted file mode 100644 index 54fadfeff9..0000000000 --- a/packages/evals/src/targets/trace.ts +++ /dev/null @@ -1,121 +0,0 @@ -// Reads tool calls straight off the platform's own `inference_turn` / -// `turn_part` tables — the same rows `@intx/hub-sessions`' event -// collector writes for every turn (vendor/intx/hub-sessions/src/event- -// collector.ts) and `@corbits/insights`' `createDrizzleRunTraceReader` -// (packages/insights/src/trace-reader.ts) reads to build a run's -// trace. -// -// This target reads the raw table instead of going through Insights' -// `/runs/:runId/trace` HTTP route on purpose: that route's -// `RunTraceSpan` intentionally drops a tool call's arguments and a -// *successful* call's result content (only a *failed* call's error -// text survives onto a span — see queries.ts's `RunTraceSpan` doc -// comment). A scorer like `memoryWritten` needs to see what was -// actually written, not just that `memory_add` fired and didn't -// error, so this reads `turn_part.metadata` directly — the full -// `{kind:"call", arguments}` / `{kind:"result", content}` pair the -// event collector already recorded, before Insights' read layer -// summarizes it away. -// -// [Observability gap worth flagging upstream: the Insights trace -// route has no query-param to opt into full arguments/content — today -// the only way to get them is this direct table read, which requires -// the eval harness to hold its own Postgres connection rather than -// going through the hub's own HTTP surface like every other target -// action does.] -// -// `inference_turn.runId` is stored as the DB column `instance_id` -// (see vendor/intx/db/src/schema/messages.ts) — the same id as the -// workflow run itself, which for a `kind: "chat"` channel launched -// directly against a definition (as this target's Myra channel is) -// equals the channel id returned from `POST /channels`. -import type { ToolCall } from "../types.ts"; - -export interface SqlClientLike { - unsafe(query: string, params?: unknown[]): Promise[]>; -} - -type ToolPartMetadata = - | { kind: "call"; callId: string; name: string; arguments: unknown } - | { kind: "result"; callId: string; content: unknown; isError: boolean }; - -function isToolPartMetadata(value: unknown): value is ToolPartMetadata { - if (typeof value !== "object" || value === null) return false; - const kind = (value as { kind?: unknown }).kind; - return kind === "call" || kind === "result"; -} - -function asRecord(value: unknown): Record { - return typeof value === "object" && value !== null ? (value as Record) : {}; -} - -/** - * Every completed tool call (call paired with its result) recorded so - * far for the given run, in the order the platform recorded them. A - * call with no result yet (the turn is still in flight) is omitted — - * never a fabricated in-progress record. - */ -export async function readAllToolCalls( - sql: SqlClientLike, - tenantId: string, - runId: string, -): Promise { - const rows = await sql.unsafe( - `SELECT tp.metadata AS metadata, tp.ordinal AS ordinal, it.started_at AS started_at - FROM turn_part tp - JOIN inference_turn it ON it.id = tp.turn_id - WHERE it.instance_id = $1 AND it.tenant_id = $2 - ORDER BY it.started_at ASC, tp.ordinal ASC`, - [runId, tenantId], - ); - - const calls = new Map }>(); - const results = new Map(); - const order: string[] = []; - - for (const row of rows) { - const metadata: unknown = - typeof row["metadata"] === "string" ? JSON.parse(row["metadata"] as string) : row["metadata"]; - if (!isToolPartMetadata(metadata)) continue; - if (metadata.kind === "call") { - calls.set(metadata.callId, { - name: metadata.name, - arguments: asRecord(metadata.arguments), - }); - order.push(metadata.callId); - } else { - results.set(metadata.callId, { - content: metadata.content, - isError: metadata.isError, - }); - } - } - - const out: ToolCall[] = []; - for (const callId of order) { - const call = calls.get(callId); - const callResult = results.get(callId); - if (call === undefined || callResult === undefined) continue; - const content = - typeof callResult.content === "string" - ? callResult.content - : JSON.stringify(callResult.content); - out.push({ - name: call.name, - arguments: call.arguments, - isError: callResult.isError, - result: content, - }); - } - return out; -} - -/** Splits `all` into calls already reported (`alreadyConsumed`) and - * calls new since the last poll — turn_part rows are append-only per - * run, so "new" is simply "beyond the count already consumed". */ -export function newToolCallsSince( - all: readonly ToolCall[], - alreadyConsumed: number, -): { newCalls: ToolCall[]; consumed: number } { - return { newCalls: all.slice(alreadyConsumed), consumed: all.length }; -} diff --git a/packages/evals/src/targets/world-snapshot.test.ts b/packages/evals/src/targets/world-snapshot.test.ts deleted file mode 100644 index f70baabc47..0000000000 --- a/packages/evals/src/targets/world-snapshot.test.ts +++ /dev/null @@ -1,261 +0,0 @@ -import { expect, test } from "bun:test"; - -import type { DB } from "@intx/db"; -import type { AssetService } from "@intx/hub-sessions"; - -import { agentDefinitionSourceTree } from "@corbits/agent-directory"; -import { webhookTrigger as webhookTriggerTable } from "@corbits/webhook-triggers"; - -import { captureWorldSnapshot, type WorldSnapshotInfra } from "./world-snapshot.ts"; - -type FakeTables = { - workflowDefinitions: { - id: string; - name: string; - description: string | null; - assetId: string | null; - }[]; - providers: { - id: string; - tenantId: string; - name: string; - apiBaseUrl: string | null; - }[]; - credentials: { - id: string; - tenantId: string; - providerId: string; - name: string; - status: string; - }[]; - webhookTriggers: { - id: string; - tenantId: string; - name: string; - workflowDefinitionId: string; - enabled: boolean; - }[]; - workflowBlobs: Record; -}; - -function fakeDb(tables: FakeTables): DB["db"] { - return { - query: { - workflowDefinition: { - findMany: async () => tables.workflowDefinitions, - }, - tenant: { - findFirst: async () => ({ parentId: null }), - }, - provider: { - findMany: async () => tables.providers, - }, - credential: { - findMany: async () => tables.credentials, - }, - }, - select: () => ({ - from: (table: unknown) => ({ - where: async () => (table === webhookTriggerTable ? tables.webhookTriggers : []), - }), - }), - } as unknown as DB["db"]; -} - -/** Fake of the ancestor-walking credential resolver: a plain name - * lookup over the fixture's credential rows. */ -function fakeResolveCredentialByName(tables: FakeTables) { - return (async (_db: unknown, _tenantId: string, name: string) => - tables.credentials.find((credential) => credential.name === name) ?? - null) as unknown as NonNullable; -} - -/** Answers each asset's entry module out of the source tree its - * definition renders into — the shape the snapshot reads through. */ -function fakeAssetService(blobs: Record): AssetService { - return { - readAssetBlob: async ({ assetId, path }: { assetId: string; path: string }) => - new TextEncoder().encode( - agentDefinitionSourceTree({ - handle: assetId, - workflowJson: blobs[assetId] ?? "{}", - })[path], - ), - } as unknown as AssetService; -} - -function workflowJsonWith(toolPackagePins: { name: string; version: string }[], model?: string) { - return JSON.stringify({ - steps: { - agent: { - agent: { - systemPrompt: "You are a helpful agent.", - toolPackagePins, - ...(model === undefined - ? {} - : { inference: { sources: [{ provider: "anthropic", model }] } }), - }, - }, - }, - }); -} - -function emptyTables(): FakeTables { - return { - workflowDefinitions: [], - providers: [], - credentials: [], - webhookTriggers: [], - workflowBlobs: {}, - }; -} - -test("captureWorldSnapshot reads agent definitions with their capabilities", async () => { - const tables = emptyTables(); - tables.workflowDefinitions = [ - { - id: "def-1", - name: "ai-daily-research", - description: "AI Daily researcher", - assetId: "asset-1", - }, - ]; - const infra: WorldSnapshotInfra = { - db: fakeDb(tables), - resolveCredentialByNameFn: fakeResolveCredentialByName(tables), - assetService: fakeAssetService({ - "asset-1": workflowJsonWith( - [{ name: "@corbits/web-search-tools", version: "0.0.1" }], - "claude-3-5-sonnet", - ), - }), - }; - const world = await captureWorldSnapshot(infra, "tenant-1"); - expect(world.agentDefinitions).toEqual([ - { - id: "def-1", - name: "ai-daily-research", - displayName: "AI Daily researcher", - toolPackagePins: ["@corbits/web-search-tools"], - skills: [], - model: "claude-3-5-sonnet", - }, - ]); -}); - -test("captureWorldSnapshot skips a definition with no materialized asset", async () => { - const tables = emptyTables(); - tables.workflowDefinitions = [{ id: "def-1", name: "draft", description: null, assetId: null }]; - const infra: WorldSnapshotInfra = { - db: fakeDb(tables), - resolveCredentialByNameFn: fakeResolveCredentialByName(tables), - assetService: fakeAssetService({}), - }; - const world = await captureWorldSnapshot(infra, "tenant-1"); - expect(world.agentDefinitions).toEqual([]); -}); - -test("captureWorldSnapshot reads live MCP connections", async () => { - const tables = emptyTables(); - tables.providers = [ - { - id: "p-1", - tenantId: "tenant-1", - name: "mcp:github", - apiBaseUrl: "https://fake/mcp", - }, - ]; - tables.credentials = [ - { - id: "c-1", - tenantId: "tenant-1", - providerId: "p-1", - name: "mcp:github", - status: "active", - }, - ]; - const infra: WorldSnapshotInfra = { - db: fakeDb(tables), - resolveCredentialByNameFn: fakeResolveCredentialByName(tables), - assetService: fakeAssetService({}), - }; - const world = await captureWorldSnapshot(infra, "tenant-1"); - expect(world.connections).toEqual([ - { slug: "github", name: "mcp:github", url: "https://fake/mcp", live: true }, - ]); -}); - -test("captureWorldSnapshot reads a connector credential (a GitHub PAT) as a live connection", async () => { - const tables = emptyTables(); - tables.credentials = [ - { - id: "c-1", - tenantId: "tenant-1", - providerId: "p-1", - name: "GitHub", - status: "active", - }, - ]; - const infra: WorldSnapshotInfra = { - db: fakeDb(tables), - resolveCredentialByNameFn: fakeResolveCredentialByName(tables), - assetService: fakeAssetService({}), - }; - const world = await captureWorldSnapshot(infra, "tenant-1"); - expect(world.connections).toEqual([{ slug: "github", name: "GitHub", url: "", live: true }]); -}); - -test("captureWorldSnapshot reads webhook trigger rows", async () => { - const tables = emptyTables(); - tables.webhookTriggers = [ - { - id: "wt-1", - tenantId: "tenant-1", - name: "abklabs/workbench pull-request-opened", - workflowDefinitionId: "def-cr", - enabled: true, - }, - ]; - const infra: WorldSnapshotInfra = { - db: fakeDb(tables), - resolveCredentialByNameFn: fakeResolveCredentialByName(tables), - assetService: fakeAssetService({}), - }; - const world = await captureWorldSnapshot(infra, "tenant-1"); - expect(world.webhookTriggers).toEqual([ - { - id: "wt-1", - name: "abklabs/workbench pull-request-opened", - workflowDefinitionId: "def-cr", - enabled: true, - }, - ]); -}); - -test("captureWorldSnapshot folds in fake receipts from the injected reader", async () => { - const tables = emptyTables(); - const infra: WorldSnapshotInfra = { - db: fakeDb(tables), - resolveCredentialByNameFn: fakeResolveCredentialByName(tables), - assetService: fakeAssetService({}), - fakeReceiptsReader: () => [{ server: "github", toolName: "list_pull_requests", arguments: {} }], - }; - const world = await captureWorldSnapshot(infra, "tenant-1"); - expect(world.fakeReceipts).toEqual([ - { server: "github", toolName: "list_pull_requests", arguments: {} }, - ]); -}); - -test("captureWorldSnapshot returns an empty, well-formed snapshot for a tenant with nothing yet", async () => { - const tables = emptyTables(); - const infra: WorldSnapshotInfra = { - db: fakeDb(tables), - resolveCredentialByNameFn: fakeResolveCredentialByName(tables), - assetService: fakeAssetService({}), - }; - const world = await captureWorldSnapshot(infra, "tenant-1"); - expect(world.agentDefinitions).toEqual([]); - expect(world.connections).toEqual([]); - expect(world.fakeReceipts).toEqual([]); - expect(typeof world.capturedAt).toBe("string"); -}); diff --git a/packages/evals/src/targets/world-snapshot.ts b/packages/evals/src/targets/world-snapshot.ts deleted file mode 100644 index 438c31f773..0000000000 --- a/packages/evals/src/targets/world-snapshot.ts +++ /dev/null @@ -1,144 +0,0 @@ -// Reads the tenant's actual state straight off the platform's own -// tables and stores — the same "read the platform's own tables -// directly" convention `./trace.ts` already established for tool -// calls, extended here to agent definitions, connections, and -// webhook triggers. A scorer that only ever saw the transcript could check -// what the agent *said/called*; this is what lets it check what -// actually exists afterward. -// -// Every read below mirrors a real, already-shipped read path instead -// of reinventing one: -// - agent definitions: `@corbits/agent-directory`'s -// `listVisibleAgentDefinitions` query shape, minus its DM-only -// filtering (a world snapshot wants every deployed definition, not -// just the conversational ones a sidebar would show), then -// `readAgentCapabilities` on the definition its asset's source tree -// carries, read back through `readAgentDefinitionWorkflowJson` — -// the same pair `GET /:definitionId/capabilities` calls. -// - connections: `@corbits/connections`' -// `listMcpServerConnections` verbatim — it already filters to an -// active credential, so everything it returns is "live". -import { and, eq } from "drizzle-orm"; -import type { DB } from "@intx/db"; -import { resolveCredentialByName, schema } from "@intx/db"; -import type { AssetService } from "@intx/hub-sessions"; -import { readAgentCapabilities, readAgentDefinitionWorkflowJson } from "@corbits/agent-directory"; -import { webhookTrigger as webhookTriggerTable } from "@corbits/webhook-triggers"; -import { connectorDescriptors, listMcpServerConnections } from "@corbits/connections"; -import { CONNECTOR_REGISTRY } from "@corbits/connections/registry"; - -import type { FakeReceipt, WorldSnapshot } from "../types.ts"; - -/** The infra `captureWorldSnapshot` reads through — a real `@intx/db` - * drizzle handle and `AssetService`, the same two things - * `agent-directory`'s own routes already depend on, so a caller - * standing up a live target has both on hand already. - * `fakeReceiptsReader` is gap 3's recording-MCP-fake feed: omitted (or - * returning `[]`) when no fake is wired for the eval. */ -export interface WorldSnapshotInfra { - readonly db: DB["db"]; - readonly assetService: AssetService; - readonly fakeReceiptsReader?: () => readonly FakeReceipt[]; - /** Test seam over `@intx/db`'s ancestor-walking credential resolver, - * defaulting to the real one — the same resolution the Plugins layer - * and the hub's own `resolveGithubConfig` binding use. */ - readonly resolveCredentialByNameFn?: typeof resolveCredentialByName; -} - -async function readAgentDefinitions(db: DB["db"], assetService: AssetService, tenantId: string) { - const rows = await db.query.workflowDefinition.findMany({ - where: and( - eq(schema.workflowDefinition.tenantId, tenantId), - eq(schema.workflowDefinition.status, "deployed"), - ), - }); - - const deployable = rows.filter( - (row): row is typeof row & { assetId: string } => row.assetId !== null, - ); - return Promise.all( - deployable.map(async (row) => { - const workflowJson = await readAgentDefinitionWorkflowJson(assetService, row.assetId); - const capabilities = readAgentCapabilities(workflowJson); - return { - id: row.id, - name: row.name, - displayName: row.description, - toolPackagePins: capabilities.toolPackagePins.map((pin) => pin.name), - skills: [] as readonly string[], - model: capabilities.model ?? null, - }; - }), - ); -} - -async function readConnections( - db: DB["db"], - tenantId: string, - resolveByName: typeof resolveCredentialByName, -) { - const mcpConnections = await listMcpServerConnections(db, tenantId); - const fromMcp = mcpConnections.map((connection) => ({ - slug: connection.slug, - name: connection.name, - url: connection.url, - live: true, - })); - - // Connector credentials (the Plugins layer): a connector's credential - // row is named after its descriptor's displayName — the exact - // resolution `@corbits/connections`' plugins module and the hub's - // own `resolveGithubConfig` binding use — so a GitHub PAT connected - // through the connections layer shows up here as slug "github", - // which is not an MCP server and was invisible to this snapshot - // before. - const fromConnectors: typeof fromMcp = []; - for (const descriptor of connectorDescriptors(CONNECTOR_REGISTRY)) { - const row = await resolveByName(db, tenantId, descriptor.displayName); - if (row === null) continue; - fromConnectors.push({ - slug: descriptor.id, - name: descriptor.displayName, - url: "", - live: true, - }); - } - return [...fromMcp, ...fromConnectors]; -} - -async function readWebhookTriggers(db: DB["db"], tenantId: string) { - const rows = await db - .select() - .from(webhookTriggerTable) - .where(eq(webhookTriggerTable.tenantId, tenantId)); - return rows.map((row) => ({ - id: row.id, - name: row.name, - workflowDefinitionId: row.workflowDefinitionId, - enabled: row.enabled, - })); -} - -/** - * Captures everything a world-snapshot scorer can check about the - * tenant right now: its deployed agent definitions (with tools/skills/ - * model), its live connections, and whatever a recording MCP fake has - * received so far. Read-only — never mutates anything. - */ -export async function captureWorldSnapshot( - infra: WorldSnapshotInfra, - tenantId: string, -): Promise { - const [agentDefinitions, connections, webhookTriggers] = await Promise.all([ - readAgentDefinitions(infra.db, infra.assetService, tenantId), - readConnections(infra.db, tenantId, infra.resolveCredentialByNameFn ?? resolveCredentialByName), - readWebhookTriggers(infra.db, tenantId), - ]); - return { - capturedAt: new Date().toISOString(), - agentDefinitions, - connections, - webhookTriggers, - fakeReceipts: infra.fakeReceiptsReader?.() ?? [], - }; -} diff --git a/packages/evals/src/types.ts b/packages/evals/src/types.ts deleted file mode 100644 index 8d2baed46a..0000000000 --- a/packages/evals/src/types.ts +++ /dev/null @@ -1,217 +0,0 @@ -// Core shapes for the Myra eval library. Modeled after -// Mastra evals / LangChain evaluators: an eval is a hardcoded expected -// scenario (`EvalDefinition.steps`), each step a scripted human turn -// plus the `Scorer`s that grade what actually happened on that turn. -// Scorers are pure functions of a recorded `Turn` — no network, no -// clock — so they're unit-testable against hand-built transcripts; -// only `runEval` (via a `Target`) ever touches a real deployment. - -/** One tool call a turn made, as recorded off the platform's own - * inference_turn/turn_part tables (see targets/real-stack.ts for how a - * live target observes these). */ -export interface ToolCall { - readonly name: string; - readonly arguments: Record; - readonly isError: boolean; - readonly result: string; -} - -/** One played turn: the human's message, the reply text, and every - * tool call the run made while producing that reply. */ -export interface Turn { - readonly human: string; - readonly replyText: string; - readonly toolCalls: readonly ToolCall[]; -} - -/** One agent definition's guided-capability state, as read off its - * `workflow.json` (the same shape `GET /:definitionId/capabilities` - * returns — see `targets/world-snapshot.ts`). */ -export interface WorldAgentDefinition { - readonly id: string; - /** The definition's stable identifier — the agent-directory handle / - * workflow asset name (`workflow_definition.name`), the same string - * `listAgentHandles` reads at install time. Scorers matching a - * template's handles match this, never the friendly label. */ - readonly name: string; - /** The human-facing label (`workflow_definition.description`, stamped - * from the asset's displayName) — "Architecture reviewer" to `name`'s - * "architecture-reviewer". */ - readonly displayName: string | null; - readonly toolPackagePins: readonly string[]; - readonly skills: readonly string[]; - readonly model: string | null; -} - -/** One connected external system — an MCP server or another connector - * — with whether its credential is currently active. */ -export interface WorldConnection { - readonly slug: string; - readonly name: string; - readonly url: string; - readonly live: boolean; -} - -/** One live `webhook_trigger` row (`@corbits/webhook-triggers`) — what - * the code-review template's start-reviewing step creates per selected - * repo, and what a fake `pull_request.opened` delivery fires. */ -export interface WorldWebhookTrigger { - readonly id: string; - readonly name: string; - readonly workflowDefinitionId: string; - readonly enabled: boolean; -} - -/** One call a recording MCP fake actually received — the eval - * harness's "what did the outside world see" channel, folded into a - * snapshot alongside the tenant's own state. Empty when no fake is - * wired for a case. */ -export interface FakeReceipt { - readonly server: string; - readonly toolName: string; - readonly arguments: Record; -} - -/** What actually exists in the tenant right now — not what the agent - * said or called, but the state those calls produced. Captured once - * per step, after that step's turn has landed (see runner.ts). */ -export interface WorldSnapshot { - readonly capturedAt: string; - readonly agentDefinitions: readonly WorldAgentDefinition[]; - readonly connections: readonly WorldConnection[]; - readonly webhookTriggers: readonly WorldWebhookTrigger[]; - readonly fakeReceipts: readonly FakeReceipt[]; -} - -/** What a scorer sees: the full transcript so far, with `turnIndex` - * naming which entry is the turn just played, and `world` — the - * tenant's actual state as of that step. A `Target` with no - * `snapshotWorld` capability yields an empty `WorldSnapshot` rather - * than an absent field, so every scorer can read `ctx.world` - * unconditionally. */ -export interface ScorerContext { - readonly transcript: readonly Turn[]; - readonly turnIndex: number; - readonly world: WorldSnapshot; -} - -export interface ScorerResult { - readonly name: string; - /** 0..1 — deterministic scorers return 0 or 1; `judge` may return a - * graded score in between. */ - readonly score: number; - readonly pass: boolean; - readonly reason: string; - /** Set when the scorer could not run (e.g. `judge` with no live - * provider key) — never counted as a failure, reported separately. */ - readonly skipped?: boolean; -} - -export type Scorer = (ctx: ScorerContext) => ScorerResult | Promise; - -/** A step whose human message is fixed in advance. `kind` defaults to - * `"scripted"` — `defineEval` fills it in, so every existing case file - * that writes a bare `{human, expect}` keeps working unchanged. */ -export interface ScriptedEvalStep { - readonly kind?: "scripted"; - readonly human: string; - readonly expect: readonly Scorer[]; -} - -/** What a simulated-human persona knows going in. `goal` is the - * persona's private objective — the opening message a scenario still - * scripts; `knownFacts` are offered only when the agent's last reply - * asks something that fairly maps to one ("answers what's asked, - * volunteers nothing else"); `tone` is a voice note only, never a - * behavioral instruction. */ -export interface PersonaBrief { - readonly name: string; - readonly goal: string; - readonly knownFacts: Readonly>; - readonly tone?: string; -} - -/** A step whose later human messages are generated turn-by-turn by a - * persona reacting to the agent's replies, instead of being scripted in - * advance. `maxTurns` is a hard bound on the sub-loop; `expect` scores - * the last turn the loop produced. */ -export interface PersonaEvalStep { - readonly kind: "persona"; - readonly opening: string; - readonly persona: PersonaBrief; - readonly maxTurns: number; - readonly expect: readonly Scorer[]; -} - -/** A harness step that fires the tenant's enabled `webhook_trigger` - * through the real ingress route (`POST /api/webhooks/:triggerId`, - * HMAC-signed) with `payload` as the delivery body. Requires a `Target` - * with `fireWebhook`; when the world snapshot carries no enabled - * trigger the step records that miss as its turn, so the step's - * scorers grade red against the honest state instead of the run - * crashing. */ -export interface FireWebhookEvalStep { - readonly kind: "fire-webhook"; - readonly payload: Readonly>; - readonly expect: readonly Scorer[]; -} - -export type EvalStep = ScriptedEvalStep | PersonaEvalStep | FireWebhookEvalStep; - -export interface EvalDefinition { - readonly name: string; - readonly description: string; - readonly steps: readonly EvalStep[]; - /** Firm-memory entries to seed before the run starts, played as - * "Please remember: " turns with no scorers before `steps` - * runs — lets a step assume "Myra already knows X" without spending - * a scripted step teaching her. */ - readonly memorySeed?: readonly string[]; -} - -/** Per-run configuration knobs a matrix varies. `toolPins` overrides - * the assistant workflow's default tool-package pin list (name only — - * version stays whatever's published); `systemPromptOverride` replaces - * the deployed system prompt outright. */ -export interface RunConfig { - readonly name: string; - readonly model?: string; - readonly systemPromptOverride?: string; - readonly toolPins?: readonly string[]; -} - -/** A target is anything that can play one scripted human turn and - * report what happened — a real Myra deployment (targets/real-stack.ts) - * or a fake for unit-testing the runner itself (runner.test.ts). - * `snapshotWorld` is an optional capability: not every target can - * read the tenant's own tables (a scripted-only fake `Target` can - * skip it), so `runEval` guards `snapshotWorld` and falls back to an - * empty `WorldSnapshot`. */ -export interface Target { - readonly configName: string; - sendTurn(human: string): Promise; - snapshotWorld?(): Promise; - /** Fires one webhook trigger through the real ingress route with a - * properly signed delivery; the returned `Turn` records the - * accepted launch (`instanceId`). */ - fireWebhook?(triggerId: string, payload: unknown): Promise; - close(): Promise; -} - -export interface ScorerReport extends ScorerResult { - readonly stepIndex: number; -} - -export interface EvalStepRecord { - readonly stepIndex: number; - readonly turn: Turn; - readonly scorerReports: ScorerReport[]; -} - -export interface EvalRunResult { - readonly evalName: string; - readonly configName: string; - readonly startedAt: string; - readonly finishedAt: string; - readonly steps: EvalStepRecord[]; -} diff --git a/packages/evals/tsconfig.json b/packages/evals/tsconfig.json deleted file mode 100644 index d7611c1229..0000000000 --- a/packages/evals/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "include": ["src", "test"], - "compilerOptions": { - "types": ["bun"], - "noEmit": true - } -} diff --git a/packages/migration-runner/README.md b/packages/migration-runner/README.md index 6dfe3ad8fb..7dcb41ab60 100644 --- a/packages/migration-runner/README.md +++ b/packages/migration-runner/README.md @@ -25,13 +25,13 @@ name rather than the name itself, so two distinct table names could in principle hash to the same key — two packages would then serialize their boot-time migrations against each other instead of running in parallel, a liveness cost (one waits its turn) and never a correctness one (each still -applies to its own schema and ledger). The nine current ledger tables -(`access_policy_migrations`, `bench_migrations`, `evals_migrations`, +applies to its own schema and ledger). The eight current ledger tables +(`access_policy_migrations`, `bench_migrations`, `inference_catalog_migrations`, `insights_migrations`, `preferences_migrations`, `webhook_triggers_migrations`, `routine_migrations`, `workflow_deploy_source_migrations`) do not collide — verified pairwise against a live PostgreSQL 17.11 `hashtext()`. Confirm the -same before naming a tenth. +same before naming a ninth. ## What it does not change