From e50e6d0d6a1f52a2c45a5801a2b6263c1bf43c1d Mon Sep 17 00:00:00 2001 From: bebiksior Date: Fri, 10 Jul 2026 01:52:07 +0200 Subject: [PATCH 01/25] =?UTF-8?q?=F0=9F=A7=B9=20Cleanup=20instance=20archi?= =?UTF-8?q?tecture?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 +- knip.config.ts | 1 - .../drizzle/0003_omniscient_maximus.sql | 20 + .../backend/drizzle/meta/0003_snapshot.json | 583 ++++++++++++++++++ packages/backend/drizzle/meta/_journal.json | 7 + packages/backend/package.json | 3 +- packages/backend/src/index.ts | 10 +- .../src/server/api/guestInstances.spec.ts | 89 +++ .../backend/src/server/api/guestInstances.ts | 70 +-- .../backend/src/server/api/instances.spec.ts | 108 ++++ packages/backend/src/server/api/instances.ts | 150 +---- .../backend/src/server/dns/server.spec.ts | 1 - .../src/server/instances/access.spec.ts | 1 - .../src/server/instances/server.spec.ts | 1 - .../backend/src/server/instances/server.ts | 11 +- .../src/server/instances/service.spec.ts | 122 ++++ .../backend/src/server/instances/service.ts | 188 ++++++ packages/backend/src/server/mcp.ts | 528 +--------------- .../backend/src/server/mcp/context.spec.ts | 36 ++ packages/backend/src/server/mcp/context.ts | 58 ++ .../backend/src/server/mcp/instanceTools.ts | 160 +++++ packages/backend/src/server/mcp/logTools.ts | 120 ++++ .../backend/src/server/smtp/server.spec.ts | 1 - packages/backend/src/storage/db.spec.ts | 125 +++- packages/backend/src/storage/db.ts | 114 ++-- .../src/storage/importLegacyDb.spec.ts | 341 ---------- .../backend/src/storage/importLegacyDb.ts | 406 ------------ packages/backend/src/storage/index.ts | 1 - packages/backend/src/storage/legacyData.ts | 65 -- packages/backend/src/storage/records.ts | 27 +- .../src/storage/repositories/instances.ts | 45 +- .../src/storage/repositories/logs.spec.ts | 67 +- packages/backend/src/storage/schema.ts | 16 +- packages/frontend/src/App.vue | 2 + .../src/agent/core/tools/createInstance.ts | 18 +- packages/frontend/src/api/client.ts | 72 +-- packages/frontend/src/api/domains/apiKeys.ts | 22 +- .../src/api/domains/guestInstances.ts | 46 +- .../frontend/src/api/domains/instances.ts | 82 +-- .../frontend/src/api/domains/userNotices.ts | 11 +- packages/frontend/src/api/domains/webhooks.ts | 28 +- packages/frontend/src/api/parseResponse.ts | 16 + .../frontend/src/pages/Home/Container.vue | 2 - .../src/pages/Home/InstanceItem/Container.vue | 35 +- packages/frontend/src/pages/Home/useLogic.ts | 4 +- .../components/InstanceData/Container.vue | 13 +- .../components/InstanceData/useFileUpload.ts | 9 +- .../InstanceData/useInstanceEditor.spec.ts | 83 +++ .../InstanceData/useInstanceEditor.ts | 120 ++++ .../components/InstanceData/useLogic.ts | 168 +---- .../useInstanceLogStream.spec.ts | 189 ++++++ .../InstanceDetail/useInstanceLogStream.ts | 267 ++++++++ .../src/pages/InstanceDetail/useLogic.ts | 333 +--------- .../src/pages/PoCBuilder/Container.vue | 11 +- .../src/pages/PoCBuilder/useBuilderPage.ts | 33 +- .../frontend/src/pages/Settings/Container.vue | 2 - .../src/queries/domains/useApiKeys.ts | 8 +- .../queries/domains/useInstanceDetail.spec.ts | 20 +- .../src/queries/domains/useInstanceDetail.ts | 40 +- .../src/queries/domains/useInstances.ts | 70 +-- .../src/queries/domains/useWebhooks.ts | 10 +- packages/frontend/src/queries/keys.ts | 20 +- packages/frontend/src/stores/agents.ts | 5 +- packages/frontend/src/stores/auth.ts | 111 +--- .../frontend/src/stores/responseEditor.ts | 3 - packages/frontend/src/utils/httpResponse.ts | 5 + packages/shared/src/api/instances.spec.ts | 43 ++ packages/shared/src/api/instances.ts | 30 +- packages/shared/src/schemas.ts | 22 +- 69 files changed, 2779 insertions(+), 2653 deletions(-) create mode 100644 packages/backend/drizzle/0003_omniscient_maximus.sql create mode 100644 packages/backend/drizzle/meta/0003_snapshot.json create mode 100644 packages/backend/src/server/api/guestInstances.spec.ts create mode 100644 packages/backend/src/server/api/instances.spec.ts create mode 100644 packages/backend/src/server/instances/service.spec.ts create mode 100644 packages/backend/src/server/instances/service.ts create mode 100644 packages/backend/src/server/mcp/context.spec.ts create mode 100644 packages/backend/src/server/mcp/context.ts create mode 100644 packages/backend/src/server/mcp/instanceTools.ts create mode 100644 packages/backend/src/server/mcp/logTools.ts delete mode 100644 packages/backend/src/storage/importLegacyDb.spec.ts delete mode 100644 packages/backend/src/storage/importLegacyDb.ts delete mode 100644 packages/backend/src/storage/legacyData.ts create mode 100644 packages/frontend/src/api/parseResponse.ts create mode 100644 packages/frontend/src/pages/InstanceDetail/components/InstanceData/useInstanceEditor.spec.ts create mode 100644 packages/frontend/src/pages/InstanceDetail/components/InstanceData/useInstanceEditor.ts create mode 100644 packages/frontend/src/pages/InstanceDetail/useInstanceLogStream.spec.ts create mode 100644 packages/frontend/src/pages/InstanceDetail/useInstanceLogStream.ts create mode 100644 packages/shared/src/api/instances.spec.ts diff --git a/README.md b/README.md index 163f704..0d2b620 100644 --- a/README.md +++ b/README.md @@ -73,11 +73,11 @@ curl https://httpworkbench.com/api/instances \ The full OpenAPI specification is served at `/api/openapi.json` ```bash -# Create a static instance +# Create an instance curl -X POST https://httpworkbench.com/api/instances \ -H "Authorization: Bearer hwb__" \ -H "Content-Type: application/json" \ - -d '{"kind":"static","raw":"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\nhello"}' + -d '{"raw":"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\nhello"}' # Read its logs (paginate with nextCursor) curl "https://httpworkbench.com/api/instances//logs?limit=50" \ diff --git a/knip.config.ts b/knip.config.ts index 2d43588..25ebc70 100644 --- a/knip.config.ts +++ b/knip.config.ts @@ -16,7 +16,6 @@ const config: KnipConfig = { vue: { enabled: true, }, - ignore: ["**/*.d.ts"], ignoreDependencies: [ "@types/dompurify", "tailwindcss", diff --git a/packages/backend/drizzle/0003_omniscient_maximus.sql b/packages/backend/drizzle/0003_omniscient_maximus.sql new file mode 100644 index 0000000..c527bbc --- /dev/null +++ b/packages/backend/drizzle/0003_omniscient_maximus.sql @@ -0,0 +1,20 @@ +DELETE FROM `instanceWebhooks` WHERE `instanceId` IN (SELECT `id` FROM `instances` WHERE `kind` = 'dynamic' OR `raw` IS NULL);--> statement-breakpoint +DELETE FROM `logs` WHERE `instanceId` IN (SELECT `id` FROM `instances` WHERE `kind` = 'dynamic' OR `raw` IS NULL);--> statement-breakpoint +DELETE FROM `instanceModerations` WHERE `instanceId` IN (SELECT `id` FROM `instances` WHERE `kind` = 'dynamic' OR `raw` IS NULL);--> statement-breakpoint +DELETE FROM `instances` WHERE `kind` = 'dynamic' OR `raw` IS NULL;--> statement-breakpoint +CREATE TABLE `__new_instances` ( + `id` text PRIMARY KEY NOT NULL, + `ownerId` text NOT NULL, + `createdAt` integer NOT NULL, + `expiresAt` integer, + `label` text, + `isPublic` integer NOT NULL, + `isLocked` integer NOT NULL, + `raw` text NOT NULL +); +--> statement-breakpoint +INSERT INTO `__new_instances`("id", "ownerId", "createdAt", "expiresAt", "label", "isPublic", "isLocked", "raw") SELECT "id", "ownerId", "createdAt", "expiresAt", "label", "isPublic", "isLocked", "raw" FROM `instances`;--> statement-breakpoint +DROP TABLE `instances`;--> statement-breakpoint +ALTER TABLE `__new_instances` RENAME TO `instances`;--> statement-breakpoint +CREATE INDEX `instances_by_owner` ON `instances` (`ownerId`);--> statement-breakpoint +CREATE INDEX `instances_by_expiration` ON `instances` (`expiresAt`); diff --git a/packages/backend/drizzle/meta/0003_snapshot.json b/packages/backend/drizzle/meta/0003_snapshot.json new file mode 100644 index 0000000..3c69a5d --- /dev/null +++ b/packages/backend/drizzle/meta/0003_snapshot.json @@ -0,0 +1,583 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "658a85e6-4c05-4e5a-bbe2-a38300ddb275", + "prevId": "d041d703-56d8-4495-905b-c83c6b7e2a2c", + "tables": { + "apiKeys": { + "name": "apiKeys", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secretHash": { + "name": "secretHash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scopesJson": { + "name": "scopesJson", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lastUsedAt": { + "name": "lastUsedAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "apiKeys_prefix_unique": { + "name": "apiKeys_prefix_unique", + "columns": ["prefix"], + "isUnique": true + }, + "apiKeys_by_user_createdAt": { + "name": "apiKeys_by_user_createdAt", + "columns": ["userId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": { + "apiKeys_userId_users_id_fk": { + "name": "apiKeys_userId_users_id_fk", + "tableFrom": "apiKeys", + "tableTo": "users", + "columnsFrom": ["userId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "instanceModerations": { + "name": "instanceModerations", + "columns": { + "instanceId": { + "name": "instanceId", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "window5mStartMs": { + "name": "window5mStartMs", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "requestsInWindow5m": { + "name": "requestsInWindow5m", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "strikeCommittedForWindow": { + "name": "strikeCommittedForWindow", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "strikeTimestampsJson": { + "name": "strikeTimestampsJson", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "discordMutedUntilMs": { + "name": "discordMutedUntilMs", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "window15mStartMs": { + "name": "window15mStartMs", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "requestsInWindow15m": { + "name": "requestsInWindow15m", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lastMinuteBucketStartMs": { + "name": "lastMinuteBucketStartMs", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "requestsInCurrentMinute": { + "name": "requestsInCurrentMinute", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "instanceModerations_instanceId_instances_id_fk": { + "name": "instanceModerations_instanceId_instances_id_fk", + "tableFrom": "instanceModerations", + "tableTo": "instances", + "columnsFrom": ["instanceId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "instanceWebhooks": { + "name": "instanceWebhooks", + "columns": { + "instanceId": { + "name": "instanceId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "webhookId": { + "name": "webhookId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "position": { + "name": "position", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "instanceWebhooks_instance_webhook_unique": { + "name": "instanceWebhooks_instance_webhook_unique", + "columns": ["instanceId", "webhookId"], + "isUnique": true + }, + "instanceWebhooks_by_instance": { + "name": "instanceWebhooks_by_instance", + "columns": ["instanceId", "position"], + "isUnique": false + } + }, + "foreignKeys": { + "instanceWebhooks_instanceId_instances_id_fk": { + "name": "instanceWebhooks_instanceId_instances_id_fk", + "tableFrom": "instanceWebhooks", + "tableTo": "instances", + "columnsFrom": ["instanceId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "instanceWebhooks_webhookId_webhooks_id_fk": { + "name": "instanceWebhooks_webhookId_webhooks_id_fk", + "tableFrom": "instanceWebhooks", + "tableTo": "webhooks", + "columnsFrom": ["webhookId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "instanceWebhooks_instanceId_position_pk": { + "columns": ["instanceId", "position"], + "name": "instanceWebhooks_instanceId_position_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "instances": { + "name": "instances", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "isPublic": { + "name": "isPublic", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "isLocked": { + "name": "isLocked", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "raw": { + "name": "raw", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "instances_by_owner": { + "name": "instances_by_owner", + "columns": ["ownerId"], + "isUnique": false + }, + "instances_by_expiration": { + "name": "instances_by_expiration", + "columns": ["expiresAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "logs": { + "name": "logs", + "columns": { + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "instanceId": { + "name": "instanceId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "timestamp": { + "name": "timestamp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "raw": { + "name": "raw", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "logs_id_unique": { + "name": "logs_id_unique", + "columns": ["id"], + "isUnique": true + }, + "logs_by_instance_timestamp_id": { + "name": "logs_by_instance_timestamp_id", + "columns": ["instanceId", "timestamp", "id"], + "isUnique": false + }, + "logs_by_instance_seq": { + "name": "logs_by_instance_seq", + "columns": ["instanceId", "seq"], + "isUnique": false + } + }, + "foreignKeys": { + "logs_instanceId_instances_id_fk": { + "name": "logs_instanceId_instances_id_fk", + "tableFrom": "logs", + "tableTo": "instances", + "columnsFrom": ["instanceId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "userNotices": { + "name": "userNotices", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "instanceId": { + "name": "instanceId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "acknowledgedAt": { + "name": "acknowledgedAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "userNotices_by_user_createdAt": { + "name": "userNotices_by_user_createdAt", + "columns": ["userId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "googleId": { + "name": "googleId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "users_googleId_unique": { + "name": "users_googleId_unique", + "columns": ["googleId"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webhooks": { + "name": "webhooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "webhooks_by_owner": { + "name": "webhooks_by_owner", + "columns": ["ownerId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/packages/backend/drizzle/meta/_journal.json b/packages/backend/drizzle/meta/_journal.json index 04a33ff..d6d0165 100644 --- a/packages/backend/drizzle/meta/_journal.json +++ b/packages/backend/drizzle/meta/_journal.json @@ -22,6 +22,13 @@ "when": 1777119692214, "tag": "0002_rapid_nomad", "breakpoints": true + }, + { + "idx": 3, + "version": "6", + "when": 1783639493291, + "tag": "0003_omniscient_maximus", + "breakpoints": true } ] } diff --git a/packages/backend/package.json b/packages/backend/package.json index 0ff3c2f..966e86f 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -14,8 +14,7 @@ "scripts": { "dev": "bun --env-file=../../.env src/index.ts", "typecheck": "tsgo --noEmit", - "drizzle:generate": "drizzle-kit generate --config drizzle.config.ts", - "migrate:sqlite": "bun --env-file=../../.env src/storage/importLegacyDb.ts" + "drizzle:generate": "drizzle-kit generate --config drizzle.config.ts" }, "dependencies": { "@elysiajs/openapi": "^1.4.15", diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 5c49a36..2962a6e 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -1,18 +1,10 @@ import { initServer } from "./server"; -import { closeDb, initDb, maybeAutoImportLegacyDb } from "./storage"; +import { closeDb, initDb } from "./storage"; import { version } from "./version"; async function init() { console.log("init", version); - const importResult = await maybeAutoImportLegacyDb(); - if (importResult.kind === "imported") { - console.log( - "Imported legacy JSON database into SQLite", - importResult.result, - ); - } - const result = initDb(); if (result.kind === "error") { console.error("Failed to initialize database", result.error); diff --git a/packages/backend/src/server/api/guestInstances.spec.ts b/packages/backend/src/server/api/guestInstances.spec.ts new file mode 100644 index 0000000..1d8b0d0 --- /dev/null +++ b/packages/backend/src/server/api/guestInstances.spec.ts @@ -0,0 +1,89 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { Elysia } from "elysia"; +import { instancePolicies } from "../../config"; +import { closeDb, initDb } from "../../storage"; +import { guestInstancesRoutes } from "./guestInstances"; + +const app = new Elysia().use(guestInstancesRoutes); +const originalAllowGuest = instancePolicies.allowGuest; +let dataDir = ""; + +const call = (pathName: string, method: string, body: unknown) => + app.handle( + new Request(`http://localhost${pathName}`, { + method, + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }), + ); + +describe("guest instance routes", () => { + beforeEach(() => { + dataDir = mkdtempSync(path.join(tmpdir(), "httpworkbench-guest-route-")); + const result = initDb({ dataDir, reset: true }); + if (result.kind === "error") { + throw result.error; + } + instancePolicies.allowGuest = true; + }); + + afterEach(() => { + instancePolicies.allowGuest = originalAllowGuest; + closeDb(); + rmSync(dataDir, { recursive: true, force: true }); + }); + + test("creates and updates static-only responses without exposing kind", async () => { + const createdResponse = await call("/api/guest/instances", "POST", { + raw: "HTTP/1.1 200 OK\nContent-Type: text/plain\n\ncreated", + webhookIds: ["guests-cannot-select-webhooks"], + }); + expect(createdResponse.status).toBe(201); + const created = (await createdResponse.json()) as { + id: string; + raw: string; + webhookIds: string[]; + kind?: unknown; + }; + expect(created.raw).toBe( + "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\ncreated", + ); + expect(created.webhookIds).toEqual([]); + expect(created.kind).toBeUndefined(); + + const updatedResponse = await call( + `/api/guest/instances/${created.id}`, + "PUT", + { raw: "HTTP/1.1 204 No Content\n\n" }, + ); + expect(updatedResponse.status).toBe(200); + const updated = (await updatedResponse.json()) as { + raw: string; + kind?: unknown; + }; + expect(updated.raw).toBe("HTTP/1.1 204 No Content\r\n\r\n"); + expect(updated.kind).toBeUndefined(); + }); + + test("rejects legacy dynamic create and update payloads", async () => { + const createResponse = await call("/api/guest/instances", "POST", { + kind: "dynamic", + processors: [], + }); + expect(createResponse.status).toBeGreaterThanOrEqual(400); + + const validResponse = await call("/api/guest/instances", "POST", { + raw: "HTTP/1.1 200 OK\n\nok", + }); + const valid = (await validResponse.json()) as { id: string }; + const updateResponse = await call( + `/api/guest/instances/${valid.id}`, + "PUT", + { kind: "dynamic", processors: [] }, + ); + expect(updateResponse.status).toBeGreaterThanOrEqual(400); + }); +}); diff --git a/packages/backend/src/server/api/guestInstances.ts b/packages/backend/src/server/api/guestInstances.ts index a3c5b22..38212cf 100644 --- a/packages/backend/src/server/api/guestInstances.ts +++ b/packages/backend/src/server/api/guestInstances.ts @@ -1,7 +1,6 @@ import { Elysia, status } from "elysia"; import { CreateInstanceSchema, - GUEST_INSTANCE_TTL_MS, GUEST_OWNER_ID, InstanceDetailResponseSchema, SetInstanceLockedSchema, @@ -9,14 +8,16 @@ import { } from "shared"; import { instancePolicies } from "../../config"; import { - addInstance, clearLogsForInstance, deleteInstance, getInstanceById, getLogsForInstance, updateInstance, } from "../../storage"; -import { generateInstanceID, validateStaticRaw } from "../utils"; +import { + createGuestInstance, + replaceGuestInstance, +} from "../instances/service"; const loadGuestInstance = (id: string) => { const instance = getInstanceById(id); @@ -36,35 +37,10 @@ export const guestInstancesRoutes = new Elysia({ name: "routes/guest" }) .post( "/api/guest/instances", ({ body }) => { - let staticRaw: string | undefined; - if (body.kind === "static") { - const check = validateStaticRaw(body.raw); - if (!check.ok) { - return status(check.status, { error: check.error }); - } - staticRaw = check.raw; - } - - const now = Date.now(); - const base = { - id: generateInstanceID(), - ownerId: GUEST_OWNER_ID, - createdAt: now, - expiresAt: now + GUEST_INSTANCE_TTL_MS, - webhookIds: [] as string[], - public: false, - locked: false, - }; - - const created = - body.kind === "static" - ? addInstance({ kind: "static", ...base, raw: staticRaw ?? body.raw }) - : addInstance({ - kind: "dynamic", - ...base, - processors: body.processors, - }); - return status(201, created); + const result = createGuestInstance(body.raw); + return result.ok + ? status(201, result.value) + : status(result.error.status, { error: result.error.message }); }, { body: CreateInstanceSchema }, ) @@ -86,32 +62,10 @@ export const guestInstancesRoutes = new Elysia({ name: "routes/guest" }) if (!loaded.ok) { return loaded.error; } - if (body.kind !== loaded.instance.kind) { - return status(400, { error: "Kind mismatch" }); - } - - let staticRaw: string | undefined; - if (body.kind === "static") { - const check = validateStaticRaw(body.raw); - if (!check.ok) { - return status(check.status, { error: check.error }); - } - staticRaw = check.raw; - } - - const updated = updateInstance(loaded.instance.id, (inst) => { - if (inst.kind === "static" && body.kind === "static") { - return { ...inst, raw: staticRaw ?? body.raw, webhookIds: [] }; - } - if (inst.kind === "dynamic" && body.kind === "dynamic") { - return { ...inst, processors: body.processors, webhookIds: [] }; - } - return inst; - }); - if (updated === undefined) { - return status(404, { error: "Not found" }); - } - return updated; + const result = replaceGuestInstance(loaded.instance.id, body.raw); + return result.ok + ? result.value + : status(result.error.status, { error: result.error.message }); }, { body: UpdateInstanceSchema }, ) diff --git a/packages/backend/src/server/api/instances.spec.ts b/packages/backend/src/server/api/instances.spec.ts new file mode 100644 index 0000000..98a9626 --- /dev/null +++ b/packages/backend/src/server/api/instances.spec.ts @@ -0,0 +1,108 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { Elysia } from "elysia"; +import { addUser, addWebhook, closeDb, initDb } from "../../storage"; + +process.env.JWT_SECRET = "instance-routes-test-secret"; + +const { issueAuthToken } = await import("../auth"); +const { instancesRoutes } = await import("./instances"); +const app = new Elysia().use(instancesRoutes); +let dataDir = ""; + +const call = async ( + pathName: string, + method: string, + userId: string, + body: unknown, +) => { + const token = await issueAuthToken(userId); + return app.handle( + new Request(`http://localhost${pathName}`, { + method, + headers: { + authorization: `Bearer ${token}`, + "content-type": "application/json", + }, + body: JSON.stringify(body), + }), + ); +}; + +describe("authenticated instance routes", () => { + beforeEach(() => { + dataDir = mkdtempSync(path.join(tmpdir(), "httpworkbench-route-")); + const result = initDb({ dataDir, reset: true }); + if (result.kind === "error") { + throw result.error; + } + addUser({ id: "owner-1", googleId: "google-1", createdAt: 1 }); + addUser({ id: "owner-2", googleId: "google-2", createdAt: 1 }); + }); + + afterEach(() => { + closeDb(); + rmSync(dataDir, { recursive: true, force: true }); + }); + + test("creates and updates a static-only instance", async () => { + const createdResponse = await call("/api/instances", "POST", "owner-1", { + raw: "HTTP/1.1 200 OK\n\ncreated", + }); + expect(createdResponse.status).toBe(201); + const created = (await createdResponse.json()) as { + id: string; + raw: string; + kind?: unknown; + }; + expect(created.raw).toBe("HTTP/1.1 200 OK\r\n\r\ncreated"); + expect(created.kind).toBeUndefined(); + + const updatedResponse = await call( + `/api/instances/${created.id}`, + "PUT", + "owner-1", + { raw: "HTTP/1.1 204 No Content\n\n" }, + ); + expect(updatedResponse.status).toBe(200); + const updated = (await updatedResponse.json()) as { + raw: string; + kind?: unknown; + }; + expect(updated.raw).toBe("HTTP/1.1 204 No Content\r\n\r\n"); + expect(updated.kind).toBeUndefined(); + }); + + test("returns domain errors for duplicate webhooks and wrong owners", async () => { + addWebhook({ + id: "webhook-1", + ownerId: "owner-1", + name: "Alerts", + url: "https://discord.com/api/webhooks/123/token", + createdAt: 1, + }); + const duplicateResponse = await call("/api/instances", "POST", "owner-1", { + raw: "HTTP/1.1 200 OK\n\nok", + webhookIds: ["webhook-1", "webhook-1"], + }); + expect(duplicateResponse.status).toBe(400); + expect(await duplicateResponse.json()).toEqual({ + error: "Invalid webhook selection", + }); + + const ownerTwoResponse = await call("/api/instances", "POST", "owner-2", { + raw: "HTTP/1.1 200 OK\n\nowner two", + }); + const ownerTwo = (await ownerTwoResponse.json()) as { id: string }; + const forbiddenResponse = await call( + `/api/instances/${ownerTwo.id}`, + "PUT", + "owner-1", + { raw: "HTTP/1.1 200 OK\n\nforbidden" }, + ); + expect(forbiddenResponse.status).toBe(403); + expect(await forbiddenResponse.json()).toEqual({ error: "Forbidden" }); + }); +}); diff --git a/packages/backend/src/server/api/instances.ts b/packages/backend/src/server/api/instances.ts index 919b362..70a4325 100644 --- a/packages/backend/src/server/api/instances.ts +++ b/packages/backend/src/server/api/instances.ts @@ -9,14 +9,12 @@ import { UpdateInstanceSchema, } from "shared"; import { - addInstance, clearLogsForInstance, deleteInstance, getInstanceById, getInstancesByOwner, getLogsForInstance, getLogsForInstancePage, - getWebhooksByOwner, updateInstance, } from "../../storage"; import { instancePolicies } from "../../config"; @@ -30,12 +28,12 @@ import { } from "../auth"; import { canReadInstance } from "../instances/access"; import { - clampLogLimit, - decodeLogsCursor, - encodeLogsCursor, - generateInstanceID, - validateStaticRaw, -} from "../utils"; + createInstance, + getOwnedInstance, + replaceInstance, + type InstanceServiceError, +} from "../instances/service"; +import { clampLogLimit, decodeLogsCursor, encodeLogsCursor } from "../utils"; const LogsQuerySchema = z.object({ limit: z.coerce.number().int().min(1).max(500).optional(), @@ -44,33 +42,15 @@ const LogsQuerySchema = z.object({ sinceTimestamp: z.coerce.number().int().min(0).optional(), }); -const ensureOwnedWebhookIds = ( - ownerId: string, - webhookIds: string[], -): { ok: true; ids: string[] } | { ok: false } => { - if (webhookIds.length === 0) { - return { ok: true, ids: [] }; - } - const ownedIds = new Set( - getWebhooksByOwner(ownerId).map((webhook) => webhook.id), - ); - for (const webhookId of webhookIds) { - if (!ownedIds.has(webhookId)) { - return { ok: false }; - } - } - return { ok: true, ids: webhookIds }; -}; +const serviceErrorResponse = (error: InstanceServiceError) => + status(error.status, { error: error.message }); const loadOwnedInstance = (id: string, userId: string) => { - const current = getInstanceById(id); - if (current === undefined) { - return { ok: false as const, res: status(404, { error: "Not found" }) }; + const result = getOwnedInstance(id, userId); + if (!result.ok) { + return { ok: false as const, res: serviceErrorResponse(result.error) }; } - if (current.ownerId !== userId) { - return { ok: false as const, res: status(403, { error: "Forbidden" }) }; - } - return { ok: true as const, instance: current }; + return { ok: true as const, instance: result.value }; }; export const instancesRoutes = new Elysia({ name: "routes/instances" }) @@ -86,51 +66,14 @@ export const instancesRoutes = new Elysia({ name: "routes/instances" }) .post( "/api/instances", ({ body, user }) => { - let staticRaw: string | undefined; - if (body.kind === "static") { - const check = validateStaticRaw(body.raw); - if (!check.ok) { - return status(check.status, { error: check.error }); - } - staticRaw = check.raw; - } - - const webhooks = ensureOwnedWebhookIds(user.id, body.webhookIds ?? []); - if (!webhooks.ok) { - return status(400, { error: "Invalid webhook selection" }); - } - - if ( - instancePolicies.maxInstancesPerOwner !== undefined && - getInstancesByOwner(user.id).length >= - instancePolicies.maxInstancesPerOwner - ) { - return status(403, { error: "Instance limit reached" }); - } - - const now = Date.now(); - const base = { - id: generateInstanceID(), + const result = createInstance({ ownerId: user.id, - createdAt: now, - public: false, - locked: false, - expiresAt: - instancePolicies.defaultTtlMs === undefined - ? undefined - : now + instancePolicies.defaultTtlMs, - webhookIds: webhooks.ids, - } as const; - - const created = - body.kind === "static" - ? addInstance({ kind: "static", ...base, raw: staticRaw ?? body.raw }) - : addInstance({ - kind: "dynamic", - ...base, - processors: body.processors, - }); - return status(201, created); + raw: body.raw, + webhookIds: body.webhookIds, + }); + return result.ok + ? status(201, result.value) + : serviceErrorResponse(result.error); }, { scope: "instances:write", @@ -184,54 +127,13 @@ export const instancesRoutes = new Elysia({ name: "routes/instances" }) .put( "/api/instances/:id", ({ params, body, user }) => { - const loaded = loadOwnedInstance(params.id, user.id); - if (!loaded.ok) { - return loaded.res; - } - if (body.kind !== loaded.instance.kind) { - return status(400, { error: "Kind mismatch" }); - } - - let staticRaw: string | undefined; - if (body.kind === "static") { - const check = validateStaticRaw(body.raw); - if (!check.ok) { - return status(check.status, { error: check.error }); - } - staticRaw = check.raw; - } - - let nextWebhookIds: string[] | undefined; - if (body.webhookIds !== undefined) { - const webhooks = ensureOwnedWebhookIds(user.id, body.webhookIds); - if (!webhooks.ok) { - return status(400, { error: "Invalid webhook selection" }); - } - nextWebhookIds = webhooks.ids; - } - - const updated = updateInstance(params.id, (inst) => { - if (inst.kind === "static" && body.kind === "static") { - return { - ...inst, - raw: staticRaw ?? body.raw, - webhookIds: nextWebhookIds ?? inst.webhookIds, - }; - } - if (inst.kind === "dynamic" && body.kind === "dynamic") { - return { - ...inst, - processors: body.processors, - webhookIds: nextWebhookIds ?? inst.webhookIds, - }; - } - return inst; + const result = replaceInstance({ + instanceId: params.id, + ownerId: user.id, + raw: body.raw, + webhookIds: body.webhookIds, }); - - if (updated === undefined) { - return status(404, { error: "Not found" }); - } - return updated; + return result.ok ? result.value : serviceErrorResponse(result.error); }, { scope: "instances:write", @@ -239,7 +141,7 @@ export const instancesRoutes = new Elysia({ name: "routes/instances" }) detail: { tags: ["Instances"], summary: "Replace an instance", - description: "Replace an owned instance. The kind must match.", + description: "Replace an owned instance response.", }, }, ) diff --git a/packages/backend/src/server/dns/server.spec.ts b/packages/backend/src/server/dns/server.spec.ts index 6b993e8..fcb5fe9 100644 --- a/packages/backend/src/server/dns/server.spec.ts +++ b/packages/backend/src/server/dns/server.spec.ts @@ -17,7 +17,6 @@ const createInstance = (id: string) => ({ webhookIds: [] as string[], public: false, locked: false, - kind: "static" as const, raw: "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n", }); diff --git a/packages/backend/src/server/instances/access.spec.ts b/packages/backend/src/server/instances/access.spec.ts index 28ec23a..dcc4370 100644 --- a/packages/backend/src/server/instances/access.spec.ts +++ b/packages/backend/src/server/instances/access.spec.ts @@ -17,7 +17,6 @@ const otherUser: User = { const privateInstance: Instance = { id: "instance-1", - kind: "static", ownerId: user.id, createdAt: 1, raw: "HTTP/1.1 200 OK\r\n\r\nok", diff --git a/packages/backend/src/server/instances/server.spec.ts b/packages/backend/src/server/instances/server.spec.ts index bb1985f..4079152 100644 --- a/packages/backend/src/server/instances/server.spec.ts +++ b/packages/backend/src/server/instances/server.spec.ts @@ -47,7 +47,6 @@ const createRawRequest = (headers: string[], body = "") => { const createStaticInstance = (raw: string) => ({ id: "demo", - kind: "static" as const, ownerId: "user-1", createdAt: 1, raw, diff --git a/packages/backend/src/server/instances/server.ts b/packages/backend/src/server/instances/server.ts index 68a0ecf..179a43d 100644 --- a/packages/backend/src/server/instances/server.ts +++ b/packages/backend/src/server/instances/server.ts @@ -156,15 +156,8 @@ export const createInstancesServer = ( broadcastLog(log); didLog = true; - switch (instance.kind) { - case "static": { - const adjustedResponse = adjustContentLength(instance.raw); - respond(socket, new TextEncoder().encode(adjustedResponse)); - break; - } - case "dynamic": - throw new Error("Dynamic instances are not supported yet"); - } + const adjustedResponse = adjustContentLength(instance.raw); + respond(socket, new TextEncoder().encode(adjustedResponse)); } catch (error) { console.error(error); if (!didLog) { diff --git a/packages/backend/src/server/instances/service.spec.ts b/packages/backend/src/server/instances/service.spec.ts new file mode 100644 index 0000000..bbcc46b --- /dev/null +++ b/packages/backend/src/server/instances/service.spec.ts @@ -0,0 +1,122 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { instancePolicies } from "../../config"; +import { addWebhook, closeDb, initDb } from "../../storage"; +import { createInstance, replaceInstance } from "./service"; + +const validRaw = "HTTP/1.1 200 OK\nContent-Type: text/plain\n\nhello"; +const originalInstanceLimit = instancePolicies.maxInstancesPerOwner; +let dataDir = ""; + +const addOwnedWebhook = (id: string, ownerId: string) => + addWebhook({ + id, + ownerId, + name: id, + url: "https://discord.com/api/webhooks/123/token", + createdAt: 1, + }); + +describe("instance service", () => { + beforeEach(() => { + dataDir = mkdtempSync(path.join(tmpdir(), "httpworkbench-service-")); + const result = initDb({ dataDir, reset: true }); + if (result.kind === "error") { + throw result.error; + } + instancePolicies.maxInstancesPerOwner = undefined; + }); + + afterEach(() => { + instancePolicies.maxInstancesPerOwner = originalInstanceLimit; + closeDb(); + rmSync(dataDir, { recursive: true, force: true }); + }); + + test("normalizes raw responses and persists owned webhooks", () => { + addOwnedWebhook("webhook-1", "owner-1"); + + const result = createInstance({ + ownerId: "owner-1", + raw: validRaw, + webhookIds: ["webhook-1"], + }); + + expect(result.ok).toBe(true); + if (!result.ok) { + return; + } + expect(result.value.raw).toBe( + "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\nhello", + ); + expect(result.value.webhookIds).toEqual(["webhook-1"]); + expect("kind" in result.value).toBe(false); + }); + + test("rejects webhooks owned by another user and duplicate ids", () => { + addOwnedWebhook("owned", "owner-1"); + addOwnedWebhook("foreign", "owner-2"); + + const foreign = createInstance({ + ownerId: "owner-1", + raw: validRaw, + webhookIds: ["foreign"], + }); + expect(foreign).toEqual({ + ok: false, + error: { + code: "invalid_webhooks", + message: "Invalid webhook selection", + status: 400, + }, + }); + + const duplicate = createInstance({ + ownerId: "owner-1", + raw: validRaw, + webhookIds: ["owned", "owned"], + }); + expect(duplicate).toEqual({ + ok: false, + error: { + code: "invalid_webhooks", + message: "Invalid webhook selection", + status: 400, + }, + }); + }); + + test("enforces the active instance quota without hydrating instances", () => { + instancePolicies.maxInstancesPerOwner = 1; + expect(createInstance({ ownerId: "owner-1", raw: validRaw }).ok).toBe(true); + + expect(createInstance({ ownerId: "owner-1", raw: validRaw })).toEqual({ + ok: false, + error: { + code: "instance_limit", + message: "Instance limit reached", + status: 403, + }, + }); + }); + + test("returns a typed forbidden result for wrong-owner updates", () => { + const created = createInstance({ ownerId: "owner-1", raw: validRaw }); + if (!created.ok) { + throw new Error(created.error.message); + } + + expect( + replaceInstance({ + instanceId: created.value.id, + ownerId: "owner-2", + raw: "HTTP/1.1 204 No Content\n\n", + }), + ).toEqual({ + ok: false, + error: { code: "forbidden", message: "Forbidden", status: 403 }, + }); + }); +}); diff --git a/packages/backend/src/server/instances/service.ts b/packages/backend/src/server/instances/service.ts new file mode 100644 index 0000000..7290bd3 --- /dev/null +++ b/packages/backend/src/server/instances/service.ts @@ -0,0 +1,188 @@ +import { GUEST_INSTANCE_TTL_MS, GUEST_OWNER_ID, type Instance } from "shared"; +import { instancePolicies } from "../../config"; +import { + addInstance, + countActiveInstancesByOwner, + getInstanceById, + getWebhooksByOwner, + updateInstance, +} from "../../storage"; +import { generateInstanceID, validateStaticRaw } from "../utils"; + +type InstanceServiceErrorCode = + | "invalid_raw" + | "invalid_webhooks" + | "instance_limit" + | "not_found" + | "forbidden"; + +export type InstanceServiceError = { + code: InstanceServiceErrorCode; + message: string; + status: number; +}; + +export type InstanceServiceResult = + | { ok: true; value: T } + | { ok: false; error: InstanceServiceError }; + +const fail = ( + code: InstanceServiceErrorCode, + message: string, + status: number, +): InstanceServiceResult => ({ + ok: false, + error: { code, message, status }, +}); + +const validateWebhookIds = ( + ownerId: string, + webhookIds: string[], +): InstanceServiceResult => { + if (webhookIds.length === 0) { + return { ok: true, value: [] }; + } + + if (new Set(webhookIds).size !== webhookIds.length) { + return fail("invalid_webhooks", "Invalid webhook selection", 400); + } + + const ownedIds = new Set( + getWebhooksByOwner(ownerId).map((webhook) => webhook.id), + ); + if (webhookIds.some((webhookId) => !ownedIds.has(webhookId))) { + return fail("invalid_webhooks", "Invalid webhook selection", 400); + } + return { ok: true, value: webhookIds }; +}; + +const validateRaw = (raw: string): InstanceServiceResult => { + const result = validateStaticRaw(raw); + return result.ok + ? { ok: true, value: result.raw } + : fail("invalid_raw", result.error, result.status); +}; + +export const getOwnedInstance = ( + instanceId: string, + ownerId: string, +): InstanceServiceResult => { + const instance = getInstanceById(instanceId); + if (instance === undefined) { + return fail("not_found", "Not found", 404); + } + if (instance.ownerId !== ownerId) { + return fail("forbidden", "Forbidden", 403); + } + return { ok: true, value: instance }; +}; + +type CreateInstanceInput = { + ownerId: string; + raw: string; + webhookIds?: string[]; + label?: string; + guest?: boolean; +}; + +export const createInstance = ( + input: CreateInstanceInput, +): InstanceServiceResult => { + const raw = validateRaw(input.raw); + if (!raw.ok) { + return raw; + } + + const webhookIds = + input.guest === true + ? ({ ok: true, value: [] as string[] } as const) + : validateWebhookIds(input.ownerId, input.webhookIds ?? []); + if (!webhookIds.ok) { + return webhookIds; + } + + if ( + input.guest !== true && + instancePolicies.maxInstancesPerOwner !== undefined && + countActiveInstancesByOwner(input.ownerId) >= + instancePolicies.maxInstancesPerOwner + ) { + return fail("instance_limit", "Instance limit reached", 403); + } + + const now = Date.now(); + const expiresAt = + input.guest === true + ? now + GUEST_INSTANCE_TTL_MS + : instancePolicies.defaultTtlMs === undefined + ? undefined + : now + instancePolicies.defaultTtlMs; + + return { + ok: true, + value: addInstance({ + id: generateInstanceID(), + ownerId: input.ownerId, + createdAt: now, + expiresAt, + webhookIds: webhookIds.value, + label: input.label, + public: false, + locked: false, + raw: raw.value, + }), + }; +}; + +type UpdateInstanceInput = { + instanceId: string; + ownerId: string; + raw: string; + webhookIds?: string[]; + guest?: boolean; +}; + +export const replaceInstance = ( + input: UpdateInstanceInput, +): InstanceServiceResult => { + const owned = getOwnedInstance(input.instanceId, input.ownerId); + if (!owned.ok) { + return owned; + } + + const raw = validateRaw(input.raw); + if (!raw.ok) { + return raw; + } + + const webhookIds = + input.guest === true + ? ({ ok: true, value: [] as string[] } as const) + : input.webhookIds === undefined + ? ({ ok: true, value: owned.value.webhookIds } as const) + : validateWebhookIds(input.ownerId, input.webhookIds); + if (!webhookIds.ok) { + return webhookIds; + } + + const updated = updateInstance(input.instanceId, (instance) => ({ + ...instance, + raw: raw.value, + webhookIds: webhookIds.value, + })); + + return updated === undefined + ? fail("not_found", "Not found", 404) + : { ok: true, value: updated }; +}; + +export const createGuestInstance = (raw: string) => + createInstance({ ownerId: GUEST_OWNER_ID, raw, guest: true }); + +export const replaceGuestInstance = (instanceId: string, raw: string) => + replaceInstance({ + instanceId, + ownerId: GUEST_OWNER_ID, + raw, + guest: true, + }); diff --git a/packages/backend/src/server/mcp.ts b/packages/backend/src/server/mcp.ts index 29550b8..55fba93 100644 --- a/packages/backend/src/server/mcp.ts +++ b/packages/backend/src/server/mcp.ts @@ -1,66 +1,37 @@ -import type { AuthInfo } from "@modelcontextprotocol/sdk/server/auth/types.js"; import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js"; -import * as z from "zod/v4"; -import { instancePolicies } from "../config"; -import { - addInstance, - clearLogsForInstance, - deleteInstance, - getInstanceById, - getInstancesByOwner, - getLogsForInstancePage, - getRecentLogsForInstance, - updateInstance, -} from "../storage"; -import type { ApiKeyAuthContext } from "./apiKeyAuth"; -import { authenticateApiKeyRequest, hasApiKeyScope } from "./apiKeyAuth"; +import { authenticateApiKeyRequest } from "./apiKeyAuth"; +import { registerInstanceTools } from "./mcp/instanceTools"; +import { registerLogTools } from "./mcp/logTools"; import { createFixedWindowRateLimiter } from "./rateLimit"; -import { - clampLogLimit, - decodeLogsCursor, - encodeLogsCursor, - generateInstanceID, - MAX_LOG_LIMIT, - validateStaticRaw, -} from "./utils"; -const mcpLogPreviewLimit = 10; const mcpRateLimitWindowMs = 60_000; -const mcpRateLimitMaxRequests = 120; -const unauthMcpRateLimitMaxRequests = 600; +const mcpRateLimiter = createFixedWindowRateLimiter({ + maxRequests: 120, + windowMs: mcpRateLimitWindowMs, +}); +const unauthMcpRateLimiter = createFixedWindowRateLimiter({ + maxRequests: 600, + windowMs: mcpRateLimitWindowMs, +}); + const mcpServerInstructions = ` HTTP Workbench helps pentesters and security researchers create temporary hosted HTTP proof-of-concept pages and observe HTTP/DNS/SMTP interactions for authorized security testing. -An instance is a hosted subdomain under the configured instances domain. Static instances serve an exact raw HTTP response and record incoming HTTP requests. Deployments with DNS enabled also record DNS lookups for instance hostnames. - -Use these tools when the user needs a hosted page, a separate origin, a callback/redirect receiver, a custom raw HTTP response, or interaction logs for a specific HTTP Workbench instance. - -Instance URLs returned by tools are public network endpoints; anyone who can reach the URL can trigger logs and receive the configured response. +An instance is a hosted subdomain under the configured instances domain. It serves an exact raw HTTP response and records incoming HTTP requests. Deployments with DNS enabled also record DNS lookups for instance hostnames. Create and update tools require a complete raw HTTP response: status line, headers, a blank line, and body. They normalize line endings, enforce response size limits, and reject invalid HTTP. Use watch_instance_logs by polling again with nextCursor after pollAfterMs when the user is actively waiting for new interactions. `.trim(); -const mcpRateLimiter = createFixedWindowRateLimiter({ - maxRequests: mcpRateLimitMaxRequests, - windowMs: mcpRateLimitWindowMs, -}); -const unauthMcpRateLimiter = createFixedWindowRateLimiter({ - maxRequests: unauthMcpRateLimitMaxRequests, - windowMs: mcpRateLimitWindowMs, -}); - const frontendUrl = Bun.env.FRONTEND_URL; const configuredDomain = Bun.env.DOMAIN; - const allowedOrigins = new Set( [frontendUrl, "http://localhost:5173", "http://127.0.0.1:5173"].filter( (value): value is string => value !== undefined && value !== "", ), ); - const allowedHosts = new Set( [ configuredDomain, @@ -74,483 +45,45 @@ const allowedHosts = new Set( ].filter((value): value is string => value !== undefined && value !== ""), ); -const jsonToolResult = (value: Record) => ({ - content: [{ type: "text" as const, text: JSON.stringify(value, null, 2) }], - structuredContent: value, -}); - -const toolError = (message: string) => ({ - content: [{ type: "text" as const, text: message }], - isError: true, -}); - -const unauthorizedResponse = () => - new Response("Unauthorized", { - status: 401, - headers: { - "WWW-Authenticate": 'Bearer realm="httpworkbench-mcp"', - }, - }); - -const validateMcpHeaders = (req: Request): Response | undefined => { - const host = req.headers.get("host"); +const validateMcpHeaders = (request: Request): Response | undefined => { + const host = request.headers.get("host"); if (host === null || !allowedHosts.has(host)) { return new Response("Invalid host", { status: 403 }); } - const origin = req.headers.get("origin"); + const origin = request.headers.get("origin"); if (origin !== null && !allowedOrigins.has(origin)) { return new Response("Invalid origin", { status: 403 }); } - return undefined; }; -const getAuthContext = (extra: { authInfo?: AuthInfo }): ApiKeyAuthContext => { - const context = extra.authInfo?.extra?.httpworkbenchAuth; - if (context === undefined) { - throw new Error("Unauthorized"); - } - return context as ApiKeyAuthContext; -}; - -const requireScope = ( - auth: ApiKeyAuthContext, - scope: - | "instances:read" - | "instances:write" - | "instances:delete" - | "logs:read" - | "logs:stream", -) => { - if (!hasApiKeyScope(auth.apiKey, scope)) { - throw new Error(`Missing API key scope: ${scope}`); - } -}; - -const getOwnedInstance = (instanceId: string, auth: ApiKeyAuthContext) => { - const instance = getInstanceById(instanceId); - if (instance?.ownerId !== auth.user.id) { - throw new Error("Instance not found"); - } - return instance; -}; - -const serializeInstance = (instance: ReturnType) => { - if (instance === undefined) { - return undefined; - } - return { - ...instance, - url: `${instance.id}.${Bun.env.INSTANCES_DOMAIN ?? ""}`, - }; -}; - -const serializeInstanceSummary = ( - instance: ReturnType, -) => { - if (instance === undefined) { - return undefined; - } - return { - id: instance.id, - kind: instance.kind, - ownerId: instance.ownerId, - createdAt: instance.createdAt, - expiresAt: instance.expiresAt, - label: instance.label, - public: instance.public, - locked: instance.locked, - url: `${instance.id}.${Bun.env.INSTANCES_DOMAIN ?? ""}`, - }; -}; - -const readLogsPage = (input: { - auth: ApiKeyAuthContext; - instanceId: string; - limit?: number; - cursor?: string; - type?: "http" | "dns" | "smtp"; - sinceTimestamp?: number; -}) => { - getOwnedInstance(input.instanceId, input.auth); - const limit = clampLogLimit(input.limit); - const cursor = - input.cursor === undefined ? undefined : decodeLogsCursor(input.cursor); - if (input.cursor !== undefined && cursor === undefined) { - throw new Error("Invalid cursor"); - } - - const page = getLogsForInstancePage({ - instanceId: input.instanceId, - limit, - cursor, - type: input.type, - sinceTimestamp: input.sinceTimestamp, - }); - - return { - logs: page.logs, - nextCursor: - page.nextCursor === undefined - ? undefined - : encodeLogsCursor(page.nextCursor), - }; -}; - const createMcpServer = () => { const server = new McpServer( - { - name: "httpworkbench", - version: "1.0.0", - }, - { - instructions: mcpServerInstructions, - }, - ); - - server.registerTool( - "list_instances", - { - title: "List Instances", - description: - "List HTTP Workbench instances owned by the authenticated API key user. Use this first when the user refers to an existing instance but has not provided an instanceId. This is read-only and returns instance metadata plus public URLs, not full response bodies or logs.", - annotations: { - readOnlyHint: true, - idempotentHint: true, - openWorldHint: false, - }, - }, - async (extra) => { - const auth = getAuthContext(extra); - requireScope(auth, "instances:read"); - return jsonToolResult({ - instances: getInstancesByOwner(auth.user.id).map( - serializeInstanceSummary, - ), - }); - }, - ); - - server.registerTool( - "create_instance", - { - title: "Create Instance", - description: - "Create a new static HTTP Workbench instance that serves the provided raw HTTP response from a public instance URL. Use this when the user needs a new hosted PoC page, callback receiver, redirect target, CORS test endpoint, separate origin, or other temporary hosted surface. Side effects: persists a new instance owned by the API key user, consumes instance quota, creates a reachable public endpoint, and future requests or DNS lookups for that endpoint may be logged. The raw input must be a complete valid HTTP response with status line, headers, blank line, and body. Consider cleaning up instance once you are done testing", - inputSchema: { - raw: z - .string() - .describe( - "Complete raw HTTP response to serve, including status line, headers, blank line, and body. Example: HTTP/1.1 200 OK\\r\\nContent-Type: text/html\\r\\n\\r\\n...", - ), - label: z - .string() - .max(100) - .optional() - .describe( - "Optional human-readable label for the instance, used only for organization.", - ), - }, - annotations: { - readOnlyHint: false, - destructiveHint: false, - idempotentHint: false, - openWorldHint: true, - }, - }, - async ({ raw, label }, extra) => { - const auth = getAuthContext(extra); - requireScope(auth, "instances:write"); - - const check = validateStaticRaw(raw); - if (!check.ok) { - return toolError(check.error); - } - const normalizedRaw = check.raw; - - if (instancePolicies.maxInstancesPerOwner !== undefined) { - const ownedInstances = getInstancesByOwner(auth.user.id); - if (ownedInstances.length >= instancePolicies.maxInstancesPerOwner) { - return toolError("Instance limit reached"); - } - } - - const now = Date.now(); - const instance = addInstance({ - id: generateInstanceID(), - ownerId: auth.user.id, - createdAt: now, - expiresAt: - instancePolicies.defaultTtlMs === undefined - ? undefined - : now + instancePolicies.defaultTtlMs, - webhookIds: [], - label, - public: false, - locked: false, - kind: "static", - raw: normalizedRaw, - }); - - return jsonToolResult({ instance: serializeInstance(instance) }); - }, - ); - - server.registerTool( - "get_instance", - { - title: "Get Instance", - description: - "Get one owned HTTP Workbench instance, including its full configuration and a small recent log preview when the API key also has logs:read. Use this to inspect the current hosted response, URL, lock/public flags, expiration, or recent interactions before deciding whether to update, delete, or read more logs. This is read-only and only returns instances owned by the API key user.", - inputSchema: { - instanceId: z - .string() - .min(1) - .describe( - "ID of the HTTP Workbench instance to inspect, usually the leftmost label of its instance URL.", - ), - }, - annotations: { - readOnlyHint: true, - idempotentHint: true, - openWorldHint: false, - }, - }, - async ({ instanceId }, extra) => { - const auth = getAuthContext(extra); - requireScope(auth, "instances:read"); - const instance = getOwnedInstance(instanceId, auth); - const logs = hasApiKeyScope(auth.apiKey, "logs:read") - ? getRecentLogsForInstance(instance.id, mcpLogPreviewLimit) - : []; - return jsonToolResult({ instance: serializeInstance(instance), logs }); - }, - ); - - server.registerTool( - "update_instance", - { - title: "Update Instance", - description: - "Replace the raw HTTP response served by an owned static instance. Use this when the user wants to revise an existing hosted PoC or change the behavior of an existing instance URL instead of creating a new one. Side effects: overwrites the response seen by future visitors to that public URL; existing logs remain unchanged. Only static instances can be updated through MCP. The raw input must be a complete valid HTTP response with status line, headers, blank line, and body.", - inputSchema: { - instanceId: z - .string() - .min(1) - .describe("ID of the owned static instance to update."), - raw: z - .string() - .describe( - "Complete replacement raw HTTP response to serve, including status line, headers, blank line, and body.", - ), - }, - annotations: { - readOnlyHint: false, - destructiveHint: true, - idempotentHint: true, - openWorldHint: true, - }, - }, - async ({ instanceId, raw }, extra) => { - const auth = getAuthContext(extra); - requireScope(auth, "instances:write"); - const current = getOwnedInstance(instanceId, auth); - if (current.kind !== "static") { - return toolError("Only static instances can be updated through MCP"); - } - - const check = validateStaticRaw(raw); - if (!check.ok) { - return toolError(check.error); - } - const normalizedRaw = check.raw; - - const updated = updateInstance(instanceId, (instance) => ({ - ...instance, - raw: normalizedRaw, - })); - return jsonToolResult({ instance: serializeInstance(updated) }); - }, - ); - - server.registerTool( - "delete_instance", - { - title: "Delete Instance", - description: - "Delete an owned, unlocked HTTP Workbench instance. Use this only when the user wants to remove a hosted endpoint and its associated data from HTTP Workbench. Side effects: removes the instance, stops serving its public URL, and cascades deletion of related instance data such as logs according to storage rules. Locked instances cannot be deleted.", - inputSchema: { - instanceId: z - .string() - .min(1) - .describe("ID of the owned, unlocked instance to delete."), - }, - annotations: { - readOnlyHint: false, - destructiveHint: true, - idempotentHint: false, - openWorldHint: false, - }, - }, - async ({ instanceId }, extra) => { - const auth = getAuthContext(extra); - requireScope(auth, "instances:delete"); - const instance = getOwnedInstance(instanceId, auth); - if (instance.locked) { - return toolError("Instance is locked"); - } - deleteInstance(instanceId); - return jsonToolResult({ deleted: true }); - }, - ); - - server.registerTool( - "clear_instance_logs", - { - title: "Clear Instance Logs", - description: - "Clear all recorded HTTP/DNS/SMTP logs for an owned instance while leaving the hosted instance itself intact. Use this when the user wants a clean slate before reproducing an interaction. Side effects: permanently removes stored logs for the instance; it does not change the served response or public URL.", - inputSchema: { - instanceId: z - .string() - .min(1) - .describe("ID of the owned instance whose logs should be cleared."), - }, - annotations: { - readOnlyHint: false, - destructiveHint: true, - idempotentHint: true, - openWorldHint: false, - }, - }, - async ({ instanceId }, extra) => { - const auth = getAuthContext(extra); - requireScope(auth, "instances:write"); - getOwnedInstance(instanceId, auth); - clearLogsForInstance(instanceId); - return jsonToolResult({ cleared: true }); - }, - ); - - server.registerTool( - "read_instance_logs", - { - title: "Read Instance Logs", - description: - "Read historical paginated HTTP/DNS/SMTP interaction logs for an owned instance. Use this when the user asks what requests, callbacks, DNS lookups, headers, paths, bodies, or source addresses reached an instance. This is read-only, but returned logs can contain sensitive request data. Use nextCursor to fetch additional pages.", - inputSchema: { - instanceId: z - .string() - .min(1) - .describe("ID of the owned instance whose logs should be read."), - limit: z - .number() - .int() - .min(1) - .max(MAX_LOG_LIMIT) - .optional() - .describe( - `Maximum number of logs to return, up to ${MAX_LOG_LIMIT}.`, - ), - cursor: z - .string() - .optional() - .describe("Opaque cursor returned by a previous log read."), - type: z - .enum(["http", "dns", "smtp"]) - .optional() - .describe("Optional log type filter."), - sinceTimestamp: z - .number() - .optional() - .describe( - "Optional Unix timestamp in milliseconds; only newer logs are returned.", - ), - }, - annotations: { - readOnlyHint: true, - idempotentHint: true, - openWorldHint: false, - }, - }, - async (input, extra) => { - const auth = getAuthContext(extra); - requireScope(auth, "logs:read"); - return jsonToolResult(readLogsPage({ ...input, auth })); - }, + { name: "httpworkbench", version: "1.0.0" }, + { instructions: mcpServerInstructions }, ); - - server.registerTool( - "watch_instance_logs", - { - title: "Watch Instance Logs", - description: - "Poll for new HTTP/DNS/SMTP logs from an owned instance while the user is waiting for an interaction to arrive. This is read-only, but returned logs can contain sensitive request data. Call again after pollAfterMs using the returned nextCursor to continue watching without replaying older entries.", - inputSchema: { - instanceId: z - .string() - .min(1) - .describe("ID of the owned instance whose logs should be watched."), - limit: z - .number() - .int() - .min(1) - .max(MAX_LOG_LIMIT) - .optional() - .describe( - `Maximum number of new logs to return, up to ${MAX_LOG_LIMIT}.`, - ), - cursor: z - .string() - .optional() - .describe( - "Opaque cursor returned by a previous log read or watch call.", - ), - type: z - .enum(["http", "dns", "smtp"]) - .optional() - .describe("Optional log type filter."), - sinceTimestamp: z - .number() - .optional() - .describe( - "Optional Unix timestamp in milliseconds; only newer logs are returned.", - ), - }, - annotations: { - readOnlyHint: true, - idempotentHint: false, - openWorldHint: false, - }, - }, - async (input, extra) => { - const auth = getAuthContext(extra); - requireScope(auth, "logs:stream"); - return jsonToolResult({ - ...readLogsPage({ ...input, auth }), - pollAfterMs: 1000, - }); - }, - ); - + registerInstanceTools(server); + registerLogTools(server); return server; }; -export const handleMcpRequest = async (req: Request): Promise => { - const headerError = validateMcpHeaders(req); +export const handleMcpRequest = async (request: Request): Promise => { + const headerError = validateMcpHeaders(request); if (headerError !== undefined) { return headerError; } - const auth = authenticateApiKeyRequest(req); + const auth = authenticateApiKeyRequest(request); if (auth === undefined) { if (!unauthMcpRateLimiter.check("unauthenticated")) { return new Response("Rate limit exceeded", { status: 429 }); } - return unauthorizedResponse(); + return new Response("Unauthorized", { + status: 401, + headers: { "WWW-Authenticate": 'Bearer realm="httpworkbench-mcp"' }, + }); } - if (!mcpRateLimiter.check(auth.apiKey.id)) { return new Response("Rate limit exceeded", { status: 429 }); } @@ -560,10 +93,9 @@ export const handleMcpRequest = async (req: Request): Promise => { sessionIdGenerator: undefined, enableJsonResponse: true, }); - try { await server.connect(transport); - return await transport.handleRequest(req, { + return await transport.handleRequest(request, { authInfo: { token: auth.apiKey.id, clientId: auth.apiKey.id, @@ -572,9 +104,7 @@ export const handleMcpRequest = async (req: Request): Promise => { auth.apiKey.expiresAt === undefined ? undefined : Math.floor(auth.apiKey.expiresAt / 1000), - extra: { - httpworkbenchAuth: auth, - }, + extra: { httpworkbenchAuth: auth }, }, }); } finally { diff --git a/packages/backend/src/server/mcp/context.spec.ts b/packages/backend/src/server/mcp/context.spec.ts new file mode 100644 index 0000000..8b67561 --- /dev/null +++ b/packages/backend/src/server/mcp/context.spec.ts @@ -0,0 +1,36 @@ +import { describe, expect, test } from "bun:test"; +import type { Instance } from "shared"; + +process.env.JWT_SECRET = "mcp-context-test-secret"; + +const { serializeInstance, serializeInstanceSummary } = + await import("./context"); + +const instance: Instance = { + id: "instance-1", + ownerId: "owner-1", + createdAt: 1, + webhookIds: [], + public: false, + locked: false, + raw: "HTTP/1.1 200 OK\r\n\r\nok", +}; + +describe("MCP instance serialization", () => { + test("serializes static-only instance details", () => { + const serialized = serializeInstance(instance); + + expect(serialized.raw).toBe(instance.raw); + expect("kind" in serialized).toBe(false); + expect("processors" in serialized).toBe(false); + }); + + test("omits response content and legacy fields from summaries", () => { + const serialized = serializeInstanceSummary(instance); + + expect(serialized.id).toBe(instance.id); + expect("raw" in serialized).toBe(false); + expect("kind" in serialized).toBe(false); + expect("processors" in serialized).toBe(false); + }); +}); diff --git a/packages/backend/src/server/mcp/context.ts b/packages/backend/src/server/mcp/context.ts new file mode 100644 index 0000000..0e9ccfd --- /dev/null +++ b/packages/backend/src/server/mcp/context.ts @@ -0,0 +1,58 @@ +import type { AuthInfo } from "@modelcontextprotocol/sdk/server/auth/types.js"; +import type { ApiKeyScope, Instance } from "shared"; +import type { ApiKeyAuthContext } from "../apiKeyAuth"; +import { hasApiKeyScope } from "../apiKeyAuth"; +import { getOwnedInstance } from "../instances/service"; + +export const jsonToolResult = (value: Record) => ({ + content: [{ type: "text" as const, text: JSON.stringify(value, null, 2) }], + structuredContent: value, +}); + +export const toolError = (message: string) => ({ + content: [{ type: "text" as const, text: message }], + isError: true, +}); + +export const getAuthContext = (extra: { + authInfo?: AuthInfo; +}): ApiKeyAuthContext => { + const context = extra.authInfo?.extra?.httpworkbenchAuth; + if (context === undefined) { + throw new Error("Unauthorized"); + } + return context as ApiKeyAuthContext; +}; + +export const requireScope = (auth: ApiKeyAuthContext, scope: ApiKeyScope) => { + if (!hasApiKeyScope(auth.apiKey, scope)) { + throw new Error(`Missing API key scope: ${scope}`); + } +}; + +export const requireOwnedInstance = ( + instanceId: string, + auth: ApiKeyAuthContext, +) => { + const result = getOwnedInstance(instanceId, auth.user.id); + if (!result.ok) { + throw new Error("Instance not found"); + } + return result.value; +}; + +export const serializeInstance = (instance: Instance) => ({ + ...instance, + url: `${instance.id}.${Bun.env.INSTANCES_DOMAIN ?? ""}`, +}); + +export const serializeInstanceSummary = (instance: Instance) => ({ + id: instance.id, + ownerId: instance.ownerId, + createdAt: instance.createdAt, + expiresAt: instance.expiresAt, + label: instance.label, + public: instance.public, + locked: instance.locked, + url: `${instance.id}.${Bun.env.INSTANCES_DOMAIN ?? ""}`, +}); diff --git a/packages/backend/src/server/mcp/instanceTools.ts b/packages/backend/src/server/mcp/instanceTools.ts new file mode 100644 index 0000000..d18e498 --- /dev/null +++ b/packages/backend/src/server/mcp/instanceTools.ts @@ -0,0 +1,160 @@ +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import * as z from "zod/v4"; +import { + deleteInstance, + getInstancesByOwner, + getRecentLogsForInstance, +} from "../../storage"; +import { createInstance, replaceInstance } from "../instances/service"; +import { hasApiKeyScope } from "../apiKeyAuth"; +import { + getAuthContext, + jsonToolResult, + requireOwnedInstance, + requireScope, + serializeInstance, + serializeInstanceSummary, + toolError, +} from "./context"; + +const mcpLogPreviewLimit = 10; + +export const registerInstanceTools = (server: McpServer) => { + server.registerTool( + "list_instances", + { + title: "List Instances", + description: + "List HTTP Workbench instances owned by the authenticated API key user. Use this first when the user refers to an existing instance but has not provided an instanceId. This is read-only and returns instance metadata plus public URLs, not full response bodies or logs.", + annotations: { + readOnlyHint: true, + idempotentHint: true, + openWorldHint: false, + }, + }, + async (extra) => { + const auth = getAuthContext(extra); + requireScope(auth, "instances:read"); + return jsonToolResult({ + instances: getInstancesByOwner(auth.user.id).map( + serializeInstanceSummary, + ), + }); + }, + ); + + server.registerTool( + "create_instance", + { + title: "Create Instance", + description: + "Create a new HTTP Workbench instance that serves the provided raw HTTP response from a public instance URL. The raw input must be a complete valid HTTP response with status line, headers, blank line, and body.", + inputSchema: { + raw: z + .string() + .describe( + "Complete raw HTTP response to serve, including status line, headers, blank line, and body.", + ), + label: z.string().max(100).optional(), + }, + annotations: { + readOnlyHint: false, + destructiveHint: false, + idempotentHint: false, + openWorldHint: true, + }, + }, + async ({ raw, label }, extra) => { + const auth = getAuthContext(extra); + requireScope(auth, "instances:write"); + const result = createInstance({ + ownerId: auth.user.id, + raw, + label, + }); + return result.ok + ? jsonToolResult({ instance: serializeInstance(result.value) }) + : toolError(result.error.message); + }, + ); + + server.registerTool( + "get_instance", + { + title: "Get Instance", + description: + "Get one owned HTTP Workbench instance, including its full configuration and a small recent log preview when the API key also has logs:read.", + inputSchema: { instanceId: z.string().min(1) }, + annotations: { + readOnlyHint: true, + idempotentHint: true, + openWorldHint: false, + }, + }, + async ({ instanceId }, extra) => { + const auth = getAuthContext(extra); + requireScope(auth, "instances:read"); + const instance = requireOwnedInstance(instanceId, auth); + const logs = hasApiKeyScope(auth.apiKey, "logs:read") + ? getRecentLogsForInstance(instance.id, mcpLogPreviewLimit) + : []; + return jsonToolResult({ instance: serializeInstance(instance), logs }); + }, + ); + + server.registerTool( + "update_instance", + { + title: "Update Instance", + description: + "Replace the raw HTTP response served by an owned instance. Existing logs remain unchanged.", + inputSchema: { + instanceId: z.string().min(1), + raw: z.string(), + }, + annotations: { + readOnlyHint: false, + destructiveHint: true, + idempotentHint: true, + openWorldHint: true, + }, + }, + async ({ instanceId, raw }, extra) => { + const auth = getAuthContext(extra); + requireScope(auth, "instances:write"); + const result = replaceInstance({ + instanceId, + ownerId: auth.user.id, + raw, + }); + return result.ok + ? jsonToolResult({ instance: serializeInstance(result.value) }) + : toolError(result.error.message); + }, + ); + + server.registerTool( + "delete_instance", + { + title: "Delete Instance", + description: "Delete an owned, unlocked HTTP Workbench instance.", + inputSchema: { instanceId: z.string().min(1) }, + annotations: { + readOnlyHint: false, + destructiveHint: true, + idempotentHint: false, + openWorldHint: false, + }, + }, + async ({ instanceId }, extra) => { + const auth = getAuthContext(extra); + requireScope(auth, "instances:delete"); + const instance = requireOwnedInstance(instanceId, auth); + if (instance.locked) { + return toolError("Instance is locked"); + } + deleteInstance(instanceId); + return jsonToolResult({ deleted: true }); + }, + ); +}; diff --git a/packages/backend/src/server/mcp/logTools.ts b/packages/backend/src/server/mcp/logTools.ts new file mode 100644 index 0000000..df01617 --- /dev/null +++ b/packages/backend/src/server/mcp/logTools.ts @@ -0,0 +1,120 @@ +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import * as z from "zod/v4"; +import { clearLogsForInstance, getLogsForInstancePage } from "../../storage"; +import type { ApiKeyAuthContext } from "../apiKeyAuth"; +import { + clampLogLimit, + decodeLogsCursor, + encodeLogsCursor, + MAX_LOG_LIMIT, +} from "../utils"; +import { + getAuthContext, + jsonToolResult, + requireOwnedInstance, + requireScope, +} from "./context"; + +const readLogsPage = (input: { + auth: ApiKeyAuthContext; + instanceId: string; + limit?: number; + cursor?: string; + type?: "http" | "dns" | "smtp"; + sinceTimestamp?: number; +}) => { + requireOwnedInstance(input.instanceId, input.auth); + const cursor = + input.cursor === undefined ? undefined : decodeLogsCursor(input.cursor); + if (input.cursor !== undefined && cursor === undefined) { + throw new Error("Invalid cursor"); + } + + const page = getLogsForInstancePage({ + instanceId: input.instanceId, + limit: clampLogLimit(input.limit), + cursor, + type: input.type, + sinceTimestamp: input.sinceTimestamp, + }); + return { + logs: page.logs, + nextCursor: + page.nextCursor === undefined + ? undefined + : encodeLogsCursor(page.nextCursor), + }; +}; + +const logsInputSchema = { + instanceId: z.string().min(1), + limit: z.number().int().min(1).max(MAX_LOG_LIMIT).optional(), + cursor: z.string().optional(), + type: z.enum(["http", "dns", "smtp"]).optional(), + sinceTimestamp: z.number().optional(), +}; + +export const registerLogTools = (server: McpServer) => { + server.registerTool( + "clear_instance_logs", + { + title: "Clear Instance Logs", + description: "Clear all recorded interaction logs for an owned instance.", + inputSchema: { instanceId: z.string().min(1) }, + annotations: { + readOnlyHint: false, + destructiveHint: true, + idempotentHint: true, + openWorldHint: false, + }, + }, + async ({ instanceId }, extra) => { + const auth = getAuthContext(extra); + requireScope(auth, "instances:write"); + requireOwnedInstance(instanceId, auth); + clearLogsForInstance(instanceId); + return jsonToolResult({ cleared: true }); + }, + ); + + server.registerTool( + "read_instance_logs", + { + title: "Read Instance Logs", + description: "Read paginated HTTP/DNS/SMTP logs for an owned instance.", + inputSchema: logsInputSchema, + annotations: { + readOnlyHint: true, + idempotentHint: true, + openWorldHint: false, + }, + }, + async (input, extra) => { + const auth = getAuthContext(extra); + requireScope(auth, "logs:read"); + return jsonToolResult(readLogsPage({ ...input, auth })); + }, + ); + + server.registerTool( + "watch_instance_logs", + { + title: "Watch Instance Logs", + description: "Poll for new logs while waiting for an interaction.", + inputSchema: logsInputSchema, + annotations: { + readOnlyHint: true, + idempotentHint: false, + openWorldHint: false, + }, + }, + async (input, extra) => { + const auth = getAuthContext(extra); + requireScope(auth, "logs:stream"); + return jsonToolResult({ + ...readLogsPage({ ...input, auth }), + pollAfterMs: 1000, + }); + }, + ); +}; diff --git a/packages/backend/src/server/smtp/server.spec.ts b/packages/backend/src/server/smtp/server.spec.ts index c15f18e..d96b8ed 100644 --- a/packages/backend/src/server/smtp/server.spec.ts +++ b/packages/backend/src/server/smtp/server.spec.ts @@ -20,7 +20,6 @@ const createInstance = (id: string): Instance => ({ webhookIds: [], public: false, locked: false, - kind: "static", raw: "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n", }); diff --git a/packages/backend/src/storage/db.spec.ts b/packages/backend/src/storage/db.spec.ts index b9dcc2f..f595367 100644 --- a/packages/backend/src/storage/db.spec.ts +++ b/packages/backend/src/storage/db.spec.ts @@ -3,8 +3,13 @@ import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { Database } from "bun:sqlite"; +import { readMigrationFiles } from "drizzle-orm/migrator"; import type { Instance, Log, UserRecord, Webhook } from "shared"; -import { closeDb, initDb, replaceData, resolveSqliteDbPath } from "./db"; +import { closeDb, getDb, initDb, resolveSqliteDbPath } from "./db"; +import { addInstance } from "./repositories/instances"; +import { addUser } from "./repositories/users"; +import { addWebhook } from "./repositories/webhooks"; +import { logs } from "./schema"; const createUser = (): UserRecord => ({ id: "user-1", @@ -27,7 +32,6 @@ const createInstance = (): Instance => ({ webhookIds: ["webhook-1"], public: false, locked: false, - kind: "static", raw: "HTTP/1.1 200 OK\r\n\r\nok", }); @@ -95,14 +99,10 @@ describe("storage db migrations", () => { const first = initDb({ dataDir, reset: true }); expect(first.kind).toBe("ok"); - replaceData({ - users: [createUser()], - instances: [createInstance()], - logs: [createLog()], - webhooks: [createWebhook()], - instanceModerations: [], - userNotices: [], - }); + addUser(createUser()); + addWebhook(createWebhook()); + addInstance(createInstance()); + getDb().insert(logs).values(createLog()).run(); closeDb(); const reopened = initDb({ dataDir }); @@ -124,14 +124,10 @@ describe("storage db migrations", () => { const first = initDb({ dataDir, reset: true }); expect(first.kind).toBe("ok"); - replaceData({ - users: [createUser()], - instances: [createInstance()], - logs: [createLog()], - webhooks: [createWebhook()], - instanceModerations: [], - userNotices: [], - }); + addUser(createUser()); + addWebhook(createWebhook()); + addInstance(createInstance()); + getDb().insert(logs).values(createLog()).run(); const reset = initDb({ dataDir, reset: true }); @@ -147,4 +143,97 @@ describe("storage db migrations", () => { }, }); }); + + test("reopening an already migrated database does not revalidate historical orphans", () => { + const first = initDb({ dataDir, reset: true }); + expect(first.kind).toBe("ok"); + closeDb(); + + const sqlite = new Database(resolveSqliteDbPath(dataDir)); + sqlite.run("PRAGMA foreign_keys = OFF"); + sqlite + .query( + "INSERT INTO logs (id, instanceId, type, timestamp, address, raw) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + ) + .run("orphan", "missing", "http", 1, "127.0.0.1", "GET /"); + sqlite.close(); + + const reopened = initDb({ dataDir }); + expect(reopened.kind).toBe("ok"); + if (reopened.kind === "ok") { + expect(reopened.stats.logsLength).toBe(1); + } + }); + + test("static-only migration removes unusable instances and preserves valid associations", () => { + const dbPath = resolveSqliteDbPath(dataDir); + const sqlite = new Database(dbPath); + const migrations = readMigrationFiles({ + migrationsFolder: path.join(import.meta.dir, "../../drizzle"), + }); + for (const migration of migrations.slice(0, 3)) { + for (const statement of migration.sql) { + sqlite.exec(statement); + } + } + sqlite.exec(` + CREATE TABLE __drizzle_migrations ( + id SERIAL PRIMARY KEY, + hash TEXT NOT NULL, + created_at NUMERIC + ); + INSERT INTO webhooks (id, name, url, message, ownerId, createdAt) + VALUES ('webhook-1', 'Alerts', 'https://example.com', NULL, 'owner', 1); + INSERT INTO instances VALUES + ('valid', 'owner', 1, NULL, NULL, 0, 0, 'static', 'HTTP/1.1 200 OK\r\n\r\nvalid', NULL), + ('dynamic', 'owner', 2, NULL, NULL, 0, 0, 'dynamic', NULL, '[]'), + ('null-raw', 'owner', 3, NULL, NULL, 0, 0, 'static', NULL, NULL); + INSERT INTO instanceWebhooks VALUES ('valid', 'webhook-1', 0); + INSERT INTO instanceWebhooks VALUES ('dynamic', 'webhook-1', 0); + INSERT INTO logs (id, instanceId, type, timestamp, address, raw) VALUES + ('log-valid', 'valid', 'http', 1, '127.0.0.1', 'valid'), + ('log-dynamic', 'dynamic', 'http', 2, '127.0.0.1', 'dynamic'), + ('log-null', 'null-raw', 'http', 3, '127.0.0.1', 'null'); + INSERT INTO instanceModerations VALUES + ('valid', 0, 0, 0, '[]', NULL, 0, 0, 0, 0), + ('dynamic', 0, 0, 0, '[]', NULL, 0, 0, 0, 0); + `); + for (const migration of migrations.slice(0, 3)) { + sqlite + .query( + "INSERT INTO __drizzle_migrations (hash, created_at) VALUES (?1, ?2)", + ) + .run(migration.hash, migration.folderMillis); + } + sqlite.close(); + + const upgraded = initDb({ dataDir }); + expect(upgraded.kind).toBe("ok"); + closeDb(); + + const migrated = new Database(dbPath); + expect(migrated.query("SELECT id, raw FROM instances").all()).toEqual([ + { id: "valid", raw: "HTTP/1.1 200 OK\r\n\r\nvalid" }, + ]); + expect( + migrated.query("SELECT instanceId FROM instanceWebhooks").all(), + ).toEqual([{ instanceId: "valid" }]); + expect(migrated.query("SELECT instanceId FROM logs").all()).toEqual([ + { instanceId: "valid" }, + ]); + expect( + migrated.query("SELECT instanceId FROM instanceModerations").all(), + ).toEqual([{ instanceId: "valid" }]); + + const columns = migrated + .query("PRAGMA table_info(instances)") + .all() as Array<{ name: string; notnull: number }>; + expect(columns.some((column) => column.name === "kind")).toBe(false); + expect(columns.some((column) => column.name === "processorsJson")).toBe( + false, + ); + expect(columns.find((column) => column.name === "raw")?.notnull).toBe(1); + expect(migrated.query("PRAGMA foreign_key_check").all()).toEqual([]); + migrated.close(); + }); }); diff --git a/packages/backend/src/storage/db.ts b/packages/backend/src/storage/db.ts index fc810de..edce3fd 100644 --- a/packages/backend/src/storage/db.ts +++ b/packages/backend/src/storage/db.ts @@ -1,13 +1,10 @@ import { existsSync, mkdirSync, rmSync } from "node:fs"; +import path from "node:path"; import { Database } from "bun:sqlite"; import { count } from "drizzle-orm"; import { drizzle } from "drizzle-orm/bun-sqlite"; import { migrate } from "drizzle-orm/bun-sqlite/migrator"; -import type { LegacyStorageData } from "./legacyData"; -import { resolveDataDir } from "./legacyData"; import { - instanceWebhooks, - instanceModerations, instances, apiKeys, logs, @@ -16,16 +13,13 @@ import { users, webhooks, } from "./schema"; -import { - toInstanceRow, - toModerationRow, - toUserNoticeRow, - toWebhookRow, -} from "./records"; const SQLITE_DB_FILENAME = "db.sqlite"; const MIGRATIONS_FOLDER = `${import.meta.dir}/../../drizzle`; +const resolveDataDir = (dataDir?: string) => + dataDir ?? Bun.env.DATA_DIR ?? path.join(process.cwd(), "data"); + type InitDbOptions = { dataDir?: string; reset?: boolean; @@ -46,15 +40,54 @@ type DbState = { let dbState: DbState | undefined; +const countAppliedMigrations = (sqlite: Database) => { + const migrationsTable = sqlite + .query( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = '__drizzle_migrations'", + ) + .get(); + if (migrationsTable === null) { + return 0; + } + + return ( + sqlite + .query< + { count: number }, + [] + >("SELECT COUNT(*) AS count FROM __drizzle_migrations") + .get()?.count ?? 0 + ); +}; + const openDatabase = (dbPath: string) => { const sqlite = new Database(dbPath, { strict: true }); - sqlite.run("PRAGMA foreign_keys = ON"); sqlite.run("PRAGMA journal_mode = WAL"); sqlite.run("PRAGMA synchronous = NORMAL"); sqlite.run("PRAGMA busy_timeout = 5000"); const db = createDrizzleDb(sqlite); - migrate(db, { migrationsFolder: MIGRATIONS_FOLDER }); - return { sqlite, db }; + try { + // Table-rebuild migrations must run with foreign keys disabled at the + // connection level; changing this pragma inside Drizzle's transaction is + // ignored by SQLite. + sqlite.run("PRAGMA foreign_keys = OFF"); + const migrationCountBefore = countAppliedMigrations(sqlite); + migrate(db, { migrationsFolder: MIGRATIONS_FOLDER }); + sqlite.run("PRAGMA foreign_keys = ON"); + const migrationCountAfter = countAppliedMigrations(sqlite); + if (migrationCountAfter > migrationCountBefore) { + const violations = sqlite.query("PRAGMA foreign_key_check").all(); + if (violations.length > 0) { + throw new Error( + `Database migration produced foreign key violations: ${JSON.stringify(violations)}`, + ); + } + } + return { sqlite, db }; + } catch (error) { + sqlite.close(); + throw error; + } }; const getStats = (db: DrizzleDb) => { @@ -151,58 +184,3 @@ export const getDb = () => { return initialized.db; }; - -export const replaceData = (data: LegacyStorageData) => { - const db = getDb(); - - db.transaction( - (tx) => { - tx.delete(logs).run(); - tx.delete(instanceWebhooks).run(); - tx.delete(instanceModerations).run(); - tx.delete(userNotices).run(); - tx.delete(apiKeys).run(); - tx.delete(webhooks).run(); - tx.delete(instances).run(); - tx.delete(users).run(); - - for (const user of data.users) { - tx.insert(users).values(user).run(); - } - - for (const webhook of data.webhooks) { - tx.insert(webhooks).values(toWebhookRow(webhook)).run(); - } - - for (const instance of data.instances) { - tx.insert(instances).values(toInstanceRow(instance)).run(); - instance.webhookIds.forEach((webhookId, position) => { - tx.insert(instanceWebhooks) - .values({ - instanceId: instance.id, - webhookId, - position, - }) - .run(); - }); - } - - for (const log of data.logs) { - tx.insert(logs).values(log).run(); - } - - for (const moderation of data.instanceModerations) { - tx.insert(instanceModerations) - .values(toModerationRow(moderation)) - .run(); - } - - for (const notice of data.userNotices) { - tx.insert(userNotices).values(toUserNoticeRow(notice)).run(); - } - }, - { - behavior: "immediate", - }, - ); -}; diff --git a/packages/backend/src/storage/importLegacyDb.spec.ts b/packages/backend/src/storage/importLegacyDb.spec.ts deleted file mode 100644 index dfcee03..0000000 --- a/packages/backend/src/storage/importLegacyDb.spec.ts +++ /dev/null @@ -1,341 +0,0 @@ -import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { mkdtempSync, rmSync } from "node:fs"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { gzipSync } from "node:zlib"; -import type { - Instance, - InstanceModeration, - Log, - UserNotice, - UserRecord, - Webhook, -} from "shared"; -import { closeDb, initDb, replaceData } from "./db"; -import { maybeAutoImportLegacyDb, runImportLegacyDb } from "./importLegacyDb"; - -const createUser = (): UserRecord => ({ - id: "user-1", - googleId: "google-1", - createdAt: 1, -}); - -const createWebhook = (): Webhook => ({ - id: "webhook-1", - name: "Alerts", - url: "https://discord.com/api/webhooks/123/token", - ownerId: "user-1", - createdAt: 2, -}); - -const createInstance = (): Instance => ({ - id: "inst-1", - ownerId: "user-1", - createdAt: 3, - webhookIds: ["webhook-1"], - public: false, - locked: false, - kind: "static", - raw: "HTTP/1.1 200 OK\r\n\r\nok", -}); - -const createLog = (): Log => ({ - id: "log-1", - instanceId: "inst-1", - type: "http", - timestamp: 4, - address: "127.0.0.1", - raw: "GET / HTTP/1.1", -}); - -const createNotice = (): UserNotice => ({ - id: "notice-1", - userId: "user-1", - kind: "instance_removed_noise", - instanceId: "inst-1", - createdAt: 5, -}); - -const createLegacyData = (): { - users: UserRecord[]; - instances: Instance[]; - logs: Log[]; - webhooks: Webhook[]; - instanceModerations: InstanceModeration[]; - userNotices: UserNotice[]; -} => ({ - users: [createUser()], - instances: [createInstance()], - logs: [createLog()], - webhooks: [createWebhook()], - instanceModerations: [ - { - instanceId: "inst-1", - window5mStartMs: 10, - requestsInWindow5m: 1, - strikeCommittedForWindow: false, - strikeTimestampsMs: [], - window15mStartMs: 10, - requestsInWindow15m: 1, - lastMinuteBucketStartMs: 10, - requestsInCurrentMinute: 1, - }, - ], - userNotices: [createNotice()], -}); - -const writeLegacyJson = async ( - dataDir: string, - data: ReturnType = createLegacyData(), -) => { - await Bun.write(`${dataDir}/db.json`, JSON.stringify(data)); -}; - -const writeLegacyGzip = async ( - dataDir: string, - data: ReturnType = createLegacyData(), -) => { - await Bun.write( - `${dataDir}/db.json.gz`, - gzipSync(JSON.stringify(data)).toString("base64"), - ); -}; - -let dataDir = ""; - -describe("runImportLegacyDb", () => { - beforeEach(() => { - dataDir = mkdtempSync(path.join(tmpdir(), "httpworkbench-import-")); - }); - - afterEach(() => { - closeDb(); - rmSync(dataDir, { recursive: true, force: true }); - }); - - test("imports a plain legacy db.json into sqlite", async () => { - await writeLegacyJson(dataDir); - - const result = await runImportLegacyDb({ dataDir, resetTarget: true }); - - expect(result).toEqual({ - sqlitePath: `${dataDir}/db.sqlite`, - users: 1, - instances: 1, - logs: 1, - webhooks: 1, - instanceModerations: 1, - userNotices: 1, - }); - }); - - test("imports a gzipped legacy db.json.gz into sqlite", async () => { - await writeLegacyGzip(dataDir); - - const result = await runImportLegacyDb({ dataDir, resetTarget: true }); - - expect(result.sqlitePath).toBe(`${dataDir}/db.sqlite`); - expect(result.logs).toBe(1); - expect(result.instances).toBe(1); - }); - - test("imports expired instances without filtering them during verification", async () => { - await writeLegacyJson(dataDir, { - ...createLegacyData(), - instances: [ - { - ...createInstance(), - id: "inst-expired", - expiresAt: 1, - webhookIds: [], - }, - ], - logs: [ - { - ...createLog(), - instanceId: "inst-expired", - }, - ], - instanceModerations: [ - { - instanceId: "inst-expired", - window5mStartMs: 10, - requestsInWindow5m: 1, - strikeCommittedForWindow: false, - strikeTimestampsMs: [], - window15mStartMs: 10, - requestsInWindow15m: 1, - lastMinuteBucketStartMs: 10, - requestsInCurrentMinute: 1, - }, - ], - userNotices: [ - { - ...createNotice(), - instanceId: "inst-expired", - }, - ], - }); - - await expect( - runImportLegacyDb({ dataDir, resetTarget: true }), - ).resolves.toEqual({ - sqlitePath: `${dataDir}/db.sqlite`, - users: 1, - instances: 1, - logs: 1, - webhooks: 1, - instanceModerations: 1, - userNotices: 1, - }); - }); - - test("drops invalid and duplicate legacy references", async () => { - await writeLegacyJson(dataDir, { - ...createLegacyData(), - instances: [ - { - ...createInstance(), - webhookIds: ["webhook-1", "webhook-1", "webhook-missing"], - }, - ], - logs: [ - createLog(), - { - ...createLog(), - id: "log-orphan", - instanceId: "inst-missing", - }, - ], - instanceModerations: [ - { - instanceId: "inst-1", - window5mStartMs: 10, - requestsInWindow5m: 1, - strikeCommittedForWindow: false, - strikeTimestampsMs: [], - window15mStartMs: 10, - requestsInWindow15m: 1, - lastMinuteBucketStartMs: 10, - requestsInCurrentMinute: 1, - }, - { - instanceId: "inst-1", - window5mStartMs: 20, - requestsInWindow5m: 2, - strikeCommittedForWindow: true, - strikeTimestampsMs: Array.of(20), - window15mStartMs: 20, - requestsInWindow15m: 2, - lastMinuteBucketStartMs: 20, - requestsInCurrentMinute: 2, - }, - { - instanceId: "inst-missing", - window5mStartMs: 10, - requestsInWindow5m: 1, - strikeCommittedForWindow: false, - strikeTimestampsMs: [], - window15mStartMs: 10, - requestsInWindow15m: 1, - lastMinuteBucketStartMs: 10, - requestsInCurrentMinute: 1, - }, - ], - }); - - await expect( - runImportLegacyDb({ dataDir, resetTarget: true }), - ).resolves.toEqual({ - sqlitePath: `${dataDir}/db.sqlite`, - users: 1, - instances: 1, - logs: 1, - webhooks: 1, - instanceModerations: 1, - userNotices: 1, - }); - }); - - test("auto-imports legacy data when sqlite is still empty", async () => { - await writeLegacyJson(dataDir); - - await expect(maybeAutoImportLegacyDb({ dataDir })).resolves.toEqual({ - kind: "imported", - result: { - sqlitePath: `${dataDir}/db.sqlite`, - users: 1, - instances: 1, - logs: 1, - webhooks: 1, - instanceModerations: 1, - userNotices: 1, - }, - }); - - expect(initDb({ dataDir })).toEqual({ - kind: "ok", - stats: { - instancesLength: 1, - logsLength: 1, - userNoticesLength: 1, - usersLength: 1, - webhooksLength: 1, - apiKeysLength: 0, - }, - }); - }); - - test("skips auto-import when sqlite already has only notices", async () => { - await writeLegacyJson(dataDir); - - const initResult = initDb({ dataDir, reset: true }); - expect(initResult.kind).toBe("ok"); - replaceData({ - users: [], - instances: [], - logs: [], - webhooks: [], - instanceModerations: [], - userNotices: [createNotice()], - }); - - await expect(maybeAutoImportLegacyDb({ dataDir })).resolves.toEqual({ - kind: "skipped", - reason: "sqlite-has-data", - }); - - expect(initDb({ dataDir })).toEqual({ - kind: "ok", - stats: { - instancesLength: 0, - logsLength: 0, - userNoticesLength: 1, - usersLength: 0, - webhooksLength: 0, - apiKeysLength: 0, - }, - }); - }); - - test("refuses to overwrite an existing sqlite database with only notices without reset", async () => { - await writeLegacyJson(dataDir); - - const initResult = initDb({ dataDir, reset: true }); - expect(initResult.kind).toBe("ok"); - replaceData({ - users: [], - instances: [], - logs: [], - webhooks: [], - instanceModerations: [], - userNotices: [createNotice()], - }); - - await expect( - runImportLegacyDb({ dataDir, resetTarget: false }), - ).rejects.toThrow( - `Target SQLite database already contains data at ${dataDir}/db.sqlite. Set RESET_SQLITE=1 to overwrite it.`, - ); - }); -}); diff --git a/packages/backend/src/storage/importLegacyDb.ts b/packages/backend/src/storage/importLegacyDb.ts deleted file mode 100644 index a3f55d3..0000000 --- a/packages/backend/src/storage/importLegacyDb.ts +++ /dev/null @@ -1,406 +0,0 @@ -import { isDeepStrictEqual } from "node:util"; -import { asc } from "drizzle-orm"; -import { closeDb, getDb, initDb, replaceData, resolveSqliteDbPath } from "./db"; -import { - type LegacyStorageData, - readLegacyStorageData, - resolveDataDir, - resolveLegacyDbPath, -} from "./legacyData"; -import { - toDomainInstance, - toDomainModeration, - toDomainUserNotice, - toDomainWebhook, -} from "./records"; -import { - instanceModerations, - instanceWebhooks, - instances, - logs, - userNotices, - users, - webhooks, -} from "./schema"; - -const normalizeForComparison = (value: T) => { - return JSON.parse(JSON.stringify(value)) as T; -}; - -const compareStrings = (left: string, right: string) => { - return left.localeCompare(right); -}; - -const sortById = (values: T[]) => { - return [...values].sort((left, right) => compareStrings(left.id, right.id)); -}; - -const sortByInstanceId = (values: T[]) => { - return [...values].sort((left, right) => - compareStrings(left.instanceId, right.instanceId), - ); -}; - -const hasStoredData = ({ - instancesLength, - logsLength, - userNoticesLength, - usersLength, - webhooksLength, -}: { - instancesLength: number; - logsLength: number; - userNoticesLength: number; - usersLength: number; - webhooksLength: number; -}) => { - return ( - instancesLength > 0 || - logsLength > 0 || - userNoticesLength > 0 || - usersLength > 0 || - webhooksLength > 0 - ); -}; - -const assertImportedSectionMatches = ( - label: string, - actual: T, - expected: T, -) => { - if ( - !isDeepStrictEqual( - normalizeForComparison(actual), - normalizeForComparison(expected), - ) - ) { - throw new Error(`Imported ${label} mismatch`); - } -}; - -const getWebhookIdsByInstanceId = () => { - const rows = getDb() - .select({ - instanceId: instanceWebhooks.instanceId, - webhookId: instanceWebhooks.webhookId, - }) - .from(instanceWebhooks) - .orderBy(asc(instanceWebhooks.instanceId), asc(instanceWebhooks.position)) - .all(); - - const webhookIdsByInstanceId = new Map(); - for (const row of rows) { - const existing = webhookIdsByInstanceId.get(row.instanceId); - if (existing === undefined) { - webhookIdsByInstanceId.set(row.instanceId, [row.webhookId]); - continue; - } - existing.push(row.webhookId); - } - - return webhookIdsByInstanceId; -}; - -const readImportedInstances = () => { - const webhookIdsByInstanceId = getWebhookIdsByInstanceId(); - return getDb() - .select() - .from(instances) - .all() - .map((row) => - toDomainInstance(row, webhookIdsByInstanceId.get(row.id) ?? []), - ); -}; - -const readImportedLogs = () => { - return getDb() - .select({ - id: logs.id, - instanceId: logs.instanceId, - type: logs.type, - timestamp: logs.timestamp, - address: logs.address, - raw: logs.raw, - }) - .from(logs) - .all(); -}; - -const readImportedModerations = () => { - return getDb() - .select() - .from(instanceModerations) - .all() - .map((row) => toDomainModeration(row)); -}; - -const readImportedUserNotices = () => { - return getDb() - .select() - .from(userNotices) - .all() - .map((row) => toDomainUserNotice(row)); -}; - -type DroppedLegacyRecords = { - duplicateInstanceWebhookReferences: number; - duplicateModerations: number; - missingWebhookReferences: number; - orphanLogs: number; - orphanModerations: number; -}; - -const hasDroppedLegacyRecords = (dropped: DroppedLegacyRecords) => { - return ( - dropped.duplicateInstanceWebhookReferences > 0 || - dropped.duplicateModerations > 0 || - dropped.missingWebhookReferences > 0 || - dropped.orphanLogs > 0 || - dropped.orphanModerations > 0 - ); -}; - -const dedupeByKey = (values: T[], getKey: (value: T) => string) => { - const seen = new Set(); - const deduped: T[] = []; - - for (let index = values.length - 1; index >= 0; index -= 1) { - const value = values[index]; - if (value === undefined) { - continue; - } - - const key = getKey(value); - if (seen.has(key)) { - continue; - } - - seen.add(key); - deduped.unshift(value); - } - - return deduped; -}; - -const sanitizeLegacyStorageData = ( - data: LegacyStorageData, -): { - data: LegacyStorageData; - dropped: DroppedLegacyRecords; -} => { - const existingWebhookIds = new Set( - data.webhooks.map((webhook) => webhook.id), - ); - const existingInstanceIds = new Set( - data.instances.map((instance) => instance.id), - ); - - let missingWebhookReferences = 0; - let duplicateInstanceWebhookReferences = 0; - const instances = data.instances.map((instance) => { - const seenWebhookIds = new Set(); - const webhookIds = instance.webhookIds.filter((webhookId) => { - if (!existingWebhookIds.has(webhookId)) { - missingWebhookReferences += 1; - return false; - } - - if (seenWebhookIds.has(webhookId)) { - duplicateInstanceWebhookReferences += 1; - return false; - } - - seenWebhookIds.add(webhookId); - return true; - }); - - if (webhookIds.length === instance.webhookIds.length) { - return instance; - } - - return { - ...instance, - webhookIds, - }; - }); - - const logs = data.logs.filter((log) => - existingInstanceIds.has(log.instanceId), - ); - const validModerations = data.instanceModerations.filter((moderation) => - existingInstanceIds.has(moderation.instanceId), - ); - const instanceModerations = dedupeByKey( - validModerations, - (moderation) => moderation.instanceId, - ); - - return { - data: { - ...data, - instances, - logs, - instanceModerations, - }, - dropped: { - duplicateInstanceWebhookReferences, - duplicateModerations: - validModerations.length - instanceModerations.length, - missingWebhookReferences, - orphanLogs: data.logs.length - logs.length, - orphanModerations: - data.instanceModerations.length - validModerations.length, - }, - }; -}; - -export const verifyImportedData = ( - data: LegacyStorageData, - dataDir?: string, -) => { - const statsResult = initDb({ dataDir }); - if (statsResult.kind === "error") { - throw statsResult.error; - } - - const { stats } = statsResult; - if (stats.usersLength !== data.users.length) { - throw new Error("Imported users count mismatch"); - } - if (stats.instancesLength !== data.instances.length) { - throw new Error("Imported instances count mismatch"); - } - if (stats.logsLength !== data.logs.length) { - throw new Error("Imported logs count mismatch"); - } - if (stats.webhooksLength !== data.webhooks.length) { - throw new Error("Imported webhooks count mismatch"); - } - if (stats.userNoticesLength !== data.userNotices.length) { - throw new Error("Imported user notices count mismatch"); - } - - assertImportedSectionMatches( - "users", - sortById(getDb().select().from(users).all()), - sortById(data.users), - ); - assertImportedSectionMatches( - "instances", - sortById(readImportedInstances()), - sortById(data.instances), - ); - assertImportedSectionMatches( - "logs", - sortById(readImportedLogs()), - sortById(data.logs), - ); - assertImportedSectionMatches( - "webhooks", - sortById( - getDb() - .select() - .from(webhooks) - .all() - .map((row) => toDomainWebhook(row)), - ), - sortById(data.webhooks), - ); - assertImportedSectionMatches( - "instance moderations", - sortByInstanceId(readImportedModerations()), - sortByInstanceId(data.instanceModerations), - ); - assertImportedSectionMatches( - "user notices", - sortById(readImportedUserNotices()), - sortById(data.userNotices), - ); -}; - -export const runImportLegacyDb = async ({ - dataDir = resolveDataDir(), - resetTarget = Bun.env.RESET_SQLITE === "1", -}: { - dataDir?: string; - resetTarget?: boolean; -} = {}) => { - const sqlitePath = resolveSqliteDbPath(dataDir); - - if (!resetTarget) { - const initResult = initDb({ dataDir }); - if (initResult.kind === "error") { - throw initResult.error; - } - - if (hasStoredData(initResult.stats)) { - throw new Error( - `Target SQLite database already contains data at ${sqlitePath}. Set RESET_SQLITE=1 to overwrite it.`, - ); - } - } - - const legacyData = await readLegacyStorageData(dataDir); - const sanitized = sanitizeLegacyStorageData(legacyData); - const initResult = initDb({ dataDir, reset: resetTarget }); - if (initResult.kind === "error") { - throw initResult.error; - } - - replaceData(sanitized.data); - verifyImportedData(sanitized.data, dataDir); - - if (hasDroppedLegacyRecords(sanitized.dropped)) { - console.warn( - "Skipped invalid legacy references during import", - sanitized.dropped, - ); - } - - return { - sqlitePath, - users: sanitized.data.users.length, - instances: sanitized.data.instances.length, - logs: sanitized.data.logs.length, - webhooks: sanitized.data.webhooks.length, - instanceModerations: sanitized.data.instanceModerations.length, - userNotices: sanitized.data.userNotices.length, - }; -}; - -export const maybeAutoImportLegacyDb = async ({ - dataDir = resolveDataDir(), -}: { - dataDir?: string; -} = {}) => { - if (resolveLegacyDbPath(dataDir) === undefined) { - return { kind: "skipped" as const, reason: "no-legacy-db" as const }; - } - - const initResult = initDb({ dataDir }); - if (initResult.kind === "error") { - throw initResult.error; - } - - if (hasStoredData(initResult.stats)) { - return { kind: "skipped" as const, reason: "sqlite-has-data" as const }; - } - - return { - kind: "imported" as const, - result: await runImportLegacyDb({ dataDir, resetTarget: false }), - }; -}; - -if (import.meta.main) { - runImportLegacyDb() - .then((result) => { - console.log(JSON.stringify(result, null, 2)); - closeDb(); - }) - .catch((error) => { - closeDb(); - console.error(error); - process.exit(1); - }); -} diff --git a/packages/backend/src/storage/index.ts b/packages/backend/src/storage/index.ts index 5e32482..8ad8118 100644 --- a/packages/backend/src/storage/index.ts +++ b/packages/backend/src/storage/index.ts @@ -1,3 +1,2 @@ export * from "./db"; -export * from "./importLegacyDb"; export * from "./repositories"; diff --git a/packages/backend/src/storage/legacyData.ts b/packages/backend/src/storage/legacyData.ts deleted file mode 100644 index 6d024ea..0000000 --- a/packages/backend/src/storage/legacyData.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { existsSync } from "node:fs"; -import path from "node:path"; -import { gunzipSync } from "node:zlib"; -import { - InstanceModerationSchema, - InstanceSchema, - LogSchema, - UserNoticeSchema, - UserRecordSchema, - WebhookSchema, -} from "shared"; -import z from "zod"; - -const LEGACY_DB_JSON_FILENAME = "db.json"; -const LEGACY_DB_GZIP_FILENAME = "db.json.gz"; - -const LegacyStorageDataSchema = z.object({ - users: z.array(UserRecordSchema), - instances: z.array(InstanceSchema), - logs: z.array(LogSchema), - webhooks: z.array(WebhookSchema), - instanceModerations: z.array(InstanceModerationSchema).default([]), - userNotices: z.array(UserNoticeSchema).default([]), -}); - -export type LegacyStorageData = z.infer; - -export const resolveDataDir = (dataDir?: string) => { - return dataDir ?? Bun.env.DATA_DIR ?? path.join(process.cwd(), "data"); -}; - -export const resolveLegacyDbPath = (dataDir?: string) => { - const resolvedDataDir = resolveDataDir(dataDir); - const gzipPath = path.join(resolvedDataDir, LEGACY_DB_GZIP_FILENAME); - if (existsSync(gzipPath)) { - return gzipPath; - } - - const jsonPath = path.join(resolvedDataDir, LEGACY_DB_JSON_FILENAME); - if (existsSync(jsonPath)) { - return jsonPath; - } - - return undefined; -}; - -export const readLegacyStorageData = async ( - dataDir?: string, -): Promise => { - const legacyDbPath = resolveLegacyDbPath(dataDir); - if (legacyDbPath === undefined) { - throw new Error("Legacy db.json or db.json.gz file not found"); - } - - const encoded = await Bun.file(legacyDbPath).text(); - const decoded = legacyDbPath.endsWith(".gz") - ? gunzipSync(Buffer.from(encoded, "base64")).toString() - : encoded; - const parsed = LegacyStorageDataSchema.safeParse(JSON.parse(decoded)); - if (!parsed.success) { - throw new Error(`Invalid legacy database format: ${parsed.error.message}`); - } - - return parsed.data; -}; diff --git a/packages/backend/src/storage/records.ts b/packages/backend/src/storage/records.ts index bcd051a..94579ea 100644 --- a/packages/backend/src/storage/records.ts +++ b/packages/backend/src/storage/records.ts @@ -27,23 +27,8 @@ type WebhookRow = typeof webhooks.$inferSelect; export const toDomainInstance = ( row: InstanceRow, webhookIds: string[], -): Instance => { - if (row.kind === "static") { - return InstanceSchema.parse({ - id: row.id, - ownerId: row.ownerId, - createdAt: row.createdAt, - expiresAt: row.expiresAt ?? undefined, - label: row.label ?? undefined, - webhookIds, - public: row.isPublic, - locked: row.isLocked, - kind: "static", - raw: row.raw ?? "", - }); - } - - return InstanceSchema.parse({ +): Instance => + InstanceSchema.parse({ id: row.id, ownerId: row.ownerId, createdAt: row.createdAt, @@ -52,10 +37,8 @@ export const toDomainInstance = ( webhookIds, public: row.isPublic, locked: row.isLocked, - kind: "dynamic", - processors: row.processorsJson ?? [], + raw: row.raw, }); -}; export const toInstanceRow = (instance: Instance): InstanceInsertRow => { const parsed = InstanceSchema.parse(instance); @@ -67,9 +50,7 @@ export const toInstanceRow = (instance: Instance): InstanceInsertRow => { label: parsed.label ?? null, isPublic: parsed.public, isLocked: parsed.locked, - kind: parsed.kind, - raw: parsed.kind === "static" ? parsed.raw : null, - processorsJson: parsed.kind === "dynamic" ? parsed.processors : null, + raw: parsed.raw, }; }; diff --git a/packages/backend/src/storage/repositories/instances.ts b/packages/backend/src/storage/repositories/instances.ts index e401253..c56ee56 100644 --- a/packages/backend/src/storage/repositories/instances.ts +++ b/packages/backend/src/storage/repositories/instances.ts @@ -1,13 +1,23 @@ import type { Instance } from "shared"; import { InstanceSchema } from "shared"; -import { and, asc, eq, inArray, isNotNull, lte } from "drizzle-orm"; +import { + and, + asc, + count, + eq, + gt, + inArray, + isNotNull, + isNull, + lte, + or, +} from "drizzle-orm"; import { getDb } from "../db"; import { toDomainInstance, toInstanceRow } from "../records"; import { instances, instanceWebhooks } from "../schema"; -const isInstanceExpired = (instance: Instance, now: number) => { - return instance.expiresAt !== undefined && instance.expiresAt <= now; -}; +const activeInstanceCondition = (now: number) => + or(isNull(instances.expiresAt), gt(instances.expiresAt, now)); type WebhookMutationStore = Pick, "delete" | "insert">; @@ -58,13 +68,9 @@ const getActiveInstanceById = ( const row = getDb() .select() .from(instances) - .where(eq(instances.id, id)) + .where(and(eq(instances.id, id), activeInstanceCondition(now))) .get(); - const instance = row === undefined ? undefined : hydrateInstances([row])[0]; - if (instance === undefined) { - return undefined; - } - return isInstanceExpired(instance, now) ? undefined : instance; + return row === undefined ? undefined : hydrateInstances([row])[0]; }; const replaceInstanceWebhooks = ( @@ -102,10 +108,25 @@ export function getInstancesByOwner(ownerId: string): Instance[] { getDb() .select() .from(instances) - .where(eq(instances.ownerId, ownerId)) + .where(and(eq(instances.ownerId, ownerId), activeInstanceCondition(now))) .orderBy(asc(instances.createdAt), asc(instances.id)) .all(), - ).filter((instance) => !isInstanceExpired(instance, now)); + ); +} + +export function countActiveInstancesByOwner(ownerId: string): number { + return ( + getDb() + .select({ count: count() }) + .from(instances) + .where( + and( + eq(instances.ownerId, ownerId), + activeInstanceCondition(Date.now()), + ), + ) + .get()?.count ?? 0 + ); } export function getInstanceById(id: string): Instance | undefined { diff --git a/packages/backend/src/storage/repositories/logs.spec.ts b/packages/backend/src/storage/repositories/logs.spec.ts index 7243f6c..0e53e1c 100644 --- a/packages/backend/src/storage/repositories/logs.spec.ts +++ b/packages/backend/src/storage/repositories/logs.spec.ts @@ -4,9 +4,15 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { count, eq } from "drizzle-orm"; import type { Instance, Log, Webhook } from "shared"; -import { closeDb, getDb, initDb, replaceData } from "../db"; -import { toDomainModeration } from "../records"; -import { instanceModerations, instances, userNotices } from "../schema"; +import { closeDb, getDb, initDb } from "../db"; +import { toDomainModeration, toModerationRow, toWebhookRow } from "../records"; +import { + instanceModerations, + instances, + logs as logsTable, + userNotices, + webhooks as webhooksTable, +} from "../schema"; import { addLog, clearLogsForInstance, @@ -14,20 +20,21 @@ import { getLogsForInstance, getLogsForInstancePage, } from "./logs"; -import { deleteInstance, removeExpiredInstances } from "./instances"; - -type StaticInstance = Extract; - -const createInstance = ( - overrides?: Partial, -): StaticInstance => ({ +import { + addInstance, + countActiveInstancesByOwner, + deleteInstance, + getInstancesByOwner, + removeExpiredInstances, +} from "./instances"; + +const createInstance = (overrides?: Partial): Instance => ({ id: "inst-1", ownerId: "user-1", createdAt: 1, webhookIds: [], public: false, locked: false, - kind: "static", raw: "HTTP/1.1 200 OK\r\n\r\nok", ...overrides, }); @@ -72,14 +79,21 @@ const seedStorage = ( }>; }>, ) => { - replaceData({ - users: [], - instances: overrides?.instances ?? [], - logs: overrides?.logs ?? [], - webhooks: overrides?.webhooks ?? [], - instanceModerations: overrides?.instanceModerations ?? [], - userNotices: [], - }); + for (const webhook of overrides?.webhooks ?? []) { + getDb().insert(webhooksTable).values(toWebhookRow(webhook)).run(); + } + for (const instance of overrides?.instances ?? []) { + addInstance(instance); + } + for (const log of overrides?.logs ?? []) { + getDb().insert(logsTable).values(log).run(); + } + for (const moderation of overrides?.instanceModerations ?? []) { + getDb() + .insert(instanceModerations) + .values(toModerationRow(moderation)) + .run(); + } }; const getModerationForInstance = (instanceId: string) => { @@ -342,4 +356,19 @@ describe("addLog", () => { createLog({ id: "log-2", instanceId: "inst-2" }), ]); }); + + test("filters expired instances and counts active rows in SQL", () => { + seedStorage({ + instances: [ + createInstance({ expiresAt: 10 }), + createInstance({ id: "inst-2", expiresAt: Date.now() + 60_000 }), + createInstance({ id: "inst-3", ownerId: "user-2" }), + ], + }); + + expect( + getInstancesByOwner("user-1").map((instance) => instance.id), + ).toEqual(["inst-2"]); + expect(countActiveInstancesByOwner("user-1")).toBe(1); + }); }); diff --git a/packages/backend/src/storage/schema.ts b/packages/backend/src/storage/schema.ts index ac6ce35..61d03c0 100644 --- a/packages/backend/src/storage/schema.ts +++ b/packages/backend/src/storage/schema.ts @@ -1,10 +1,4 @@ -import type { - ApiKeyScope, - InstanceKind, - LogType, - Processor, - UserNoticeKind, -} from "shared"; +import type { ApiKeyScope, LogType, UserNoticeKind } from "shared"; import { index, integer, @@ -34,13 +28,7 @@ export const instances = sqliteTable( label: text("label"), isPublic: integer("isPublic", { mode: "boolean" }).notNull(), isLocked: integer("isLocked", { mode: "boolean" }).notNull(), - kind: text("kind", { enum: ["static", "dynamic"] }) - .$type() - .notNull(), - raw: text("raw"), - processorsJson: text("processorsJson", { mode: "json" }).$type< - Processor[] | null - >(), + raw: text("raw").notNull(), }, (table) => [ index("instances_by_owner").on(table.ownerId), diff --git a/packages/frontend/src/App.vue b/packages/frontend/src/App.vue index d2827d9..353b3ce 100644 --- a/packages/frontend/src/App.vue +++ b/packages/frontend/src/App.vue @@ -1,5 +1,6 @@