diff --git a/apps/hub/package.json b/apps/hub/package.json index c39cedd8..d9134010 100644 --- a/apps/hub/package.json +++ b/apps/hub/package.json @@ -11,7 +11,10 @@ "./migrate": "./src/migrate.ts", "./workflows/install-for-tenant": "./src/workflows/install-for-tenant.ts", "./config/models": "./src/config/models.ts", - "./settings/policy": "./src/settings/policy.ts" + "./settings/policy": "./src/settings/policy.ts", + "./db/schema": "./src/db/schema.ts", + "./mail/hub-mail": "./src/mail/hub-mail.ts", + "./projects/specialist-ended": "./src/projects/specialist-ended.ts" }, "dependencies": { "@anthropic-ai/sandbox-runtime": "0.0.81", diff --git a/apps/hub/src/db/migrations/0036_ended_run_retry.sql b/apps/hub/src/db/migrations/0036_ended_run_retry.sql new file mode 100644 index 00000000..95264b4a --- /dev/null +++ b/apps/hub/src/db/migrations/0036_ended_run_retry.sql @@ -0,0 +1,2 @@ +ALTER TABLE "sb_ended_run" ADD COLUMN "next_attempt_at" timestamp with time zone;--> statement-breakpoint +ALTER TABLE "sb_ended_run" ADD COLUMN "attempts" integer DEFAULT 0 NOT NULL; \ No newline at end of file diff --git a/apps/hub/src/db/migrations/meta/0036_snapshot.json b/apps/hub/src/db/migrations/meta/0036_snapshot.json new file mode 100644 index 00000000..394140cf --- /dev/null +++ b/apps/hub/src/db/migrations/meta/0036_snapshot.json @@ -0,0 +1,1108 @@ +{ + "id": "2f982fda-3e4f-44b4-b563-97b290e1c723", + "prevId": "d7ba8602-2460-4b8e-bdbc-d17af7642440", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.sb_artifact_approval": { + "name": "sb_artifact_approval", + "schema": "", + "columns": { + "approval_id": { + "name": "approval_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "artifact_id": { + "name": "artifact_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool": { + "name": "tool", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sb_artifact_approval_artifact_idx": { + "name": "sb_artifact_approval_artifact_idx", + "columns": [ + { + "expression": "tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sb_artifact_approval_approval_id_approval_id_fk": { + "name": "sb_artifact_approval_approval_id_approval_id_fk", + "tableFrom": "sb_artifact_approval", + "tableTo": "approval", + "columnsFrom": ["approval_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "sb_artifact_approval_tenant_id_tenant_id_fk": { + "name": "sb_artifact_approval_tenant_id_tenant_id_fk", + "tableFrom": "sb_artifact_approval", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "sb_artifact_approval_project_id_project_id_fk": { + "name": "sb_artifact_approval_project_id_project_id_fk", + "tableFrom": "sb_artifact_approval", + "tableTo": "project", + "columnsFrom": ["project_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "sb_artifact_approval_approval_id_artifact_id_pk": { + "name": "sb_artifact_approval_approval_id_artifact_id_pk", + "columns": ["approval_id", "artifact_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sb_artifact_import_approval": { + "name": "sb_artifact_import_approval", + "schema": "", + "columns": { + "artifact_id": { + "name": "artifact_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sb_artifact_import_approval_tenant_idx": { + "name": "sb_artifact_import_approval_tenant_idx", + "columns": [ + { + "expression": "tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sb_artifact_import_approval_tenant_id_tenant_id_fk": { + "name": "sb_artifact_import_approval_tenant_id_tenant_id_fk", + "tableFrom": "sb_artifact_import_approval", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "sb_artifact_import_approval_project_id_project_id_fk": { + "name": "sb_artifact_import_approval_project_id_project_id_fk", + "tableFrom": "sb_artifact_import_approval", + "tableTo": "project", + "columnsFrom": ["project_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "sb_artifact_import_approval_principal_id_principal_id_fk": { + "name": "sb_artifact_import_approval_principal_id_principal_id_fk", + "tableFrom": "sb_artifact_import_approval", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "sb_artifact_import_approval_artifact_id_version_pk": { + "name": "sb_artifact_import_approval_artifact_id_version_pk", + "columns": ["artifact_id", "version"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sb_design_concept": { + "name": "sb_design_concept", + "schema": "", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "direction": { + "name": "direction", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sb_design_concept_tenant_idx": { + "name": "sb_design_concept_tenant_idx", + "columns": [ + { + "expression": "tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "label", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sb_design_concept_run_id_workflow_run_id_fk": { + "name": "sb_design_concept_run_id_workflow_run_id_fk", + "tableFrom": "sb_design_concept", + "tableTo": "workflow_run", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "sb_design_concept_tenant_id_tenant_id_fk": { + "name": "sb_design_concept_tenant_id_tenant_id_fk", + "tableFrom": "sb_design_concept", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "sb_design_concept_project_id_project_id_fk": { + "name": "sb_design_concept_project_id_project_id_fk", + "tableFrom": "sb_design_concept", + "tableTo": "project", + "columnsFrom": ["project_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.device_code": { + "name": "device_code", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "device_code": { + "name": "device_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_code": { + "name": "user_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_polled_at": { + "name": "last_polled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "polling_interval": { + "name": "polling_interval", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "device_code_device_code_unique": { + "name": "device_code_device_code_unique", + "nullsNotDistinct": false, + "columns": ["device_code"] + }, + "device_code_user_code_unique": { + "name": "device_code_user_code_unique", + "nullsNotDistinct": false, + "columns": ["user_code"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.device_session": { + "name": "device_session", + "schema": "", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": true, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "device_session_session_id_session_id_fk": { + "name": "device_session_session_id_session_id_fk", + "tableFrom": "device_session", + "tableTo": "session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sb_ended_run": { + "name": "sb_ended_run", + "schema": "", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "notified": { + "name": "notified", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sb_ended_run_run_id_workflow_run_id_fk": { + "name": "sb_ended_run_run_id_workflow_run_id_fk", + "tableFrom": "sb_ended_run", + "tableTo": "workflow_run", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "sb_ended_run_project_id_project_id_fk": { + "name": "sb_ended_run_project_id_project_id_fk", + "tableFrom": "sb_ended_run", + "tableTo": "project", + "columnsFrom": ["project_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sb_invitation": { + "name": "sb_invitation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_id": { + "name": "role_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_sha256": { + "name": "role_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "invited_by": { + "name": "invited_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload_sha256": { + "name": "payload_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "accepted_at": { + "name": "accepted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "accepted_principal_id": { + "name": "accepted_principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_by": { + "name": "revoked_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sb_invitation_recipient_idx": { + "name": "sb_invitation_recipient_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sb_invitation_tenant_id_tenant_id_fk": { + "name": "sb_invitation_tenant_id_tenant_id_fk", + "tableFrom": "sb_invitation", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sb_invitation_tenant_id_invited_by_idempotency_key_unique": { + "name": "sb_invitation_tenant_id_invited_by_idempotency_key_unique", + "nullsNotDistinct": false, + "columns": ["tenant_id", "invited_by", "idempotency_key"] + } + }, + "policies": {}, + "checkConstraints": { + "sb_invitation_terminal_check": { + "name": "sb_invitation_terminal_check", + "value": "NOT (\"sb_invitation\".\"accepted_at\" IS NOT NULL AND \"sb_invitation\".\"revoked_at\" IS NOT NULL)" + }, + "sb_invitation_acceptance_check": { + "name": "sb_invitation_acceptance_check", + "value": "(\"sb_invitation\".\"accepted_at\" IS NULL) = (\"sb_invitation\".\"accepted_principal_id\" IS NULL)" + }, + "sb_invitation_revocation_check": { + "name": "sb_invitation_revocation_check", + "value": "(\"sb_invitation\".\"revoked_at\" IS NULL) = (\"sb_invitation\".\"revoked_by\" IS NULL)" + } + }, + "isRLSEnabled": false + }, + "public.mutation_receipt": { + "name": "mutation_receipt", + "schema": "", + "columns": { + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_principal_id": { + "name": "actor_principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "command": { + "name": "command", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload_sha256": { + "name": "payload_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "result_body": { + "name": "result_body", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "mutation_receipt_tenant_id_tenant_id_fk": { + "name": "mutation_receipt_tenant_id_tenant_id_fk", + "tableFrom": "mutation_receipt", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "mutation_receipt_actor_principal_id_principal_id_fk": { + "name": "mutation_receipt_actor_principal_id_principal_id_fk", + "tableFrom": "mutation_receipt", + "tableTo": "principal", + "columnsFrom": ["actor_principal_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "mutation_receipt_tenant_id_actor_principal_id_command_idempotency_key_pk": { + "name": "mutation_receipt_tenant_id_actor_principal_id_command_idempotency_key_pk", + "columns": [ + "tenant_id", + "actor_principal_id", + "command", + "idempotency_key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.outbox": { + "name": "outbox", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sb_outbox_project_time_id_idx": { + "name": "sb_outbox_project_time_id_idx", + "columns": [ + { + "expression": "project_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "outbox_project_id_project_id_fk": { + "name": "outbox_project_id_project_id_fk", + "tableFrom": "outbox", + "tableTo": "project", + "columnsFrom": ["project_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.project": { + "name": "project", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "version_of_project_id": { + "name": "version_of_project_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "shipped_at": { + "name": "shipped_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "project_tenant_id_tenant_id_fk": { + "name": "project_tenant_id_tenant_id_fk", + "tableFrom": "project", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "project_version_of_project_id_project_id_fk": { + "name": "project_version_of_project_id_project_id_fk", + "tableFrom": "project", + "tableTo": "project", + "columnsFrom": ["version_of_project_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "project_tenant_id_unique": { + "name": "project_tenant_id_unique", + "nullsNotDistinct": false, + "columns": ["tenant_id"] + } + }, + "policies": {}, + "checkConstraints": { + "project_status_check": { + "name": "project_status_check", + "value": "\"project\".\"status\" in ('active', 'archived')" + } + }, + "isRLSEnabled": false + }, + "public.project_coordinator": { + "name": "project_coordinator", + "schema": "", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "active": { + "name": "active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "registered_by": { + "name": "registered_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "project_coordinator_active_project_idx": { + "name": "project_coordinator_active_project_idx", + "columns": [ + { + "expression": "project_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"project_coordinator\".\"active\"", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "project_coordinator_run_id_workflow_run_id_fk": { + "name": "project_coordinator_run_id_workflow_run_id_fk", + "tableFrom": "project_coordinator", + "tableTo": "workflow_run", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "project_coordinator_project_id_project_id_fk": { + "name": "project_coordinator_project_id_project_id_fk", + "tableFrom": "project_coordinator", + "tableTo": "project", + "columnsFrom": ["project_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "project_coordinator_registered_by_principal_id_fk": { + "name": "project_coordinator_registered_by_principal_id_fk", + "tableFrom": "project_coordinator", + "tableTo": "principal", + "columnsFrom": ["registered_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sb_settings": { + "name": "sb_settings", + "schema": "", + "columns": { + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scope_key": { + "name": "scope_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "fields": { + "name": "fields", + "type": "jsonb", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "sb_settings_tenant_id_tenant_id_fk": { + "name": "sb_settings_tenant_id_tenant_id_fk", + "tableFrom": "sb_settings", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "sb_settings_project_id_project_id_fk": { + "name": "sb_settings_project_id_project_id_fk", + "tableFrom": "sb_settings", + "tableTo": "project", + "columnsFrom": ["project_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "sb_settings_tenant_id_scope_key_pk": { + "name": "sb_settings_tenant_id_scope_key_pk", + "columns": ["tenant_id", "scope_key"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/apps/hub/src/db/migrations/meta/_journal.json b/apps/hub/src/db/migrations/meta/_journal.json index ef002f3b..4479ff9c 100644 --- a/apps/hub/src/db/migrations/meta/_journal.json +++ b/apps/hub/src/db/migrations/meta/_journal.json @@ -253,6 +253,13 @@ "when": 1793900000000, "tag": "0035_drop_deck_looks", "breakpoints": true + }, + { + "idx": 36, + "version": "7", + "when": 1794000000000, + "tag": "0036_ended_run_retry", + "breakpoints": true } ] } diff --git a/apps/hub/src/db/schema.ts b/apps/hub/src/db/schema.ts index bfe669c4..cc55cfef 100644 --- a/apps/hub/src/db/schema.ts +++ b/apps/hub/src/db/schema.ts @@ -199,6 +199,9 @@ export const endedRun = pgTable("sb_ended_run", { .notNull() .references(() => project.id), notified: boolean("notified").notNull(), + // Set while the notice still waits to be delivered; null once it is settled. + nextAttemptAt: timestamp("next_attempt_at", { withTimezone: true }), + attempts: integer("attempts").notNull().default(0), createdAt: timestamp("created_at", { withTimezone: true }) .notNull() .defaultNow(), diff --git a/apps/hub/src/projects/specialist-ended.ts b/apps/hub/src/projects/specialist-ended.ts index 12ee1fbb..fed84195 100644 --- a/apps/hub/src/projects/specialist-ended.ts +++ b/apps/hub/src/projects/specialist-ended.ts @@ -1,5 +1,5 @@ import { join } from "node:path"; -import { and, eq, inArray, isNull } from "drizzle-orm"; +import { and, eq, inArray, isNotNull, isNull, lte, or, sql } from "drizzle-orm"; import type { DB } from "@intx/db"; import { workflowRun, workflowRunLaunchSpec } from "@intx/db/schema"; import type { WorkflowRunReader } from "@intx/hub-sessions"; @@ -74,19 +74,31 @@ export async function liveSpecialistRuns( .map((run) => run.runId); } +// Mirrors defaultRetryDelay in Interchange's hub-sessions workflow-dispatch-service.ts, which it does +// not export. +const mailRetryDelay = (attempt: number) => + Math.min(500 * 2 ** Math.min(attempt, 6), 30_000); + +// A coordinator can take minutes to come back, so waiting for one backs off further. +const coordinatorWaitDelay = (attempt: number) => + Math.min(500 * 2 ** Math.min(attempt, 10), 300_000); + /** * Mails a project's coordinator once when one of its specialists ends failed or cancelled with an * assignment it never answered. The terminal status is read from the run row on the native scheduler - * whose reconcilers write it. + * whose reconcilers write it. A notice that cannot be delivered yet waits for its own next attempt, + * so it never holds back another project's. */ export function createSpecialistEndNotices(deps: { db: DB["db"]; runs: WorkflowRunReader; mailDirectory: string; mail: HubMail; + now?: () => Date; }) { const { db } = deps; return async function reconcileNext(): Promise { + const now = deps.now?.() ?? new Date(); const [ended] = await db .select({ runId: workflowRun.id, @@ -97,6 +109,7 @@ export function createSpecialistEndNotices(deps: { cancellationReason: workflowRun.cancellationReason, projectId: project.id, bundle: workflowRunLaunchSpec.frozenApprovalBundle, + attempts: endedRun.attempts, }) .from(workflowRun) .innerJoin(project, eq(project.tenantId, workflowRun.tenantId)) @@ -108,17 +121,39 @@ export function createSpecialistEndNotices(deps: { .where( and( inArray(workflowRun.status, ["failed", "cancelled"]), - isNull(endedRun.runId), + or(isNull(endedRun.runId), lte(endedRun.nextAttemptAt, now)), ), ) - .orderBy(workflowRun.endedAt) + .orderBy( + sql`${endedRun.nextAttemptAt} asc nulls first`, + workflowRun.endedAt, + ) .limit(1); if (!ended) return false; - const record = (notified: boolean) => + const settled = (notified: boolean) => ({ notified, nextAttemptAt: null }); + const retry = (delay: (attempt: number) => number) => { + const attempts = (ended.attempts ?? 0) + 1; + return { + notified: false, + attempts, + nextAttemptAt: new Date(now.getTime() + delay(attempts)), + }; + }; + const undelivered = (status: number) => { + if (status === 409) return retry(coordinatorWaitDelay); + if (status < 500) return settled(false); + return retry(mailRetryDelay); + }; + // A notice settled already stays settled. + const record = (state: ReturnType) => db .insert(endedRun) - .values({ runId: ended.runId, projectId: ended.projectId, notified }) - .onConflictDoNothing() + .values({ runId: ended.runId, projectId: ended.projectId, ...state }) + .onConflictDoUpdate({ + target: endedRun.runId, + set: state, + where: isNotNull(endedRun.nextAttemptAt), + }) .returning({ runId: endedRun.runId }); const role = spawnedWorkflowRole(ended.bundle); const correlationId = role @@ -146,15 +181,22 @@ export function createSpecialistEndNotices(deps: { and( eq(projectCoordinator.projectId, ended.projectId), eq(projectCoordinator.active, true), + inArray(workflowRun.status, ["deployed", "running"]), ), ); // The hub cancels unpicked concept designers itself after the person's pick; nothing is lost. const unpicked = ended.cancellationReason === "design_concept_not_picked"; - if (!role || correlationId === undefined || !coordinator || unpicked) { - await record(false); + if (!role || correlationId === undefined || unpicked) { + await record(settled(false)); + return true; + } + // Between an upgrade's release and its bind, or while an ended coordinator stays bound until the + // person's next message starts another. + if (!coordinator) { + await record(retry(coordinatorWaitDelay)); return true; } - if ((await record(true)).length === 0) return true; + if ((await record(settled(true))).length === 0) return true; // A writer's or reviewer's assignment is correlated as `--` by deploy_writer // and review_artifact. const prefix = role + "-"; @@ -190,14 +232,13 @@ export function createSpecialistEndNotices(deps: { coordinatorRunId: coordinator.runId, ...requestErrorMetadata(error), }); - // A refusal stands; anything else is tried again on a later pass. - if (error instanceof HubMailError && error.status < 500) - await db - .update(endedRun) - .set({ notified: false }) - .where(eq(endedRun.runId, ended.runId)); - else await db.delete(endedRun).where(eq(endedRun.runId, ended.runId)); - return false; + // A refusal stands. A coordinator that is stopping or no longer routable (409) is waited out + // like a missing one; anything else is tried again soon. + const status = error instanceof HubMailError ? error.status : 500; + await db + .update(endedRun) + .set(undelivered(status)) + .where(eq(endedRun.runId, ended.runId)); } return true; }; diff --git a/test/e2e/specialist-end-notices.test.ts b/test/e2e/specialist-end-notices.test.ts new file mode 100644 index 00000000..fe30f1cc --- /dev/null +++ b/test/e2e/specialist-end-notices.test.ts @@ -0,0 +1,285 @@ +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + expect, + test, +} from "bun:test"; +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { and, eq, inArray } from "drizzle-orm"; +import postgres from "postgres"; +import { createDB, type DB } from "@intx/db"; +import { + principal, + tenant, + workflowDefinition, + workflowRun, + workflowRunLaunchSpec, +} from "@intx/db/schema"; +import type { WorkflowRunReader } from "@intx/hub-sessions"; +import { + createIsogitStore, + createMailAuditStore, +} from "@intx/storage-isogit/node"; +import { + endedRun, + project, + projectCoordinator, +} from "@corbits-builder/hub/db/schema"; +import { HubMailError } from "@corbits-builder/hub/mail/hub-mail"; +import { migrateDatabase } from "@corbits-builder/hub/migrate"; +import { createSpecialistEndNotices } from "@corbits-builder/hub/projects/specialist-ended"; + +function required(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`Missing required e2e environment: ${name}`); + return value; +} + +let handle: DB; +let admin: postgres.Sql; +let mailDirectory: string; +let dbName: string; + +beforeAll(() => { + admin = postgres({ + host: required("E2E_DB_HOST"), + port: Number(required("E2E_DB_PORT")), + username: required("E2E_DB_USER"), + password: required("E2E_DB_PASSWORD"), + database: "postgres", + max: 1, + onnotice: () => undefined, + }); +}); + +afterAll(async () => { + await admin?.end(); +}); + +// Each test gets its own database and mail store, so one test's pending notices never reach another. +beforeEach(async () => { + dbName = `sb_notice_${randomUUID().replaceAll("-", "")}`; + await admin.unsafe(`create database ${dbName}`); + const database = { + host: required("E2E_DB_HOST"), + port: Number(required("E2E_DB_PORT")), + user: required("E2E_DB_USER"), + password: required("E2E_DB_PASSWORD"), + }; + await migrateDatabase({ + DB_HOST: database.host, + DB_PORT: String(database.port), + DB_USER: database.user, + DB_PASSWORD: database.password, + DB_NAME: dbName, + DB_SSL: "false", + }); + handle = createDB({ ...database, database: dbName, ssl: false }); + mailDirectory = await mkdtemp(join(tmpdir(), "sb-notice-mail-")); +}); + +afterEach(async () => { + await handle?.close(); + await admin.unsafe(`drop database if exists ${dbName} with (force)`); + await rm(mailDirectory, { recursive: true, force: true }); +}); + +const writerBundle = { + projection: { + steps: { write: { agent: { tags: { "writer.version": "1" } } } }, + }, +}; + +// A tenant with one project, and its specialist that ended cancelled with an unanswered assignment. +async function endedSpecialist(name: string, endedAt: Date) { + const db = handle.db; + const tenantId = `${name}-${randomUUID()}`; + await db.insert(tenant).values({ + id: tenantId, + name: tenantId, + slug: tenantId, + domain: `${tenantId}.test`, + }); + const principalId = `${tenantId}-person`; + await db.insert(principal).values({ + id: principalId, + tenantId, + kind: "user", + refId: principalId, + status: "active", + }); + const definitionId = `${tenantId}-definition`; + await db + .insert(workflowDefinition) + .values({ id: definitionId, tenantId, name: definitionId }); + const projectId = `${tenantId}-project`; + await db.insert(project).values({ id: projectId, tenantId, title: name }); + const run = async ( + id: string, + values: Partial, + bundle: object, + ) => { + await db.insert(workflowRun).values({ + id, + definitionId, + anchorRunId: id, + tenantId, + address: `${id}@${tenantId}.test`, + ...values, + }); + await db.insert(workflowRunLaunchSpec).values({ + anchorRunId: id, + sessionId: id, + deploymentDomain: `${tenantId}.test`, + sourceAuthorityPrincipalId: principalId, + frozenApprovalBundle: bundle, + sourceOfferingIds: [], + defaultSourceOfferingId: "offering", + deployContent: {}, + }); + }; + const specialistId = `${tenantId}-writer`; + await run( + specialistId, + { status: "cancelled", cancellationReason: "stopped", endedAt }, + writerBundle, + ); + const directory = join(mailDirectory, specialistId); + await createIsogitStore(directory); + const journal = await createMailAuditStore(directory); + await journal.commitMail( + new TextEncoder().encode( + [ + `From: coordinator@${tenantId}.test`, + `To: ${specialistId}@${tenantId}.test`, + `Date: ${new Date(endedAt.getTime() - 60_000).toUTCString()}`, + `Message-ID: <${specialistId}-assignment@${tenantId}.test>`, + `Interchange-Correlation-ID: writer-draft-${specialistId}`, + "Subject: draft", + "", + "Write the draft.", + "", + ].join("\r\n"), + ), + "in", + ); + // Binds a new coordinator in place of the project's current one. + const bindCoordinator = async (status: "running" | "failed" = "running") => { + const coordinatorId = `${tenantId}-coordinator-${randomUUID()}`; + await run(coordinatorId, { status }, {}); + await db + .update(projectCoordinator) + .set({ active: false }) + .where(eq(projectCoordinator.projectId, projectId)); + await db.insert(projectCoordinator).values({ + runId: coordinatorId, + projectId, + registeredBy: principalId, + }); + return coordinatorId; + }; + return { specialistId, bindCoordinator }; +} + +async function notice(runId: string) { + const [row] = await handle.db + .select() + .from(endedRun) + .where(eq(endedRun.runId, runId)); + return row; +} + +// Mails like the native run mail route: a 409 for a coordinator run that has ended, and a 503 for +// the ones listed as unreachable. +function reconciler(unreachable: string[] = []) { + let clock = Date.now(); + const delivered: string[] = []; + const reconcileNext = createSpecialistEndNotices({ + db: handle.db, + runs: {} as WorkflowRunReader, + mailDirectory, + mail: async ({ runId }) => { + const [ended] = await handle.db + .select({ id: workflowRun.id }) + .from(workflowRun) + .where( + and( + eq(workflowRun.id, runId), + inArray(workflowRun.status, ["completed", "failed", "cancelled"]), + ), + ); + if (ended) throw new HubMailError(409, "workflow_run_terminal"); + if (unreachable.includes(runId)) + throw new HubMailError(503, "unavailable"); + delivered.push(runId); + }, + now: () => new Date(clock), + }); + // Works through every notice due now. + const drain = async () => { + for (let pass = 0; pass < 20; pass += 1) + if (!(await reconcileNext())) return; + throw new Error("Notices never settled"); + }; + const later = () => { + clock += 60 * 60_000; + }; + return { delivered, drain, later }; +} + +test("an unreachable coordinator holds back only its own notice", async () => { + const blocked = await endedSpecialist( + "blocked", + new Date(Date.now() - 2_000), + ); + const other = await endedSpecialist("other", new Date(Date.now() - 1_000)); + const unreachable = await blocked.bindCoordinator(); + const reachable = await other.bindCoordinator(); + const { delivered, drain } = reconciler([unreachable]); + + await drain(); + expect(delivered).toEqual([reachable]); + const pending = await notice(blocked.specialistId); + expect(pending?.notified).toBe(false); + expect(pending?.nextAttemptAt).not.toBeNull(); +}); + +test("a notice for a project without a coordinator is delivered once one binds", async () => { + const unbound = await endedSpecialist("unbound", new Date()); + const { delivered, drain, later } = reconciler(); + + await drain(); + expect(delivered).toEqual([]); + expect((await notice(unbound.specialistId))?.nextAttemptAt).not.toBeNull(); + const coordinator = await unbound.bindCoordinator(); + later(); + await drain(); + expect(delivered).toEqual([coordinator]); + expect(await notice(unbound.specialistId)).toMatchObject({ + notified: true, + nextAttemptAt: null, + }); +}); + +test("a notice for a project whose bound coordinator has ended is delivered to the next one", async () => { + const stale = await endedSpecialist("stale", new Date()); + await stale.bindCoordinator("failed"); + const { delivered, drain, later } = reconciler(); + + await drain(); + expect(delivered).toEqual([]); + expect((await notice(stale.specialistId))?.nextAttemptAt).not.toBeNull(); + const coordinator = await stale.bindCoordinator(); + later(); + await drain(); + expect(delivered).toEqual([coordinator]); + expect(await notice(stale.specialistId)).toMatchObject({ + notified: true, + nextAttemptAt: null, + }); +});