feat(artifacts): file approval_package with source pins and secret scan

Author: TheGreatAxios

apps/hub/src/artifacts/approval-package.test.ts

@@ -0,0 +1,183 @@
+import { type } from "arktype";
+import {
+  getArtifact,
+  getArtifactVersion,
+  type ArtifactDb,
+} from "@corbits/artifacts";
+import {
+  ApprovalPackageBundle,
+  type ApprovalPackageSource,
+} from "@corbits-builder/schemas/artifact";
+import { parseJson } from "../lib/json.ts";
+import { ProjectError, type ProjectErrorIssue } from "../projects/errors.ts";
+
+// An approval package summarizes planning and design documents for stakeholders; the built app and
+// earlier approval packages are never pinned as package sources.
+export const EXCLUDED_PACKAGE_SOURCE_KINDS = new Set([
+  "app",
+  "approval_package",
+]);
+
+const SECRET_PATTERNS: readonly { label: string; pattern: RegExp }[] = [
+  {
+    label: "api_key",
+    pattern:
+      /(?:sk-ant-[A-Za-z0-9_-]{20,}|sk-[A-Za-z0-9_-]{20,}|AKIA[0-9A-Z]{16}|ghp_[A-Za-z0-9]{30,}|github_pat_[A-Za-z0-9_]{30,}|xai-[A-Za-z0-9_-]{20,}|AIza[0-9A-Za-z_-]{35})/,
+  },
+  {
+    label: "private_key",
+    pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----/,
+  },
+  {
+    label: "demo_login",
+    pattern: /demoLogin/,
+  },
+  {
+    label: "live_app_url",
+    pattern: /http:\/\/(?:127\.0\.0\.1|localhost):/,
+  },
+];
+
+const UUID_RE =
+  /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
+export type StoredPackageSource = {
+  kind: string;
+  version: number;
+  sha256: string;
+};
+
+export type StoredSourceReader = (
+  artifactId: string,
+  version: number,
+) => Promise<StoredPackageSource | undefined>;
+
+function scanSectionForSecrets(
+  path: string,
+  text: string,
+): ProjectErrorIssue[] {
+  const issues: ProjectErrorIssue[] = [];
+  for (const { label, pattern } of SECRET_PATTERNS) {
+    if (pattern.test(text)) issues.push({ path, message: label });
+  }
+  return issues;
+}
+
+export function scanApprovalPackageForSecrets(
+  bundle: ApprovalPackageBundle,
+): void {
+  const manifestText = [
+    bundle.manifest.audience,
+    bundle.manifest.decisionNeeded,
+    bundle.manifest.tone,
+    bundle.manifest.length,
+    ...bundle.manifest.sources.map((source) => source.title),
+  ].join("\n");
+  const issues: ProjectErrorIssue[] = [
+    ...scanSectionForSecrets("manifest", manifestText),
+    ...scanSectionForSecrets("deck", JSON.stringify(bundle.deck)),
+  ];
+  if (bundle.documents !== undefined) {
+    for (const [name, content] of Object.entries(bundle.documents)) {
+      issues.push(...scanSectionForSecrets(`documents/${name}`, content));
+    }
+  }
+  if (issues.length > 0)
+    throw new ProjectError(422, "package_contains_secret", issues);
+}
+
+export function parseApprovalPackageBundle(
+  content: string,
+): ApprovalPackageBundle {
+  const bundle = ApprovalPackageBundle(parseJson(content));
+  if (bundle instanceof type.errors)
+    throw new ProjectError(400, "invalid_approval_package", {
+      issues: bundle.summary,
+    });
+  return bundle;
+}
+
+async function verifySourcePin(
+  source: ApprovalPackageSource,
+  readSource: StoredSourceReader,
+): Promise<void> {
+  if (EXCLUDED_PACKAGE_SOURCE_KINDS.has(source.kind))
+    throw new ProjectError(422, "excluded_package_source_kind", [
+      { path: "manifest.sources", message: source.kind },
+    ]);
+  const stored = await readSource(source.artifactId, source.version);
+  if (
+    stored === undefined ||
+    stored.kind !== source.kind ||
+    stored.version !== source.version ||
+    stored.sha256 !== source.sha256
+  )
+    throw new ProjectError(422, "invalid_package_source", [
+      {
+        path: "manifest.sources",
+        message: `${source.artifactId}@${String(source.version)}`,
+      },
+    ]);
+}
+
+export async function validateApprovalPackage(
+  input: string | ApprovalPackageBundle,
+  readSource: StoredSourceReader,
+): Promise<ApprovalPackageBundle> {
+  const bundle =
+    typeof input === "string" ? parseApprovalPackageBundle(input) : input;
+  for (const source of bundle.manifest.sources) {
+    if (EXCLUDED_PACKAGE_SOURCE_KINDS.has(source.kind))
+      throw new ProjectError(422, "excluded_package_source_kind", [
+        { path: "manifest.sources", message: source.kind },
+      ]);
+  }
+  scanApprovalPackageForSecrets(bundle);
+  for (const source of bundle.manifest.sources)
+    await verifySourcePin(source, readSource);
+  return bundle;
+}
+
+export function readProjectPackageSource(
+  db: ArtifactDb,
+  tenantId: string,
+): StoredSourceReader {
+  return async (artifactId, version) => {
+    if (!UUID_RE.test(artifactId)) return undefined;
+    const row = await getArtifact(db, artifactId);
+    if (!row || row.tenantId !== tenantId) return undefined;
+    const stored = await getArtifactVersion(db, artifactId, version);
+    if (!stored?.contentSha256) return undefined;
+    return {
+      kind: row.kind,
+      version: stored.version,
+      sha256: stored.contentSha256,
+    };
+  };
+}
+
+function base64PaddingBytes(encoded: string): number {
+  if (encoded.endsWith("==")) return 2;
+  if (encoded.endsWith("=")) return 1;
+  return 0;
+}
+
+function base64ByteLength(encoded: string): number {
+  return Math.max(
+    0,
+    Math.floor((encoded.length * 3) / 4) - base64PaddingBytes(encoded),
+  );
+}
+
+// Agents reading an approval_package need its manifest, deck structure and markdown documents, not
+// megabytes of base64 binary that would trip the tool-output size cap.
+export function stripApprovalPackageBinaries(content: string): string {
+  const bundle = ApprovalPackageBundle(parseJson(content));
+  if (bundle instanceof type.errors) return content;
+  const { pptxBase64, pdfBase64, ...rest } = bundle;
+  return JSON.stringify({
+    ...rest,
+    pptxBytes: base64ByteLength(pptxBase64),
+    pdfBytes: base64ByteLength(pdfBase64),
+  });
+}

apps/hub/src/artifacts/approval-package.ts

@@ -24,6 +24,7 @@ import {
   DesignTweaks,
   DesignTweaksReceipt,
   LiveApp,
+  MAX_APPROVAL_PACKAGE_BYTES,
   MaterialReceipt,
 } from "@corbits-builder/schemas/artifact";
 import { ProjectActivity } from "@corbits-builder/schemas/activity";
@@ -123,6 +124,10 @@ export function mountArtifactRoutes(
   );
   app.post(
     "/:projectId/artifacts",
+    bodyLimit({
+      maxSize: MAX_APPROVAL_PACKAGE_BYTES + 1024 * 1024,
+      onError: (c) => c.json({ code: "artifact_too_large" }, 413),
+    }),
     describeRoute({
       tags: ["Artifacts"],
       operationId: "writeProjectArtifact",

apps/hub/src/artifacts/routes.ts

@@ -41,6 +41,7 @@ import {
   AppVersionMetadata,
   DocumentContent,
   MAX_APP_BYTES,
+  MAX_APPROVAL_PACKAGE_BYTES,
   MATERIAL_KIND,
   ArtifactKindWrite,
   DesignBundle,
@@ -138,6 +139,11 @@ import {
   materialContent,
   materialMimeType,
 } from "./materials.ts";
+import {
+  parseApprovalPackageBundle,
+  readProjectPackageSource,
+  validateApprovalPackage,
+} from "./approval-package.ts";
 import { requestErrorMetadata } from "../request-error.ts";
 
 const log = getLogger(["corbits-builder", "hub", "artifacts"]);
@@ -151,8 +157,8 @@ const APPROVAL_COVERS: Record<string, (kind: string) => boolean> = {
     kind !== "app" && kind !== "design" && kind !== "design_option",
   instruct_coder: (kind) => kind !== "app" && kind !== "design_option",
 };
-// Whole documents, the filed app among them (up to 1 MiB of source).
-const MAX_READ_CHARS = 1024 * 1024;
+// Whole documents, the filed app and approval package among them.
+const MAX_READ_CHARS = MAX_APPROVAL_PACKAGE_BYTES;
 const RECENT_FAILURES = 5;
 const RunError = type({ "+": "delete", message: "string", "code?": "string" });
 const UUID = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
@@ -184,6 +190,68 @@ async function isDirectory(path: string) {
 
 const artifactResource = (kind: string) => "sb-artifact:" + kind;
 
+function validateArtifactWriteContent(
+  input: typeof ArtifactKindWrite.infer,
+): boolean {
+  const bundled =
+    input.kind === "design" ||
+    input.kind === "design_option" ||
+    input.kind === "app" ||
+    input.kind === "approval_package";
+  if (bundled && input.openQuestions)
+    throw new ProjectError(400, "invalid_artifact_write", {
+      issues: "openQuestions belong to documents, not a " + input.kind,
+    });
+  if (input.approaches) {
+    if (input.kind !== "approaches")
+      throw new ProjectError(400, "invalid_artifact_write", {
+        issues:
+          "approaches belong to the approaches document, not a " + input.kind,
+      });
+    const issue = approachesIssue(input.approaches);
+    if (issue)
+      throw new ProjectError(400, "invalid_artifact_write", {
+        issues: issue,
+      });
+  }
+  if (
+    input.kind === "design" ||
+    input.kind === "design_option" ||
+    input.kind === "app"
+  ) {
+    const bundle = (input.kind === "app" ? AppBundle : DesignBundle)(
+      parseJson(input.content),
+    );
+    if (bundle instanceof type.errors)
+      throw new ProjectError(400, "invalid_" + input.kind, {
+        issues: bundle.summary,
+      });
+  }
+  if (input.kind === "app") {
+    const bytes = Buffer.byteLength(input.content);
+    if (bytes > MAX_APP_BYTES)
+      throw new ProjectError(413, "app_too_large", {
+        bytes,
+        limit: MAX_APP_BYTES,
+      });
+  } else if (input.kind === "approval_package") {
+    const bytes = Buffer.byteLength(input.content);
+    if (bytes > MAX_APPROVAL_PACKAGE_BYTES)
+      throw new ProjectError(413, "approval_package_too_large", {
+        bytes,
+        limit: MAX_APPROVAL_PACKAGE_BYTES,
+      });
+    parseApprovalPackageBundle(input.content);
+  } else {
+    const document = DocumentContent(input.content);
+    if (document instanceof type.errors)
+      throw new ProjectError(400, "invalid_artifact_write", {
+        issues: document.summary,
+      });
+  }
+  return bundled;
+}
+
 // A document version records the open questions it leaves for the person; a version filed without
 // them, before they were recorded or by an import, leaves none. An approaches version may also
 // record its approaches for a side-by-side comparison.
@@ -1254,55 +1322,17 @@ export function createArtifactService(
         throw new ProjectError(400, "invalid_artifact_write", {
           issues: input.summary,
         });
-      const bundled =
-        input.kind === "design" ||
-        input.kind === "design_option" ||
-        input.kind === "app";
-      if (bundled && input.openQuestions)
-        throw new ProjectError(400, "invalid_artifact_write", {
-          issues: "openQuestions belong to documents, not a " + input.kind,
-        });
-      if (input.approaches) {
-        if (input.kind !== "approaches")
-          throw new ProjectError(400, "invalid_artifact_write", {
-            issues:
-              "approaches belong to the approaches document, not a " +
-              input.kind,
-          });
-        const issue = approachesIssue(input.approaches);
-        if (issue)
-          throw new ProjectError(400, "invalid_artifact_write", {
-            issues: issue,
-          });
-      }
-      if (bundled) {
-        const bundle = (input.kind === "app" ? AppBundle : DesignBundle)(
-          parseJson(input.content),
-        );
-        if (bundle instanceof type.errors)
-          throw new ProjectError(400, "invalid_" + input.kind, {
-            issues: bundle.summary,
-          });
-      }
-      if (input.kind === "app") {
-        const bytes = Buffer.byteLength(input.content);
-        if (bytes > MAX_APP_BYTES)
-          throw new ProjectError(413, "app_too_large", {
-            bytes,
-            limit: MAX_APP_BYTES,
-          });
-      } else {
-        const document = DocumentContent(input.content);
-        if (document instanceof type.errors)
-          throw new ProjectError(400, "invalid_artifact_write", {
-            issues: document.summary,
-          });
-      }
+      const bundled = validateArtifactWriteContent(input);
       const allowed = await requireProject(actor, projectId, "read");
       if (!(await allowed(artifactResource(input.kind), "write")))
         throw new ProjectError(403, "artifact_write_refused", {
           kind: input.kind,
         });
+      if (input.kind === "approval_package")
+        await validateApprovalPackage(
+          input.content,
+          readProjectPackageSource(db, actor.tenantId),
+        );
       const [active] = await rawDB
         .select({ kind: principal.kind, refId: principal.refId })
         .from(principal)