feat: import a folder as a project

README.md

@@ -65,7 +65,7 @@ the app or in `.env.hub`
 - [Self-hosting](https://github.com/corbitsdev/corbits-builder/blob/main/docs/SELF_HOST.md): running it with Docker.
 - [Development](https://github.com/corbitsdev/corbits-builder/blob/main/docs/DEV.md) and [Contributing](https://github.com/corbitsdev/corbits-builder/blob/main/CONTRIBUTING.md): working on the code.
 - [AGENTS.md](https://github.com/corbitsdev/corbits-builder/blob/main/AGENTS.md): conventions for coding agents.
-- [skills/](https://github.com/corbitsdev/corbits-builder/tree/main/skills): agent skills to install Builder and drive it through its API.
+- [skills/](https://github.com/corbitsdev/corbits-builder/tree/main/skills): agent skills to install Builder, drive it through its API and import a repository as a project.
 
 Builder runs on [Interchange](https://github.com/faremeter/interchange), which
 runs the agents and handles their messages, approvals and credentials.

apps/hub/src/builds/app-snapshot.ts

@@ -1,4 +1,3 @@
-import { type } from "arktype";
 import ignore, { type Ignore } from "ignore";
 import { constants } from "node:fs";
 import { open, readdir, realpath } from "node:fs/promises";
@@ -11,28 +10,25 @@ import {
   ArtifactReceipt,
   DesignDraft,
 } from "@corbits-builder/schemas/artifact";
+import {
+  EXCLUDED_DIRECTORIES,
+  addBundleFile,
+  bundleContent,
+  excludedFile,
+} from "@corbits-builder/schemas/app-files";
 import { WorkflowHttpError, workflowHttpJson } from "../workflows/http-json.ts";
-import { INPUTS_DIRECTORY } from "../config/inputs.ts";
 import { spliceDesignScreens } from "./starter.ts";
 
-// Installs and builds are rebuilt by setup and check-app, .builder and the inputs come from filed
-// documents, and .scratch holds the coder's throwaway files, such as the screenshots it takes.
-export const EXCLUDED_DIRECTORIES: ReadonlySet<string> = new Set([
-  "node_modules",
-  "dist",
-  ".git",
-  ".builder",
-  ".scratch",
-  INPUTS_DIRECTORY,
-]);
-// Secrets (environment files, keys, registry credentials) and local databases never leave the
-// workspace; an example environment file holds no secret and documents the app's settings.
-const EXCLUDED_FILE =
-  /^(\.env(\..*)?|\.npmrc|\.dev\.vars|.*\.(db|db-wal|db-shm|pem|key|p12)|.*\.sqlite.*)$/;
-export const excludedFile = (name: string) => {
-  const lower = name.toLowerCase();
-  return lower !== ".env.example" && EXCLUDED_FILE.test(lower);
-};
+// The rules for which files are the app's and how they pack live with the schemas, so the UI's
+// repository import and the import script pack a person's code the way the hub packs a workspace.
+export {
+  EXCLUDED_DIRECTORIES,
+  addBundleFile,
+  bundleContent,
+  bundleFiles,
+  excludedFile,
+} from "@corbits-builder/schemas/app-files";
+
 // Byte-order marks stay in the text, so a file reads back exactly as it was.
 const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
 
@@ -160,60 +156,11 @@ export async function collectApp(
   return bundle;
 }
 
-// Keeps a UTF-8 file as text and any other base64-encoded under binary.
-export function addBundleFile(
-  bundle: Required<AppBundle>,
-  path: string,
-  bytes: Uint8Array,
-) {
-  try {
-    bundle.files[path] = utf8.decode(bytes);
-  } catch (error) {
-    if (!(error instanceof TypeError)) throw error;
-    bundle.binary[path] = Buffer.from(bytes).toString("base64");
-  }
-}
-
-// A bundle's files as they are written to disk: text as is, binaries decoded.
-export function bundleFiles(bundle: {
-  files: Readonly<Record<string, string>>;
-  binary?: Readonly<Record<string, string>>;
-}): Record<string, string | Uint8Array> {
-  return {
-    ...bundle.files,
-    ...Object.fromEntries(
-      Object.entries(bundle.binary ?? {}).map(([path, data]) => [
-        path,
-        Buffer.from(data, "base64"),
-      ]),
-    ),
-  };
-}
-
-// Keys in order, so the same files always make the same content and an unchanged app files nothing.
-const sorted = (record: Record<string, string>) =>
-  Object.fromEntries(
-    Object.entries(record).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)),
-  );
-
 // The workspace as app document content, checked against the app limit.
 export async function appContent(workspace: string) {
   return bundleContent(await collectApp(workspace)).content;
 }
 
-// Collected files as an app bundle and its document content. An app is as large as it is: a
-// resource that runs out on the way, such as memory or a request body, reports its own error.
-export function bundleContent(collected: Required<AppBundle>) {
-  const files = sorted(collected.files);
-  const binary = sorted(collected.binary);
-  const bundle = AppBundle(
-    Object.keys(binary).length ? { files, binary } : { files },
-  );
-  if (bundle instanceof type.errors)
-    throw new Error("invalid_app: " + bundle.summary);
-  return { bundle, content: JSON.stringify(bundle) };
-}
-
 const projectPath = (scope: { tenantId: string; projectId: string }) =>
   "/api/tenants/" +
   encodeURIComponent(scope.tenantId) +

apps/hub/src/config/import.ts

@@ -4,14 +4,36 @@ const IMPORT_APP_MESSAGE =
 export const IMPORTED_PROJECT_LEAD =
   "I just imported this project from a file. Acknowledge the import, then tell me in a few plain sentences what the imported documents establish and what happens next.";
 
+// The documents a project has before its app, in Builder's order.
+export const PLANNED_DOCUMENT_KINDS = [
+  "problem_brief",
+  "constraints",
+  "approaches",
+  "product_requirements",
+  "design",
+  "build_plan",
+] as const;
+
 // The importing person's opening message to a newly imported project's coordinator, naming the
-// imported documents with no kind of their own so it reads them as the project's materials.
+// imported documents with no kind of their own so it reads them as the project's materials, and,
+// when the app came in without some of its documents (a person's own repository), which ones
+// the coordinator writes with them from the code.
 export function importedProjectMessage(
   materials: readonly string[],
   withApp: boolean,
+  missing: readonly string[] = [],
 ): string {
   const filed = materials.length
     ? ` The documents with no place of their own here are filed as materials: ${materials.join(", ")}.`
     : "";
-  return IMPORTED_PROJECT_LEAD + filed + (withApp ? IMPORT_APP_MESSAGE : "");
+  const unwritten =
+    withApp && missing.length
+      ? ` The code came from my own repository, so these documents were never written: ${missing.join(", ")}. Read the code with read_artifact, its package.json and README first, and write the brief yourself from what it shows, asking me only what the code cannot tell you; then have the writers produce the rest in order, each instruction carrying what you learned from the code, and have me approve each one before the next.`
+      : "";
+  return (
+    IMPORTED_PROJECT_LEAD +
+    filed +
+    (withApp ? IMPORT_APP_MESSAGE : "") +
+    unwritten
+  );
 }

apps/hub/src/config/inputs.ts

@@ -1,7 +1,8 @@
 // A warmed designer or coder workspace holds the filed documents it works from as read-only files,
 // and its hub-built assignment opens by naming them, so it starts from them instead of fetching
 // each one (a kept trace's coder made 54 document reads in 107 turns).
-export const INPUTS_DIRECTORY = ".inputs";
+import { INPUTS_DIRECTORY } from "@corbits-builder/schemas/app-files";
+export { INPUTS_DIRECTORY };
 
 const INPUTS = {
   problem_brief: {

apps/hub/src/projects/routes.ts

@@ -41,7 +41,10 @@ import {
 } from "./next-step.ts";
 import type { Actor } from "../lib/actor.ts";
 import type { LiveApps } from "../live-app/runner.ts";
-import { importedProjectMessage } from "../config/import.ts";
+import {
+  PLANNED_DOCUMENT_KINDS,
+  importedProjectMessage,
+} from "../config/import.ts";
 import { MODEL_CHOICE_REFUSALS } from "../config/models.ts";
 import { mailRunAsCaller } from "../mail/caller-mail.ts";
 import { readJson } from "../lib/json.ts";
@@ -322,6 +325,7 @@ export function createProjectRoutes(
             importedProjectMessage(
               materials.map(({ title }) => title),
               kinds.includes("app"),
+              PLANNED_DOCUMENT_KINDS.filter((kind) => !kinds.includes(kind)),
             ),
           );
         } catch (error) {

apps/web/src/api/repository.test.ts

@@ -0,0 +1,47 @@
+import { expect, test } from "bun:test";
+import { describeBytes, listedDirectory, readDirectory } from "./repository.ts";
+
+const listed = (path: string, text: string) => {
+  const blob = new Blob([text]);
+  return {
+    name: path.slice(path.lastIndexOf("/") + 1),
+    size: blob.size,
+    webkitRelativePath: path,
+    arrayBuffer: () => blob.arrayBuffer(),
+  };
+};
+
+test("a folder input's files lose the folder's name and never include dependencies", async () => {
+  const picked = listedDirectory([
+    listed("rtv7/package.json", '{"name":"rtv7"}'),
+    listed("rtv7/src/main.ts", "export {};"),
+    listed("rtv7/node_modules/left/index.js", ""),
+    listed("rtv7/.git/HEAD", "ref: refs/heads/main"),
+  ])!;
+  expect(picked.name).toBe("rtv7");
+  expect(picked.files.map((file) => file.path)).toEqual([
+    "package.json",
+    "src/main.ts",
+  ]);
+  const { file, report } = await readDirectory(picked, new Set());
+  expect(report.title).toBe("Rtv7");
+  expect(file.documents.map((document) => document.kind)).toEqual(["app"]);
+  expect(report.missing).toEqual([
+    "problem_brief",
+    "constraints",
+    "approaches",
+    "product_requirements",
+    "design",
+    "build_plan",
+  ]);
+});
+
+test("an empty listing is no folder", () => {
+  expect(listedDirectory([])).toBeUndefined();
+});
+
+test("sizes read as people do", () => {
+  expect(describeBytes(512)).toBe("512 B");
+  expect(describeBytes(20 * 1024)).toBe("20 KB");
+  expect(describeBytes(1.5 * 1024 * 1024)).toBe("1.5 MB");
+});

apps/web/src/api/repository.ts

@@ -0,0 +1,101 @@
+import { EXCLUDED_DIRECTORIES } from "@corbits-builder/schemas/app-files";
+import {
+  readRepository,
+  type RepositoryFile,
+  type RepositoryImport,
+} from "@corbits-builder/schemas/repository-import";
+
+/** A folder the person picked: its name and its files, read lazily. */
+export type PickedDirectory = { name: string; files: RepositoryFile[] };
+
+// The File System Access API, which the DOM typings leave out: Chromium's folder picker hands
+// over a handle the page walks itself, so node_modules and .git are never listed.
+type FileHandle = { kind: "file"; name: string; getFile(): Promise<File> };
+type DirectoryHandle = {
+  kind: "directory";
+  name: string;
+  values(): AsyncIterable<FileHandle | DirectoryHandle>;
+};
+type DirectoryPicker = (options: { mode: "read" }) => Promise<DirectoryHandle>;
+
+const directoryPicker = () =>
+  (window as { showDirectoryPicker?: DirectoryPicker }).showDirectoryPicker;
+
+/** Whether the browser has a folder picker; otherwise a folder input lists the files. */
+export const canPickDirectory = () => directoryPicker() !== undefined;
+
+const lazily = (file: File, path: string): RepositoryFile => ({
+  path,
+  size: file.size,
+  bytes: async () => new Uint8Array(await file.arrayBuffer()),
+});
+
+async function walk(
+  handle: DirectoryHandle,
+  prefix: string,
+): Promise<RepositoryFile[]> {
+  const files: RepositoryFile[] = [];
+  for await (const entry of handle.values()) {
+    if (entry.kind === "directory") {
+      if (!EXCLUDED_DIRECTORIES.has(entry.name))
+        files.push(...(await walk(entry, `${prefix}${entry.name}/`)));
+    } else files.push(lazily(await entry.getFile(), prefix + entry.name));
+  }
+  return files;
+}
+
+/** Opens the folder picker; undefined when the person closes it without choosing. */
+export async function pickDirectory(): Promise<PickedDirectory | undefined> {
+  const picker = directoryPicker();
+  if (!picker) return undefined;
+  try {
+    const handle = await picker({ mode: "read" });
+    return { name: handle.name, files: await walk(handle, "") };
+  } catch (cause) {
+    if (cause instanceof DOMException && cause.name === "AbortError")
+      return undefined;
+    throw cause;
+  }
+}
+
+/** What a folder input lists: each file under the chosen folder's name. */
+export type ListedFile = {
+  name: string;
+  size: number;
+  webkitRelativePath: string;
+  arrayBuffer(): Promise<ArrayBuffer>;
+};
+
+/**
+ * The files a folder input lists, with the chosen folder's name taken off each path and the
+ * directories the app never keeps left out.
+ */
+export function listedDirectory(
+  listed: Iterable<ListedFile>,
+): PickedDirectory | undefined {
+  let name: string | undefined;
+  const files: RepositoryFile[] = [];
+  for (const file of listed) {
+    const parts = file.webkitRelativePath.split("/");
+    name ??= parts[0];
+    const path = parts.slice(1);
+    if (path.length === 0) continue;
+    if (path.some((part) => EXCLUDED_DIRECTORIES.has(part))) continue;
+    files.push(lazily(file as unknown as File, path.join("/")));
+  }
+  return name === undefined ? undefined : { name, files };
+}
+
+/** Reads a picked folder as a project, leaving out the paths the person unticked. */
+export const readDirectory = (
+  picked: PickedDirectory,
+  exclude: ReadonlySet<string>,
+): Promise<RepositoryImport> =>
+  readRepository(picked.files, { name: picked.name, exclude });
+
+/** A size as a person reads it. */
+export function describeBytes(bytes: number): string {
+  if (bytes < 1024) return `${bytes} B`;
+  if (bytes < 1024 * 1024) return `${Math.round(bytes / 1024)} KB`;
+  return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}

apps/web/src/config.ts

@@ -272,6 +272,25 @@ export const wording = {
     source: `A project ${productName} or the original Solution Builder exported`,
     invalid: `This file is not a project ${productName} or the original Solution Builder exported.`,
     omitted: (kinds: string[]) => `Not carried over: ${kinds.join(", ")}.`,
+    folderAction: "Import a folder",
+    folderSource:
+      "A folder of code, usually a git repository, to make a project from",
+    folderTitle: (title: string, folder: string) =>
+      `Import ${title} from ${folder}?`,
+    folderCode: (files: number, size: string) =>
+      `Code: ${files} files, ${size}. It is filed as the app, and the app builder starts from it.`,
+    folderNoCode:
+      "No package.json was found, so no code is filed; only the documents found are.",
+    folderNothing: "Nothing to import: no code and no documents were found.",
+    folderFound: (names: string) => `Found in the folder: ${names}.`,
+    folderToWrite: (names: string) =>
+      `${productName} writes with you, for your approval: ${names}.`,
+    folderLeaveOut: "Leave out:",
+    folderReasons: {
+      secret: "looks like a secret",
+      lockfile: `a lockfile ${productName} does not use`,
+    },
+    folderImport: "Import",
   },
   shell: {
     back: "Back",

apps/web/src/pages/home-layout.css

@@ -89,3 +89,20 @@
   gap: 8px;
   margin-top: 12px;
 }
+
+.home-page .import-folder-notes {
+  margin: 8px 0 0;
+  padding-inline-start: 1.25em;
+  font-size: 0.8125rem;
+  color: var(--wb-muted-foreground);
+}
+
+.home-page .import-folder-files {
+  display: grid;
+  gap: 4px;
+  margin-top: 12px;
+}
+
+.home-page .import-folder-files > p {
+  margin: 0;
+}