diff --git a/README.md b/README.md index 3bbd9e9a..5fead68a 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ the app or in `.env.hub` - [Self-hosting](https://github.com/corbitsdev/corbits-builder/blob/main/docs/SELF_HOST.md): running it with Docker. - [Development](https://github.com/corbitsdev/corbits-builder/blob/main/docs/DEV.md) and [Contributing](https://github.com/corbitsdev/corbits-builder/blob/main/CONTRIBUTING.md): working on the code. - [AGENTS.md](https://github.com/corbitsdev/corbits-builder/blob/main/AGENTS.md): conventions for coding agents. -- [skills/](https://github.com/corbitsdev/corbits-builder/tree/main/skills): agent skills to install Builder and drive it through its API. +- [skills/](https://github.com/corbitsdev/corbits-builder/tree/main/skills): agent skills to install Builder, drive it through its API and import a repository as a project. Builder runs on [Interchange](https://github.com/faremeter/interchange), which runs the agents and handles their messages, approvals and credentials. diff --git a/apps/hub/src/builds/app-snapshot.ts b/apps/hub/src/builds/app-snapshot.ts index c12d1799..218707e5 100644 --- a/apps/hub/src/builds/app-snapshot.ts +++ b/apps/hub/src/builds/app-snapshot.ts @@ -1,4 +1,3 @@ -import { type } from "arktype"; import ignore, { type Ignore } from "ignore"; import { constants } from "node:fs"; import { open, readdir, realpath } from "node:fs/promises"; @@ -11,28 +10,25 @@ import { ArtifactReceipt, DesignDraft, } from "@corbits-builder/schemas/artifact"; +import { + EXCLUDED_DIRECTORIES, + addBundleFile, + bundleContent, + excludedFile, +} from "@corbits-builder/schemas/app-files"; import { WorkflowHttpError, workflowHttpJson } from "../workflows/http-json.ts"; -import { INPUTS_DIRECTORY } from "../config/inputs.ts"; import { spliceDesignScreens } from "./starter.ts"; -// Installs and builds are rebuilt by setup and check-app, .builder and the inputs come from filed -// documents, and .scratch holds the coder's throwaway files, such as the screenshots it takes. -export const EXCLUDED_DIRECTORIES: ReadonlySet = new Set([ - "node_modules", - "dist", - ".git", - ".builder", - ".scratch", - INPUTS_DIRECTORY, -]); -// Secrets (environment files, keys, registry credentials) and local databases never leave the -// workspace; an example environment file holds no secret and documents the app's settings. -const EXCLUDED_FILE = - /^(\.env(\..*)?|\.npmrc|\.dev\.vars|.*\.(db|db-wal|db-shm|pem|key|p12)|.*\.sqlite.*)$/; -export const excludedFile = (name: string) => { - const lower = name.toLowerCase(); - return lower !== ".env.example" && EXCLUDED_FILE.test(lower); -}; +// The rules for which files are the app's and how they pack live with the schemas, so the UI's +// repository import and the import script pack a person's code the way the hub packs a workspace. +export { + EXCLUDED_DIRECTORIES, + addBundleFile, + bundleContent, + bundleFiles, + excludedFile, +} from "@corbits-builder/schemas/app-files"; + // Byte-order marks stay in the text, so a file reads back exactly as it was. const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); @@ -160,60 +156,11 @@ export async function collectApp( return bundle; } -// Keeps a UTF-8 file as text and any other base64-encoded under binary. -export function addBundleFile( - bundle: Required, - path: string, - bytes: Uint8Array, -) { - try { - bundle.files[path] = utf8.decode(bytes); - } catch (error) { - if (!(error instanceof TypeError)) throw error; - bundle.binary[path] = Buffer.from(bytes).toString("base64"); - } -} - -// A bundle's files as they are written to disk: text as is, binaries decoded. -export function bundleFiles(bundle: { - files: Readonly>; - binary?: Readonly>; -}): Record { - return { - ...bundle.files, - ...Object.fromEntries( - Object.entries(bundle.binary ?? {}).map(([path, data]) => [ - path, - Buffer.from(data, "base64"), - ]), - ), - }; -} - -// Keys in order, so the same files always make the same content and an unchanged app files nothing. -const sorted = (record: Record) => - Object.fromEntries( - Object.entries(record).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)), - ); - // The workspace as app document content, checked against the app limit. export async function appContent(workspace: string) { return bundleContent(await collectApp(workspace)).content; } -// Collected files as an app bundle and its document content. An app is as large as it is: a -// resource that runs out on the way, such as memory or a request body, reports its own error. -export function bundleContent(collected: Required) { - const files = sorted(collected.files); - const binary = sorted(collected.binary); - const bundle = AppBundle( - Object.keys(binary).length ? { files, binary } : { files }, - ); - if (bundle instanceof type.errors) - throw new Error("invalid_app: " + bundle.summary); - return { bundle, content: JSON.stringify(bundle) }; -} - const projectPath = (scope: { tenantId: string; projectId: string }) => "/api/tenants/" + encodeURIComponent(scope.tenantId) + diff --git a/apps/hub/src/config/import.ts b/apps/hub/src/config/import.ts index 88586e3e..d4e70711 100644 --- a/apps/hub/src/config/import.ts +++ b/apps/hub/src/config/import.ts @@ -4,14 +4,36 @@ const IMPORT_APP_MESSAGE = export const IMPORTED_PROJECT_LEAD = "I just imported this project from a file. Acknowledge the import, then tell me in a few plain sentences what the imported documents establish and what happens next."; +// The documents a project has before its app, in Builder's order. +export const PLANNED_DOCUMENT_KINDS = [ + "problem_brief", + "constraints", + "approaches", + "product_requirements", + "design", + "build_plan", +] as const; + // The importing person's opening message to a newly imported project's coordinator, naming the -// imported documents with no kind of their own so it reads them as the project's materials. +// imported documents with no kind of their own so it reads them as the project's materials, and, +// when the app came in without some of its documents (a person's own repository), which ones +// the coordinator writes with them from the code. export function importedProjectMessage( materials: readonly string[], withApp: boolean, + missing: readonly string[] = [], ): string { const filed = materials.length ? ` The documents with no place of their own here are filed as materials: ${materials.join(", ")}.` : ""; - return IMPORTED_PROJECT_LEAD + filed + (withApp ? IMPORT_APP_MESSAGE : ""); + const unwritten = + withApp && missing.length + ? ` The code came from my own repository, so these documents were never written: ${missing.join(", ")}. Read the code with read_artifact, its package.json and README first, and write the brief yourself from what it shows, asking me only what the code cannot tell you; then have the writers produce the rest in order, each instruction carrying what you learned from the code, and have me approve each one before the next.` + : ""; + return ( + IMPORTED_PROJECT_LEAD + + filed + + (withApp ? IMPORT_APP_MESSAGE : "") + + unwritten + ); } diff --git a/apps/hub/src/config/inputs.ts b/apps/hub/src/config/inputs.ts index ddf069d3..c993f4fd 100644 --- a/apps/hub/src/config/inputs.ts +++ b/apps/hub/src/config/inputs.ts @@ -1,7 +1,8 @@ // A warmed designer or coder workspace holds the filed documents it works from as read-only files, // and its hub-built assignment opens by naming them, so it starts from them instead of fetching // each one (a kept trace's coder made 54 document reads in 107 turns). -export const INPUTS_DIRECTORY = ".inputs"; +import { INPUTS_DIRECTORY } from "@corbits-builder/schemas/app-files"; +export { INPUTS_DIRECTORY }; const INPUTS = { problem_brief: { diff --git a/apps/hub/src/projects/routes.ts b/apps/hub/src/projects/routes.ts index e2e39967..52c492ae 100644 --- a/apps/hub/src/projects/routes.ts +++ b/apps/hub/src/projects/routes.ts @@ -41,7 +41,10 @@ import { } from "./next-step.ts"; import type { Actor } from "../lib/actor.ts"; import type { LiveApps } from "../live-app/runner.ts"; -import { importedProjectMessage } from "../config/import.ts"; +import { + PLANNED_DOCUMENT_KINDS, + importedProjectMessage, +} from "../config/import.ts"; import { MODEL_CHOICE_REFUSALS } from "../config/models.ts"; import { mailRunAsCaller } from "../mail/caller-mail.ts"; import { readJson } from "../lib/json.ts"; @@ -322,6 +325,7 @@ export function createProjectRoutes( importedProjectMessage( materials.map(({ title }) => title), kinds.includes("app"), + PLANNED_DOCUMENT_KINDS.filter((kind) => !kinds.includes(kind)), ), ); } catch (error) { diff --git a/apps/web/src/api/repository.test.ts b/apps/web/src/api/repository.test.ts new file mode 100644 index 00000000..0970da89 --- /dev/null +++ b/apps/web/src/api/repository.test.ts @@ -0,0 +1,47 @@ +import { expect, test } from "bun:test"; +import { describeBytes, listedDirectory, readDirectory } from "./repository.ts"; + +const listed = (path: string, text: string) => { + const blob = new Blob([text]); + return { + name: path.slice(path.lastIndexOf("/") + 1), + size: blob.size, + webkitRelativePath: path, + arrayBuffer: () => blob.arrayBuffer(), + }; +}; + +test("a folder input's files lose the folder's name and never include dependencies", async () => { + const picked = listedDirectory([ + listed("rtv7/package.json", '{"name":"rtv7"}'), + listed("rtv7/src/main.ts", "export {};"), + listed("rtv7/node_modules/left/index.js", ""), + listed("rtv7/.git/HEAD", "ref: refs/heads/main"), + ])!; + expect(picked.name).toBe("rtv7"); + expect(picked.files.map((file) => file.path)).toEqual([ + "package.json", + "src/main.ts", + ]); + const { file, report } = await readDirectory(picked, new Set()); + expect(report.title).toBe("Rtv7"); + expect(file.documents.map((document) => document.kind)).toEqual(["app"]); + expect(report.missing).toEqual([ + "problem_brief", + "constraints", + "approaches", + "product_requirements", + "design", + "build_plan", + ]); +}); + +test("an empty listing is no folder", () => { + expect(listedDirectory([])).toBeUndefined(); +}); + +test("sizes read as people do", () => { + expect(describeBytes(512)).toBe("512 B"); + expect(describeBytes(20 * 1024)).toBe("20 KB"); + expect(describeBytes(1.5 * 1024 * 1024)).toBe("1.5 MB"); +}); diff --git a/apps/web/src/api/repository.ts b/apps/web/src/api/repository.ts new file mode 100644 index 00000000..40c0e12d --- /dev/null +++ b/apps/web/src/api/repository.ts @@ -0,0 +1,101 @@ +import { EXCLUDED_DIRECTORIES } from "@corbits-builder/schemas/app-files"; +import { + readRepository, + type RepositoryFile, + type RepositoryImport, +} from "@corbits-builder/schemas/repository-import"; + +/** A folder the person picked: its name and its files, read lazily. */ +export type PickedDirectory = { name: string; files: RepositoryFile[] }; + +// The File System Access API, which the DOM typings leave out: Chromium's folder picker hands +// over a handle the page walks itself, so node_modules and .git are never listed. +type FileHandle = { kind: "file"; name: string; getFile(): Promise }; +type DirectoryHandle = { + kind: "directory"; + name: string; + values(): AsyncIterable; +}; +type DirectoryPicker = (options: { mode: "read" }) => Promise; + +const directoryPicker = () => + (window as { showDirectoryPicker?: DirectoryPicker }).showDirectoryPicker; + +/** Whether the browser has a folder picker; otherwise a folder input lists the files. */ +export const canPickDirectory = () => directoryPicker() !== undefined; + +const lazily = (file: File, path: string): RepositoryFile => ({ + path, + size: file.size, + bytes: async () => new Uint8Array(await file.arrayBuffer()), +}); + +async function walk( + handle: DirectoryHandle, + prefix: string, +): Promise { + const files: RepositoryFile[] = []; + for await (const entry of handle.values()) { + if (entry.kind === "directory") { + if (!EXCLUDED_DIRECTORIES.has(entry.name)) + files.push(...(await walk(entry, `${prefix}${entry.name}/`))); + } else files.push(lazily(await entry.getFile(), prefix + entry.name)); + } + return files; +} + +/** Opens the folder picker; undefined when the person closes it without choosing. */ +export async function pickDirectory(): Promise { + const picker = directoryPicker(); + if (!picker) return undefined; + try { + const handle = await picker({ mode: "read" }); + return { name: handle.name, files: await walk(handle, "") }; + } catch (cause) { + if (cause instanceof DOMException && cause.name === "AbortError") + return undefined; + throw cause; + } +} + +/** What a folder input lists: each file under the chosen folder's name. */ +export type ListedFile = { + name: string; + size: number; + webkitRelativePath: string; + arrayBuffer(): Promise; +}; + +/** + * The files a folder input lists, with the chosen folder's name taken off each path and the + * directories the app never keeps left out. + */ +export function listedDirectory( + listed: Iterable, +): PickedDirectory | undefined { + let name: string | undefined; + const files: RepositoryFile[] = []; + for (const file of listed) { + const parts = file.webkitRelativePath.split("/"); + name ??= parts[0]; + const path = parts.slice(1); + if (path.length === 0) continue; + if (path.some((part) => EXCLUDED_DIRECTORIES.has(part))) continue; + files.push(lazily(file as unknown as File, path.join("/"))); + } + return name === undefined ? undefined : { name, files }; +} + +/** Reads a picked folder as a project, leaving out the paths the person unticked. */ +export const readDirectory = ( + picked: PickedDirectory, + exclude: ReadonlySet, +): Promise => + readRepository(picked.files, { name: picked.name, exclude }); + +/** A size as a person reads it. */ +export function describeBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${Math.round(bytes / 1024)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} diff --git a/apps/web/src/config.ts b/apps/web/src/config.ts index 1dd99f80..acf8bc9d 100644 --- a/apps/web/src/config.ts +++ b/apps/web/src/config.ts @@ -272,6 +272,25 @@ export const wording = { source: `A project ${productName} or the original Solution Builder exported`, invalid: `This file is not a project ${productName} or the original Solution Builder exported.`, omitted: (kinds: string[]) => `Not carried over: ${kinds.join(", ")}.`, + folderAction: "Import a folder", + folderSource: + "A folder of code, usually a git repository, to make a project from", + folderTitle: (title: string, folder: string) => + `Import ${title} from ${folder}?`, + folderCode: (files: number, size: string) => + `Code: ${files} files, ${size}. It is filed as the app, and the app builder starts from it.`, + folderNoCode: + "No package.json was found, so no code is filed; only the documents found are.", + folderNothing: "Nothing to import: no code and no documents were found.", + folderFound: (names: string) => `Found in the folder: ${names}.`, + folderToWrite: (names: string) => + `${productName} writes with you, for your approval: ${names}.`, + folderLeaveOut: "Leave out:", + folderReasons: { + secret: "looks like a secret", + lockfile: `a lockfile ${productName} does not use`, + }, + folderImport: "Import", }, shell: { back: "Back", diff --git a/apps/web/src/pages/home-layout.css b/apps/web/src/pages/home-layout.css index b3f90042..cfdf64e6 100644 --- a/apps/web/src/pages/home-layout.css +++ b/apps/web/src/pages/home-layout.css @@ -89,3 +89,20 @@ gap: 8px; margin-top: 12px; } + +.home-page .import-folder-notes { + margin: 8px 0 0; + padding-inline-start: 1.25em; + font-size: 0.8125rem; + color: var(--wb-muted-foreground); +} + +.home-page .import-folder-files { + display: grid; + gap: 4px; + margin-top: 12px; +} + +.home-page .import-folder-files > p { + margin: 0; +} diff --git a/apps/web/src/pages/projects.tsx b/apps/web/src/pages/projects.tsx index f07df24e..5eca16a2 100644 --- a/apps/web/src/pages/projects.tsx +++ b/apps/web/src/pages/projects.tsx @@ -8,6 +8,7 @@ import { wording } from "../config.ts"; import { toast } from "sonner"; import { + Checkbox, PromptInput, PromptInputSubmit, PromptInputTextarea, @@ -21,6 +22,14 @@ import { importProject, readImportFile, } from "../api/projects.ts"; +import { + canPickDirectory, + describeBytes, + listedDirectory, + pickDirectory, + readDirectory, + type PickedDirectory, +} from "../api/repository.ts"; import { Banner, Button } from "../components.jsx"; import { ConnectProvider } from "./providers.jsx"; import { NewProjectProfile } from "./profile-select.jsx"; @@ -30,6 +39,29 @@ import { COMPOSER_BOX_CLASS } from "./workspace/pane-classes.ts"; import { HOME_COMPOSER_PLACEHOLDER, canStartProject } from "./home-view.ts"; import "./home-layout.css"; import type { ProjectImport } from "@corbits-builder/schemas/project-import"; +import { + DOCUMENT_LABELS, + type RepositoryImport, + type SuggestedReason, +} from "@corbits-builder/schemas/repository-import"; + +// A folder read as a project, with the files the reading suggests leaving out, ticked from the +// start, which the person can keep. +type PendingRepository = { + picked: PickedDirectory; + reading: RepositoryImport; + choices: Map; + excluded: Set; +}; + +function repositoryChoices( + previous: PendingRepository["choices"], + { report }: RepositoryImport, +): PendingRepository["choices"] { + const choices = new Map(previous); + for (const { path, reason } of report.suggested) choices.set(path, reason); + return choices; +} export function Projects({ onOpen, @@ -82,6 +114,9 @@ export function Projects({ name: string; body: ProjectImport; } | null>(null); + const folderInput = useRef(null); + const [pendingRepository, setPendingRepository] = + useState(null); const failImport = (name: string, cause: unknown, id?: string | number) => toast.error( cause instanceof ApiFailure ? cause.detail.message : String(cause), @@ -99,6 +134,55 @@ export function Projects({ if (importInput.current) importInput.current.value = ""; } }; + // The folder is read in the browser and shown before anything is created; a file the reading + // suggests leaving out starts out left out, and the person can leave out more until the code fits. + const readFolder = async ( + picked: PickedDirectory, + excluded: Set, + choices: PendingRepository["choices"], + ) => { + const reading = await readDirectory(picked, excluded); + const next = repositoryChoices(choices, reading); + const fresh = reading.report.suggested + .map(({ path }) => path) + .filter((path) => !choices.has(path)); + if (fresh.length > 0) + return readFolder(picked, new Set([...excluded, ...fresh]), next); + setPendingRepository({ picked, reading, choices: next, excluded }); + }; + const chooseFolder = async (picked: PickedDirectory | undefined) => { + if (!picked) return; + setPendingImport(null); + setBusy("import"); + try { + await readFolder(picked, new Set(), new Map()); + } catch (cause) { + failImport(picked.name, cause); + } finally { + setBusy(null); + if (folderInput.current) folderInput.current.value = ""; + } + }; + const pickFolder = async () => chooseFolder(await pickDirectory()); + const toggleExcluded = async (path: string, excluded: boolean) => { + if (!pendingRepository) return; + const next = new Set(pendingRepository.excluded); + if (excluded) next.add(path); + else next.delete(path); + setBusy("import"); + try { + await readFolder( + pendingRepository.picked, + next, + pendingRepository.choices, + ); + } catch (cause) { + failImport(pendingRepository.picked.name, cause); + } finally { + setBusy(null); + } + }; + const importFile = async ( name: string, { file, omitted }: ProjectImport, @@ -205,6 +289,30 @@ export function Projects({ if (file) void chooseImport(file); }} /> + + { + const listed = event.target.files; + if (listed) void chooseFolder(listedDirectory(listed)); + }} + /> {pendingImport === null ? null : ( )} + {pendingRepository === null ? null : ( + void toggleExcluded(path, excluded)} + onImport={() => + void importFile( + pendingRepository.picked.name, + { file: pendingRepository.reading.file, omitted: [] }, + false, + ) + } + onCancel={() => setPendingRepository(null)} + /> + )} ); } + +function RepositoryBanner({ + pending: { picked, reading, choices, excluded }, + busy, + onExclude, + onImport, + onCancel, +}: { + pending: PendingRepository; + busy: boolean; + onExclude: (path: string, excluded: boolean) => void; + onImport: () => void; + onCancel: () => void; +}) { + const { report, file } = reading; + const text = wording.importProject; + const labels = (kinds: readonly string[]) => + kinds + .map((kind) => DOCUMENT_LABELS[kind as keyof typeof DOCUMENT_LABELS]) + .join(", "); + const found = file.documents + .filter((document) => document.kind !== "app") + .map((document) => document.kind); + const toWrite = report.missing.filter((kind) => kind !== "app"); + const nothing = file.documents.length === 0; + const code = report.app + ? text.folderCode(report.app.files, describeBytes(report.app.bytes)) + : text.folderNoCode; + return ( + +

{nothing ? text.folderNothing : code}

+ {found.length > 0 ?

{text.folderFound(labels(found))}

: null} + {toWrite.length > 0 && !nothing ? ( +

{text.folderToWrite(labels(toWrite))}

+ ) : null} + {report.warnings.length > 0 ? ( +
    + {report.warnings.map((warning) => ( +
  • {warning}
    • + ))} +
+ ) : null} + {choices.size > 0 ? ( +
+

{text.folderLeaveOut}

+ {[...choices].map(([path, reason]) => ( + onExclude(path, checked === true)} + /> + ))} +
+ ) : null} +
+ + +
+ + ); +} diff --git a/bun.lock b/bun.lock index 0c3bd480..f339c17a 100644 --- a/bun.lock +++ b/bun.lock @@ -166,6 +166,7 @@ "version": "0.1.0-rc.3", "dependencies": { "arktype": "catalog:", + "ignore": "5.3.2", }, }, "packages/slidedeck": { @@ -630,7 +631,7 @@ }, "workflows/coordinator": { "name": "@corbits-builder/coordinator", - "version": "0.35.0", + "version": "0.35.1", "dependencies": { "@intx/agent": "workspace:*", "@intx/workflow": "workspace:*", @@ -638,7 +639,7 @@ }, "workflows/designer": { "name": "@corbits-builder/designer", - "version": "0.1.0", + "version": "0.2.0", "dependencies": { "@intx/agent": "workspace:*", "@intx/workflow": "workspace:*", @@ -662,7 +663,7 @@ }, "workflows/writer": { "name": "@corbits-builder/writer", - "version": "0.7.0", + "version": "0.7.1", "dependencies": { "@intx/agent": "workspace:*", "@intx/workflow": "workspace:*", diff --git a/docs/DEV.md b/docs/DEV.md index 4d410784..c2dd88d1 100644 --- a/docs/DEV.md +++ b/docs/DEV.md @@ -15,7 +15,7 @@ running with Docker Compose is in | `apps/web` | The UI: Vite and React over that client, with its own small router in `src/routes.ts`. | | `workflows/*` | One package per role: its agent, prompt and workflow definition; the hub supplies its tools. | | `packages/schemas` | Types and schemas shared by the hub, the client and the workflows. | -| `scripts/` | Dev and verification scripts: pin and vendor checks, the commit check, the Google sign-in emulator and React UI API generation. | +| `scripts/` | Dev and verification scripts: pin and vendor checks, the commit check, the Google sign-in emulator, React UI API generation and the repository import (`bun run import:repo`, which writes a directory as a project file the import reads). | | `deploy/` | The Dockerfile and Compose files ([SELF_HOST.md](https://github.com/corbitsdev/corbits-builder/blob/main/docs/SELF_HOST.md)). | The roles: @@ -94,6 +94,7 @@ real API ([CONTRIBUTING.md](https://github.com/corbitsdev/corbits-builder/blob/m | `bun run auth:bootstrap` | Migrates and creates the `BOOTSTRAP_*` machine account and workspace. | | `bun run auth:emulate` | Starts a loopback Google sign-in emulator (`GOOGLE_EMULATE_URL` in `.env.hub.example`). | | `bun run dev:seed` | Seeds dummy projects (`habit-tracker`, `field-service-dispatch`, `clinical-trial-intake`) at a chosen lifecycle stage. | +| `bun run import:repo` | Writes a directory, usually a git repository, as the project file **Import project** reads and names the Builder documents it lacks (`--help` lists the options). | | `bun run vendor:verify` | Checks `vendor/interchange` matches its pinned upstream commit. | | `bun run skills:react-ui` | Regenerates the designer's `@corbits/react-ui` API reference. | diff --git a/docs/USER_MANUAL.md b/docs/USER_MANUAL.md index 9e1832db..77942985 100644 --- a/docs/USER_MANUAL.md +++ b/docs/USER_MANUAL.md @@ -329,6 +329,18 @@ pick the `project.json` file from an export. If the file marks documents as approved, you choose **Import and approve** (records your own approval of exactly those versions) or **Import without approvals**. +To make a project from code you already have, choose **Import a folder** on +the new-project page and pick the folder, usually a git repository. Builder +reads it the way it reads its own exports: the code becomes the app, and any +planning documents or design kept in the folder are filed with it. Dependencies, +build output, environment files and whatever the folder's `.gitignore` files +leave out are never read. A card then shows what was found, what Builder still +has to write with you, and files you may want to leave out: those whose names +suggest a secret, and lockfiles of package managers Builder does not use, start +out left out, and you can keep any of them. Choose **Import**. The coordinator reads the code, asks you what you want to do and +what the code cannot tell it, and writes the missing documents for your +approval, one at a time, before any building starts. + ## 9. Manage projects ### The projects list diff --git a/package.json b/package.json index c971c109..19f22d42 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "auth:bootstrap": "bun --env-file=.env.hub --conditions=intx-src apps/hub/src/setup.ts", "auth:emulate": "bun --env-file=.env.hub scripts/google-emulate.ts", "dev:seed": "bun --env-file=.env.hub --conditions=intx-src scripts/seed-projects.ts", + "import:repo": "bun --conditions=intx-src scripts/import-repo.ts", "lint": "bun --bun run oxlint --deny-warnings", "format": "bun fmt --no-script", "format:check": "bun fmt --no-script --check", diff --git a/packages/schemas/package.json b/packages/schemas/package.json index d66d8971..88335fce 100644 --- a/packages/schemas/package.json +++ b/packages/schemas/package.json @@ -25,9 +25,12 @@ "./audit": "./src/audit.ts", "./questions": "./src/questions.ts", "./approaches": "./src/approaches.ts", - "./approval-round": "./src/approval-round.ts" + "./approval-round": "./src/approval-round.ts", + "./app-files": "./src/app-files.ts", + "./repository-import": "./src/repository-import.ts" }, "dependencies": { - "arktype": "catalog:" + "arktype": "catalog:", + "ignore": "5.3.2" } } diff --git a/packages/schemas/src/app-files.ts b/packages/schemas/src/app-files.ts new file mode 100644 index 00000000..faf4b502 --- /dev/null +++ b/packages/schemas/src/app-files.ts @@ -0,0 +1,99 @@ +import { type } from "arktype"; +import { AppBundle } from "./artifact.ts"; + +// The rules for which of a workspace's or a repository's files are the app's, and how they are +// packed as an app document. The hub packs a coder's workspace with them, the UI and the import +// script pack a person's own repository with them, so the three agree on what leaves out. + +// A warmed designer or coder workspace holds the filed documents it works from here. +export const INPUTS_DIRECTORY = ".inputs"; + +// Installs and builds are rebuilt by setup and check-app, .builder and the inputs come from filed +// documents, and .scratch holds the coder's throwaway files, such as the screenshots it takes. +export const EXCLUDED_DIRECTORIES: ReadonlySet = new Set([ + "node_modules", + "dist", + ".git", + ".builder", + ".scratch", + INPUTS_DIRECTORY, +]); + +// Secrets (environment files, keys, registry credentials) and local databases never leave the +// workspace; an example environment file holds no secret and documents the app's settings. +const EXCLUDED_FILE = + /^(\.env(\..*)?|\.npmrc|\.dev\.vars|.*\.(db|db-wal|db-shm|pem|key|p12)|.*\.sqlite.*)$/; +export const excludedFile = (name: string) => { + const lower = name.toLowerCase(); + return lower !== ".env.example" && EXCLUDED_FILE.test(lower); +}; + +// Byte-order marks stay in the text, so a file reads back exactly as it was. +const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +const encoder = new TextEncoder(); + +/** The bytes a string takes as UTF-8. */ +export const utf8Length = (text: string) => encoder.encode(text).length; + +// Base64 without Buffer, so a browser packs an app the same way the hub does. +const CHUNK = 0x8000; +export function base64Encode(bytes: Uint8Array): string { + let binary = ""; + for (let at = 0; at < bytes.length; at += CHUNK) + binary += String.fromCharCode(...bytes.subarray(at, at + CHUNK)); + return btoa(binary); +} +export function base64Decode(data: string): Uint8Array { + const binary = atob(data); + const bytes = new Uint8Array(binary.length); + for (let at = 0; at < binary.length; at++) bytes[at] = binary.charCodeAt(at); + return bytes; +} + +// Keeps a UTF-8 file as text and any other base64-encoded under binary. +export function addBundleFile( + bundle: Required, + path: string, + bytes: Uint8Array, +) { + try { + bundle.files[path] = utf8.decode(bytes); + } catch (error) { + if (!(error instanceof TypeError)) throw error; + bundle.binary[path] = base64Encode(bytes); + } +} + +// A bundle's files as they are written to disk: text as is, binaries decoded. +export function bundleFiles(bundle: { + files: Readonly>; + binary?: Readonly>; +}): Record { + const files: Record = { ...bundle.files }; + if (bundle.binary) + for (const [path, data] of Object.entries(bundle.binary)) + files[path] = base64Decode(data); + return files; +} + +// Keys in order, so the same files always make the same content and an unchanged app files nothing. +const sorted = (record: Record) => + Object.fromEntries( + Object.entries(record).sort(([a], [b]) => { + if (a < b) return -1; + return a > b ? 1 : 0; + }), + ); + +// Collected files as an app bundle and its document content. An app is as large as it is: a +// resource that runs out on the way, such as memory or a request body, reports its own error. +export function bundleContent(collected: Required) { + const files = sorted(collected.files); + const binary = sorted(collected.binary); + const bundle = AppBundle( + Object.keys(binary).length ? { files, binary } : { files }, + ); + if (bundle instanceof type.errors) + throw new Error("invalid_app: " + bundle.summary); + return { bundle, content: JSON.stringify(bundle) }; +} diff --git a/packages/schemas/src/repository-import.test.ts b/packages/schemas/src/repository-import.test.ts new file mode 100644 index 00000000..0f77384f --- /dev/null +++ b/packages/schemas/src/repository-import.test.ts @@ -0,0 +1,308 @@ +import { describe, expect, test } from "bun:test"; +import { AppBundle, DesignBundle } from "./artifact.ts"; +import { ProjectExport } from "./project.ts"; +import { + describeRepositoryImport, + documentTitle, + humanize, + looksSecret, + readRepository, + suggestedReason, + type RepositoryFile, +} from "./repository-import.ts"; + +const encoder = new TextEncoder(); +const memory = (files: Record): RepositoryFile[] => + Object.entries(files).map(([path, content]) => { + const bytes = + typeof content === "string" ? encoder.encode(content) : content; + return { path, size: bytes.length, bytes: () => Promise.resolve(bytes) }; + }); + +// A person's own app: code, a README, a nested .gitignore, a secret, a binary and the planning +// documents of Builder's seed layout. +const APP = { + "package.json": JSON.stringify({ + name: "@acme/habit-tracker", + scripts: { dev: "bun run src/main.ts", build: "bun build src/main.ts" }, + }), + "README.md": "# Habit tracker\n", + ".gitignore": "*.log\nbuild-output/\n", + "src/.gitignore": "generated/\n", + "src/main.ts": "console.log('hi');\n", + "src/generated/types.ts": "ignored by the nested file\n", + "build-output/bundle.js": "ignored\n", + "debug.log": "ignored\n", + "node_modules/left/index.js": "never the app's\n", + ".env": "SECRET=1\n", + ".env.example": "SECRET=\n", + "config/deepl-key.txt": "k3y\n", + "assets/dot.png": new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0, 0xff, 0xfe]), + "docs/problem_brief.md": "# Problem Brief: Habit tracker\n\n## In short\n", + "docs/build_plan.md": "# Build Plan: Habit tracker\n\n## Phases\n", +}; + +describe("readRepository", () => { + test("files the code as the app, honours .gitignore files and names what is missing", async () => { + const { file, report } = await readRepository(memory(APP), { + name: "habit-tracker", + }); + expect(ProjectExport(file)).toEqual(file); + expect(report.title).toBe("Habit tracker"); + expect(report.found).toEqual({ + problem_brief: "docs/problem_brief.md", + build_plan: "docs/build_plan.md", + app: ".", + }); + expect(report.missing).toEqual([ + "constraints", + "approaches", + "product_requirements", + "design", + ]); + const app = file.documents.find((document) => document.kind === "app")!; + const bundle = AppBundle.assert(JSON.parse(app.content)); + expect(Object.keys(bundle.files)).toEqual([ + ".env.example", + ".gitignore", + "README.md", + "config/deepl-key.txt", + "docs/build_plan.md", + "docs/problem_brief.md", + "package.json", + "src/.gitignore", + "src/main.ts", + ]); + expect(Object.keys(bundle.binary ?? {})).toEqual(["assets/dot.png"]); + expect(report.suggested).toEqual([ + { path: "config/deepl-key.txt", reason: "secret" }, + ]); + expect(report.app).toEqual({ + root: ".", + files: 10, + bytes: report.app!.bytes, + }); + expect(report.warnings).toEqual([ + expect.stringContaining('no "typecheck" script'), + expect.stringContaining("no bun.lock"), + ]); + expect(file.documents.every((document) => !document.approved)).toBe(true); + const brief = file.documents.find( + (document) => document.kind === "problem_brief", + )!; + expect(brief.title).toBe("Problem Brief: Habit tracker"); + const described = describeRepositoryImport(report); + expect(described).toContain("constraints: missing"); + expect(described).toContain( + "consider leaving out config/deepl-key.txt: looks like a secret", + ); + }); + + test("leaves out the paths the person chose and trusts git's own listing", async () => { + const { file, report } = await readRepository(memory(APP), { + name: "habit-tracker", + exclude: new Set(["config/deepl-key.txt", "assets/dot.png"]), + gitignore: false, + }); + const bundle = AppBundle.assert( + JSON.parse(file.documents.find((d) => d.kind === "app")!.content), + ); + expect(bundle.binary).toBeUndefined(); + expect(Object.keys(bundle.files)).toContain("debug.log"); + expect(Object.keys(bundle.files)).not.toContain("config/deepl-key.txt"); + expect(Object.keys(bundle.files)).not.toContain(".env"); + expect(report.suggested).toEqual([]); + }); + + test("takes drafts over the directory's files, approves named kinds and lays app files over", async () => { + const drafts = memory({ + "problem_brief.md": "# Brief, revised\n\ntext\n", + "planning/constraints.md": "# Constraints\n\n- Web only\n", + "approaches.md": "# Approaches\n\n## Option 1\n", + "approaches.json": JSON.stringify({ + recommended: "a", + approaches: [ + { + id: "a", + name: "A", + summary: "s", + bestFor: "b", + pros: ["p"], + cons: ["c"], + effort: "low", + risks: [], + }, + { + id: "b", + name: "B", + summary: "s", + bestFor: "b", + pros: ["p"], + cons: ["c"], + effort: "high", + risks: ["r"], + }, + ], + }), + "design/DESIGN.md": '---\nname: "Calm"\n---\n# Design\n', + "design/designs/index.html": '
', + "design/designs/screens/home.html": + '
', + "app/bun.lock": "{}\n", + }); + const { file, report } = await readRepository(memory(APP), { + name: "habit-tracker", + drafts, + title: " Streaks ", + approve: ["problem_brief", "design", "product_requirements"], + }); + expect(report.title).toBe("Streaks"); + expect(report.missing).toEqual(["product_requirements"]); + expect(report.found.problem_brief).toBe("drafts/problem_brief.md"); + expect(report.found.design).toBe("drafts/design/DESIGN.md"); + expect(report.warnings).toEqual([ + expect.stringContaining('no "typecheck" script'), + "product_requirements was to be approved but was not found", + ]); + const byKind = Object.fromEntries( + file.documents.map((document) => [document.kind, document]), + ); + expect(byKind.problem_brief!.content).toBe("# Brief, revised\n\ntext\n"); + expect(byKind.problem_brief!.approved).toBe(true); + expect(byKind.constraints!.approved).toBe(false); + expect(byKind.approaches!.approaches?.recommended).toBe("a"); + expect(byKind.design!.title).toBe("Calm"); + expect(byKind.design!.approved).toBe(true); + expect( + Object.keys( + DesignBundle.assert(JSON.parse(byKind.design!.content)).files, + ).sort(), + ).toEqual(["DESIGN.md", "designs/index.html", "designs/screens/home.html"]); + const bundle = AppBundle.assert(JSON.parse(byKind.app!.content)); + expect(bundle.files["bun.lock"]).toBe("{}\n"); + }); + + test("reads an unpacked Builder export, with the code under build/ standing over the file's app", async () => { + const exported = ProjectExport.assert({ + format: "corbits-builder.project", + version: 1, + title: "Exported", + documents: [ + { + kind: "problem_brief", + title: "Brief", + content: "brief", + approved: true, + }, + { + kind: "design", + title: "Look", + content: JSON.stringify({ + files: { + "DESIGN.md": "# Look", + "designs/index.html": "
", + }, + }), + approved: true, + }, + { + kind: "app", + title: "App", + content: JSON.stringify({ + files: { "package.json": "{}", "old.ts": "" }, + }), + approved: false, + }, + ], + }); + const unpacked = { + "project.json": JSON.stringify(exported), + "planning/problem_brief.md": "# Stale copy\n", + "planning/constraints.md": "# Constraints from planning\n", + "build/package.json": JSON.stringify({ name: "exported-app" }), + "build/new.ts": "", + }; + const { file, report } = await readRepository(memory(unpacked), { + approve: "all", + }); + expect(report.title).toBe("Exported"); + expect(report.found).toEqual({ + problem_brief: "project.json", + constraints: "planning/constraints.md", + design: "project.json", + app: "build", + }); + const byKind = Object.fromEntries( + file.documents.map((document) => [document.kind, document]), + ); + expect(byKind.problem_brief!.content).toBe("brief"); + expect( + Object.keys(AppBundle.assert(JSON.parse(byKind.app!.content)).files), + ).toEqual(["new.ts", "package.json"]); + expect(file.documents.every((document) => document.approved)).toBe(true); + + // With no code beside it, the file's own app is what there is. + const { + "build/package.json": _p, + "build/new.ts": _n, + ...noCode + } = unpacked; + const again = await readRepository(memory(noCode)); + expect(again.report.found.app).toBe("project.json"); + expect(again.report.app).toEqual({ + root: "project.json", + files: 2, + bytes: again.report.app!.bytes, + }); + }); + + test("a directory with nothing Builder knows is an empty project named after itself", async () => { + const { file, report } = await readRepository( + memory({ "notes.md": "# Notes\n" }), + { name: "bare-notes" }, + ); + expect(report.title).toBe("Bare notes"); + expect(file.documents).toEqual([]); + expect(report.app).toBeUndefined(); + }); +}); + +describe("names", () => { + test("titles come from a package name, a directory name, a heading or front matter", () => { + expect(humanize("@acme/habit-tracker")).toBe("Habit tracker"); + expect(humanize("my_app.v2")).toBe("My app v2"); + expect(documentTitle("# Heading \n\ntext", "x")).toBe("Heading"); + expect(documentTitle("no heading", "Fallback")).toBe("Fallback"); + expect( + documentTitle( + '---\nname: "Warm (soft)"\ncolors:\n a: "#fff"\n---\n# Other\n', + "x", + ), + ).toBe("Warm (soft)"); + }); + + test("a file's name can say it holds a secret, unless it is source code", () => { + for (const path of [ + "DEEPL_KEY.txt", + "config/resend-key.txt", + "secrets.json", + "private.pem", + "passwords.csv", + "credentials", + ]) + expect(looksSecret(path)).toBe(true); + for (const path of [ + "lib/auth/password.ts", + "lib/token.go", + "keyboard.ts", + "src/main.ts", + "README.md", + "monkey.png", + ]) + expect(looksSecret(path)).toBe(false); + expect(suggestedReason("package-lock.json")).toBe("lockfile"); + expect(suggestedReason("yarn.lock")).toBe("lockfile"); + expect(suggestedReason("bun.lock")).toBeUndefined(); + expect(suggestedReason("api-key.txt")).toBe("secret"); + }); +}); diff --git a/packages/schemas/src/repository-import.ts b/packages/schemas/src/repository-import.ts new file mode 100644 index 00000000..17cae9b3 --- /dev/null +++ b/packages/schemas/src/repository-import.ts @@ -0,0 +1,556 @@ +import ignore, { type Ignore } from "ignore"; +import { type } from "arktype"; +import { ApproachesMeta } from "./approaches.ts"; +import { AppBundle, DesignBundle } from "./artifact.ts"; +import { ProjectExport } from "./project.ts"; +import { + EXCLUDED_DIRECTORIES, + addBundleFile, + bundleContent, + excludedFile, + utf8Length, +} from "./app-files.ts"; + +// A person's own directory, usually a git repository, read as the project file the import takes, +// as if Builder had made the project: the code becomes the app document, and the planning +// documents and the design are taken from wherever the directory keeps them. The report names +// each document Builder expects that the directory lacks, for the coordinator to write with the +// person once the project exists, or for whoever runs the import to draft first. + +export const REPOSITORY_DOCUMENT_KINDS = [ + "problem_brief", + "constraints", + "approaches", + "product_requirements", + "design", + "build_plan", + "app", +] as const; +export type RepositoryDocumentKind = (typeof REPOSITORY_DOCUMENT_KINDS)[number]; +type TextKind = Exclude; + +export const DOCUMENT_LABELS: Record = { + problem_brief: "Problem brief", + constraints: "Constraints", + approaches: "Approaches", + product_requirements: "Product requirements", + design: "Design", + build_plan: "Build plan", + app: "App", +}; + +// Where a directory may keep each text document: an unpacked Builder export (planning/), the +// layout of Builder's seed fixtures (docs/) and a coder workspace's inputs (.inputs/). +const TEXT_PATHS: Record = { + problem_brief: [ + "planning/problem_brief.md", + "docs/problem_brief.md", + ".inputs/brief.md", + ], + constraints: [ + "planning/constraints.md", + "docs/constraints.md", + ".inputs/constraints.md", + ], + approaches: [ + "planning/approaches.md", + "docs/approaches.md", + ".inputs/approaches.md", + ], + product_requirements: [ + "planning/product_requirements.md", + "docs/product_requirements.md", + ".inputs/requirements.md", + ], + build_plan: [ + "planning/build_plan.md", + "docs/build_plan.md", + ".inputs/build-plan.md", + ], +}; +const TEXT_KINDS = Object.keys(TEXT_PATHS) as TextKind[]; +// Where a directory may keep the design bundle: an unpacked export's design/ or a coder +// workspace's .builder/. Drafts keep it under design/. +const DESIGN_DIRECTORIES = ["design", ".builder"]; +// Where the app's package.json may sit: the directory itself, an unpacked export's build/ or +// the seed fixtures' app/. +const APP_DIRECTORIES = ["", "build", "app"]; +const PROJECT_FILE = "project.json"; +// The scripts the app starter gives a Builder app; the coder and check-app run them. +const STARTER_SCRIPTS = ["dev", "build", "typecheck"]; +const TITLE_MAX = 512; +// A file whose name says it holds a secret and that is not source code, which the rules above +// do not already leave out: the person decides, it is only pointed at. Source files named for +// what they handle (password.ts, token.go) are the app's own. +const SECRET_NAME = + /(secret|token|credential|password|passwd|private|(^|[-_.])key([-_.]|$))/i; +const SOURCE_EXTENSION = + /\.(c|cc|cpp|cs|css|go|h|hpp|html|java|js|jsx|kt|m|mjs|cjs|php|py|rb|rs|scss|sh|sql|svelte|swift|ts|tsx|vue)$/i; +// Lockfiles of package managers the coder's sandbox does not run; it installs from bun.lock. +const FOREIGN_LOCKFILES = new Set([ + "package-lock.json", + "npm-shrinkwrap.json", + "yarn.lock", + "pnpm-lock.yaml", +]); + +export type RepositoryDocument = ProjectExport["documents"][number]; + +/** One file of the directory: its path relative to the directory, with / between parts. */ +export type RepositoryFile = { + path: string; + size: number; + bytes: () => Promise; +}; + +export type RepositoryImportOptions = { + /** The directory's own name, the title of last resort. */ + name?: string | undefined; + /** The project's title, over the one the directory suggests. */ + title?: string | undefined; + /** The kinds the person approved, or every kind found. */ + approve?: readonly RepositoryDocumentKind[] | "all" | undefined; + /** App paths to leave out, relative to the directory. */ + exclude?: ReadonlySet | undefined; + /** + * Drafted documents, standing over the directory's: .md and approaches.json at the root + * or under planning/, a design under design/, files to lay over the app under app/. + */ + drafts?: readonly RepositoryFile[] | undefined; + /** Whether the directory's .gitignore files apply; off when git listed the files itself. */ + gitignore?: boolean | undefined; +}; + +export type RepositoryImportReport = { + title: string; + /** Each kind found, with the path it came from. */ + found: Partial>; + missing: RepositoryDocumentKind[]; + app?: { root: string; files: number; bytes: number } | undefined; + /** App files the person would likely leave out, each with why, for them to decide. */ + suggested: { path: string; reason: SuggestedReason }[]; + warnings: string[]; +}; + +/** Why a file is suggested for leaving out: it looks like a secret, or it is a lockfile of a package manager the sandbox does not run. */ +export type SuggestedReason = "secret" | "lockfile"; + +export type RepositoryImport = { + file: ProjectExport; + report: RepositoryImportReport; +}; + +type Found = { document: RepositoryDocument; source: string }; + +const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +const basename = (path: string) => path.slice(path.lastIndexOf("/") + 1); +const joined = (directory: string, path: string) => + directory ? `${directory}/${path}` : path; +const under = (directory: string, path: string) => + directory ? path.startsWith(directory + "/") : true; +const within = (directory: string, path: string) => + directory ? path.slice(directory.length + 1) : path; + +/** The first Markdown heading, or the YAML front matter's name, names a document. */ +export function documentTitle(content: string, fallback: string): string { + const name = /^---\n(?:.*\n)*?name:\s*"?([^"\n]+?)"?\s*\n(?:.*\n)*?---/.exec( + content, + ); + const heading = /^#\s+(.+?)\s*#*\s*$/m.exec(content); + const title = name?.[1]?.trim() || heading?.[1]?.trim() || fallback; + return title.slice(0, TITLE_MAX); +} + +/** A package name or directory name as a title: "@acme/habit-tracker" becomes "Habit tracker". */ +export function humanize(name: string): string { + const words = name + .replace(/^@[^/]+\//, "") + .replace(/[-_.]+/g, " ") + .trim(); + return words ? words[0]!.toUpperCase() + words.slice(1) : name; +} + +const parseJson = (text: string | undefined): unknown => { + if (text === undefined) return undefined; + try { + return JSON.parse(text); + } catch (error) { + if (error instanceof SyntaxError) return undefined; + throw error; + } +}; + +/** Whether a file's name says it holds a secret, and it is not source code. */ +export const looksSecret = (path: string) => { + const name = basename(path); + return SECRET_NAME.test(name) && !SOURCE_EXTENSION.test(name); +}; + +/** Why a file would likely be left out of the app, or undefined when there is no reason to. */ +export function suggestedReason(path: string): SuggestedReason | undefined { + if (FOREIGN_LOCKFILES.has(basename(path))) return "lockfile"; + if (looksSecret(path)) return "secret"; + return undefined; +} + +// A file's text, or undefined when it is not UTF-8. +async function decodeText(file: RepositoryFile): Promise { + const bytes = await file.bytes(); + try { + return utf8.decode(bytes); + } catch (error) { + if (error instanceof TypeError) return undefined; + throw error; + } +} + +// The files by path, with each read once as text or bytes. +class Files { + readonly byPath = new Map(); + private readonly texts = new Map>(); + constructor(files: readonly RepositoryFile[] = []) { + for (const file of files) this.byPath.set(file.path, file); + } + has(path: string) { + return this.byPath.has(path); + } + paths() { + return [...this.byPath.keys()].sort(); + } + // The file's text, or undefined when there is no such file or it is not UTF-8. + text(path: string): Promise { + const file = this.byPath.get(path); + if (!file) return Promise.resolve(undefined); + let text = this.texts.get(path); + if (!text) { + text = decodeText(file); + this.texts.set(path, text); + } + return text; + } +} + +// Which paths the directory's own .gitignore files leave out, as git would: each file's rules +// apply beneath its directory, to the path and to every directory on the way to it. +async function gitignored(files: Files): Promise<(path: string) => boolean> { + const rules: { base: string; rules: Ignore }[] = []; + for (const path of files.paths()) { + if (basename(path) !== ".gitignore") continue; + const text = await files.text(path); + if (text === undefined) continue; + const base = path.slice(0, -".gitignore".length).replace(/\/$/, ""); + rules.push({ base, rules: ignore().add(text) }); + } + return (path: string) => { + for (const { base, rules: scoped } of rules) { + if (!under(base, path)) continue; + const relativeParts = within(base, path).split("/"); + for (let depth = 1; depth <= relativeParts.length; depth++) { + const candidate = relativeParts.slice(0, depth).join("/"); + const isDirectory = depth < relativeParts.length; + if (scoped.ignores(isDirectory ? candidate + "/" : candidate)) + return true; + } + } + return false; + }; +} + +const excludedPath = (path: string) => + path.split("/").some((segment) => EXCLUDED_DIRECTORIES.has(segment)) || + excludedFile(basename(path)); + +// A root project.json in Builder's own format, as a Builder export unpacked into a repository has. +async function readProjectFile( + files: Files, +): Promise { + const file = ProjectExport(parseJson(await files.text(PROJECT_FILE))); + return file instanceof type.errors ? undefined : file; +} + +async function textDocument( + files: Files, + kind: TextKind, + path: string, +): Promise { + const content = await files.text(path); + if (!content?.trim()) return undefined; + const document: RepositoryDocument = { + kind, + title: documentTitle(content, DOCUMENT_LABELS[kind]), + content, + approved: false, + }; + if (kind === "approaches") { + const meta = parseJson(await files.text(path.replace(/\.md$/, ".json"))); + if (meta !== undefined) document.approaches = ApproachesMeta.assert(meta); + } + return { document, source: path }; +} + +// The design bundle kept under a directory: DESIGN.md and designs/ with index.html, as the +// designer files it, with every other text file beneath the directory alongside. +async function designDocument( + files: Files, + directory: string, +): Promise { + const source = joined(directory, "DESIGN.md"); + if (!files.has(source) || !files.has(joined(directory, "designs/index.html"))) + return undefined; + const bundled: Record = {}; + for (const path of files.paths()) { + if (!under(directory, path) || excludedPath(within(directory, path))) + continue; + const text = await files.text(path); + if (text !== undefined) bundled[within(directory, path)] = text; + } + const bundle = DesignBundle({ files: bundled }); + if (bundle instanceof type.errors) return undefined; + return { + document: { + kind: "design", + title: documentTitle(bundle.files["DESIGN.md"]!, DOCUMENT_LABELS.design), + content: JSON.stringify(bundle), + approved: false, + }, + source, + }; +} + +type Context = { + files: Files; + drafts: Files; + projectFile: ProjectExport | undefined; + options: RepositoryImportOptions; +}; + +const fromProjectFile = ( + { projectFile }: Context, + kind: RepositoryDocumentKind, +): Found | undefined => { + const document = projectFile?.documents.find((entry) => entry.kind === kind); + return document ? { document, source: PROJECT_FILE } : undefined; +}; + +const asDraft = (found: Found | undefined) => + found ? { ...found, source: `drafts/${found.source}` } : undefined; + +// Text documents and the design in order of standing: the person's drafts, then Builder's own +// project file, then the files kept at the paths a Builder export, a seed fixture or a coder +// workspace uses. +async function readDocuments( + context: Context, +): Promise> { + const { files, drafts } = context; + const documents = new Map(); + const first = async ( + source: Files, + kind: TextKind, + paths: readonly string[], + ) => { + for (const path of paths) { + const found = await textDocument(source, kind, path); + if (found) return found; + } + return undefined; + }; + for (const kind of TEXT_KINDS) { + const found = + asDraft( + await first(drafts, kind, [`planning/${kind}.md`, `${kind}.md`]), + ) ?? + fromProjectFile(context, kind) ?? + (await first(files, kind, TEXT_PATHS[kind])); + if (found) documents.set(kind, found); + } + let design = + asDraft(await designDocument(drafts, "design")) ?? + fromProjectFile(context, "design"); + for (const directory of DESIGN_DIRECTORIES) + design ??= await designDocument(files, directory); + if (design) documents.set("design", design); + return documents; +} + +type AppReading = { + found?: Found | undefined; + root?: string | undefined; + app?: RepositoryImportReport["app"]; + suggested: RepositoryImportReport["suggested"]; + warnings: string[]; +}; + +const count = (bundle: AppBundle) => + Object.keys(bundle.files).length + + (bundle.binary ? Object.keys(bundle.binary).length : 0); + +// What a Builder app has that this one lacks, for whoever makes the import to put right. +function appWarnings(files: Readonly>): string[] { + const warnings: string[] = []; + const manifest = parseJson(files["package.json"]) as + | { scripts?: Record } + | undefined; + const scripts = manifest?.scripts ? Object.keys(manifest.scripts) : []; + const absent = STARTER_SCRIPTS.filter((name) => !scripts.includes(name)); + const quoted = (names: readonly string[]) => + names.map((name) => `"${name}"`).join(", "); + if (absent.length) + warnings.push( + `package.json has no ${quoted(absent)} script; Builder's app starter has ${quoted(STARTER_SCRIPTS)} and the coder runs them`, + ); + if (!("bun.lock" in files)) + warnings.push( + "the app has no bun.lock; the coder's sandbox runs bun install --frozen-lockfile, so commit one made with bun install", + ); + return warnings; +} + +// The app is the code as it is now, under the first directory with a package.json, with any +// files drafted under app/ laid over it, so the directory's files stand over the project file's. +async function readApp(context: Context): Promise { + const { files, drafts, options } = context; + const root = APP_DIRECTORIES.find((directory) => + files.has(joined(directory, "package.json")), + ); + const overlayPaths = drafts.paths().filter((path) => under("app", path)); + const overlaid = overlayPaths.some((path) => path === "app/package.json"); + if (root === undefined && !overlaid) { + const filed = fromProjectFile(context, "app"); + if (!filed) return { suggested: [], warnings: [] }; + const bundle = AppBundle.assert(JSON.parse(filed.document.content)); + const bytes = utf8Length(filed.document.content); + return { + found: filed, + app: { root: PROJECT_FILE, files: count(bundle), bytes }, + suggested: [], + warnings: [], + }; + } + const ignored = + options.gitignore === false ? () => false : await gitignored(files); + const exclude = options.exclude ?? new Set(); + const collected: Required = { files: {}, binary: {} }; + const suggested: RepositoryImportReport["suggested"] = []; + if (root !== undefined) + for (const path of files.paths()) { + if (!under(root, path) || exclude.has(path)) continue; + const relative = within(root, path); + if (excludedPath(relative) || ignored(path)) continue; + const reason = suggestedReason(path); + if (reason) suggested.push({ path, reason }); + addBundleFile(collected, relative, await files.byPath.get(path)!.bytes()); + } + for (const path of overlayPaths) + addBundleFile( + collected, + within("app", path), + await drafts.byPath.get(path)!.bytes(), + ); + const source = root === undefined ? "drafts/app" : root || "."; + const { bundle, content } = bundleContent(collected); + return { + found: { + document: { + kind: "app", + title: DOCUMENT_LABELS.app, + content, + approved: false, + }, + source, + }, + root, + app: { root: source, files: count(bundle), bytes: utf8Length(content) }, + suggested, + warnings: appWarnings(collected.files), + }; +} + +// The title given, else the project file's, else the app's package name, else the directory's. +async function readTitle( + context: Context, + root: string | undefined, +): Promise { + const { files, projectFile, options } = context; + const given = options.title?.trim(); + if (given) return given.slice(0, TITLE_MAX); + if (projectFile) return projectFile.title; + const manifestPath = + root === undefined ? "package.json" : joined(root, "package.json"); + const manifest = parseJson(await files.text(manifestPath)); + const name = (manifest as { name?: unknown } | undefined)?.name; + const source = typeof name === "string" && name.trim() ? name : options.name; + return humanize(source?.trim() || "Project").slice(0, TITLE_MAX); +} + +/** Reads a directory's files as a Builder project file and reports what it holds and lacks. */ +export async function readRepository( + files: readonly RepositoryFile[], + options: RepositoryImportOptions = {}, +): Promise { + const context: Context = { + files: new Files(files), + drafts: new Files(options.drafts), + projectFile: undefined, + options, + }; + context.projectFile = await readProjectFile(context.files); + const documents = await readDocuments(context); + const app = await readApp(context); + if (app.found) documents.set("app", app.found); + const title = await readTitle(context, app.root); + const approve = options.approve; + const warnings = [...app.warnings]; + if (approve !== undefined && approve !== "all") + for (const kind of approve) + if (!documents.has(kind)) + warnings.push(`${kind} was to be approved but was not found`); + const approved = (kind: RepositoryDocumentKind) => + approve === "all" || approve?.includes(kind) === true; + const list = REPOSITORY_DOCUMENT_KINDS.flatMap((kind) => { + const entry = documents.get(kind); + if (!entry) return []; + return [{ ...entry.document, approved: approved(kind) }]; + }); + const file = ProjectExport.assert({ + format: "corbits-builder.project", + version: 1, + title, + documents: list, + }); + const found: RepositoryImportReport["found"] = {}; + for (const [kind, entry] of documents) found[kind] = entry.source; + return { + file, + report: { + title, + found, + missing: REPOSITORY_DOCUMENT_KINDS.filter((kind) => !documents.has(kind)), + app: app.app, + suggested: app.suggested, + warnings, + }, + }; +} + +/** The report as lines a person or an agent reads. */ +export function describeRepositoryImport( + report: RepositoryImportReport, +): string { + const number = (value: number) => value.toLocaleString("en-US"); + const lines = [`Title: ${report.title}`]; + for (const kind of REPOSITORY_DOCUMENT_KINDS) { + if (kind === "app" && report.app) + lines.push( + `${kind}: ${report.app.files} files, ${number(report.app.bytes)} bytes, from ${report.app.root}`, + ); + else lines.push(`${kind}: ${report.found[kind] ?? "missing"}`); + } + const why: Record = { + secret: "looks like a secret", + lockfile: "a lockfile the sandbox does not use", + }; + for (const { path, reason } of report.suggested) + lines.push(`consider leaving out ${path}: ${why[reason]}`); + for (const warning of report.warnings) lines.push(`warning: ${warning}`); + return lines.join("\n"); +} diff --git a/scripts/import-repo.ts b/scripts/import-repo.ts new file mode 100644 index 00000000..e31ea13a --- /dev/null +++ b/scripts/import-repo.ts @@ -0,0 +1,250 @@ +import { lstatSync, readFileSync, readdirSync, realpathSync } from "node:fs"; +import { basename, join, relative, resolve, sep } from "node:path"; +import { EXCLUDED_DIRECTORIES } from "@corbits-builder/schemas/app-files"; +import { + REPOSITORY_DOCUMENT_KINDS, + describeRepositoryImport, + readRepository, + type RepositoryDocumentKind, + type RepositoryFile, + type RepositoryImportOptions, +} from "@corbits-builder/schemas/repository-import"; + +// Writes a directory, usually a git repository, as the project file Builder's import reads, as +// if Builder had made the project itself, and reports which of Builder's documents it lacks so +// whoever runs this can draft them, have the person approve them and run again with --drafts. +// The reading itself is the schemas' repository import, which the UI's folder import shares; +// this script only lists the files, with git when the directory is a repository. + +const statOf = (path: string) => lstatSync(path, { throwIfNoEntry: false }); +const isRegularFile = (path: string) => statOf(path)?.isFile() === true; +const isDirectory = (path: string) => statOf(path)?.isDirectory() === true; +const posix = (path: string) => path.split(sep).join("/"); + +const fileOf = (root: string, path: string): RepositoryFile => ({ + path: posix(path), + size: lstatSync(join(root, path)).size, + bytes: () => Promise.resolve(new Uint8Array(readFileSync(join(root, path)))), +}); + +// The paths git holds for the directory, tracked or untracked and not ignored, or undefined when +// the directory is not a git work tree or git is not installed. +export function gitPaths(directory: string): string[] | undefined { + if (!Bun.which("git")) return undefined; + const inside = Bun.spawnSync( + ["git", "-C", directory, "rev-parse", "--is-inside-work-tree"], + { stdout: "pipe", stderr: "pipe" }, + ); + if (!inside.success || inside.stdout.toString().trim() !== "true") + return undefined; + const listed = Bun.spawnSync( + [ + "git", + "-C", + directory, + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-standard", + ], + { stdout: "pipe", stderr: "pipe" }, + ); + if (!listed.success) return undefined; + return [ + ...new Set(listed.stdout.toString().split("\0").filter(Boolean)), + ].sort(); +} + +// Every regular file beneath a directory, except under the directories the app never keeps. +function walkPaths(root: string, directory = ""): string[] { + const paths: string[] = []; + const entries = readdirSync(join(root, directory), { withFileTypes: true }); + for (const entry of entries) { + const path = directory ? join(directory, entry.name) : entry.name; + if (entry.isDirectory()) { + if (!EXCLUDED_DIRECTORIES.has(entry.name)) + paths.push(...walkPaths(root, path)); + } else if (entry.isFile()) paths.push(path); + } + return paths.sort(); +} + +/** The directory's files: what git holds when it is a repository, else every regular file. */ +export function listDirectory(directory: string): { + files: RepositoryFile[]; + git: boolean; +} { + const root = realpathSync(directory); + const fromGit = gitPaths(root); + // A symlink, a submodule or a file deleted since it was staged is not the directory's own file. + const paths = (fromGit ?? walkPaths(root)).filter((path) => + isRegularFile(join(root, path)), + ); + return { + files: paths.map((path) => fileOf(root, path)), + git: fromGit !== undefined, + }; +} + +export type ImportOptions = { + /** A directory of drafted documents, laid out as the planning/ and design/ of an export. */ + drafts?: string | undefined; + title?: string | undefined; + approve?: readonly RepositoryDocumentKind[] | "all" | undefined; + /** Globs of app paths to leave out, relative to the directory. */ + exclude?: readonly string[] | undefined; +}; + +/** Reads a directory as a Builder project file and reports what it holds and lacks. */ +export async function importRepository( + directory: string, + options: ImportOptions = {}, +) { + const root = realpathSync(directory); + const { files, git } = listDirectory(root); + const patterns = options.exclude; + const globs = + patterns === undefined + ? [] + : patterns.map((pattern) => new Bun.Glob(pattern)); + const exclude = new Set( + files + .map(({ path }) => path) + .filter((path) => globs.some((glob) => glob.match(path))), + ); + const reading: RepositoryImportOptions = { + name: basename(root), + title: options.title, + approve: options.approve, + exclude, + gitignore: !git, + drafts: options.drafts ? listDirectory(options.drafts).files : undefined, + }; + return readRepository(files, reading); +} + +export const usage = `Usage: bun run import:repo [options] + +Writes the directory as a Builder project file (the file "Import project" reads) and reports +which of Builder's documents it lacks. + +Options: + --out Where to write the project file (default: .project.json) + --drafts Drafted documents: .md (and approaches.json) at its root or under + planning/, a design under design/, files to add to the app under app/ + --title The project's title + --approve <kinds> Mark these kinds approved: a comma-separated list, or "all" + --exclude <glob> Leave matching app paths out; may repeat + --allow-missing Exit 0 even when documents are missing + --help Show this text + +Kinds: ${REPOSITORY_DOCUMENT_KINDS.join(", ")}`; + +export type CliArgs = { + directory: string; + out: string; + allowMissing: boolean; + options: ImportOptions; +}; + +const parseApprove = (kinds: string): ImportOptions["approve"] => { + if (kinds === "all") return "all"; + const named = kinds.split(",").map((kind) => kind.trim()); + const unknown = named.find( + (kind) => + !REPOSITORY_DOCUMENT_KINDS.includes(kind as RepositoryDocumentKind), + ); + if (unknown !== undefined) + throw new Error( + `--approve names an unknown kind: ${unknown}. Kinds: ${REPOSITORY_DOCUMENT_KINDS.join(", ")}`, + ); + return named as RepositoryDocumentKind[]; +}; + +export function parseCliArgs(argv: readonly string[]): CliArgs | "help" { + let directory: string | undefined; + let out: string | undefined; + let allowMissing = false; + const options: ImportOptions = {}; + const exclude: string[] = []; + const value = (index: number, flag: string) => { + const next = argv[index + 1]; + if (next === undefined || next.startsWith("--")) + throw new Error(`${flag} needs a value`); + return next; + }; + for (let index = 0; index < argv.length; index++) { + const argument = argv[index]!; + switch (argument) { + case "--help": + case "-h": + return "help"; + case "--out": + out = value(index++, argument); + break; + case "--drafts": + options.drafts = value(index++, argument); + break; + case "--title": + options.title = value(index++, argument); + break; + case "--exclude": + exclude.push(value(index++, argument)); + break; + case "--allow-missing": + allowMissing = true; + break; + case "--approve": + options.approve = parseApprove(value(index++, argument)); + break; + default: + if (argument.startsWith("--")) + throw new Error(`unknown option ${argument}`); + if (directory !== undefined) throw new Error("one directory at a time"); + directory = argument; + } + } + if (directory === undefined) throw new Error("a directory is needed"); + options.exclude = exclude; + return { + directory, + out: out ?? `${basename(resolve(directory))}.project.json`, + allowMissing, + options, + }; +} + +async function main() { + let args: CliArgs | "help"; + try { + args = parseCliArgs(process.argv.slice(2)); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + console.error(usage); + process.exit(1); + } + if (args === "help") { + console.log(usage); + return; + } + if (!isDirectory(args.directory)) { + console.error(`${args.directory} is not a directory`); + process.exit(1); + } + const { file, report } = await importRepository(args.directory, args.options); + await Bun.write(args.out, JSON.stringify(file, null, 2) + "\n"); + console.error(describeRepositoryImport(report)); + const approved = file.documents.filter((document) => document.approved); + console.error( + `Wrote ${relative(process.cwd(), args.out) || args.out}: ${file.documents.length} documents, ${approved.length} approved`, + ); + if (report.missing.length) { + console.error( + `Missing: ${report.missing.join(", ")}. Draft them and run again with --drafts <dir>, or import without them with --allow-missing.`, + ); + if (!args.allowMissing) process.exit(2); + } +} + +if (import.meta.main) await main(); diff --git a/skills/operate-corbits-builder/SKILL.md b/skills/operate-corbits-builder/SKILL.md index a0b1b94c..0ad6057b 100644 --- a/skills/operate-corbits-builder/SKILL.md +++ b/skills/operate-corbits-builder/SKILL.md @@ -1,6 +1,6 @@ --- name: operate-corbits-builder -description: Drives a running Builder (Corbits Builder) through its HTTP API, from creating or resuming a project and talking with its coordinator to handing document approvals to the person and downloading the built app. Use whenever someone asks you to start a Builder project, check on one, answer its questions, read its documents or fetch its app from a Builder server, even if they only give a project link or name. To install Builder first, use corbits-builder-install. +description: Drives a running Builder (Corbits Builder) through its HTTP API, from creating or resuming a project and talking with its coordinator to handing document approvals to the person and downloading the built app. Use whenever someone asks you to start a Builder project, check on one, answer its questions, read its documents, fetch its app from a Builder server or import a directory or git repository into Builder as a project, even if they only give a project link or name. To install Builder first, use corbits-builder-install. --- # Operate Builder @@ -57,3 +57,75 @@ Approvals belong to the person and are made from their own session. source as a zip (the latest when `version` is absent). `GET P/export/archive?sections=planning,design,build` exports the project: every document, the design and the app. + +## Import a directory + +When the person hands you a directory to import, most often a git repository, +import it as if Builder had made the project itself: the code as the app and +each of Builder's documents in place, so the coordinator and the coder pick it +up where Builder would have left it. Builder's documents, in order: +`problem_brief`, `constraints`, `approaches`, `product_requirements`, `design`, +`build_plan`, then `app`. The import route +(`POST /api/tenants/:tenantId/projects/import`, `importProject` in the client) +takes a `corbits-builder.project` file, files each document as +version 1 and records the importing person's approval of each one marked +approved. + +1. **Read the directory.** From the Builder checkout, run + `bun run import:repo <directory>` (`scripts/import-repo.ts`). It writes + `<name>.project.json` and reports: the title it chose, each document it + found and where (an unpacked export's `project.json` and `planning/`, + `docs/<kind>.md`, a coder workspace's `.inputs/` and `.builder/`), the app + (the files git holds under the first directory with a `package.json`, less + secrets, dependencies and build output), + the documents still missing, and what the app lacks that Builder's own apps + have (the `dev`, `build` and `typecheck` scripts, a `bun.lock`). Exit code 2 + means something is missing; `--help` lists the options. Never run it from + inside the directory you import, and never write into that directory. +2. **Learn the project before drafting.** Read what the directory says about + itself: its README and docs, `package.json`, the code's layout, and + `git log` for how it was built. What the code cannot tell you, such as who + the app is for, its budget and bounds, and what done looks like, is the + person's to say: ask them, in one round of questions, rather than invent it. +3. **Draft each missing document the way Builder's own writers do.** Each + kind has a skill describing its sections and shape in + `workflows/writer/skills/<name>/SKILL.md` (`problem-brief`, `constraints`, + `approaches`, `product-requirements`, `build-plan`), with finished examples + under `test/seed/*/docs` and a design under `test/seed/*/designs`. Write + them from the project as it is: the approaches document records the way it + was actually built as the recommended approach beside the alternatives set + aside, the build plan says what is built and what remains, the design + describes the screens the app really has. Put the drafts in a directory of + their own outside the repository: `<drafts>/<kind>.md`, the approaches' + comparison as `<drafts>/approaches.json` (an `ApproachesMeta` from + `packages/schemas/src/approaches.ts`), the design as + `<drafts>/design/DESIGN.md` and `<drafts>/design/designs/index.html` with + its screens under `designs/screens/`, and files the app needs under + `<drafts>/app/` (a `package.json` for an app with none, a `bun.lock` made + with `bun install` in a copy). Draft a design only for an app with screens + you can describe; otherwise leave it out and say that Builder's designer can + make one later. An app that is not a Bun or Node project cannot run in + Builder's sandbox: say so, and ask whether to import the documents without + it. +4. **Get the person's approval of every draft.** Show each one by path with a + short summary of what it claims, and the full text when they ask. For each, + they choose: approve it as it stands, change it (then show it again), or + leave it out. Mark approved only what they approved in so many words; a + document they left out is Builder's to write later. Do not approve on their + behalf. +5. **Write the file.** Run again with `--drafts <drafts>` and + `--approve <kinds>` naming the kinds they approved (`--title`, `--exclude` + and `--allow-missing` as needed) until the report lists nothing missing but + what they chose to leave out. Read the warnings back to them: without a + `bun.lock` the coder's `bun install --frozen-lockfile` fails. +6. **Import on their word.** Tell them what the file holds: the title, each + document and whether it is approved, the app's file count and size. On their + go-ahead, send the file as the JSON body of the import route with the + session cookie and `Origin` header; a 400 names the invalid field, a 413 the + size. The receipt's `projectId` is the new project. The person can also pick + the file themselves with **Import project** on the new-project page, or + skip the drafting and pick the folder itself with **Import a folder**, + which reads it the same way in the browser and leaves the missing + documents to the coordinator to write with them. Then + follow the procedure above from step 1: the coordinator's first message + acknowledges the import and says what the documents establish. diff --git a/test/integration/import-repo.test.ts b/test/integration/import-repo.test.ts new file mode 100644 index 00000000..e378ba2d --- /dev/null +++ b/test/integration/import-repo.test.ts @@ -0,0 +1,198 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { mkdtemp, mkdir, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { AppBundle } from "@corbits-builder/schemas/artifact"; +import { ProjectExport } from "@corbits-builder/schemas/project"; +import { + gitPaths, + importRepository, + listDirectory, + parseCliArgs, +} from "../../scripts/import-repo.ts"; + +const git = (directory: string, ...args: string[]) => { + const run = Bun.spawnSync(["git", "-C", directory, ...args], { + stdout: "pipe", + stderr: "pipe", + }); + if (!run.success) throw new Error(run.stderr.toString()); +}; + +async function writeFiles( + root: string, + files: Record<string, string | Uint8Array>, +) { + for (const [path, content] of Object.entries(files)) { + await mkdir(dirname(join(root, path)), { recursive: true }); + await writeFile(join(root, path), content); + } +} + +// A person's own app as a repository: code, dependencies and build output git ignores, a secret, +// a binary, a symlink and the planning documents of this repository's seed layout. +const APP_FILES = { + "package.json": JSON.stringify({ + name: "@acme/habit-tracker", + scripts: { dev: "bun run src/main.ts", build: "bun build src/main.ts" }, + }), + "README.md": "# Habit tracker\n", + ".gitignore": "node_modules/\n*.log\nbuild-output/\n", + "src/main.ts": "console.log('hi');\n", + "node_modules/left/index.js": "module.exports = 1;\n", + "build-output/bundle.js": "ignored\n", + "debug.log": "ignored\n", + ".env": "SECRET=1\n", + "assets/dot.png": new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0, 0xff, 0xfe]), + "docs/problem_brief.md": "# Problem Brief: Habit tracker\n\n## In short\n", +}; + +let root: string; +let repo: string; +beforeAll(async () => { + root = await mkdtemp(join(tmpdir(), "import-repo-")); + repo = join(root, "habit-tracker"); + await writeFiles(repo, APP_FILES); + await symlink("/etc/hosts", join(repo, "hosts-link")); + git(repo, "init", "-q"); +}); +afterAll(() => rm(root, { recursive: true, force: true })); + +describe("listDirectory", () => { + test("lists what git holds in a repository, regular files only", () => { + const { files, git: fromGit } = listDirectory(repo); + expect(fromGit).toBe(true); + const paths = files.map((file) => file.path); + expect(paths).toEqual([ + ".env", + ".gitignore", + "README.md", + "assets/dot.png", + "docs/problem_brief.md", + "package.json", + "src/main.ts", + ]); + expect(gitPaths(repo)).toContain("hosts-link"); + }); + + test("walks a plain directory, never into dependencies", async () => { + const plain = join(root, "plain"); + await writeFiles(plain, { + "package.json": "{}", + "node_modules/x/index.js": "", + "lib/a.ts": "", + }); + const { files, git: fromGit } = listDirectory(plain); + expect(fromGit).toBe(false); + expect(files.map((file) => file.path)).toEqual([ + "lib/a.ts", + "package.json", + ]); + expect(gitPaths(tmpdir())).toBeUndefined(); + }); +}); + +describe("importRepository", () => { + test("files a git repository's code as the app and names the documents it lacks", async () => { + const { file, report } = await importRepository(repo); + expect(ProjectExport(file)).toEqual(file); + expect(report.title).toBe("Habit tracker"); + expect(report.found).toEqual({ + problem_brief: "docs/problem_brief.md", + app: ".", + }); + expect(report.missing).toEqual([ + "constraints", + "approaches", + "product_requirements", + "design", + "build_plan", + ]); + const app = file.documents.find((document) => document.kind === "app")!; + const bundle = AppBundle.assert(JSON.parse(app.content)); + expect(Object.keys(bundle.files)).toEqual([ + ".gitignore", + "README.md", + "docs/problem_brief.md", + "package.json", + "src/main.ts", + ]); + expect(Object.keys(bundle.binary ?? {})).toEqual(["assets/dot.png"]); + }); + + test("drafts, approvals and excluded globs reach the reading", async () => { + const drafts = join(root, "drafts"); + await writeFiles(drafts, { + "constraints.md": "# Constraints\n", + "app/bun.lock": "{}\n", + }); + const { file, report } = await importRepository(repo, { + drafts, + title: "Streaks", + approve: ["constraints"], + exclude: ["assets/**", "docs/*"], + }); + expect(report.title).toBe("Streaks"); + expect(report.found.constraints).toBe("drafts/constraints.md"); + const byKind = Object.fromEntries( + file.documents.map((document) => [document.kind, document]), + ); + expect(byKind.constraints!.approved).toBe(true); + expect(byKind.problem_brief!.approved).toBe(false); + const bundle = AppBundle.assert(JSON.parse(byKind.app!.content)); + expect(Object.keys(bundle.files)).toEqual([ + ".gitignore", + "README.md", + "bun.lock", + "package.json", + "src/main.ts", + ]); + expect(bundle.binary).toBeUndefined(); + }); +}); + +describe("parseCliArgs", () => { + test("reads the directory, its options and every kind approved", () => { + expect( + parseCliArgs([ + "../repo", + "--approve", + "problem_brief,design", + "--exclude", + "a/**", + "--exclude", + "b", + "--allow-missing", + "--drafts", + "d", + "--title", + "T", + "--out", + "o.json", + ]), + ).toEqual({ + directory: "../repo", + out: "o.json", + allowMissing: true, + options: { + approve: ["problem_brief", "design"], + exclude: ["a/**", "b"], + drafts: "d", + title: "T", + }, + }); + expect(parseCliArgs(["repo", "--approve", "all"])).toMatchObject({ + out: "repo.project.json", + options: { approve: "all" }, + }); + expect(parseCliArgs(["--help"])).toBe("help"); + }); + + test("refuses what it cannot read", () => { + expect(() => parseCliArgs([])).toThrow("a directory is needed"); + expect(() => parseCliArgs(["a", "b"])).toThrow("one directory"); + expect(() => parseCliArgs(["a", "--approve", "app,nope"])).toThrow("nope"); + expect(() => parseCliArgs(["a", "--out"])).toThrow("--out needs a value"); + expect(() => parseCliArgs(["a", "--bogus"])).toThrow("unknown option"); + }); +}); diff --git a/workflows/coordinator/src/prompt.ts b/workflows/coordinator/src/prompt.ts index b03317a0..aebc514a 100644 --- a/workflows/coordinator/src/prompt.ts +++ b/workflows/coordinator/src/prompt.ts @@ -42,7 +42,7 @@ The person makes every decision; your part is a recommendation and a proposal to When their preferences let documents go ahead without their approval, tell them about each document instead. ## Imported projects -After a project is imported, ask the person what they want to do now: re-plan from scratch, redesign, rebuild, add a feature, or something else in their own words. Then request their approval of the imported documents that become the baseline for that goal, in one request_approval covering them all. +After a project is imported, ask the person what they want to do now: re-plan from scratch, redesign, rebuild, add a feature, or something else in their own words. Then request their approval of the imported documents that become the baseline for that goal, in one request_approval covering them all. When the import filed the app's code without some of its documents, the opening message names them: learn the app from its code first, then ask the person what they want to do and, in the same ask_questions, the essentials the code cannot tell you, write the brief yourself and have the writers produce the rest in Builder's order, each instruction carrying what the code shows, with the person's approval of each before the next. ## Design Once the brief, constraints, approaches and product requirements exist, give the person a quick, plain check-in on where things stand: which documents are drafted, by title, and an invitation to look them over. In the same message ask them two things: whether they want one design you suggest or to explore several different looks, and what ideas they have for the look, such as apps they like, colours or a feel. Once they answer, use design_app with their ideas in their own words: for one suggested design, the single look that best fits their idea and what they said; to explore, as many clearly different looks as the person's preferences ask for, from the catalog in design_app or of your own when the idea calls for it. The instruction states the goal; the designers decide the screens. When the person chose one of the design themes, design_app makes just that theme's concept, with their changes in its direction.