diff --git a/apps/hub/src/artifacts/routes.ts b/apps/hub/src/artifacts/routes.ts index e3fe47a2..b01f7758 100644 --- a/apps/hub/src/artifacts/routes.ts +++ b/apps/hub/src/artifacts/routes.ts @@ -25,7 +25,6 @@ import { DesignTweaks, DesignTweaksReceipt, LiveApp, - MAX_APPROVAL_PACKAGE_BYTES, MaterialReceipt, MaterialRemovedReceipt, } from "@corbits-builder/schemas/artifact"; @@ -127,10 +126,6 @@ export function mountArtifactRoutes( ); app.post( "/:projectId/artifacts", - bodyLimit({ - maxSize: MAX_APPROVAL_PACKAGE_BYTES + 1024 * 1024, - onError: (c) => c.json({ code: "artifact_too_large" }, 413), - }), describeRoute({ tags: ["Artifacts"], operationId: "writeProjectArtifact", @@ -600,10 +595,6 @@ export function mountArtifactRoutes( description: "The newest coder run's workspace cannot be read by this hub (app_workspace_unavailable, with its reason); an older run's workspace is never taken instead", }, - 413: { - description: - "The app is larger than an app version holds (app_too_large)", - }, }, }), async (c) => { diff --git a/apps/hub/src/artifacts/service.ts b/apps/hub/src/artifacts/service.ts index df08fd8d..f041d6b6 100644 --- a/apps/hub/src/artifacts/service.ts +++ b/apps/hub/src/artifacts/service.ts @@ -41,7 +41,6 @@ import { AppBundle, AppVersionMetadata, DocumentContent, - MAX_APP_BYTES, MAX_APPROVAL_PACKAGE_BYTES, MATERIAL_KIND, ArtifactKindWrite, @@ -125,11 +124,7 @@ import { warmStepWorkspace, } from "./design-draft.ts"; import type { LiveApps } from "../live-app/runner.ts"; -import { - AppTooLargeError, - appContent, - bundleFiles, -} from "../builds/app-snapshot.ts"; +import { appContent, bundleFiles } from "../builds/app-snapshot.ts"; import { MAX_DESIGN_BYTES } from "./design-tools.ts"; import { DESIGN_STYLES, setThemeTokens } from "./design-tweaks.ts"; import { @@ -177,8 +172,6 @@ const APPROVAL_COVERS: Record boolean> = { kind !== "app" && kind !== "design" && kind !== "design_option", instruct_coder: (kind) => kind !== "app" && kind !== "design_option", }; -// Whole documents, the filed app and approval package among them. -const MAX_READ_CHARS = MAX_APPROVAL_PACKAGE_BYTES; const RECENT_FAILURES = 5; // A command's progress line older than this was left by a command that died; it is not shown. const COMMAND_PROGRESS_STALE_MS = 5 * 60_000; @@ -197,14 +190,6 @@ const isUuid = (value: string) => new RegExp("^" + UUID + "$", "i").test(value); // The coder task names documents as id@version@sha256 tokens. const PINNED_REF = new RegExp("^(" + UUID + ")@(\\d+)@([a-f0-9]{64})$", "i"); -const appTooLarge = (error: unknown) => - error instanceof AppTooLargeError - ? new ProjectError(413, "app_too_large", { - bytes: error.bytes, - limit: MAX_APP_BYTES, - }) - : error; - // An imported bundle's paths and a material's filename are the importer's text: none may climb // out of the directory it is archived under. const isContainedPath = (path: string) => @@ -267,14 +252,8 @@ function validateArtifactWriteContent( issues: typeIssues(bundle, "content"), }); } - if (input.kind === "app") { - const bytes = Buffer.byteLength(input.content); - if (bytes > MAX_APP_BYTES) - throw new ProjectError(413, "app_too_large", { - bytes, - limit: MAX_APP_BYTES, - }); - } else if (input.kind === "approval_package") { + // An app is as large as it is. + if (input.kind === "approval_package") { const bytes = Buffer.byteLength(input.content); if (bytes > MAX_APPROVAL_PACKAGE_BYTES) throw new ProjectError(413, "approval_package_too_large", { @@ -282,7 +261,7 @@ function validateArtifactWriteContent( limit: MAX_APPROVAL_PACKAGE_BYTES, }); parseApprovalPackageBundle(input.content); - } else { + } else if (input.kind !== "app") { const document = DocumentContent(input.content); if (document instanceof type.errors) throw new ProjectError(400, "invalid_artifact_write", { @@ -554,16 +533,11 @@ export function createArtifactService( await listArtifacts(db, actor.tenantId, { kind: "app", limit: 1 }) ).rows; const filed = latest?.contentSha256 ? latest : undefined; - let repository; - try { - repository = await newerRepositoryApp( - actor, - projectId, - filed && (await filedAt(filed.id, filed.version)), - ); - } catch (error) { - throw appTooLarge(error); - } + const repository = await newerRepositoryApp( + actor, + projectId, + filed && (await filedAt(filed.id, filed.version)), + ); if (repository) return { name: repository.commit.slice(0, 7), @@ -1447,12 +1421,6 @@ export function createArtifactService( artifactId, version, }); - if (stored.content.length > MAX_READ_CHARS) - throw new ProjectError(409, "artifact_too_large", { - artifactId, - contentLength: stored.content.length, - limit: MAX_READ_CHARS, - }); const { openQuestions = [], approaches } = documentMetadataOf( stored.metadata, ); @@ -1714,12 +1682,7 @@ export function createArtifactService( runId: workspace.runId, reason: workspace.reason, }); - let content: string; - try { - content = await appContent(workspace.dir); - } catch (error) { - throw appTooLarge(error); - } + const content = await appContent(workspace.dir); const metadata = AppVersionMetadata.assert({ source: "snapshot", runId: workspace.runId, diff --git a/apps/hub/src/artifacts/tools.ts b/apps/hub/src/artifacts/tools.ts index 584b23e4..1066d5e5 100644 --- a/apps/hub/src/artifacts/tools.ts +++ b/apps/hub/src/artifacts/tools.ts @@ -12,7 +12,6 @@ import { ArtifactList, ArtifactRead, ArtifactReceipt, - MAX_APPROVAL_PACKAGE_BYTES, WriterDocumentFields, type DocumentWrite, type WriterDocumentKind, @@ -30,8 +29,6 @@ interface ArtifactToolEnv extends BaseEnv { capabilities: RuntimeCapabilities; } -const MAX_RESULT_BYTES = MAX_APPROVAL_PACKAGE_BYTES + 256 * 1024; - // A deployed agent keeps the tool bundle it started with while the hub moves on; a field a newer // hub adds must not fail every read of an older agent. export const HubArtifactList = ArtifactList.onDeepUndeclaredKey("ignore"); @@ -181,7 +178,8 @@ function artifactTools( handle: HUB_CREDENTIAL_HANDLE, path, ...(body === undefined ? {} : { body }), - maxResponseBytes: MAX_RESULT_BYTES, + // A document is as large as it is; the parts below keep the model's read small. + maxResponseBytes: Number.POSITIVE_INFINITY, signal, }); if (call.name === "write_artifact" && save) { diff --git a/apps/hub/src/builds/app-repository.ts b/apps/hub/src/builds/app-repository.ts index d6145afc..2f47f574 100644 --- a/apps/hub/src/builds/app-repository.ts +++ b/apps/hub/src/builds/app-repository.ts @@ -61,9 +61,6 @@ const PERMANENT = // files changed since the check. export const CHECK_PASSED = "Check passed"; export const EDIT = "Edit"; -// The repository carries the app's binary assets, such as images in public/, which the 1 MiB text -// artifact of the app leaves out. -const APP_REPOSITORY_MAX_BYTES = 25 * 1024 * 1024; // A pkt-line holds at most 65520 bytes: its 4-byte length, the band byte and the data. const SIDE_BAND_DATA = 65515; const log = getLogger(["corbits-builder", "hub", "app-repository"]); @@ -227,8 +224,6 @@ export async function signWithSidecarKey(payload: string) { ); } -class AppTooLarge extends Error {} - type Blob = { mode: string; oid: string }; type Tree = Map; type WorkspaceFile = { bytes: Buffer; executable: boolean }; @@ -283,11 +278,6 @@ export function createAppRepository(options: { password: secret, }); - const tooLarge = (filepath: string) => - new AppTooLarge( - `app_too_large: the app's files pass the ${APP_REPOSITORY_MAX_BYTES}-byte limit at ${filepath}`, - ); - // Whether a directory's real location is still the one the walk reached it by. const inPlace = async (root: string, directory: string) => (await realpath(directory)) === join(root, relative(workspace, directory)); @@ -326,7 +316,6 @@ export function createAppRepository(options: { root: string, path: string, filepath: string, - room: number, ): Promise { let file; try { @@ -347,9 +336,7 @@ export function createAppRepository(options: { if (linkOrGone(error)) return undefined; throw error; } - if (stats.size > room) throw tooLarge(filepath); const bytes = await file.readFile(); - if (bytes.length > room) throw tooLarge(filepath); if (!(await readInPlace(file.fd, root, path, stats))) throw new Error( `app_workspace_changed: ${filepath} moved while it was read`, @@ -400,17 +387,9 @@ export function createAppRepository(options: { // The workspace's own files by path. async function collect(root: string, signal: AbortSignal) { const files = new Map(); - let bytes = 0; await walk(root, workspace, signal, async (path, filepath) => { - const file = await readRegular( - root, - path, - filepath, - APP_REPOSITORY_MAX_BYTES - bytes, - ); - if (!file) return; - bytes += file.bytes.length; - files.set(filepath, file); + const file = await readRegular(root, path, filepath); + if (file) files.set(filepath, file); }); return files; } @@ -633,58 +612,64 @@ export function createAppRepository(options: { // The workspace's fingerprint at its last version that reached the hub. let versioned: string | undefined; - // The workspace's fingerprint when its files passed the size limit; it is not read again unchanged. - let refused: string | undefined; + + // The fingerprint a check passed on, or undefined once the check failed: its files are then an + // edit, not a passed check. + const passedFiles = async (checked: Promise) => { + try { + return await checked; + } catch (error) { + log.debug("The check did not pass, so the files are an edit: {reason}", { + reason: error instanceof Error ? error.message : String(error), + }); + return undefined; + } + }; // Commits the workspace when it differs from main, then pushes main if the hub lacks it. A - // workspace unchanged since its last version, as after a read-only command, is left alone, and - // one unchanged since it was refused as too large is skipped, keeping that failure. A check's - // label names only a commit of the files it passed on: unchanged since the check, and while read. + // workspace unchanged since its last version, as after a read-only command, is left alone. A + // check's label names only a commit of the files it passed on: unchanged since the check, and + // while read. const version = ( reason: string, checked: Promise | undefined, signal: AbortSignal, - ): Promise<{ commit: string | undefined } | undefined> => + ): Promise<{ commit: string | undefined }> => serial(async () => { const files = await fingerprint(signal); if (files === versioned) return { commit: undefined }; - if (files === refused) return undefined; let created: string | undefined; - try { - await attach(signal, false); - const head = await headCommit(); - const { tree, blobs } = await snapshot(signal); - if (tree !== head.tree) { - const label = - checked === undefined || - ((await checked.catch(() => undefined)) === files && - (await fingerprint(signal)) === files) - ? reason - : EDIT; - const { author } = await pushAccess(signal); - const { title, body } = summary(await flatten(head.tree), blobs); - const identity = { - ...author, - timestamp: Math.floor(Date.now() / 1000), - }; - created = await git.commit({ - fs, - gitdir, - ref: MAIN, - tree, - parent: [head.oid], - author: identity, - committer: identity, - message: `${label}: ${title}\n\n${body}\n`, - signingKey: "sshsig", - onSign: async ({ payload }) => ({ - signature: await options.sign(payload), - }), - }); - } - } catch (error) { - if (error instanceof AppTooLarge) refused = files; - throw error; + await attach(signal, false); + const head = await headCommit(); + const { tree, blobs } = await snapshot(signal); + if (tree !== head.tree) { + let label = EDIT; + if (checked === undefined) label = reason; + else if ( + (await passedFiles(checked)) === files && + (await fingerprint(signal)) === files + ) + label = reason; + const { author } = await pushAccess(signal); + const { title, body } = summary(await flatten(head.tree), blobs); + const identity = { + ...author, + timestamp: Math.floor(Date.now() / 1000), + }; + created = await git.commit({ + fs, + gitdir, + ref: MAIN, + tree, + parent: [head.oid], + author: identity, + committer: identity, + message: `${label}: ${title}\n\n${body}\n`, + signingKey: "sshsig", + onSign: async ({ payload }) => ({ + signature: await options.sign(payload), + }), + }); } await push(signal); versioned = files; @@ -759,12 +744,7 @@ export function createAppRepository(options: { true, ); } catch (error) { - if ( - attempt === CLONE_ATTEMPTS || - stopping.aborted || - error instanceof AppTooLarge - ) - throw error; + if (attempt === CLONE_ATTEMPTS || stopping.aborted) throw error; } await sleep(CLONE_RETRY_MS, undefined, { signal: stopping }); } diff --git a/apps/hub/src/builds/app-snapshot.ts b/apps/hub/src/builds/app-snapshot.ts index 85ed9693..c12d1799 100644 --- a/apps/hub/src/builds/app-snapshot.ts +++ b/apps/hub/src/builds/app-snapshot.ts @@ -10,7 +10,6 @@ import { ArtifactContent, ArtifactReceipt, DesignDraft, - MAX_APP_BYTES, } from "@corbits-builder/schemas/artifact"; import { WorkflowHttpError, workflowHttpJson } from "../workflows/http-json.ts"; import { INPUTS_DIRECTORY } from "../config/inputs.ts"; @@ -191,12 +190,6 @@ export function bundleFiles(bundle: { }; } -export class AppTooLargeError extends Error { - constructor(readonly bytes: number) { - super(`app_too_large: ${bytes} bytes; the limit is ${MAX_APP_BYTES}`); - } -} - // Keys in order, so the same files always make the same content and an unchanged app files nothing. const sorted = (record: Record) => Object.fromEntries( @@ -208,7 +201,8 @@ export async function appContent(workspace: string) { return bundleContent(await collectApp(workspace)).content; } -// Collected files as an app bundle and its document content, checked against the app limit. +// Collected files as an app bundle and its document content. An app is as large as it is: a +// resource that runs out on the way, such as memory or a request body, reports its own error. export function bundleContent(collected: Required) { const files = sorted(collected.files); const binary = sorted(collected.binary); @@ -217,10 +211,7 @@ export function bundleContent(collected: Required) { ); if (bundle instanceof type.errors) throw new Error("invalid_app: " + bundle.summary); - const content = JSON.stringify(bundle); - const bytes = Buffer.byteLength(content); - if (bytes > MAX_APP_BYTES) throw new AppTooLargeError(bytes); - return { bundle, content }; + return { bundle, content: JSON.stringify(bundle) }; } const projectPath = (scope: { tenantId: string; projectId: string }) => @@ -269,32 +260,35 @@ export async function readFiledDocument( kind: string, signal: AbortSignal, ) { - return unlessNotFound( + const body = await unlessNotFound( workflowHttpJson({ capabilities, handle: HUB_CREDENTIAL_HANDLE, path: projectPath(scope) + "/artifacts/" + encodeURIComponent(kind), - // An app bundle's JSON is escaped again inside the response. - maxResponseBytes: 2 * MAX_APP_BYTES + 16384, + // The app is as large as it is; memory is what bounds the read. + maxResponseBytes: Number.POSITIVE_INFINITY, signal, - }).then((body) => ArtifactContent.assert(body)), + }), ); + return body === undefined ? undefined : ArtifactContent.assert(body); } -const readDesignFiles = ( +const readDesignFiles = async ( capabilities: RuntimeCapabilities, path: string, signal: AbortSignal, -) => - unlessNotFound( +) => { + const body = await unlessNotFound( workflowHttpJson({ capabilities, handle: HUB_CREDENTIAL_HANDLE, path, - maxResponseBytes: 2 * MAX_APP_BYTES, + maxResponseBytes: Number.POSITIVE_INFINITY, signal, - }).then((body) => DesignDraft.assert(body).files), + }), ); + return body === undefined ? undefined : DesignDraft.assert(body).files; +}; // The draft an earlier designer run left for this run's concept, if any. export async function readPreviousDesignDraft( diff --git a/apps/hub/src/projects/app-repository.ts b/apps/hub/src/projects/app-repository.ts index c2a54ede..a0e7f2dd 100644 --- a/apps/hub/src/projects/app-repository.ts +++ b/apps/hub/src/projects/app-repository.ts @@ -24,15 +24,11 @@ import { type AssetService, type RepoStore, } from "@intx/hub-sessions"; -import { - MAX_APP_BYTES, - type AppBundle, -} from "@corbits-builder/schemas/artifact"; +import type { AppBundle } from "@corbits-builder/schemas/artifact"; import { outbox, project, projectCoordinator } from "../db/schema.ts"; import { spawnedWorkflowRole } from "../workflows/spawn-role.ts"; import { addBundleFile, - AppTooLargeError, bundleContent, EXCLUDED_DIRECTORIES, excludedFile, @@ -247,7 +243,6 @@ export async function appRepositoryMain(dir: string) { .commit; const read = async () => { const collected: Required = { files: {}, binary: {} }; - let bytes = 0; const walk = async (oid: string, prefix: string) => { for (const entry of (await git.readTree({ fs, dir, oid })).tree) { if (entry.type === "tree") { @@ -259,9 +254,6 @@ export async function appRepositoryMain(dir: string) { !excludedFile(entry.path) ) { const { blob } = await git.readBlob({ fs, dir, oid: entry.oid }); - bytes += blob.length; - // Content outgrows its raw bytes, so this stops the read before it outgrows the limit. - if (bytes > MAX_APP_BYTES) throw new AppTooLargeError(bytes); addBundleFile(collected, prefix + entry.path, blob); } } diff --git a/apps/hub/src/projects/routes.ts b/apps/hub/src/projects/routes.ts index 6144af92..e2e39967 100644 --- a/apps/hub/src/projects/routes.ts +++ b/apps/hub/src/projects/routes.ts @@ -5,7 +5,6 @@ import { createArtifactService } from "../artifacts/service.ts"; import type { AgentActivityTracker } from "../artifacts/activity.ts"; import { mountProjectAuditRoutes } from "../audit/routes.ts"; import { Hono, type Context, type MiddlewareHandler } from "hono"; -import { bodyLimit } from "hono/body-limit"; import { describeRoute, resolver } from "hono-openapi"; import { createResolveTenant, @@ -85,7 +84,6 @@ const TENANT_ERROR_STATUS = { 404: 404, 409: 409, } as const; -const IMPORT_MAX_BYTES = 20 * 1024 * 1024; // Tenant creation is native: POST /api/tenants (hub-api routes/tenants.ts // createTenantRoutes) runs as the calling user, checks `create` on `tenant:*` @@ -216,14 +214,8 @@ export function createProjectRoutes( c.header("cache-control", "no-store"); await next(); }); - // Registered ahead of the group's 3 MiB limit, which would otherwise run - // first: one file carries every document and the app. app.post( "/import", - bodyLimit({ - maxSize: IMPORT_MAX_BYTES, - onError: (c) => c.json({ code: "project_request_too_large" }, 413), - }), describeRoute({ tags: ["Projects"], operationId: "importProject", @@ -256,7 +248,6 @@ export function createProjectRoutes( 409: { description: "Tenant slug conflict or coordinator start failed", }, - 413: { description: "Request exceeds 20 MiB" }, 415: { description: "JSON content type required" }, 503: { description: "Native tenant creation unavailable" }, }, @@ -345,14 +336,6 @@ export function createProjectRoutes( } }, ); - app.use( - "*", - // Filing an app sends its 1 MiB bundle as a JSON string inside the request, escaped once more. - bodyLimit({ - maxSize: 3 * 1024 * 1024, - onError: (c) => c.json({ code: "project_request_too_large" }, 413), - }), - ); app.get( "/", describeRoute({ @@ -497,7 +480,6 @@ export function createProjectRoutes( "Project, tenant slug or idempotency conflict, or the new project's coordinator could not start; a project refused with model_provider_missing or model_profile_required is kept", }, 503: { description: "Native tenant creation unavailable" }, - 413: { description: "Request exceeds 3 MiB" }, 415: { description: "JSON content type required" }, }, }), diff --git a/apps/web/src/api/artifacts.ts b/apps/web/src/api/artifacts.ts index 8e1fcb85..362c2fbf 100644 --- a/apps/web/src/api/artifacts.ts +++ b/apps/web/src/api/artifacts.ts @@ -141,13 +141,7 @@ export async function currentAppVersion( } catch (cause) { const failure = await refusal(cause); if (!failure) throw cause; - why = - failure.code === "app_too_large" - ? versionText.why.tooLarge( - Number(failure.body["bytes"]), - Number(failure.body["limit"]), - ) - : versionText.why.failed(failure.status, failure.code); + why = versionText.why.failed(failure.status, failure.code); } } else why = versionText.why[snapshot.code]; const latest = versions[0]; diff --git a/apps/web/src/config.ts b/apps/web/src/config.ts index 9de7e4b4..1dd99f80 100644 --- a/apps/web/src/config.ts +++ b/apps/web/src/config.ts @@ -226,8 +226,6 @@ export const wording = { coder_run_not_found: "The app builder has not started on the code yet.", app_workspace_unavailable: "The app builder's working copy cannot be read from here, so its current code is not in it.", - tooLarge: (bytes: number, limit: number) => - `The current code is ${String(bytes)} bytes, over the ${String(limit)}-byte limit, so it is not in it.`, failed: (status: number, code: string) => `Saving the current code failed (HTTP ${String(status)}, ${code}), so it is not in it.`, }, diff --git a/packages/schemas/src/artifact.ts b/packages/schemas/src/artifact.ts index 86af425f..f8217d58 100644 --- a/packages/schemas/src/artifact.ts +++ b/packages/schemas/src/artifact.ts @@ -20,12 +20,10 @@ const WritableArtifactKind = DocumentKind.or( ); const content = type("string > 0").atMostLength(256 * 1024); export const DocumentContent = content; -// A working weather app packed to about 270 KB of source. -export const MAX_APP_BYTES = 1024 * 1024; export const MAX_APPROVAL_PACKAGE_BYTES = 4 * 1024 * 1024; -// A filed approval package runs up to MAX_APPROVAL_PACKAGE_BYTES and a filed app up to -// MAX_APP_BYTES, which the hub checks in bytes; it holds every other kind to DocumentContent. +// A filed approval package runs up to MAX_APPROVAL_PACKAGE_BYTES, which the hub checks in bytes, +// and a filed app is as large as the app is; the hub holds every other kind to DocumentContent. export const ArtifactKindWrite = type({ "+": "reject", kind: WritableArtifactKind,