From 44aad6889b9efc99cbbf0c901ac154e2a53eca4f Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 22:56:34 +0000 Subject: [PATCH 1/5] feat(client): Add an Objectstore client for batched uploads Port the Objectstore upload client from the Sentry CLI (getsentry/toolkit#2062) into `@sentry/api/client`, so hosts share it. `findExistingObjects` and `putObjects` group HEADs and small inserts into multipart batch requests (1000 ops, 25 MB, 4 in flight) and PUT objects over 1 MB individually (5 in flight). One request per object trips the Sentry proxy's per-request rate limit. The client uses only web-standard APIs, like the rest of the entry: hosts pass each upload's bytes through `read()` and may inject `fetch`. Failures throw `ObjectstoreError`, an `ApiError` carrying the URL. --- README.md | 19 + lib/api-client/index.ts | 1 + lib/api-client/objectstore.ts | 525 ++++++++++++++++++++++++++++ test/api-client/objectstore.test.ts | 282 +++++++++++++++ 4 files changed, 827 insertions(+) create mode 100644 lib/api-client/objectstore.ts create mode 100644 test/api-client/objectstore.test.ts diff --git a/README.md b/README.md index 3f001aed..1ed97123 100644 --- a/README.md +++ b/README.md @@ -279,6 +279,25 @@ Your transport must honor the signal while a request is in flight. The SDK canno Cancellation follows the [error handling contract](#error-handling): before dispatch the SDK uses `signal.reason`; during fetch it preserves the transport's rejection, which may differ by runtime. By default this is returned as `{ error, request, response: undefined }`, or thrown with `throwOnError: true`. Error interceptors can intentionally map it. Cancellation during response body reading rejects unchanged, even in fields mode. There is no SDK-specific cancellation error class. +### Uploading files to Objectstore + +`@sentry/api/client` exports an Objectstore client for the service that upload-options endpoints such as `snapshots/upload-options/` return. `findExistingObjects` checks keys with batched HEAD requests; `putObjects` sends uploads up to 1 MB in batched multipart requests and PUTs larger ones individually. Batches hold up to 1000 operations and 25 MB, with up to 4 in flight, so large uploads stay under the per-request rate limit of the Sentry proxy. The client uses only web-standard APIs: the host supplies each file's bytes and may supply `fetch`. + +```ts +import { findExistingObjects, putObjects } from "@sentry/api/client"; + +const existing = await findExistingObjects(objectstore, keys, { fetch }); +await putObjects( + objectstore, + files + .filter((file) => !existing.has(file.key)) + .map((file) => ({ key: file.key, size: file.size, read: () => readFile(file.path) })), + { fetch }, +); +``` + +Failures throw `ObjectstoreError`, an `ApiError` with the failing object or request `url`. + ## Runtime validation The root `@sentry/api` entry has no runtime dependencies. It provides the API client and pure TypeScript types without installing a validation library. diff --git a/lib/api-client/index.ts b/lib/api-client/index.ts index 5396a460..679efe64 100644 --- a/lib/api-client/index.ts +++ b/lib/api-client/index.ts @@ -11,6 +11,7 @@ export * from "./auth-token.js"; export * from "./core.js"; export * from "./errors.js"; export * from "./idempotent-operations.js"; +export * from "./objectstore.js"; export * from "./pagination.js"; export * from "./response.js"; export * from "./response-cache/index.js"; diff --git a/lib/api-client/objectstore.ts b/lib/api-client/objectstore.ts new file mode 100644 index 00000000..2fe0edc6 --- /dev/null +++ b/lib/api-client/objectstore.ts @@ -0,0 +1,525 @@ +/** + * Objectstore client for uploading files, such as snapshot images, through the + * service that Sentry's upload-options endpoints return + * (e.g. `snapshots/upload-options/`). + * + * Implements the subset of the `objectstore-client` protocol + * (getsentry/objectstore) that uploads need: find which objects already exist, + * then upload the missing ones. The service URL, scopes, auth token, and + * expiration policy come from the Sentry API; this client never signs tokens. + * + * Object path layout: `{url}/v1/objects/{usecase}/{scope}/{key}`, where `scope` + * is a `;`-joined list of ordered `key=value` pairs. Auth is carried in the + * `x-os-auth: Bearer ` header, not `Authorization`. + * + * Like the Rust client's `Session::many()`, HEADs and small inserts are grouped + * into multipart requests to `{url}/v1/objects:batch/{usecase}/{scope}/`. The + * Sentry proxy in front of Objectstore rate-limits each HTTP request, so one + * request per object trips it (HTTP 429) on larger uploads. + * + * Uses only web-standard APIs. Hosts supply file contents through + * {@link ObjectUpload.read} and may inject `fetch` (e.g. for custom CAs). + */ + +import { ApiError } from "./errors.js"; + +/** Header carrying the Objectstore bearer token. */ +const AUTH_HEADER = "x-os-auth"; +/** Header carrying an object's expiration policy (e.g. `ttl:30d`). */ +const EXPIRATION_HEADER = "x-sn-expiration"; + +/** Matches one or more trailing slashes (for base-URL normalization). */ +const TRAILING_SLASHES = /\/+$/; + +/** Timeout for uploading a single object. */ +const PUT_TIMEOUT_MS = 120_000; +/** Timeout for one batch request (up to {@link MAX_BATCH_BODY_SIZE} bytes). */ +const BATCH_TIMEOUT_MS = 300_000; + +/** Per-part headers of the batch protocol. */ +const BATCH_KIND_HEADER = "x-sn-batch-operation-kind"; +const BATCH_KEY_HEADER = "x-sn-batch-operation-key"; +const BATCH_INDEX_HEADER = "x-sn-batch-operation-index"; +const BATCH_STATUS_HEADER = "x-sn-batch-operation-status"; + +/** Server-enforced maximum number of operations per batch request. */ +const MAX_BATCH_OPS = 1000; +/** Server-enforced maximum body size of one batch part; larger objects are PUT individually. */ +const MAX_BATCH_PART_SIZE = 1024 * 1024; +/** + * Maximum total body size of one batch request. With {@link BATCH_CONCURRENCY} + * batches in flight, upload bodies held in memory stay around 100 MB. + */ +const MAX_BATCH_BODY_SIZE = 25 * 1024 * 1024; +/** + * Batch requests in flight at once. Each batch is one request to the Sentry + * proxy, so this stays far below its per-request rate limit. + */ +const BATCH_CONCURRENCY = 4; +/** Concurrency for objects too large to batch (matches the Rust client). */ +const INDIVIDUAL_CONCURRENCY = 5; + +/** Matches the `boundary` parameter of a multipart `Content-Type`. */ +const BOUNDARY_PARAM = /boundary=(?:"([^"]+)"|([^;\s]+))/i; + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); +const CRLF = encoder.encode("\r\n"); +const HEADER_END = encoder.encode("\r\n\r\n"); + +/** + * Objectstore upload configuration, as returned (camelCase) by the Sentry + * upload-options endpoints. + */ +export type ObjectstoreConfig = { + /** Base service URL (may include a path prefix). */ + url: string; + usecase: string; + /** Ordered scope pairs (e.g. `[["org","1"],["project","2"]]`). */ + scopes: [string, string][]; + /** Pre-signed bearer token, or null/absent for unauthenticated stores. */ + authToken?: string | null; + /** Expiration policy string applied to uploaded objects. */ + expirationPolicy: string; +}; + +/** Host capabilities for Objectstore requests. */ +export type ObjectstoreOptions = { + /** Fetch to send requests with. Defaults to `globalThis.fetch`. */ + fetch?: (input: string, init?: RequestInit) => Promise; +}; + +/** A file to upload under an object key. */ +export type ObjectUpload = { + key: string; + /** Size in bytes, used to group uploads before any file is read. */ + size: number; + /** Read the file's bytes. Called once, when its request starts. */ + read: () => Promise>; +}; + +/** + * A failed Objectstore request or operation. `url` is the object URL for a + * failed operation, or the request URL for a failed request. + */ +export class ObjectstoreError extends ApiError { + constructor( + message: string, + status: number, + detail: string, + public readonly url: string, + ) { + super(message, status, detail); + this.name = "ObjectstoreError"; + Object.setPrototypeOf(this, ObjectstoreError.prototype); + } +} + +/** Render scope pairs into the `k=v;k=v` path segment. */ +function scopeSegment(scopes: [string, string][]): string { + return scopes.map(([key, value]) => `${key}=${value}`).join(";"); +} + +/** + * Build the full URL for an object key within the configured usecase + scope. + * + * @param config - The objectstore configuration. + * @param key - The object key (e.g. `//`). + */ +function buildObjectUrl(config: ObjectstoreConfig, key: string): string { + const base = config.url.replace(TRAILING_SLASHES, ""); + return `${base}/v1/objects/${config.usecase}/${scopeSegment(config.scopes)}/${key}`; +} + +/** Build the batch endpoint URL for the configured usecase + scope. */ +function buildBatchUrl(config: ObjectstoreConfig): string { + const base = config.url.replace(TRAILING_SLASHES, ""); + return `${base}/v1/objects:batch/${config.usecase}/${scopeSegment(config.scopes)}/`; +} + +/** + * Escape a value for an Objectstore header, matching `encode_header_value` in + * `objectstore-types`: percent-encode control bytes, DEL, `%`, and non-ASCII. + */ +function encodeHeaderValue(value: string): string { + let out = ""; + for (const byte of encoder.encode(value)) { + if (byte < 0x20 || byte >= 0x7f || byte === 0x25) { + out += `%${byte.toString(16).toUpperCase().padStart(2, "0")}`; + } else { + out += String.fromCharCode(byte); + } + } + return out; +} + +/** Auth headers for an objectstore request, if a token is configured. */ +function authHeaders(config: ObjectstoreConfig): Record { + return config.authToken ? { [AUTH_HEADER]: `Bearer ${config.authToken}` } : {}; +} + +function fetchFor(options: ObjectstoreOptions | undefined) { + return options?.fetch ?? ((input: string, init?: RequestInit) => globalThis.fetch(input, init)); +} + +/** + * Upload one object's bytes, raw (no content encoding). + * + * @throws {ObjectstoreError} On a non-2xx response. + */ +async function putObject( + config: ObjectstoreConfig, + key: string, + body: Uint8Array, + options?: ObjectstoreOptions, +): Promise { + const url = buildObjectUrl(config, key); + const response = await fetchFor(options)(url, { + method: "PUT", + headers: { ...authHeaders(config), [EXPIRATION_HEADER]: config.expirationPolicy }, + body, + signal: AbortSignal.timeout(PUT_TIMEOUT_MS), + }); + if (!response.ok) { + throw new ObjectstoreError( + "Objectstore upload failed", + response.status, + response.statusText || "PUT failed", + url, + ); + } +} + +/** One operation within a batch request. */ +type BatchOperation = + | { kind: "head"; key: string } + | { kind: "insert"; key: string; body: Uint8Array }; + +/** Outcome of one batch operation, as reported in its response part. */ +type BatchOperationResult = { + /** HTTP status of this operation (e.g. 200, 201, 404, 429). */ + status: number; + /** Response part body (an error description for failed operations). */ + message: string; +}; + +/** Concatenate byte arrays into one. */ +function concatBytes(chunks: Uint8Array[]): Uint8Array { + const out = new Uint8Array(chunks.reduce((total, c) => total + c.byteLength, 0)); + let offset = 0; + for (const c of chunks) { + out.set(c, offset); + offset += c.byteLength; + } + return out; +} + +/** Serialize batch operations into a `multipart/form-data` request body. */ +function encodeBatchBody( + config: ObjectstoreConfig, + operations: BatchOperation[], + boundary: string, +): Uint8Array { + const chunks: Uint8Array[] = []; + for (const op of operations) { + let head = + `--${boundary}\r\n` + + 'content-disposition: form-data; name="part"\r\n' + + `${BATCH_KIND_HEADER}: ${op.kind}\r\n` + + `${BATCH_KEY_HEADER}: ${encodeHeaderValue(op.key)}\r\n`; + if (op.kind === "insert") { + head += `${EXPIRATION_HEADER}: ${config.expirationPolicy}\r\n`; + } + chunks.push(encoder.encode(`${head}\r\n`)); + if (op.kind === "insert") { + chunks.push(op.body); + } + chunks.push(CRLF); + } + chunks.push(encoder.encode(`--${boundary}--\r\n`)); + return concatBytes(chunks); +} + +/** Index of `needle` in `haystack` at or after `from`, or -1. */ +function indexOfBytes(haystack: Uint8Array, needle: Uint8Array, from: number): number { + const last = haystack.length - needle.length; + outer: for (let i = from; i <= last; i++) { + for (let j = 0; j < needle.length; j++) { + if (haystack[i + j] !== needle[j]) { + continue outer; + } + } + return i; + } + return -1; +} + +/** Parse the header block of one multipart part into a lowercase-keyed map. */ +function parsePartHeaders(block: string): Map { + const headers = new Map(); + for (const line of block.split("\r\n")) { + const colon = line.indexOf(":"); + if (colon > 0) { + headers.set(line.slice(0, colon).trim().toLowerCase(), line.slice(colon + 1).trim()); + } + } + return headers; +} + +/** + * Parse a batch `multipart/form-data` response into per-operation results, + * keyed by the operation index the server echoes on each part. + * + * Response parts only carry status headers and small error bodies, so the + * whole response is buffered and split on the boundary. + */ +function parseBatchResponse( + contentType: string, + body: Uint8Array, + url: string, +): Map { + const match = BOUNDARY_PARAM.exec(contentType); + const boundary = match?.[1] ?? match?.[2]; + if (!boundary) { + throw new ObjectstoreError( + "Objectstore batch response is not multipart", + 502, + `Unexpected Content-Type: ${contentType || "(none)"}`, + url, + ); + } + + const delimiter = encoder.encode(`--${boundary}`); + const results = new Map(); + let start = indexOfBytes(body, delimiter, 0); + while (start >= 0) { + let partStart = start + delimiter.length; + if (body[partStart] === 0x2d && body[partStart + 1] === 0x2d) { + break; // closing delimiter + } + if (body[partStart] === 0x0d && body[partStart + 1] === 0x0a) { + partStart += 2; + } + const next = indexOfBytes(body, delimiter, partStart); + let partEnd = next < 0 ? body.length : next; + if (body[partEnd - 2] === 0x0d && body[partEnd - 1] === 0x0a) { + partEnd -= 2; + } + const split = indexOfBytes(body, HEADER_END, partStart); + if (split >= 0 && split < partEnd) { + const headers = parsePartHeaders(decoder.decode(body.subarray(partStart, split))); + const index = Number.parseInt(headers.get(BATCH_INDEX_HEADER) ?? "", 10); + const status = Number.parseInt(headers.get(BATCH_STATUS_HEADER) ?? "", 10); + if (!Number.isNaN(index) && !Number.isNaN(status)) { + results.set(index, { + status, + message: decoder.decode(body.subarray(split + HEADER_END.length, partEnd)), + }); + } + } + start = next; + } + return results; +} + +/** + * Send operations as one batch request. + * + * @returns Results aligned with `operations` by index. + * @throws {ObjectstoreError} When the batch request itself fails or the + * response is missing a result for any operation. + */ +async function sendBatch( + config: ObjectstoreConfig, + operations: BatchOperation[], + options: ObjectstoreOptions | undefined, +): Promise { + const url = buildBatchUrl(config); + const boundary = `sentry-api-${crypto.randomUUID()}`; + const response = await fetchFor(options)(url, { + method: "POST", + headers: { + ...authHeaders(config), + "content-type": `multipart/form-data; boundary="${boundary}"`, + }, + body: encodeBatchBody(config, operations, boundary), + signal: AbortSignal.timeout(BATCH_TIMEOUT_MS), + }); + if (!response.ok) { + throw new ObjectstoreError( + "Objectstore batch request failed", + response.status, + response.statusText || "batch request failed", + url, + ); + } + + const parsed = parseBatchResponse( + response.headers.get("content-type") ?? "", + new Uint8Array(await response.arrayBuffer()), + url, + ); + return operations.map((op, index) => { + const result = parsed.get(index); + if (!result) { + throw new ObjectstoreError( + "Objectstore batch response is incomplete", + 502, + `No result for ${op.kind} of ${op.key}`, + url, + ); + } + return result; + }); +} + +/** Throw an {@link ObjectstoreError} for a failed operation inside a batch. */ +function throwOperationError( + config: ObjectstoreConfig, + message: string, + key: string, + result: BatchOperationResult, +): never { + throw new ObjectstoreError( + message, + result.status, + result.message || `HTTP ${result.status}`, + buildObjectUrl(config, key), + ); +} + +/** Split `items` into consecutive chunks of at most `size` items. */ +function chunk(items: T[], size: number): T[][] { + const chunks: T[][] = []; + for (let i = 0; i < items.length; i += size) { + chunks.push(items.slice(i, i + size)); + } + return chunks; +} + +/** + * Run `tasks` with at most `limit` in flight. Rejects with the first failure; + * tasks that have not started by then are skipped. + */ +async function runLimited(tasks: (() => Promise)[], limit: number): Promise { + let next = 0; + let failed = false; + const worker = async () => { + while (!failed && next < tasks.length) { + const task = tasks[next++] as () => Promise; + try { + await task(); + } catch (error) { + failed = true; + throw error; + } + } + }; + await Promise.all(Array.from({ length: Math.min(limit, tasks.length) }, worker)); +} + +/** + * Find which keys already exist, using batched HEAD operations (one request + * per {@link MAX_BATCH_OPS} keys, up to {@link BATCH_CONCURRENCY} at once). + * + * @returns The subset of `keys` that exist. + * @throws {ObjectstoreError} On a failed batch, or any HEAD result other than 2xx/404. + */ +export async function findExistingObjects( + config: ObjectstoreConfig, + keys: string[], + options?: ObjectstoreOptions, +): Promise> { + const existing = new Set(); + await runLimited( + chunk(keys, MAX_BATCH_OPS).map((batch) => async () => { + const results = await sendBatch( + config, + batch.map((key) => ({ kind: "head", key })), + options, + ); + results.forEach((result, i) => { + const key = batch[i] as string; + if (result.status === 404) { + return; + } + if (result.status >= 400) { + throwOperationError(config, "Objectstore HEAD failed", key, result); + } + existing.add(key); + }); + }), + BATCH_CONCURRENCY, + ); + return existing; +} + +/** Group uploads into batches within the op-count and body-size limits. */ +function groupBatches(uploads: ObjectUpload[]): ObjectUpload[][] { + const batches: ObjectUpload[][] = []; + let current: ObjectUpload[] = []; + let currentSize = 0; + for (const upload of uploads) { + if ( + current.length >= MAX_BATCH_OPS || + (current.length > 0 && currentSize + upload.size > MAX_BATCH_BODY_SIZE) + ) { + batches.push(current); + current = []; + currentSize = 0; + } + current.push(upload); + currentSize += upload.size; + } + if (current.length > 0) { + batches.push(current); + } + return batches; +} + +/** + * Upload objects. Uploads up to {@link MAX_BATCH_PART_SIZE} are sent in batch + * requests, up to {@link BATCH_CONCURRENCY} at once; larger ones are PUT + * individually, up to {@link INDIVIDUAL_CONCURRENCY} at once. Both run in + * parallel. Each upload is read only when its request starts. + * + * @throws {ObjectstoreError} On a failed batch, or any failed insert. + */ +export async function putObjects( + config: ObjectstoreConfig, + uploads: ObjectUpload[], + options?: ObjectstoreOptions, +): Promise { + const batchable = uploads.filter((u) => u.size <= MAX_BATCH_PART_SIZE); + const individual = uploads.filter((u) => u.size > MAX_BATCH_PART_SIZE); + + await Promise.all([ + runLimited( + groupBatches(batchable).map((batch) => async () => { + const operations = await Promise.all( + batch.map( + async (upload): Promise => ({ + kind: "insert", + key: upload.key, + body: await upload.read(), + }), + ), + ); + const results = await sendBatch(config, operations, options); + results.forEach((result, i) => { + if (result.status >= 400) { + const key = (batch[i] as ObjectUpload).key; + throwOperationError(config, "Objectstore upload failed", key, result); + } + }); + }), + BATCH_CONCURRENCY, + ), + runLimited( + individual.map((upload) => async () => { + await putObject(config, upload.key, await upload.read(), options); + }), + INDIVIDUAL_CONCURRENCY, + ), + ]); +} diff --git a/test/api-client/objectstore.test.ts b/test/api-client/objectstore.test.ts new file mode 100644 index 00000000..8df6eba7 --- /dev/null +++ b/test/api-client/objectstore.test.ts @@ -0,0 +1,282 @@ +/** + * Tests for the Objectstore client. A fetch is injected so URL construction, + * the `x-os-auth` header, the batch request/response protocol, and the PUT + * request shape can be checked without a network. + */ + +import { describe, expect, it, vi } from "vitest"; +import { ApiError } from "../../src/api-client/errors.ts"; +import { + findExistingObjects, + type ObjectstoreConfig, + ObjectstoreError, + type ObjectUpload, + putObjects, +} from "../../src/api-client/objectstore.ts"; + +const config: ObjectstoreConfig = { + url: "https://objectstore.example.com/", + usecase: "preprod_snapshots", + scopes: [ + ["org", "123"], + ["project", "456"], + ], + authToken: "jwt-token", + expirationPolicy: "ttl:30d", +}; + +type FetchMock = ReturnType Promise>>; + +function mockFetch(): FetchMock { + return vi.fn<(input: string, init?: RequestInit) => Promise>(); +} + +/** Build a batch response in the server's `multipart/form-data` shape. */ +function batchResponse(parts: { index: number; status: string; body?: string }[]): Response { + const boundary = "os-boundary-00000000000000000000000000000abc"; + let text = ""; + for (const part of parts) { + text += + `--${boundary}\r\n` + + `x-sn-batch-operation-index: ${part.index}\r\n` + + `x-sn-batch-operation-status: ${part.status}\r\n` + + "content-disposition: form-data; name=part\r\n\r\n" + + `${part.body ?? ""}\r\n`; + } + text += `--${boundary}--`; + return new Response(text, { + status: 200, + headers: { "content-type": `multipart/form-data; boundary="${boundary}"` }, + }); +} + +/** Decode a captured batch request body as multipart form data. */ +async function sentParts(init: RequestInit): Promise { + const headers = init.headers as Record; + const form = await new Response(init.body as BodyInit, { + headers: { "content-type": headers["content-type"] as string }, + }).formData(); + return form.getAll("part"); +} + +function rawBody(init: RequestInit): string { + return new TextDecoder("latin1").decode(init.body as Uint8Array); +} + +function upload(key: string, bytes: Uint8Array): ObjectUpload & { reads: number } { + const item = { + key, + size: bytes.byteLength, + reads: 0, + read: async () => { + item.reads += 1; + return bytes; + }, + }; + return item; +} + +const OBJECTS_URL = "https://objectstore.example.com/v1/objects/preprod_snapshots/org=123;project=456"; +/** Over the 1 MB batch part limit, so it is PUT on its own. */ +const LARGE = 1024 * 1024 + 1; + +describe("findExistingObjects", () => { + it("sends all HEADs in one batch request and returns existing keys", async () => { + const fetch = mockFetch().mockResolvedValue( + batchResponse([ + { index: 1, status: "404 Not Found" }, + { index: 0, status: "200 OK" }, + { index: 2, status: "200 OK" }, + ]), + ); + + const existing = await findExistingObjects(config, ["k0", "k1", "k2"], { fetch }); + + expect([...existing].sort()).toEqual(["k0", "k2"]); + expect(fetch).toHaveBeenCalledTimes(1); + const [url, init] = fetch.mock.calls[0] as [string, RequestInit]; + expect(url).toBe( + "https://objectstore.example.com/v1/objects:batch/preprod_snapshots/org=123;project=456/", + ); + expect(init.method).toBe("POST"); + expect((init.headers as Record)["x-os-auth"]).toBe("Bearer jwt-token"); + expect(await sentParts(init)).toHaveLength(3); + const raw = rawBody(init); + expect(raw).toContain("x-sn-batch-operation-kind: head\r\n"); + expect(raw).toContain("x-sn-batch-operation-key: k1\r\n"); + }); + + it("percent-encodes `%`, controls, and non-ASCII in operation keys", async () => { + const fetch = mockFetch().mockResolvedValue(batchResponse([{ index: 0, status: "404 Not Found" }])); + + await findExistingObjects(config, ["100% r\u00e9port\n"], { fetch }); + + const [, init] = fetch.mock.calls[0] as [string, RequestInit]; + expect(rawBody(init)).toContain("x-sn-batch-operation-key: 100%25 r%C3%A9port%0A\r\n"); + }); + + it("splits more than 1000 keys into batches sent concurrently", async () => { + let inFlight = 0; + let maxInFlight = 0; + const fetch = mockFetch().mockImplementation(async (_url, init) => { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + const count = (await sentParts(init as RequestInit)).length; + await new Promise((resolve) => setTimeout(resolve, 10)); + inFlight -= 1; + return batchResponse(Array.from({ length: count }, (_, index) => ({ index, status: "200 OK" }))); + }); + const keys = Array.from({ length: 2500 }, (_, i) => `k${i}`); + + const existing = await findExistingObjects(config, keys, { fetch }); + + expect(existing.size).toBe(2500); + expect(fetch).toHaveBeenCalledTimes(3); + expect(maxInFlight).toBe(3); + }); + + it("throws on a per-operation error other than 404", async () => { + const fetch = mockFetch().mockResolvedValue( + batchResponse([ + { index: 0, status: "200 OK" }, + { index: 1, status: "429 Too Many Requests", body: "slow down — retry" }, + ]), + ); + const error = await findExistingObjects(config, ["k0", "k1"], { fetch }).catch((e) => e); + expect(error).toBeInstanceOf(ObjectstoreError); + expect(error).toBeInstanceOf(ApiError); + expect(error.status).toBe(429); + expect(error.detail).toBe("slow down — retry"); + expect(error.url).toBe(`${OBJECTS_URL}/k1`); + }); + + it("throws when the batch request itself fails", async () => { + const fetch = mockFetch().mockResolvedValue( + new Response(null, { status: 429, statusText: "Too Many Requests" }), + ); + const error = await findExistingObjects(config, ["k0"], { fetch }).catch((e) => e); + expect(error).toBeInstanceOf(ObjectstoreError); + expect(error.status).toBe(429); + expect(error.detail).toBe("Too Many Requests"); + }); + + it("throws when the response is missing an operation", async () => { + const fetch = mockFetch().mockResolvedValue(batchResponse([{ index: 0, status: "200 OK" }])); + await expect(findExistingObjects(config, ["k0", "k1"], { fetch })).rejects.toThrow( + ObjectstoreError, + ); + }); + + it("throws when the response is not multipart", async () => { + const fetch = mockFetch().mockResolvedValue( + new Response("{}", { headers: { "content-type": "application/json" } }), + ); + const error = await findExistingObjects(config, ["k0"], { fetch }).catch((e) => e); + expect(error).toBeInstanceOf(ObjectstoreError); + expect(error.status).toBe(502); + }); +}); + +describe("putObjects", () => { + it("batches small uploads and PUTs uploads over the batch part limit", async () => { + const small = upload("1/2/small", new Uint8Array([1, 2, 3])); + const large = upload("1/2/large", new Uint8Array(LARGE)); + const fetch = mockFetch().mockImplementation(async (_url, init) => + init?.method === "POST" + ? batchResponse([{ index: 0, status: "201 Created" }]) + : new Response(null, { status: 201 }), + ); + + await putObjects(config, [small, large], { fetch }); + + expect(fetch).toHaveBeenCalledTimes(2); + expect(small.reads).toBe(1); + expect(large.reads).toBe(1); + const [, batchInit] = fetch.mock.calls.find(([, init]) => init?.method === "POST") as [ + string, + RequestInit, + ]; + const parts = await sentParts(batchInit); + expect(parts).toHaveLength(1); + // Parts without a filename decode as strings. + expect(parts[0]).toBe("\x01\x02\x03"); + const raw = rawBody(batchInit); + expect(raw).toContain("x-sn-batch-operation-kind: insert\r\n"); + expect(raw).toContain("x-sn-batch-operation-key: 1/2/small\r\n"); + expect(raw).toContain("x-sn-expiration: ttl:30d\r\n"); + + const [putUrl] = fetch.mock.calls.find(([, init]) => init?.method === "PUT") as [string]; + expect(putUrl).toBe(`${OBJECTS_URL}/1/2/large`); + }); + + it("splits uploads over 25 MB into batches sent concurrently", async () => { + let inFlight = 0; + let maxInFlight = 0; + const fetch = mockFetch().mockImplementation(async (_url, init) => { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + const count = (await sentParts(init as RequestInit)).length; + await new Promise((resolve) => setTimeout(resolve, 10)); + inFlight -= 1; + return batchResponse( + Array.from({ length: count }, (_, index) => ({ index, status: "201 Created" })), + ); + }); + // 60 uploads of 1 MB: three batches of at most 25 MB each. + const uploads = Array.from({ length: 60 }, (_, i) => upload(`k${i}`, new Uint8Array(1024 * 1024))); + + await putObjects(config, uploads, { fetch }); + + expect(fetch).toHaveBeenCalledTimes(3); + expect(maxInFlight).toBe(3); + expect(uploads.every((u) => u.reads === 1)).toBe(true); + }); + + it("throws when an insert in the batch fails", async () => { + const fetch = mockFetch().mockResolvedValue( + batchResponse([{ index: 0, status: "500 Internal Server Error" }]), + ); + await expect( + putObjects(config, [upload("k", new Uint8Array([1]))], { fetch }), + ).rejects.toThrow(ObjectstoreError); + }); +}); + +describe("individual PUTs", () => { + it("PUTs the body with auth + expiration headers", async () => { + const fetch = mockFetch().mockResolvedValue(new Response(null, { status: 200 })); + + await putObjects(config, [upload("1/2/abc", new Uint8Array(LARGE))], { fetch }); + + const [url, init] = fetch.mock.calls[0] as [string, RequestInit]; + expect(url).toBe(`${OBJECTS_URL}/1/2/abc`); + expect(init.method).toBe("PUT"); + expect((init.body as Uint8Array).byteLength).toBe(LARGE); + const headers = init.headers as Record; + expect(headers["x-os-auth"]).toBe("Bearer jwt-token"); + expect(headers["x-sn-expiration"]).toBe("ttl:30d"); + }); + + it("omits the auth header without a token", async () => { + const fetch = mockFetch().mockResolvedValue(new Response(null, { status: 200 })); + + await putObjects({ ...config, authToken: null }, [upload("k", new Uint8Array(LARGE))], { + fetch, + }); + + const [, init] = fetch.mock.calls[0] as [string, RequestInit]; + expect(init.headers as Record).not.toHaveProperty("x-os-auth"); + }); + + it("throws on a non-2xx response", async () => { + const fetch = mockFetch().mockResolvedValue( + new Response(null, { status: 403, statusText: "Forbidden" }), + ); + const error = await putObjects(config, [upload("k", new Uint8Array(LARGE))], { fetch }).catch( + (e) => e, + ); + expect(error).toBeInstanceOf(ObjectstoreError); + expect(error.status).toBe(403); + expect(error.url).toBe(`${OBJECTS_URL}/k`); + }); +}); From 929e1f60af65ed082702d2ecb24459b01430002f Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 23:08:00 +0000 Subject: [PATCH 2/5] ref(client): Limit Objectstore concurrency with p-limit Use p-limit, as the CLI does, instead of a local limiter. The client entry already has runtime dependencies (http-cache-semantics), and p-limit uses only web-standard APIs. The client bundle keeps it external, and the packed-package tests link every runtime dependency. --- build.mjs | 2 +- lib/api-client/objectstore.ts | 74 +++++++++++++---------------------- package.json | 3 +- pnpm-lock.yaml | 17 ++++++++ test/packed-package.ts | 13 +++--- 5 files changed, 55 insertions(+), 54 deletions(-) diff --git a/build.mjs b/build.mjs index 7a77ca12..53197bfc 100644 --- a/build.mjs +++ b/build.mjs @@ -109,7 +109,7 @@ await Promise.all([ bundle("zod", { external: ["zod"] }), bundle("browser"), // Keep the package entries external so consumers share the root error classes. - bundle("api-client/index", { external: ["@sentry/api", "http-cache-semantics"] }), + bundle("api-client/index", { external: ["@sentry/api", "http-cache-semantics", "p-limit"] }), bundle("api-client/endpoints/index", { external: ["@sentry/api", "@sentry/api/client"] }), bundle("operations"), bundle("source-analysis", { external: ["typescript"], platform: "node" }), diff --git a/lib/api-client/objectstore.ts b/lib/api-client/objectstore.ts index 2fe0edc6..5a04a473 100644 --- a/lib/api-client/objectstore.ts +++ b/lib/api-client/objectstore.ts @@ -21,6 +21,7 @@ * {@link ObjectUpload.read} and may inject `fetch` (e.g. for custom CAs). */ +import pLimit from "p-limit"; import { ApiError } from "./errors.js"; /** Header carrying the Objectstore bearer token. */ @@ -397,27 +398,6 @@ function chunk(items: T[], size: number): T[][] { return chunks; } -/** - * Run `tasks` with at most `limit` in flight. Rejects with the first failure; - * tasks that have not started by then are skipped. - */ -async function runLimited(tasks: (() => Promise)[], limit: number): Promise { - let next = 0; - let failed = false; - const worker = async () => { - while (!failed && next < tasks.length) { - const task = tasks[next++] as () => Promise; - try { - await task(); - } catch (error) { - failed = true; - throw error; - } - } - }; - await Promise.all(Array.from({ length: Math.min(limit, tasks.length) }, worker)); -} - /** * Find which keys already exist, using batched HEAD operations (one request * per {@link MAX_BATCH_OPS} keys, up to {@link BATCH_CONCURRENCY} at once). @@ -431,25 +411,27 @@ export async function findExistingObjects( options?: ObjectstoreOptions, ): Promise> { const existing = new Set(); - await runLimited( - chunk(keys, MAX_BATCH_OPS).map((batch) => async () => { - const results = await sendBatch( - config, - batch.map((key) => ({ kind: "head", key })), - options, - ); - results.forEach((result, i) => { - const key = batch[i] as string; - if (result.status === 404) { - return; - } - if (result.status >= 400) { - throwOperationError(config, "Objectstore HEAD failed", key, result); - } - existing.add(key); - }); - }), - BATCH_CONCURRENCY, + const limit = pLimit(BATCH_CONCURRENCY); + await Promise.all( + chunk(keys, MAX_BATCH_OPS).map((batch) => + limit(async () => { + const results = await sendBatch( + config, + batch.map((key) => ({ kind: "head", key })), + options, + ); + results.forEach((result, i) => { + const key = batch[i] as string; + if (result.status === 404) { + return; + } + if (result.status >= 400) { + throwOperationError(config, "Objectstore HEAD failed", key, result); + } + existing.add(key); + }); + }), + ), ); return existing; } @@ -493,9 +475,11 @@ export async function putObjects( const batchable = uploads.filter((u) => u.size <= MAX_BATCH_PART_SIZE); const individual = uploads.filter((u) => u.size > MAX_BATCH_PART_SIZE); + const batchLimit = pLimit(BATCH_CONCURRENCY); + const individualLimit = pLimit(INDIVIDUAL_CONCURRENCY); await Promise.all([ - runLimited( - groupBatches(batchable).map((batch) => async () => { + ...groupBatches(batchable).map((batch) => + batchLimit(async () => { const operations = await Promise.all( batch.map( async (upload): Promise => ({ @@ -513,13 +497,11 @@ export async function putObjects( } }); }), - BATCH_CONCURRENCY, ), - runLimited( - individual.map((upload) => async () => { + ...individual.map((upload) => + individualLimit(async () => { await putObject(config, upload.key, await upload.read(), options); }), - INDIVIDUAL_CONCURRENCY, ), ]); } diff --git a/package.json b/package.json index 7b0d13b5..c22a9bcc 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,8 @@ "url": "https://github.com/getsentry/sentry-api-schema/issues" }, "dependencies": { - "http-cache-semantics": "^4.2.0" + "http-cache-semantics": "^4.2.0", + "p-limit": "^7.3.3" }, "peerDependencies": { "typescript": "*", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index abf06be7..a5bcf23c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,6 +14,9 @@ importers: http-cache-semantics: specifier: ^4.2.0 version: 4.3.0 + p-limit: + specifier: ^7.3.3 + version: 7.3.3 devDependencies: '@hey-api/openapi-ts': specifier: 0.98.0 @@ -620,6 +623,10 @@ packages: resolution: {integrity: sha512-smsWv2LzFjP03xmvFoJ331ss6h+jixfA4UUV/Bsiyuu4YJPfN+FIQGOIiv4w9/+MoHkfkJ22UIaQWRVFRfH6Vw==} engines: {node: '>=20'} + p-limit@7.3.3: + resolution: {integrity: sha512-SKgVvDU5TNIxQ4r30U75BNLC6a+6GOEjngrk/ysM5+Zu3oJuk1SC5ApzIzY/7PjCEjoVPMWJ0zho/4QUiEB7TQ==} + engines: {node: '>=20'} + path-key@3.1.1: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} @@ -816,6 +823,10 @@ packages: resolution: {integrity: sha512-g/eziiSUNBSsdDJtCLB8bdYEUMj4jR7AGeUo96p/3dTafgjHhpF4RiCFPiRILwjQoDXx5MqkBr4fwWtR3Ky4Wg==} engines: {node: '>=20'} + yocto-queue@1.2.2: + resolution: {integrity: sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==} + engines: {node: '>=12.20'} + zod@4.6.7: resolution: {integrity: sha512-g8W2D2kU6a8untc053fX1uVBKzM7yqAxW/3zwk83+6RyF2+SKOuV+c0wBDhUwzV1PaGapmZH+rmcUzPf+3CGwg==} @@ -1247,6 +1258,10 @@ snapshots: powershell-utils: 0.1.0 wsl-utils: 0.3.1 + p-limit@7.3.3: + dependencies: + yocto-queue: 1.2.2 + path-key@3.1.1: {} pathe@2.0.3: {} @@ -1379,4 +1394,6 @@ snapshots: is-wsl: 3.1.1 powershell-utils: 0.1.0 + yocto-queue@1.2.2: {} + zod@4.6.7: {} diff --git a/test/packed-package.ts b/test/packed-package.ts index ff3cd93a..910a228b 100644 --- a/test/packed-package.ts +++ b/test/packed-package.ts @@ -1,5 +1,5 @@ import { execFileSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, symlinkSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -20,11 +20,12 @@ export function installPackedPackage(prefix: string): string { const installed = join(directory, "node_modules", "@sentry", "api"); mkdirSync(installed, { recursive: true }); execFileSync("tar", ["-xzf", join(directory, packed[0]!.filename), "-C", installed, "--strip-components=1"]); - symlinkSync( - join(packageRoot, "node_modules", "http-cache-semantics"), - join(directory, "node_modules", "http-cache-semantics"), - "dir", - ); + const { dependencies } = JSON.parse(readFileSync(join(packageRoot, "package.json"), "utf8")) as { + dependencies: Record; + }; + for (const name of Object.keys(dependencies)) { + symlinkSync(join(packageRoot, "node_modules", name), join(directory, "node_modules", name), "dir"); + } writeFileSync(join(directory, "package.json"), JSON.stringify({ type: "module" })); return directory; } From bdc75d78cc1dd0e5fba10ed4befc95389ff33279 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 23:11:21 +0000 Subject: [PATCH 3/5] fix(client): Bound Objectstore batch reads and encode object URL keys Read the uploads of a batch one at a time, so the batches in flight hold at most 4 files open instead of up to 1000 each (EMFILE). Percent-encode each `/`-separated key segment in object URLs with the path-segment set the Rust client uses (url crate), so keys with `%`, `?`, `#`, spaces, or non-ASCII PUT the same object that batch operations address. --- lib/api-client/objectstore.ts | 44 ++++++++++++++++++----------- test/api-client/objectstore.test.ts | 33 ++++++++++++++++++++++ 2 files changed, 61 insertions(+), 16 deletions(-) diff --git a/lib/api-client/objectstore.ts b/lib/api-client/objectstore.ts index 5a04a473..f260f07f 100644 --- a/lib/api-client/objectstore.ts +++ b/lib/api-client/objectstore.ts @@ -129,7 +129,8 @@ function scopeSegment(scopes: [string, string][]): string { */ function buildObjectUrl(config: ObjectstoreConfig, key: string): string { const base = config.url.replace(TRAILING_SLASHES, ""); - return `${base}/v1/objects/${config.usecase}/${scopeSegment(config.scopes)}/${key}`; + const path = key.split("/").map(encodePathSegment).join("/"); + return `${base}/v1/objects/${config.usecase}/${scopeSegment(config.scopes)}/${path}`; } /** Build the batch endpoint URL for the configured usecase + scope. */ @@ -138,14 +139,11 @@ function buildBatchUrl(config: ObjectstoreConfig): string { return `${base}/v1/objects:batch/${config.usecase}/${scopeSegment(config.scopes)}/`; } -/** - * Escape a value for an Objectstore header, matching `encode_header_value` in - * `objectstore-types`: percent-encode control bytes, DEL, `%`, and non-ASCII. - */ -function encodeHeaderValue(value: string): string { +/** Percent-encode the UTF-8 bytes of `value` that are controls, DEL, non-ASCII, or in `reserved`. */ +function percentEncode(value: string, reserved: string): string { let out = ""; for (const byte of encoder.encode(value)) { - if (byte < 0x20 || byte >= 0x7f || byte === 0x25) { + if (byte < 0x20 || byte >= 0x7f || reserved.includes(String.fromCharCode(byte))) { out += `%${byte.toString(16).toUpperCase().padStart(2, "0")}`; } else { out += String.fromCharCode(byte); @@ -154,6 +152,23 @@ function encodeHeaderValue(value: string): string { return out; } +/** + * Escape a value for an Objectstore header, matching `encode_header_value` in + * `objectstore-types`: percent-encode control bytes, DEL, `%`, and non-ASCII. + */ +function encodeHeaderValue(value: string): string { + return percentEncode(value, "%"); +} + +/** + * Escape one key segment for an object URL, matching the Rust client, which + * pushes each `/`-separated part of the key with the `url` crate's + * path-segment encode set. + */ +function encodePathSegment(segment: string): string { + return percentEncode(segment, ' "#%<>?`{}'); +} + /** Auth headers for an objectstore request, if a token is configured. */ function authHeaders(config: ObjectstoreConfig): Record { return config.authToken ? { [AUTH_HEADER]: `Bearer ${config.authToken}` } : {}; @@ -480,15 +495,12 @@ export async function putObjects( await Promise.all([ ...groupBatches(batchable).map((batch) => batchLimit(async () => { - const operations = await Promise.all( - batch.map( - async (upload): Promise => ({ - kind: "insert", - key: upload.key, - body: await upload.read(), - }), - ), - ); + // Read one upload at a time, so batches in flight hold at most + // BATCH_CONCURRENCY files open, not up to MAX_BATCH_OPS each. + const operations: BatchOperation[] = []; + for (const upload of batch) { + operations.push({ kind: "insert", key: upload.key, body: await upload.read() }); + } const results = await sendBatch(config, operations, options); results.forEach((result, i) => { if (result.status >= 400) { diff --git a/test/api-client/objectstore.test.ts b/test/api-client/objectstore.test.ts index 8df6eba7..779ddb62 100644 --- a/test/api-client/objectstore.test.ts +++ b/test/api-client/objectstore.test.ts @@ -232,6 +232,30 @@ describe("putObjects", () => { expect(uploads.every((u) => u.reads === 1)).toBe(true); }); + it("reads one upload at a time within a batch", async () => { + let reading = 0; + let maxReading = 0; + const uploads = Array.from({ length: 50 }, (_, i): ObjectUpload => ({ + key: `k${i}`, + size: 1, + read: async () => { + reading += 1; + maxReading = Math.max(maxReading, reading); + await new Promise((resolve) => setTimeout(resolve, 1)); + reading -= 1; + return new Uint8Array([i]); + }, + })); + const fetch = mockFetch().mockResolvedValue( + batchResponse(uploads.map((_, index) => ({ index, status: "201 Created" }))), + ); + + await putObjects(config, uploads, { fetch }); + + expect(fetch).toHaveBeenCalledTimes(1); + expect(maxReading).toBe(1); + }); + it("throws when an insert in the batch fails", async () => { const fetch = mockFetch().mockResolvedValue( batchResponse([{ index: 0, status: "500 Internal Server Error" }]), @@ -257,6 +281,15 @@ describe("individual PUTs", () => { expect(headers["x-sn-expiration"]).toBe("ttl:30d"); }); + it("percent-encodes key segments in the object URL", async () => { + const fetch = mockFetch().mockResolvedValue(new Response(null, { status: 200 })); + + await putObjects(config, [upload("1/a b%?#/r\u00e9", new Uint8Array(LARGE))], { fetch }); + + const [url] = fetch.mock.calls[0] as [string]; + expect(url).toBe(`${OBJECTS_URL}/1/a%20b%25%3F%23/r%C3%A9`); + }); + it("omits the auth header without a token", async () => { const fetch = mockFetch().mockResolvedValue(new Response(null, { status: 200 })); From 5c5b1baa7f2b972893df4daf385b1de290517e3d Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 23:16:36 +0000 Subject: [PATCH 4/5] fix(client): Stop queued Objectstore requests after the first failure p-limit keeps starting queued tasks after one rejects, so a failed batch or PUT (e.g. a 429) was followed by more requests and file reads. Clear the queues of the limiters in the same operation when a task fails, before p-limit can start the next one. --- lib/api-client/objectstore.ts | 38 +++++++++++++++++++++++------ test/api-client/objectstore.test.ts | 37 ++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 7 deletions(-) diff --git a/lib/api-client/objectstore.ts b/lib/api-client/objectstore.ts index f260f07f..0b4b1131 100644 --- a/lib/api-client/objectstore.ts +++ b/lib/api-client/objectstore.ts @@ -21,7 +21,7 @@ * {@link ObjectUpload.read} and may inject `fetch` (e.g. for custom CAs). */ -import pLimit from "p-limit"; +import pLimit, { type LimitFunction } from "p-limit"; import { ApiError } from "./errors.js"; /** Header carrying the Objectstore bearer token. */ @@ -413,6 +413,29 @@ function chunk(items: T[], size: number): T[][] { return chunks; } +/** + * Wrap `task` so that its failure clears every queue in `limits` before the + * limiter can start the next task: after the first error, no new request + * starts and no further upload is read. Cleared tasks never settle, which is + * fine: the caller's `Promise.all` has already rejected with the first error, + * and pending promises hold no timers or handles. + */ +function stopOnFailure( + limits: LimitFunction[], + task: () => Promise, +): () => Promise { + return async () => { + try { + await task(); + } catch (error) { + for (const limit of limits) { + limit.clearQueue(); + } + throw error; + } + }; +} + /** * Find which keys already exist, using batched HEAD operations (one request * per {@link MAX_BATCH_OPS} keys, up to {@link BATCH_CONCURRENCY} at once). @@ -429,7 +452,7 @@ export async function findExistingObjects( const limit = pLimit(BATCH_CONCURRENCY); await Promise.all( chunk(keys, MAX_BATCH_OPS).map((batch) => - limit(async () => { + limit(stopOnFailure([limit], async () => { const results = await sendBatch( config, batch.map((key) => ({ kind: "head", key })), @@ -445,7 +468,7 @@ export async function findExistingObjects( } existing.add(key); }); - }), + })), ), ); return existing; @@ -492,9 +515,10 @@ export async function putObjects( const batchLimit = pLimit(BATCH_CONCURRENCY); const individualLimit = pLimit(INDIVIDUAL_CONCURRENCY); + const limits = [batchLimit, individualLimit]; await Promise.all([ ...groupBatches(batchable).map((batch) => - batchLimit(async () => { + batchLimit(stopOnFailure(limits, async () => { // Read one upload at a time, so batches in flight hold at most // BATCH_CONCURRENCY files open, not up to MAX_BATCH_OPS each. const operations: BatchOperation[] = []; @@ -508,12 +532,12 @@ export async function putObjects( throwOperationError(config, "Objectstore upload failed", key, result); } }); - }), + })), ), ...individual.map((upload) => - individualLimit(async () => { + individualLimit(stopOnFailure(limits, async () => { await putObject(config, upload.key, await upload.read(), options); - }), + })), ), ]); } diff --git a/test/api-client/objectstore.test.ts b/test/api-client/objectstore.test.ts index 779ddb62..3f05f7a2 100644 --- a/test/api-client/objectstore.test.ts +++ b/test/api-client/objectstore.test.ts @@ -135,6 +135,25 @@ describe("findExistingObjects", () => { expect(maxInFlight).toBe(3); }); + it("starts no queued batch after one fails", async () => { + // Five batches, four in flight: the first fails before the others finish. + const fetch = mockFetch().mockImplementation(async (_url, init) => { + const call = fetch.mock.calls.length; + const count = (await sentParts(init as RequestInit)).length; + if (call === 1) { + return new Response(null, { status: 429 }); + } + await new Promise((resolve) => setTimeout(resolve, 10)); + return batchResponse(Array.from({ length: count }, (_, index) => ({ index, status: "200 OK" }))); + }); + const keys = Array.from({ length: 5000 }, (_, i) => `k${i}`); + + await expect(findExistingObjects(config, keys, { fetch })).rejects.toThrow(ObjectstoreError); + await new Promise((resolve) => setTimeout(resolve, 30)); + + expect(fetch).toHaveBeenCalledTimes(4); + }); + it("throws on a per-operation error other than 404", async () => { const fetch = mockFetch().mockResolvedValue( batchResponse([ @@ -256,6 +275,24 @@ describe("putObjects", () => { expect(maxReading).toBe(1); }); + it("starts and reads no queued upload after one fails", async () => { + // Six large uploads, five PUTs in flight: the first fails before the others finish. + const fetch = mockFetch().mockImplementation(async () => { + if (fetch.mock.calls.length === 1) { + return new Response(null, { status: 429 }); + } + await new Promise((resolve) => setTimeout(resolve, 10)); + return new Response(null, { status: 201 }); + }); + const uploads = Array.from({ length: 6 }, (_, i) => upload(`k${i}`, new Uint8Array(LARGE))); + + await expect(putObjects(config, uploads, { fetch })).rejects.toThrow(ObjectstoreError); + await new Promise((resolve) => setTimeout(resolve, 30)); + + expect(fetch).toHaveBeenCalledTimes(5); + expect(uploads[5]?.reads).toBe(0); + }); + it("throws when an insert in the batch fails", async () => { const fetch = mockFetch().mockResolvedValue( batchResponse([{ index: 0, status: "500 Internal Server Error" }]), From 17d78bfc21ae8cdc3d81a2b29e59a2217a48a678 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 23:19:58 +0000 Subject: [PATCH 5/5] fix(client): Percent-encode backslash and caret in object URL keys Match the url crate's SPECIAL_PATH_SEGMENT set, which the Rust client uses for http(s) object URLs: add `\` (fetch parses it as `/`, storing the object under another key) and `^`. --- lib/api-client/objectstore.ts | 5 +++-- test/api-client/objectstore.test.ts | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/api-client/objectstore.ts b/lib/api-client/objectstore.ts index 0b4b1131..1759579a 100644 --- a/lib/api-client/objectstore.ts +++ b/lib/api-client/objectstore.ts @@ -163,10 +163,11 @@ function encodeHeaderValue(value: string): string { /** * Escape one key segment for an object URL, matching the Rust client, which * pushes each `/`-separated part of the key with the `url` crate's - * path-segment encode set. + * `SPECIAL_PATH_SEGMENT` encode set (http(s) URLs). That set includes `\`, + * which those URLs would otherwise parse as `/`. */ function encodePathSegment(segment: string): string { - return percentEncode(segment, ' "#%<>?`{}'); + return percentEncode(segment, ' "#%<>?^`{}\\'); } /** Auth headers for an objectstore request, if a token is configured. */ diff --git a/test/api-client/objectstore.test.ts b/test/api-client/objectstore.test.ts index 3f05f7a2..c879125e 100644 --- a/test/api-client/objectstore.test.ts +++ b/test/api-client/objectstore.test.ts @@ -321,10 +321,11 @@ describe("individual PUTs", () => { it("percent-encodes key segments in the object URL", async () => { const fetch = mockFetch().mockResolvedValue(new Response(null, { status: 200 })); - await putObjects(config, [upload("1/a b%?#/r\u00e9", new Uint8Array(LARGE))], { fetch }); + await putObjects(config, [upload("1/a b%?#^\\c/r\u00e9", new Uint8Array(LARGE))], { fetch }); const [url] = fetch.mock.calls[0] as [string]; - expect(url).toBe(`${OBJECTS_URL}/1/a%20b%25%3F%23/r%C3%A9`); + expect(url).toBe(`${OBJECTS_URL}/1/a%20b%25%3F%23%5E%5Cc/r%C3%A9`); + expect(new URL(url).pathname).toMatch(/\/1\/a%20b%25%3F%23%5E%5Cc\/r%C3%A9$/); }); it("omits the auth header without a token", async () => {