From c15fabf6d7597d3d7045427eacbef565bf577903 Mon Sep 17 00:00:00 2001 From: JPeer264 Date: Fri, 2 Oct 2026 13:21:15 +0200 Subject: [PATCH] test(e2e): Add a node-pi-durable end-to-end application Covers what the node integration test cannot: the packed SDK, real model requests through pi-ai's `@anthropic-ai/sdk` path, SQLite storage, and a run that resumes after the server crashes during a tool call. Co-Authored-By: Claude Opus 5.5 --- .../node-pi-durable/.gitignore | 5 + .../node-pi-durable/instrument.mjs | 8 + .../node-pi-durable/package.json | 41 ++++ .../node-pi-durable/playwright.config.mjs | 10 + .../node-pi-durable/src/server.mjs | 162 +++++++++++++++ .../node-pi-durable/src/start.mjs | 35 ++++ .../node-pi-durable/start-event-proxy.mjs | 6 + .../node-pi-durable/tests/pi-durable.test.ts | 188 ++++++++++++++++++ .../node-pi-durable/tests/utils.ts | 47 +++++ 9 files changed, 502 insertions(+) create mode 100644 dev-packages/e2e-tests/test-applications/node-pi-durable/.gitignore create mode 100644 dev-packages/e2e-tests/test-applications/node-pi-durable/instrument.mjs create mode 100644 dev-packages/e2e-tests/test-applications/node-pi-durable/package.json create mode 100644 dev-packages/e2e-tests/test-applications/node-pi-durable/playwright.config.mjs create mode 100644 dev-packages/e2e-tests/test-applications/node-pi-durable/src/server.mjs create mode 100644 dev-packages/e2e-tests/test-applications/node-pi-durable/src/start.mjs create mode 100644 dev-packages/e2e-tests/test-applications/node-pi-durable/start-event-proxy.mjs create mode 100644 dev-packages/e2e-tests/test-applications/node-pi-durable/tests/pi-durable.test.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-pi-durable/tests/utils.ts diff --git a/dev-packages/e2e-tests/test-applications/node-pi-durable/.gitignore b/dev-packages/e2e-tests/test-applications/node-pi-durable/.gitignore new file mode 100644 index 000000000000..6054d28d34f9 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-pi-durable/.gitignore @@ -0,0 +1,5 @@ +node_modules +.data +results.junit.xml +test-results +playwright-report diff --git a/dev-packages/e2e-tests/test-applications/node-pi-durable/instrument.mjs b/dev-packages/e2e-tests/test-applications/node-pi-durable/instrument.mjs new file mode 100644 index 000000000000..b774d9306950 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-pi-durable/instrument.mjs @@ -0,0 +1,8 @@ +import * as Sentry from '@sentry/node'; + +Sentry.init({ + environment: 'qa', + dsn: process.env.E2E_TEST_DSN, + tunnel: 'http://localhost:3031/', // proxy server + tracesSampleRate: 1.0, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-pi-durable/package.json b/dev-packages/e2e-tests/test-applications/node-pi-durable/package.json new file mode 100644 index 000000000000..75c0994b4834 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-pi-durable/package.json @@ -0,0 +1,41 @@ +{ + "name": "node-pi-durable", + "version": "0.0.0", + "private": true, + "type": "module", + "scripts": { + "start": "node src/start.mjs", + "clean": "npx rimraf node_modules .data pnpm-lock.yaml", + "test:build": "pnpm install", + "test:build-latest": "pnpm install && pnpm add @earendil-works/pi-durable@latest @earendil-works/pi-ai@latest @earendil-works/chord@latest", + "test:assert": "OPENROUTER_API_KEY=$E2E_OPENROUTER_API_KEY playwright test" + }, + "dependencies": { + "@earendil-works/chord": "1.0.0", + "@earendil-works/pi-ai": "1.0.0", + "@earendil-works/pi-durable": "1.0.0", + "@sentry/node": "file:../../packed/sentry-node-packed.tgz" + }, + "devDependencies": { + "@playwright/test": "~1.63.0", + "@sentry-internal/test-utils": "link:../../../test-utils", + "@sentry/core": "file:../../packed/sentry-core-packed.tgz", + "@types/node": "24.x" + }, + "engines": { + "node": "24.x" + }, + "volta": { + "node": "24.15.0", + "extends": "../../package.json" + }, + "sentryTest": { + "optional": true, + "optionalVariants": [ + { + "build-command": "pnpm test:build-latest", + "label": "node-pi-durable (latest)" + } + ] + } +} diff --git a/dev-packages/e2e-tests/test-applications/node-pi-durable/playwright.config.mjs b/dev-packages/e2e-tests/test-applications/node-pi-durable/playwright.config.mjs new file mode 100644 index 000000000000..ad0caf7da2c9 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-pi-durable/playwright.config.mjs @@ -0,0 +1,10 @@ +import { getPlaywrightConfig } from '@sentry-internal/test-utils'; + +const config = getPlaywrightConfig( + { startCommand: 'pnpm start' }, + // Each test drives real OpenRouter tool-calling turns, and one restarts the server mid-run, which + // does not fit the default 30s timeout when the provider is slow. + { timeout: 120_000 }, +); + +export default config; diff --git a/dev-packages/e2e-tests/test-applications/node-pi-durable/src/server.mjs b/dev-packages/e2e-tests/test-applications/node-pi-durable/src/server.mjs new file mode 100644 index 000000000000..56e37cb54082 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-pi-durable/src/server.mjs @@ -0,0 +1,162 @@ +import { mkdirSync, writeFileSync } from 'node:fs'; +import { createServer } from 'node:http'; +import { fileURLToPath } from 'node:url'; +import { BACKGROUND_CONTEXT } from '@earendil-works/chord/context'; +import { Type } from '@earendil-works/pi-ai'; +import { createModels } from '@earendil-works/pi-ai/models'; +import { openrouterProvider } from '@earendil-works/pi-ai/providers/openrouter'; +import { + AssistantEntry, + createRegistry, + defineExtension, + defineTool, + Harness, + section, +} from '@earendil-works/pi-durable'; +import { openNodeSqliteStorage } from '@earendil-works/pi-durable/storage/sqlite/node'; +import * as Sentry from '@sentry/node'; + +const context = BACKGROUND_CONTEXT; +const MODEL = { provider: 'openrouter', modelId: 'anthropic/claude-haiku-4.5' }; +const CRASH_EXIT_CODE = 75; + +if (!process.env.OPENROUTER_API_KEY) { + throw new Error('OPENROUTER_API_KEY is not set (E2E_OPENROUTER_API_KEY in the e2e-tests .env)'); +} + +const dataDir = fileURLToPath(new URL('../.data/', import.meta.url)); +mkdirSync(dataDir, { recursive: true }); + +const models = createModels(); +models.setProvider(openrouterProvider()); // reads OPENROUTER_API_KEY + +const tools = [ + defineTool({ + name: 'get_weather', + description: 'Get the current weather for a city.', + parameters: Type.Object({ city: Type.String() }), + // The manual span should nest under the SDK's `execute_tool` span. + execute: async args => + Sentry.startSpan({ name: 'resolve-weather', attributes: { 'weather.city': args.city } }, () => ({ + content: [{ type: 'text', text: `It is 21 degrees and sunny in ${args.city}.` }], + })), + }), + defineTool({ + name: 'fail_now', + description: 'Always throws an error. Call this when the user asks to trigger a failure.', + parameters: Type.Object({}), + execute: async () => { + throw new Error('Intentional pi-durable tool failure'); + }, + }), + defineTool({ + name: 'flaky_step', + description: 'Runs one step of a job. Call this when the user asks to run the flaky step.', + parameters: Type.Object({ job: Type.String() }), + // Replay-safe, so pi-durable reruns the call after the crash instead of failing it. + replay: 'safe', + execute: async args => { + // The marker is created once, atomically; its presence means the first attempt already crashed. + try { + writeFileSync(`${dataDir}flaky-${args.job}`, 'crashed once', { flag: 'wx' }); + } catch { + return { content: [{ type: 'text', text: `Step of job ${args.job} completed.` }] }; + } + process.exit(CRASH_EXIT_CODE); + }, + }), + defineTool({ + name: 'delegate', + description: 'Delegate a self-contained task to a subagent and get its answer back.', + parameters: Type.Object({ task: Type.String() }), + replay: 'safe', + execute: async (args, api, toolContext) => { + const childId = await api.commit(async tx => { + const existing = (await tx.scanConversations({ ownerTaskId: api.taskId }, 1)).items[0]; + if (existing !== undefined) return existing.id; + return (await tx.createConversation({ ownership: { kind: 'task', taskId: api.taskId } })).id; + }, toolContext); + const child = await api.conversation(childId, toolContext); + const request = { type: 'input', content: args.task, requestId: `delegate:${api.taskId}` }; + const settled = await (await child.submit(request, toolContext)).wait(toolContext); + if (settled.status !== 'done') { + return { isError: true, content: [{ type: 'text', text: `Subagent ended: ${settled.reason}` }] }; + } + const answer = await api.commit(tx => tx.entry(AssistantEntry, settled.answer), toolContext); + const text = (answer?.model?.[0]?.content ?? []) + .filter(part => part.type === 'text') + .map(part => part.text) + .join(''); + return { content: [{ type: 'text', text }] }; + }, + }), +]; + +const registry = createRegistry(); +registry.install( + defineExtension({ + name: 'e2e', + sections: [ + section('preamble', () => 'You are a test assistant. Use the tools exactly as asked. Keep answers short.', { + tag: false, + }), + ], + tools, + }), +); + +const storage = await openNodeSqliteStorage(`${dataDir}pi.sqlite`); +const harness = await Harness.open(storage, { models, registry }, context); +// Continue whatever the previous process left unfinished, such as a run interrupted by `flaky_step`. +harness.resume(); + +async function readJson(req) { + let body = ''; + for await (const chunk of req) body += chunk; + return body ? JSON.parse(body) : {}; +} + +async function handle(req, res) { + const url = new URL(req.url, 'http://localhost'); + const parts = url.pathname.split('/').filter(Boolean); + + if (req.method === 'POST' && url.pathname === '/conversations') { + const conversation = await harness.createConversation( + { ownership: { kind: 'ownerless' }, agent: { model: MODEL } }, + context, + ); + return { id: conversation.id }; + } + + // Returns as soon as the input is admitted. The run continues in the background, possibly in a + // later process, so clients poll `GET /submissions/:id` for the outcome. + if (req.method === 'POST' && parts[0] === 'conversations' && parts[2] === 'messages') { + const conversation = await harness.conversation(Number(parts[1]), context); + if (!conversation) return undefined; + const { content } = await readJson(req); + const submission = await conversation.submit({ type: 'input', content }, context); + return { submissionId: submission.id }; + } + + if (req.method === 'GET' && parts[0] === 'submissions') { + const submission = await harness.submission(Number(parts[1]), context); + if (!submission) return undefined; + const record = await submission.status(context); + return { status: record.status, reason: record.reason, detail: record.detail }; + } + + return undefined; +} + +createServer((req, res) => { + handle(req, res).then( + result => { + res.writeHead(result ? 200 : 404, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(result ?? { error: 'not found' })); + }, + error => { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: String(error) })); + }, + ); +}).listen(3030); diff --git a/dev-packages/e2e-tests/test-applications/node-pi-durable/src/start.mjs b/dev-packages/e2e-tests/test-applications/node-pi-durable/src/start.mjs new file mode 100644 index 000000000000..8c31deebda2b --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-pi-durable/src/start.mjs @@ -0,0 +1,35 @@ +import { spawn } from 'node:child_process'; +import { rmSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +// The `flaky_step` tool exits the server mid-call with this code to simulate a crash. Restarting the +// server on the same storage, as a process manager would, is what lets the run resume. +const CRASH_EXIT_CODE = 75; + +const appDir = fileURLToPath(new URL('..', import.meta.url)); +rmSync(new URL('../.data', import.meta.url), { recursive: true, force: true }); + +let server; + +function startServer() { + server = spawn(process.execPath, ['--import', './instrument.mjs', 'src/server.mjs'], { + cwd: appDir, + stdio: 'inherit', + }); + server.on('exit', code => { + if (code === CRASH_EXIT_CODE) { + startServer(); + } else { + process.exit(code ?? 1); + } + }); +} + +for (const signal of ['SIGINT', 'SIGTERM']) { + process.on(signal, () => { + server.kill(signal); + process.exit(0); + }); +} + +startServer(); diff --git a/dev-packages/e2e-tests/test-applications/node-pi-durable/start-event-proxy.mjs b/dev-packages/e2e-tests/test-applications/node-pi-durable/start-event-proxy.mjs new file mode 100644 index 000000000000..dd9d4d2b9da8 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-pi-durable/start-event-proxy.mjs @@ -0,0 +1,6 @@ +import { startEventProxyServer } from '@sentry-internal/test-utils'; + +startEventProxyServer({ + port: 3031, + proxyServerName: 'node-pi-durable', +}); diff --git a/dev-packages/e2e-tests/test-applications/node-pi-durable/tests/pi-durable.test.ts b/dev-packages/e2e-tests/test-applications/node-pi-durable/tests/pi-durable.test.ts new file mode 100644 index 000000000000..ee748fbad297 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-pi-durable/tests/pi-durable.test.ts @@ -0,0 +1,188 @@ +import { expect, test } from '@playwright/test'; +import { collectStreamedSpans, getSpanOp, waitForError } from '@sentry-internal/test-utils'; +import { createConversation, sendMessage } from './utils'; + +const APP = 'node-pi-durable'; + +test('traces a run as invoke_agent with chat, execute_tool and provider spans', async ({ baseURL }) => { + const conversationId = await createConversation(baseURL!); + // A run is its own trace, and its `invoke_agent` segment ends last. + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + spansOfTrace.some( + span => + span.is_segment && String(span.attributes['gen_ai.conversation.id']?.value).endsWith(`:${conversationId}`), + ) && spansOfTrace.some(span => span.attributes['gen_ai.tool.name']?.value === 'get_weather'), + ); + + const settled = await sendMessage( + baseURL!, + conversationId, + 'What is the weather in Vienna? Use the get_weather tool.', + ); + expect(settled.status, settled.detail).toBe('done'); + + const spans = await spansPromise; + const agent = spans.find(span => span.is_segment)!; + const chats = spans.filter(span => getSpanOp(span) === 'gen_ai.chat'); + // The manual span of the tool is the one way to pick the call that ran, should the model call twice. + const manualSpan = spans.find(span => span.name === 'resolve-weather')!; + const tool = spans.find(span => span.span_id === manualSpan.parent_span_id)!; + const providerCalls = spans.filter(span => getSpanOp(span) === 'http.client'); + + expect(getSpanOp(agent)).toBe('gen_ai.invoke_agent'); + expect(agent.attributes['sentry.origin']?.value).toBe('auto.ai.pi_durable'); + // The request that submitted the input is a trace of its own: the scheduler runs the work later. + expect(spans.some(span => getSpanOp(span) === 'http.server')).toBe(false); + // pi-ai sends the requests through `@anthropic-ai/sdk`, whose own integration must stay out so + // each request is reported once. + expect(spans.filter(span => String(span.attributes['sentry.origin']?.value).startsWith('auto.ai.'))).toEqual( + spans.filter(span => span.attributes['sentry.origin']?.value === 'auto.ai.pi_durable'), + ); + + // One tool-calling response, then the answer. + expect(chats.length).toBeGreaterThanOrEqual(2); + for (const chat of chats) { + expect(chat.parent_span_id).toBe(agent.span_id); + expect(chat.attributes['sentry.origin']?.value).toBe('auto.ai.pi_durable'); + expect(chat.attributes['gen_ai.provider.name']?.value).toBe('openrouter'); + expect(chat.attributes['gen_ai.request.model']?.value).toBe('anthropic/claude-haiku-4.5'); + // pi-durable retries a provider error inside the run; such a request has no usage and no HTTP span + // of its own to assert on. + if (chat.status === 'ok') { + expect(typeof chat.attributes['gen_ai.usage.input_tokens']?.value).toBe('number'); + expect(typeof chat.attributes['gen_ai.usage.output_tokens']?.value).toBe('number'); + expect(typeof chat.attributes['gen_ai.cost.total_tokens']?.value).toBe('number'); + expect(providerCalls.some(providerCall => providerCall.parent_span_id === chat.span_id)).toBe(true); + } + } + + // The request after the tool round carries the tool call and its result in the conventions shape. + const answerInput = JSON.parse( + String( + chats.find(chat => chat.attributes['gen_ai.response.finish_reasons']?.value === '["stop"]')!.attributes[ + 'gen_ai.input.messages' + ]?.value, + ), + ) as { role: string; parts: { type: string }[] }[]; + expect(answerInput.map(message => message.role)).toContain('tool'); + expect(answerInput.flatMap(message => message.parts.map(part => part.type))).toEqual( + expect.arrayContaining(['tool_call', 'tool_call_response']), + ); + + expect(tool.attributes['gen_ai.tool.name']?.value).toBe('get_weather'); + expect(tool.parent_span_id).toBe(agent.span_id); + expect(tool.status).toBe('ok'); + expect(tool.attributes['gen_ai.tool.call.arguments']?.value).toContain('Vienna'); + expect(tool.attributes['gen_ai.tool.call.result']?.value).toContain('21 degrees and sunny in'); + + // The provider's HTTP calls nest inside the `chat` span that sent them. + expect(providerCalls.length).toBeGreaterThan(0); + for (const providerCall of providerCalls) { + expect(chats.map(chat => chat.span_id)).toContain(providerCall.parent_span_id); + expect(providerCall.attributes['server.address']?.value).toBe('openrouter.ai'); + } +}); + +test('reports a throwing tool as an error on its execute_tool span', async ({ baseURL }) => { + const conversationId = await createConversation(baseURL!); + const errorPromise = waitForError( + APP, + event => event.exception?.values?.[0]?.value === 'Intentional pi-durable tool failure', + ); + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + spansOfTrace.some( + span => + span.is_segment && String(span.attributes['gen_ai.conversation.id']?.value).endsWith(`:${conversationId}`), + ) && spansOfTrace.some(span => span.attributes['gen_ai.tool.name']?.value === 'fail_now'), + ); + + const settled = await sendMessage(baseURL!, conversationId, 'Call the fail_now tool, then tell me what happened.'); + expect(settled.status, settled.detail).toBe('done'); + + const [error, spans] = await Promise.all([errorPromise, spansPromise]); + // The error is reported on the span of the call that threw. + const tool = spans.find(span => span.span_id === error.contexts?.trace?.span_id)!; + + expect(tool.attributes['gen_ai.tool.name']?.value).toBe('fail_now'); + expect(tool.status).toBe('error'); + expect(tool.trace_id).toBe(error.contexts?.trace?.trace_id); + expect(error.exception?.values?.[0]?.mechanism).toEqual({ type: 'auto.ai.pi_durable', handled: true }); +}); + +test('nests a subagent run under the tool call that delegated to it', async ({ baseURL }) => { + const conversationId = await createConversation(baseURL!); + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + spansOfTrace.some( + span => + span.is_segment && String(span.attributes['gen_ai.conversation.id']?.value).endsWith(`:${conversationId}`), + ) && spansOfTrace.some(span => !span.is_segment && getSpanOp(span) === 'gen_ai.invoke_agent'), + ); + + const settled = await sendMessage( + baseURL!, + conversationId, + 'Use the delegate tool to ask a subagent: "What is 2 + 2? Reply with only the number." Then reply with its answer.', + ); + expect(settled.status, settled.detail).toBe('done'); + + const spans = await spansPromise; + const agent = spans.find(span => span.is_segment)!; + const delegate = spans.find(span => span.attributes['gen_ai.tool.name']?.value === 'delegate')!; + const subagent = spans.find(span => !span.is_segment && getSpanOp(span) === 'gen_ai.invoke_agent')!; + const subagentChat = spans.find( + span => getSpanOp(span) === 'gen_ai.chat' && span.parent_span_id === subagent.span_id, + ); + + expect(delegate.parent_span_id).toBe(agent.span_id); + expect(subagent.parent_span_id).toBe(delegate.span_id); + expect(subagentChat).toBeDefined(); + + // The subagent has a conversation of its own, in the same Harness. + const [agentHarness, agentConversation] = String(agent.attributes['gen_ai.conversation.id']?.value).split(':'); + const [subagentHarness, subagentConversation] = String(subagent.attributes['gen_ai.conversation.id']?.value).split( + ':', + ); + expect(subagentHarness).toBe(agentHarness); + expect(subagentConversation).not.toBe(agentConversation); +}); + +test('resumes a run in a new trace after the server crashes during a tool call', async ({ baseURL }) => { + const conversationId = await createConversation(baseURL!); + // The tool span of the crashed process never ends, so a finished `flaky_step` span can only come + // from the rerun in the restarted server. + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + spansOfTrace.some( + span => + span.is_segment && String(span.attributes['gen_ai.conversation.id']?.value).endsWith(`:${conversationId}`), + ) && + spansOfTrace.some(span => span.attributes['gen_ai.tool.name']?.value === 'flaky_step' && span.status === 'ok'), + ); + + const settled = await sendMessage( + baseURL!, + conversationId, + 'Run the flaky step for job "nightly" with the flaky_step tool, then reply with the word DONE.', + ); + expect(settled.status, settled.detail).toBe('done'); + + const spans = await spansPromise; + const agent = spans.find(span => span.is_segment)!; + const tool = spans.find(span => span.attributes['gen_ai.tool.name']?.value === 'flaky_step')!; + const chats = spans.filter(span => getSpanOp(span) === 'gen_ai.chat'); + const answer = chats.find(span => span.attributes['gen_ai.response.finish_reasons']?.value === '["stop"]'); + + expect(getSpanOp(agent)).toBe('gen_ai.invoke_agent'); + expect(tool.parent_span_id).toBe(agent.span_id); + expect(answer?.parent_span_id).toBe(agent.span_id); + // The request that called the tool ran in the crashed process, so this trace starts with the rerun + // of the tool. A run that never crashed would start with that request. + expect(chats.every(chat => chat.start_timestamp >= tool.start_timestamp)).toBe(true); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-pi-durable/tests/utils.ts b/dev-packages/e2e-tests/test-applications/node-pi-durable/tests/utils.ts new file mode 100644 index 000000000000..8f2754169431 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-pi-durable/tests/utils.ts @@ -0,0 +1,47 @@ +import { expect } from '@playwright/test'; + +export async function createConversation(baseURL: string): Promise { + const res = await fetch(`${baseURL}/conversations`, { method: 'POST' }); + expect(res.status).toBe(200); + return ((await res.json()) as { id: number }).id; +} + +/** + * Send a message and wait until pi-durable settles it. + * + * The server only admits the input; the run continues in the background, and in the crash test in a + * second server process. Polling the submission until it settles keeps each test to its own run, and + * failed requests while the server restarts are retried. + */ +export async function sendMessage( + baseURL: string, + conversationId: number, + content: string, +): Promise<{ status: string; reason?: string; detail?: string }> { + const res = await fetch(`${baseURL}/conversations/${conversationId}/messages`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ content }), + }); + expect(res.status).toBe(200); + const { submissionId } = (await res.json()) as { submissionId: number }; + + const deadline = Date.now() + 90_000; + while (Date.now() < deadline) { + try { + const submission = (await (await fetch(`${baseURL}/submissions/${submissionId}`)).json()) as { + status: string; + reason?: string; + detail?: string; + }; + if (submission.status === 'done' || submission.status === 'unanswered') { + return submission; + } + } catch { + // The server is restarting. + } + await new Promise(resolve => setTimeout(resolve, 250)); + } + + throw new Error(`Submission ${submissionId} did not settle within 90s`); +}