{
await waitFor(() => {
expect(apiClient.get).toHaveBeenCalledWith("/files", {
- params: { parent: "root" },
+ params: { parent: "root", offset: 0, limit: 200 },
});
});
expect(apiClient.get).not.toHaveBeenCalledWith("/files");
});
+ it("loads additional bounded pages only when requested", async () => {
+ let filePage = 0;
+ apiClient.get.mockImplementation((url) => {
+ if (url === "/files/project-suggestions") {
+ return Promise.resolve({ data: [] });
+ }
+ if (url !== "/files") {
+ return Promise.resolve({ data: { exists: false, profile: null } });
+ }
+ filePage += 1;
+ if (filePage === 1) {
+ return Promise.resolve({
+ data: {
+ items: [
+ {
+ id: 1,
+ name: "first-volume.tif",
+ path: "root",
+ is_folder: false,
+ size: "1KB",
+ type: "image/tiff",
+ },
+ ],
+ total: 201,
+ offset: 0,
+ limit: 200,
+ has_more: true,
+ },
+ });
+ }
+ return Promise.resolve({
+ data: {
+ items: [
+ {
+ id: 2,
+ name: "last-volume.tif",
+ path: "root",
+ is_folder: false,
+ size: "1KB",
+ type: "image/tiff",
+ },
+ ],
+ total: 201,
+ offset: 200,
+ limit: 200,
+ has_more: false,
+ },
+ });
+ });
+
+ renderFilesManager();
+ fireEvent.click(await screen.findByText("Load more files"));
+
+ await waitFor(() =>
+ expect(apiClient.get).toHaveBeenCalledWith("/files", {
+ params: { parent: "root", offset: 200, limit: 200 },
+ }),
+ );
+ expect(await screen.findByText("last-volume.tif")).toBeTruthy();
+ expect(screen.queryByText("Load more files")).toBeNull();
+ });
+
it("opens a confirmation modal before registering a suggested smoke project", async () => {
mockProjectSuggestionResponses([smokeSuggestion]);
apiClient.post.mockResolvedValue({
@@ -506,7 +568,7 @@ describe("FilesManager", () => {
);
});
expect(apiClient.get).toHaveBeenCalledWith("/files", {
- params: { parent: "7" },
+ params: { parent: "7", offset: 0, limit: 200 },
});
await continueWithProjectContext();
await waitFor(() => {
@@ -1049,7 +1111,7 @@ describe("FilesManager", () => {
await screen.findByText("Confirm project basics");
await waitFor(() => {
expect(apiClient.get).toHaveBeenCalledWith("/files", {
- params: { parent: "7" },
+ params: { parent: "7", offset: 0, limit: 200 },
});
expect(mockWorkflowContext.consumeRuntimeAction).toHaveBeenCalledWith(
"choose-data-action",
diff --git a/client/src/views/ProjectProgress.js b/client/src/views/ProjectProgress.js
index 1a4eca99..c5c99066 100644
--- a/client/src/views/ProjectProgress.js
+++ b/client/src/views/ProjectProgress.js
@@ -19,6 +19,7 @@ import {
} from "@ant-design/icons";
import { useWorkflow } from "../contexts/WorkflowContext";
import { logClientEvent } from "../logging/appEventLog";
+import WorkflowOperationsPanel from "../components/workflow/WorkflowOperationsPanel";
const { Text, Title } = Typography;
@@ -362,6 +363,10 @@ function ProjectProgress() {
+
+
+
+
{(blockers.length > 0 || actions.length > 0) && (
({
logClientEvent: jest.fn(),
}));
+jest.mock("../components/workflow/WorkflowOperationsPanel", () => () => (
+
Operations
+));
+
describe("ProjectProgress", () => {
beforeEach(() => {
jest.clearAllMocks();
diff --git a/client/src/views/Views.js b/client/src/views/Views.js
index 8cd60b5c..1f0c6006 100644
--- a/client/src/views/Views.js
+++ b/client/src/views/Views.js
@@ -7,7 +7,9 @@ import {
ThunderboltOutlined,
BugOutlined,
MessageOutlined,
+ PlusOutlined,
ProjectOutlined,
+ QuestionCircleOutlined,
} from "@ant-design/icons";
import FilesManager from "./FilesManager";
import Visualization from "./Visualization";
@@ -59,7 +61,7 @@ const workflowPhaseTone = {
evaluate: "#7c3aed",
};
-function WorkflowOverviewStrip({ overview, workflow, onNavigate }) {
+function WorkflowOverviewStrip({ overview, workflow, onNavigate, onAction }) {
const summary = overview?.volume_summary || {};
const phase = overview?.phase || workflow?.stage || "setup";
const phaseLabel =
@@ -156,7 +158,9 @@ function WorkflowOverviewStrip({ overview, workflow, onNavigate }) {
{action && (
-
{
BugOutlined: Icon,
ProjectOutlined: Icon,
MessageOutlined: Icon,
+ PlusOutlined: Icon,
+ QuestionCircleOutlined: Icon,
};
});
@@ -106,6 +108,7 @@ describe("Views", () => {
},
loading: false,
startNewWorkflow: jest.fn().mockResolvedValue({}),
+ runClientEffects: jest.fn().mockResolvedValue(),
consumeClientEffects: jest.fn(),
};
});
@@ -188,4 +191,29 @@ describe("Views", () => {
expect(screen.getByText("Chatbot")).toBeTruthy();
expect(screen.getByText("What should I do next?")).toBeTruthy();
});
+
+ it("applies overview action targets before navigating", async () => {
+ const clientEffects = {
+ navigate_to: "mask-proofreading",
+ set_proofreading_dataset_path: "/tmp/review-image.h5",
+ set_proofreading_mask_path: "/tmp/review-mask.h5",
+ };
+ mockWorkflowContext.workflowOverview.recommended_next_actions = [
+ {
+ id: "proofread-draft-masks",
+ label: "Proofread draft masks",
+ target_view: "mask-proofreading",
+ client_effects: clientEffects,
+ },
+ ];
+
+ render();
+ fireEvent.click(screen.getByText("Proofread draft masks"));
+
+ await waitFor(() => {
+ expect(mockWorkflowContext.runClientEffects).toHaveBeenCalledWith(
+ clientEffects,
+ );
+ });
+ });
});
diff --git a/client/src/views/Visualization.js b/client/src/views/Visualization.js
index b740f63b..7e8ec3ac 100644
--- a/client/src/views/Visualization.js
+++ b/client/src/views/Visualization.js
@@ -46,6 +46,11 @@ const getDisplay = (val) => {
return val.display || val.path || "";
};
+const looksLikeVolumeFile = (value) => {
+ const path = getPath(value).toLowerCase();
+ return /\.(?:h5|hdf5|tif|tiff|npy|npz|nii|nii\.gz|mrc|mrcs)$/.test(path);
+};
+
const getViewerTitle = (imageValue, labelValue) => {
const getImageName = (val) => {
const display = getDisplay(val);
@@ -96,10 +101,24 @@ function Visualization(props) {
const globalImage = appContext?.currentImage || null;
const globalLabel = appContext?.currentLabel || null;
const globalScales = appContext?.visualizationScales || "";
+ const workflow = workflowContext?.workflow;
+ const observedVolumeSets =
+ workflow?.metadata?.project_observation?.volume_sets;
+ const observedVolumeSet = Array.isArray(observedVolumeSets)
+ ? observedVolumeSets.find((item) => item?.is_current) ||
+ observedVolumeSets[0]
+ : null;
+ const workflowImage =
+ observedVolumeSet?.image_path ||
+ (looksLikeVolumeFile(workflow?.image_path) ? workflow.image_path : null);
+ const workflowLabel =
+ observedVolumeSet?.label_path ||
+ (looksLikeVolumeFile(workflow?.label_path) ? workflow.label_path : null);
const workflowScales =
- workflowContext?.workflow?.metadata?.visualization_scales ||
- workflowContext?.workflow?.metadata?.project_context?.voxel_size_nm ||
+ workflow?.metadata?.visualization_scales ||
+ workflow?.metadata?.project_context?.voxel_size_nm ||
null;
+ const hydratedWorkflowRef = useRef(null);
const handledRuntimeActionRef = useRef(null);
const viewerRefreshTimersRef = useRef([]);
const [activeKey, setActiveKey] = useState(
@@ -107,13 +126,48 @@ function Visualization(props) {
);
// Input states
- const [currentImage, setCurrentImage] = useState(globalImage);
- const [currentLabel, setCurrentLabel] = useState(globalLabel);
+ const [currentImage, setCurrentImage] = useState(
+ globalImage || workflowImage || null,
+ );
+ const [currentLabel, setCurrentLabel] = useState(
+ globalLabel || workflowLabel || null,
+ );
const [scales, setScales] = useState(
- formatScales(globalScales) || formatScales(workflowScales) || "",
+ formatScales(workflowScales) || formatScales(globalScales) || "",
);
const [isLoading, setIsLoading] = useState(false);
+ useEffect(() => {
+ const workflowId = workflow?.id;
+ if (!workflowId || hydratedWorkflowRef.current === workflowId) {
+ return;
+ }
+ const isWorkflowChange = hydratedWorkflowRef.current !== null;
+ hydratedWorkflowRef.current = workflowId;
+
+ if ((isWorkflowChange || !globalImage) && workflowImage) {
+ setCurrentImage(workflowImage);
+ appContext?.setCurrentImage?.(workflowImage);
+ }
+ if ((isWorkflowChange || !globalLabel) && workflowLabel) {
+ setCurrentLabel(workflowLabel);
+ appContext?.setCurrentLabel?.(workflowLabel);
+ }
+ if ((isWorkflowChange || !globalImage) && workflowScales) {
+ const nextScales = formatScales(workflowScales);
+ setScales(nextScales);
+ appContext?.setVisualizationScales?.(nextScales);
+ }
+ }, [
+ appContext,
+ globalImage,
+ globalLabel,
+ workflow?.id,
+ workflowImage,
+ workflowLabel,
+ workflowScales,
+ ]);
+
useEffect(() => {
return () => {
viewerRefreshTimersRef.current.forEach((timerId) => {
@@ -134,23 +188,27 @@ function Visualization(props) {
};
useEffect(() => {
- if (globalImage !== currentImage) {
- setCurrentImage(globalImage);
+ const nextImage = globalImage || workflowImage || null;
+ if (nextImage !== currentImage) {
+ setCurrentImage(nextImage);
}
- }, [globalImage, currentImage]);
+ }, [globalImage, workflowImage, currentImage]);
useEffect(() => {
- if (globalLabel !== currentLabel) {
- setCurrentLabel(globalLabel);
+ const nextLabel = globalLabel || workflowLabel || null;
+ if (nextLabel !== currentLabel) {
+ setCurrentLabel(nextLabel);
}
- }, [globalLabel, currentLabel]);
+ }, [globalLabel, workflowLabel, currentLabel]);
useEffect(() => {
- const nextScales = formatScales(globalScales || workflowScales);
+ const nextScales = globalImage
+ ? formatScales(globalScales || workflowScales)
+ : formatScales(workflowScales) || formatScales(globalScales);
if (nextScales !== scales) {
setScales(nextScales);
}
- }, [globalScales, workflowScales, scales]);
+ }, [globalImage, globalScales, workflowScales, scales]);
const handleInputScales = (event) => {
const nextScales = event.target.value;
diff --git a/client/src/views/Visualization.test.js b/client/src/views/Visualization.test.js
new file mode 100644
index 00000000..59b71f49
--- /dev/null
+++ b/client/src/views/Visualization.test.js
@@ -0,0 +1,134 @@
+import React from "react";
+import { render, screen, waitFor } from "@testing-library/react";
+import Visualization from "./Visualization";
+import { AppContext } from "../contexts/GlobalContext";
+import { useWorkflow } from "../contexts/WorkflowContext";
+import { QueryClientProvider } from "@tanstack/react-query";
+import { createAppQueryClient } from "../queryClient";
+
+jest.mock("../contexts/WorkflowContext", () => ({
+ useWorkflow: jest.fn(),
+}));
+
+jest.mock("../api", () => ({
+ apiClient: {
+ get: jest.fn(),
+ defaults: { baseURL: "http://localhost:4242" },
+ },
+ getNeuroglancerViewer: jest.fn(),
+}));
+
+describe("Visualization workflow defaults", () => {
+ const renderWithQueryClient = (ui) => {
+ const queryClient = createAppQueryClient();
+ return render(ui, {
+ wrapper: ({ children }) => (
+
+ {children}
+
+ ),
+ });
+ };
+
+ it("hydrates the canonical volume pair and voxel scales", async () => {
+ const setCurrentImage = jest.fn();
+ const setCurrentLabel = jest.fn();
+ const setVisualizationScales = jest.fn();
+ useWorkflow.mockReturnValue({
+ workflow: {
+ id: 1,
+ image_path: "/project/data/raw",
+ label_path: "/project/data/seg",
+ metadata: {
+ visualization_scales: [40, 8, 8],
+ project_observation: {
+ volume_sets: [
+ {
+ is_current: true,
+ image_path: "/project/data/raw/train-01_image.h5",
+ label_path:
+ "/project/data/seg/ground_truth/train-01_ground_truth.h5",
+ },
+ ],
+ },
+ },
+ },
+ });
+
+ renderWithQueryClient(
+
+
+ ,
+ );
+
+ expect(
+ screen.getByPlaceholderText("Please select or input image path").value,
+ ).toBe("/project/data/raw/train-01_image.h5");
+ expect(
+ screen.getByPlaceholderText("Please select or input label path").value,
+ ).toBe("/project/data/seg/ground_truth/train-01_ground_truth.h5");
+ expect(screen.getByPlaceholderText("z,y,x nm").value).toBe("40,8,8");
+ await waitFor(() =>
+ expect(setCurrentImage).toHaveBeenCalledWith(
+ "/project/data/raw/train-01_image.h5",
+ ),
+ );
+ expect(setCurrentLabel).toHaveBeenCalledWith(
+ "/project/data/seg/ground_truth/train-01_ground_truth.h5",
+ );
+ expect(setVisualizationScales).toHaveBeenCalledWith("40,8,8");
+ });
+
+ it("does not restore old workflow defaults while context is being reset", () => {
+ const setCurrentImage = jest.fn();
+ const setCurrentLabel = jest.fn();
+ const setVisualizationScales = jest.fn();
+ useWorkflow.mockReturnValue({
+ workflow: {
+ id: 7,
+ image_path: "/project/train-01_image.h5",
+ label_path: "/project/train-01_ground_truth.h5",
+ metadata: { visualization_scales: [40, 8, 8] },
+ },
+ });
+ const context = {
+ currentImage: "/project/custom-image.h5",
+ currentLabel: "/project/custom-label.h5",
+ visualizationScales: "20,4,4",
+ setCurrentImage,
+ setCurrentLabel,
+ setVisualizationScales,
+ };
+ const { rerender } = renderWithQueryClient(
+
+
+ ,
+ );
+
+ rerender(
+
+
+ ,
+ );
+
+ expect(setCurrentImage).not.toHaveBeenCalled();
+ expect(setCurrentLabel).not.toHaveBeenCalled();
+ expect(setVisualizationScales).not.toHaveBeenCalled();
+ });
+});
diff --git a/docs/decisions/dbos-durable-operations-spike.md b/docs/decisions/dbos-durable-operations-spike.md
new file mode 100644
index 00000000..a7227fab
--- /dev/null
+++ b/docs/decisions/dbos-durable-operations-spike.md
@@ -0,0 +1,116 @@
+# DBOS Durable Operations Spike
+
+- **Date:** 2026-07-21
+- **Status:** Conditional go for a Postgres-backed integration prototype; no-go for production-route integration
+- **Library evaluated:** `dbos[aiosqlite]==2.28.0`
+- **Production code changed:** None
+
+## Question
+
+Can DBOS replace the process-global job state used by training and inference while
+preserving the product-facing `WorkflowOperation` lifecycle: idempotent submission,
+durable progress, cancellation, and restart recovery?
+
+## Prototype
+
+The isolated implementation lives in `spikes/dbos_operation`. It models a heavy
+operation as a queued DBOS workflow containing checkpointed steps. Each step writes
+an idempotent external marker representing a compute-chunk side effect. The workflow
+publishes an `operation_progress` DBOS event containing the workflow ID, correlation
+ID, completed and total step counts, normalized progress, and status.
+
+The runner supports four operations without registering a production route:
+
+```bash
+uv run python scripts/run_dbos_operation_spike.py execute \
+ --database /tmp/pytc-dbos.sqlite \
+ --workspace /tmp/pytc-dbos-work \
+ --workflow-id workflow-1 \
+ --correlation-id request-1 \
+ --duplicate-submission
+
+uv run python scripts/run_dbos_operation_spike.py status \
+ --database /tmp/pytc-dbos.sqlite --workflow-id workflow-1
+
+uv run python scripts/run_dbos_operation_spike.py cancel \
+ --database /tmp/pytc-dbos.sqlite --workflow-id workflow-1
+
+uv run python scripts/run_dbos_operation_spike.py recover \
+ --database /tmp/pytc-dbos.sqlite --workflow-id workflow-1
+```
+
+Automated evidence is produced by:
+
+```bash
+uv run pytest -q tests/test_dbos_operation_spike.py
+```
+
+Current result: **4 passed in 21.75 seconds** on Python 3.11 with a local SQLite
+system database.
+
+## Gates
+
+| Gate | Required evidence | Result |
+| --- | --- | --- |
+| Runtime compatibility | Installs on the repository's Python 3.10-3.11 range | **Pass.** DBOS 2.28.0 declares Python >=3.10; the spike ran on 3.11. |
+| Idempotent submission | Submitting the same workflow ID twice executes one workflow and one set of external markers | **Pass.** Both handles have the same ID; every marker is written once. |
+| Durable progress | Progress is queryable outside the worker and remains available after completion or process death | **Pass.** `DBOS.set_event` progress is read through `DBOSClient`. |
+| Queued cancellation | Cancelling enqueued work removes it before any external effect | **Pass.** Status becomes `CANCELLED`; no marker directory is created. |
+| Running cancellation | Cancellation stops work at a documented durable boundary | **Pass with constraint.** Cancellation preempts at the next step boundary; it does not interrupt an ordinary blocking synchronous step. |
+| Single-server restart | A killed process resumes from its last completed step without repeating that step's external effect | **Pass.** A replacement with the same executor identity recovers the `PENDING` workflow and completes the remaining markers. |
+| Mid-step crash safety | Killing a process during a non-transactional external side effect cannot duplicate or corrupt that effect | **Not proven.** The test kills after the step and progress event are durable. Production steps still require idempotent outputs or transactional integration. |
+| Postgres and multiple executors | Recovery, queue concurrency, and cancellation work with the intended production topology | **Not run; production gate fails.** SQLite is explicitly a development/test backend. |
+| PyTC subprocess control | Training/inference subprocesses are killed, reaped, and reconciled correctly on cancel/restart | **Not run; production gate fails.** A blocking `Popen` step is not sufficient. |
+| Product-state projection | DBOS state and `WorkflowOperation` cannot diverge under crashes | **Not designed; production gate fails.** A single source of truth and projection strategy is required. |
+
+## Findings
+
+1. DBOS workflow IDs directly provide the idempotency behavior currently implemented
+ around `WorkflowOperation.idempotency_key`.
+2. DBOS events are a good fit for durable progress and correlation data. They avoid
+ writing a high-frequency progress stream into the main workflow table.
+3. Queue concurrency maps well to GPU/CPU resource limits without a separate broker.
+4. Recovery ownership is operationally significant. In the local single-server
+ setup, the replacement process must reuse the interrupted executor identity.
+ Multiple live executors require a production recovery strategy rather than PID-
+ derived identities.
+5. DBOS cancellation is cooperative at workflow/step boundaries. Immediate training
+ cancellation needs a preemptible async step or a short polling supervisor around
+ the subprocess, including process-group termination and output reconciliation.
+6. DBOS does not make arbitrary filesystem or model-training side effects exactly
+ once. Those effects must remain idempotent. The spike demonstrates this with
+ exclusive marker creation.
+
+## Decision
+
+**Do not mount DBOS into the FastAPI production routes on this branch.** The local
+prototype passes the behavioral gates it can honestly exercise, but the Postgres,
+multi-executor, subprocess cancellation, and state-projection gates remain open.
+
+Proceed with one additional, bounded integration phase only if Postgres is accepted
+as backend infrastructure:
+
+1. Run this suite against Postgres with two executor processes and forced worker
+ loss.
+2. Implement a supervised synthetic subprocess step and prove cancel/restart process
+ cleanup before using real training.
+3. Keep `WorkflowOperation` as the user-facing read model while DBOS owns execution,
+ and update that read model through an idempotent projection keyed by DBOS workflow
+ ID. Do not dual-write independent lifecycle transitions.
+4. Use stable IDs such as
+ `workflow:{workflow_id}:operation:{operation_id}:attempt:{attempt}` and retain the
+ request correlation ID in progress events.
+5. Integrate one operation type behind a feature flag, starting with evidence export
+ or evaluation rather than GPU training.
+
+The go/no-go after that phase is straightforward: all four currently failing gates
+must pass before a production endpoint submits DBOS work.
+
+## Primary References
+
+- [DBOS Python workflows and workflow-ID idempotency](https://docs.dbos.dev/python/tutorials/workflow-tutorial)
+- [DBOS queues and concurrency](https://docs.dbos.dev/python/reference/queues)
+- [DBOS cancellation and resume semantics](https://docs.dbos.dev/python/tutorials/workflow-management)
+- [DBOS workflow events for progress](https://docs.dbos.dev/python/tutorials/workflow-communication)
+- [DBOS SQLite and Postgres guidance](https://docs.dbos.dev/python/tutorials/database-connection)
+- [DBOS workflow recovery](https://docs.dbos.dev/production/workflow-recovery)
diff --git a/docs/platform-hardening-branch-scope.md b/docs/platform-hardening-branch-scope.md
new file mode 100644
index 00000000..bd3be898
--- /dev/null
+++ b/docs/platform-hardening-branch-scope.md
@@ -0,0 +1,123 @@
+# Platform Hardening Branch Scope
+
+Branch: `agent/platform-hardening`
+
+Base: `origin/main` at `a991b1b`
+
+## Objective
+
+Establish the shared technical substrate needed for a recoverable, resource-bounded,
+agent-mediated segmentation loop. This branch hardens existing workflow behavior;
+it does not redesign the participant workflow or replace PyTorch Connectomics.
+
+## In Scope
+
+1. A structured API error contract with stable machine codes, user-facing recovery
+ guidance, request correlation, and a consistent frontend representation.
+2. Application-level frontend failure boundaries and explicit retry/recovery states
+ for failures that currently disappear into transient notifications.
+3. A bounded volume access contract that separates metadata inspection and region
+ reads from full materialization across supported volume formats.
+4. Persisted operation records for long-running work, including terminal failure and
+ cancellation states, idempotency, and correlation to workflow evidence.
+5. A typed agent action registry that defines allowed inputs, risk, approval policy,
+ and execution ownership independently of language-model intent routing.
+6. Focused tests and compatibility shims that preserve the current closed-loop smoke
+ path while new primitives are adopted incrementally.
+7. Bounded file-workspace listing and preview behavior, plus a deterministic browser
+ acceptance check for the synthetic core project.
+
+## Out of Scope
+
+- Database sharding or horizontal multi-tenant scaling.
+- A mandatory bulk conversion of existing datasets.
+- Replacing Neuroglancer, PyTorch Connectomics, or every existing API route.
+- Full migration of all subprocess execution into a distributed worker system.
+- A visual redesign unrelated to loading, failure, recovery, or task state.
+- Removing existing workflow evidence records or approval history.
+
+## Architectural Invariants
+
+- Base image and imported label artifacts remain immutable by default.
+- Volume callers must be able to inspect metadata and request a bounded region without
+ materializing the complete volume.
+- Risky agent actions remain approval-gated and must resolve through a registered,
+ typed action definition.
+- The browser may perform navigation and presentation effects; domain execution and
+ durable state transitions belong to the server.
+- Operation state must survive a server restart even before subprocess reattachment is
+ implemented.
+- User-visible failures include a stable code, recovery guidance, and request ID while
+ technical details remain available for diagnosis.
+
+## Acceptance Criteria
+
+- Backend tests cover error serialization, volume metadata/region reads, operation
+ lifecycle transitions, and registered agent action validation.
+- Frontend tests cover normalized API failures, the application error boundary, and a
+ user-triggered recovery path.
+- Existing workflow, EHTool, runtime, and agent proposal tests remain green or receive
+ narrowly justified compatibility updates.
+- The deterministic synthetic project mounts in a real browser, exposes a bounded
+ volume preview, reports the expected `4 / 2 / 1 / 1` progress state, and survives a
+ page reload without API 5xx or browser errors.
+- The production frontend build completes.
+- No new critical path depends on full-volume NumPy materialization when only metadata
+ or a subvolume is required.
+- New agent proposal types cannot silently bypass registry validation or approval
+ policy.
+
+## Delivered
+
+- Shared FastAPI error envelopes preserve legacy `detail` values while adding stable
+ codes, categories, retry guidance, recovery actions, and request IDs. The React
+ client normalizes transport and API failures and has a root recovery boundary.
+- `VolumeStore` separates metadata and region access from NumPy materialization for
+ HDF5, TIFF/OME-TIFF, NPY/NPZ, Zarr/N5, NIfTI, and MRC sources. Neuroglancer now
+ serves 3D image and label chunks from those stores and closes backing resources
+ when retained viewers expire or are evicted.
+- `WorkflowOperation` persists queued, running, succeeded, failed, and cancelled
+ work with idempotency, correlation, leases, heartbeats, progress, and cancellation
+ requests. Approved training commands create a per-attempt operation and commit
+ command/operation state together.
+- Agent effects resolve through strict, discriminated runtime and workflow action
+ schemas. The registry owns risk, approval, execution-owner, and specialist policy;
+ unknown proposal types, effect keys, and nested action parameters are rejected.
+- File listings support opt-in, server-filtered pagination while preserving the legacy
+ array response. The file picker uses cached paged queries and virtualized rows.
+ Preview decoding requests a sampled middle-plane region and is isolated behind a
+ two-request AnyIO capacity limiter instead of materializing complete volumes.
+- Persisted operations are visible in the Workflow UI with reconnect-aware polling,
+ cancellation, and constrained retry controls. The first approval-bound server action
+ executor computes and persists before/after evaluation evidence with a typed receipt.
+- GitHub Actions now creates a clean synthetic fixture, starts FastAPI and React without
+ Electron, runs the Chromium browser smoke, tears services down, and retains its JSON
+ report and service logs for diagnosis.
+
+## Verification
+
+- Backend: full suite passes (`334 passed`, plus five subtests), including an actual
+ on-demand Neuroglancer chunk read from HDF5-backed image and label sources.
+- Frontend: all 29 suites pass (`154 passed`).
+- Browser: the deterministic Chromium smoke passes all six checks locally, including
+ bounded file-picker navigation, decoded volume preview, progress verification,
+ reload continuity, and rendered durable operation history. CI uploads the report
+ screenshot, and both service logs on every run.
+- Production frontend build succeeds. Existing React hook and bundle-size warnings
+ remain unchanged.
+
+## Follow-On Work
+
+This branch intentionally creates migration points. A subsequent branch should:
+
+1. Synchronize worker completion, failure, and cancellation into long-running training
+ and inference operations; current training-command success means the worker accepted
+ submission.
+2. Connect worker-side cancellation checkpoints and completion callbacks to the task
+ panel so accepted training and inference submissions expose their full lifecycle.
+3. Materialize multiscale OME-Zarr for repeated large-volume access and preprocess 4D
+ prediction tensors into 3D label volumes before Neuroglancer launch.
+4. Convert EHTool proofreading persistence from retained full arrays to chunk-aligned
+ correction overlays; its editing model still requires complete mutable volumes.
+5. Move remaining browser-orchestrated runtime launches behind server command handlers
+ while keeping navigation and form-prefill effects browser-owned.
diff --git a/docs/synthetic-core-project.md b/docs/synthetic-core-project.md
new file mode 100644
index 00000000..a137f308
--- /dev/null
+++ b/docs/synthetic-core-project.md
@@ -0,0 +1,57 @@
+# Synthetic Core Project
+
+The local app now starts with a deterministic synthetic segmentation project by default. It formalizes one fixed development paradigm across the actual file workspace, project scanner, chunked volume reader, workflow state, progress tracker, viewer, proofreading flow, agent context, training handoff, and evaluation surfaces.
+
+## Canonical State
+
+The generated project lives at `.pytc/synthetic-core-project` and contains four compressed, chunked HDF5 image volumes:
+
+| Volume | Initial state | Intended workflow role |
+| --- | --- | --- |
+| `train-01` | Ground truth | Training source |
+| `train-02` | Ground truth | Training source |
+| `review-01` | Imperfect draft | Proofreading and correction |
+| `target-01` | Image only | Inference target |
+
+The expected progress state is always **4 total / 2 ground truth / 1 needs proofreading / 1 missing segmentation**. Baseline and corrected candidate predictions are prepopulated for comparison. The data are only for interaction and systems testing; they are not scientific evidence or a model-quality benchmark.
+
+## Run
+
+From the repository root:
+
+```bash
+./scripts/start.sh
+```
+
+Startup creates the fixture if absent, uses `.pytc/synthetic-core.db`, seeds the current workflow, mounts the project into the file browser, builds the client, and launches Electron.
+
+Normal startup preserves edits made inside the generated project. Restore the exact canonical files and state with:
+
+```bash
+PYTC_SYNTHETIC_PROJECT_RESET=1 ./scripts/start.sh
+```
+
+Generate or reset the fixture without starting the app:
+
+```bash
+uv run python scripts/create_synthetic_project.py --reset
+```
+
+Disable the development fixture and use the previous project/database behavior:
+
+```bash
+PYTC_SYNTHETIC_PROJECT=0 ./scripts/start.sh
+```
+
+Supplying `PYTC_INITIAL_PROJECT_ROOT` also takes precedence over synthetic mode.
+
+## Core Test Sequence
+
+1. Confirm the project is already mounted and the progress tracker reports `2 / 1 / 1`.
+2. Open a training image and ground-truth label in the storage-backed viewer.
+3. Open `review-01`, compare the baseline and candidate, and proofread the draft.
+4. Ask the workflow agent for project status and the next recommended action.
+5. Stage training from only the two confirmed ground-truth volumes.
+6. Select `target-01` for inference and verify that the output is registered.
+7. Inspect operation progress, cancellation, failure, and retry states.
+8. Reset the fixture before repeating a canonical acceptance run.
diff --git a/pyproject.toml b/pyproject.toml
index 3d8a6e7b..17e38681 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -25,6 +25,7 @@ dependencies = [
"requests>=2.31",
"scikit-image>=0.25",
"scipy>=1.11",
+ "sqlalchemy>=2.0,<3",
"tensorboard==2.20.0",
"tensorboard-data-server==0.7.2",
"tifffile>=2025.5.10",
@@ -42,5 +43,7 @@ dependencies = [
[dependency-groups]
dev = [
"black==26.5.1",
+ "dbos[aiosqlite]==2.28.0",
+ "playwright>=1.54,<2",
"pytest==9.0.2",
]
diff --git a/scripts/browser_synthetic_core_smoke.py b/scripts/browser_synthetic_core_smoke.py
new file mode 100644
index 00000000..1a1df16a
--- /dev/null
+++ b/scripts/browser_synthetic_core_smoke.py
@@ -0,0 +1,340 @@
+#!/usr/bin/env python3
+"""Browser smoke for the deterministic synthetic core project."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+from urllib.parse import urlparse
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+ sys.path.insert(0, str(REPO_ROOT))
+
+from scripts.browser_yixiao_case_study_smoke import (
+ DEFAULT_BASE_URL,
+ VIEWPORT_DEFAULT,
+ _extract_progress_snapshot,
+ _parse_viewport,
+ _playwright_import_error,
+ _raise_playwright_error,
+ _resolve_browser_playwright,
+)
+
+PROJECT_TITLE = "Synthetic Segmentation Core Loop"
+PROJECT_MOUNT_LABEL = PROJECT_TITLE
+EXPECTED_PROGRESS = {
+ "tracked": 4,
+ "good": 2,
+ "proofread": 1,
+ "missing": 1,
+}
+DEFAULT_REPORT = "/tmp/synthetic-core-browser-smoke.json"
+DEFAULT_API_BASE_URL = "http://127.0.0.1:4242"
+
+
+def _open_synthetic_tab(page, label: str, timeout_ms: int) -> None:
+ menu_item = page.locator(".pytc-top-menu .ant-menu-item").filter(has_text=label)
+ if menu_item.count() and menu_item.first.is_visible():
+ menu_item.first.click()
+ page.wait_for_timeout(250)
+ return
+
+ overflow = page.locator(".pytc-top-menu .ant-menu-overflow-item-rest")
+ if not overflow.count() or not overflow.first.is_visible():
+ raise AssertionError(f"Top-level tab '{label}' not found.")
+ overflow.first.click()
+ popup_item = page.locator(".ant-menu-submenu-popup .ant-menu-item").filter(
+ has_text=label
+ )
+ popup_item.first.wait_for(state="visible", timeout=timeout_ms)
+ popup_item.first.click()
+ page.keyboard.press("Escape")
+ page.wait_for_timeout(250)
+
+
+def _build_arg_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
+ parser.add_argument("--api-base-url", default=DEFAULT_API_BASE_URL)
+ parser.add_argument("--timeout-ms", type=int, default=30_000)
+ parser.add_argument(
+ "--viewport",
+ default=f"{VIEWPORT_DEFAULT[0]}x{VIEWPORT_DEFAULT[1]}",
+ )
+ parser.add_argument("--no-headless", action="store_true")
+ parser.add_argument("--skip-reload", action="store_true")
+ parser.add_argument("--report", default=DEFAULT_REPORT)
+ parser.add_argument("--screenshot")
+ return parser
+
+
+def _assert_progress(page) -> Dict[str, Any]:
+ snapshot = _extract_progress_snapshot(page)
+ for key, expected in EXPECTED_PROGRESS.items():
+ actual = (snapshot.get("metrics") or {}).get(key)
+ if actual != expected:
+ raise AssertionError(
+ f"Synthetic progress metric {key!r} expected {expected}, got {actual}"
+ )
+ return snapshot
+
+
+def _open_file_picker(page, timeout_ms: int) -> None:
+ _open_synthetic_tab(page, "Visualize", timeout_ms)
+ page.get_by_role("button", name="Browse", exact=True).first.click()
+ dialog = page.get_by_role("dialog", name="Select File")
+ dialog.wait_for(state="visible", timeout=timeout_ms)
+ if dialog.get_by_text("Files unavailable").count():
+ raise AssertionError("Browse modal reported that files were unavailable")
+
+ project_row = dialog.locator(".file-picker-item").filter(
+ has_text=PROJECT_MOUNT_LABEL
+ )
+ project_row.first.wait_for(state="visible", timeout=timeout_ms)
+ project_row.first.click()
+ data_row = dialog.locator(".file-picker-item").filter(has_text="data")
+ data_row.first.wait_for(state="visible", timeout=timeout_ms)
+ data_row.first.click()
+ raw_row = dialog.locator(".file-picker-item").filter(has_text="raw")
+ raw_row.first.wait_for(state="visible", timeout=timeout_ms)
+ raw_row.first.click()
+ preview = dialog.get_by_role("img", name="train-01_image.h5")
+ preview.wait_for(state="attached", timeout=timeout_ms)
+ page.wait_for_function(
+ "image => image.complete && image.naturalWidth > 0",
+ arg=preview.element_handle(),
+ timeout=timeout_ms,
+ )
+ if dialog.get_by_text("Files unavailable").count():
+ raise AssertionError("Browse modal failed after opening the project folder")
+ dialog.get_by_role("button", name="Close").click()
+
+
+def _exercise_core_ui(page, timeout_ms: int) -> Dict[str, Any]:
+ page.get_by_text(PROJECT_TITLE, exact=True).first.wait_for(
+ state="visible", timeout=timeout_ms
+ )
+ _open_synthetic_tab(page, "Files", timeout_ms)
+ page.get_by_text(PROJECT_MOUNT_LABEL, exact=True).first.wait_for(
+ state="visible", timeout=timeout_ms
+ )
+ _open_file_picker(page, timeout_ms)
+ _open_synthetic_tab(page, "Workflow", timeout_ms)
+ page.wait_for_timeout(300)
+ return _assert_progress(page)
+
+
+def _exercise_operation_history(
+ context, page, *, api_base_url: str, timeout_ms: int
+) -> None:
+ current_response = context.request.get(
+ f"{api_base_url}/api/workflows/current", timeout=timeout_ms
+ )
+ if not current_response.ok:
+ raise AssertionError(
+ f"Workflow lookup failed with HTTP {current_response.status}"
+ )
+ workflow_id = current_response.json()["workflow"]["id"]
+ operation_response = context.request.post(
+ f"{api_base_url}/api/workflows/{workflow_id}/operations",
+ data={
+ "operation_type": "browser_smoke",
+ "idempotency_key": "browser-smoke:operation-history:v1",
+ "correlation_id": "browser-smoke-operation-history",
+ "actor": "system",
+ "input": {"source": "synthetic_browser_smoke"},
+ "metadata": {"source": "synthetic_browser_smoke"},
+ },
+ timeout=timeout_ms,
+ )
+ if not operation_response.ok:
+ raise AssertionError(
+ f"Operation creation failed with HTTP {operation_response.status}"
+ )
+ operation = operation_response.json()
+ operation_id = operation["id"]
+ transition_url = (
+ f"{api_base_url}/api/workflows/{workflow_id}/operations/"
+ f"{operation_id}/transitions"
+ )
+ transition_payloads = []
+ if operation["status"] == "queued":
+ transition_payloads.append(
+ {"status": "running", "expected_status": "queued", "progress": 0.5}
+ )
+ if operation["status"] in {"queued", "running"}:
+ transition_payloads.append(
+ {
+ "status": "succeeded",
+ "expected_status": "running",
+ "result": {"source": "synthetic_browser_smoke"},
+ }
+ )
+ elif operation["status"] != "succeeded":
+ raise AssertionError(
+ "Browser smoke operation is unexpectedly terminal: "
+ f"{operation['status']}"
+ )
+ for payload in transition_payloads:
+ transition_response = context.request.post(
+ transition_url,
+ data=payload,
+ timeout=timeout_ms,
+ )
+ if not transition_response.ok:
+ raise AssertionError(
+ "Operation transition failed with HTTP " f"{transition_response.status}"
+ )
+
+ _open_synthetic_tab(page, "Workflow", timeout_ms)
+ operations_panel = page.locator(".ant-card").filter(
+ has=page.get_by_text("Operations", exact=True)
+ )
+ operations_panel.first.wait_for(state="visible", timeout=timeout_ms)
+ operations_panel.first.get_by_role("button", name="Refresh operations").click()
+ operation_row = operations_panel.first.locator(".workflow-operation-row").filter(
+ has_text="Browser smoke"
+ )
+ operation_row.first.wait_for(state="visible", timeout=timeout_ms)
+ operation_row.first.get_by_text("Succeeded", exact=True).wait_for(
+ state="visible", timeout=timeout_ms
+ )
+ operation_row.first.get_by_text(
+ "Ref browser-smoke-operation-history", exact=True
+ ).wait_for(state="visible", timeout=timeout_ms)
+ page.mouse.move(5, 600)
+ page.wait_for_timeout(500)
+
+
+def run_smoke(
+ *,
+ base_url: str,
+ api_base_url: str,
+ timeout_ms: int,
+ viewport: Tuple[int, int],
+ headless: bool,
+ skip_reload: bool,
+ screenshot_path: Optional[str],
+) -> Dict[str, Any]:
+ playwright_error = _playwright_import_error()
+ if playwright_error is not None:
+ raise _raise_playwright_error(str(playwright_error))
+
+ from playwright.sync_api import sync_playwright
+
+ result: Dict[str, Any] = {"passed": True, "base_url": base_url, "checks": []}
+ with sync_playwright() as playwright:
+ try:
+ browser = _resolve_browser_playwright(playwright, headless=headless)
+ except RuntimeError as exc:
+ raise _raise_playwright_error(str(exc)) from exc
+ context = browser.new_context(
+ viewport={"width": viewport[0], "height": viewport[1]}
+ )
+ page = context.new_page()
+ page.set_default_timeout(timeout_ms)
+ page_errors: List[str] = []
+ api_failures: List[str] = []
+ page.on("pageerror", lambda error: page_errors.append(str(error)))
+ page.on(
+ "response",
+ lambda response: (
+ api_failures.append(
+ f"{response.status} {response.request.method} {response.url}"
+ )
+ if response.status >= 500 and "/api" in response.url
+ else None
+ ),
+ )
+ try:
+ page.goto(base_url, wait_until="domcontentloaded", timeout=timeout_ms)
+ snapshot = _exercise_core_ui(page, timeout_ms)
+ result["checks"].extend(
+ [
+ "synthetic project loaded",
+ "bounded file picker navigated",
+ "bounded volume preview decoded",
+ "progress counts verified",
+ ]
+ )
+ result["progress_snapshot"] = snapshot
+
+ if not skip_reload:
+ page.reload(wait_until="domcontentloaded", timeout=timeout_ms)
+ _exercise_core_ui(page, timeout_ms)
+ result["checks"].append("reload continuity verified")
+
+ _exercise_operation_history(
+ context,
+ page,
+ api_base_url=api_base_url,
+ timeout_ms=timeout_ms,
+ )
+ result["checks"].append("durable operation history rendered")
+ if screenshot_path:
+ screenshot = Path(screenshot_path)
+ screenshot.parent.mkdir(parents=True, exist_ok=True)
+ page.screenshot(path=str(screenshot), full_page=True)
+ result["screenshot"] = str(screenshot)
+
+ if page_errors:
+ raise AssertionError(f"Browser page errors: {page_errors}")
+ if api_failures:
+ raise AssertionError(f"API 5xx responses: {api_failures}")
+ finally:
+ context.close()
+ browser.close()
+ return result
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+ args = _build_arg_parser().parse_args(argv)
+ base_url = args.base_url.rstrip("/")
+ api_base_url = args.api_base_url.rstrip("/")
+ if urlparse(base_url).scheme not in {"http", "https"}:
+ raise ValueError("base-url must include an http or https scheme")
+ if urlparse(api_base_url).scheme not in {"http", "https"}:
+ raise ValueError("api-base-url must include an http or https scheme")
+
+ report: Dict[str, Any] = {
+ "generated_at_unix": time.time(),
+ "script": str(Path(__file__).resolve()),
+ }
+ try:
+ report.update(
+ run_smoke(
+ base_url=base_url,
+ api_base_url=api_base_url,
+ timeout_ms=max(5_000, args.timeout_ms),
+ viewport=_parse_viewport(args.viewport),
+ headless=not args.no_headless,
+ skip_reload=args.skip_reload,
+ screenshot_path=args.screenshot,
+ )
+ )
+ except Exception as exc:
+ report.update(
+ passed=False,
+ error=str(exc),
+ error_type=exc.__class__.__name__,
+ )
+
+ Path(args.report).write_text(
+ json.dumps(report, indent=2, sort_keys=True), encoding="utf-8"
+ )
+ print(
+ "Synthetic browser smoke passed."
+ if report.get("passed")
+ else f"Synthetic browser smoke failed: {report.get('error')}"
+ )
+ print(f"Report: {args.report}")
+ return 0 if report.get("passed") else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/browser_yixiao_case_study_smoke.py b/scripts/browser_yixiao_case_study_smoke.py
index be50f066..17914a1a 100755
--- a/scripts/browser_yixiao_case_study_smoke.py
+++ b/scripts/browser_yixiao_case_study_smoke.py
@@ -174,18 +174,17 @@ def _playwright_import_error() -> Optional[BaseException]:
return exc if isinstance(exc, ImportError) else ImportError(str(exc))
-def _resolve_browser_playwright(playwright) -> None:
+def _resolve_browser_playwright(playwright, *, headless: bool = True):
browsers = [
- ("chromium", playwright.chromium),
- ("firefox", playwright.firefox),
- ("webkit", playwright.webkit),
+ ("chromium", playwright.chromium, {}),
+ ("chromium-channel", playwright.chromium, {"channel": "chromium"}),
+ ("firefox", playwright.firefox, {}),
+ ("webkit", playwright.webkit, {}),
]
errors: List[str] = []
- for name, launcher in browsers:
+ for name, launcher, options in browsers:
try:
- browser = launcher.launch(headless=True)
- browser.close()
- return
+ return launcher.launch(headless=headless, **options)
except Exception as exc: # pragma: no cover - engine-specific runtime behavior
errors.append(f"{name}: {exc}")
raise RuntimeError(
@@ -522,12 +521,10 @@ def run_smoke(
from playwright.sync_api import sync_playwright
with sync_playwright() as p:
- # Pick a working engine once to avoid launch failures at runtime.
try:
- _resolve_browser_playwright(p)
+ browser = _resolve_browser_playwright(p, headless=headless)
except RuntimeError as exc:
raise _raise_playwright_error(str(exc))
- browser = p.chromium.launch(headless=headless)
context = browser.new_context(viewport=viewport_config)
page = context.new_page()
page.set_default_timeout(timeout_ms)
diff --git a/scripts/create_synthetic_project.py b/scripts/create_synthetic_project.py
new file mode 100644
index 00000000..17cad7b6
--- /dev/null
+++ b/scripts/create_synthetic_project.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""Create the deterministic local project used by the development app."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+ sys.path.insert(0, str(REPO_ROOT))
+
+from server_api.synthetic_project import create_synthetic_project
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument(
+ "--output-dir",
+ default=str(REPO_ROOT / ".pytc/synthetic-core-project"),
+ help="Project directory (default: .pytc/synthetic-core-project)",
+ )
+ parser.add_argument(
+ "--reset",
+ action="store_true",
+ help="Delete and restore the generated project to its canonical state.",
+ )
+ args = parser.parse_args()
+ print(
+ json.dumps(
+ create_synthetic_project(args.output_dir, reset=args.reset),
+ sort_keys=True,
+ )
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/run_browser_synthetic_core_smoke_ci.sh b/scripts/run_browser_synthetic_core_smoke_ci.sh
new file mode 100755
index 00000000..9be34407
--- /dev/null
+++ b/scripts/run_browser_synthetic_core_smoke_ci.sh
@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+ARTIFACT_DIR="${PYTC_BROWSER_SMOKE_ARTIFACT_DIR:-${ROOT_DIR}/.ci/synthetic-browser-smoke}"
+PROJECT_ROOT="${PYTC_SYNTHETIC_PROJECT_ROOT:-${ROOT_DIR}/.pytc/synthetic-core-project-ci}"
+API_PORT="${PYTC_CI_API_PORT:-4242}"
+REACT_PORT="${PYTC_CI_REACT_PORT:-3000}"
+API_BASE_URL="http://127.0.0.1:${API_PORT}"
+REACT_BASE_URL="http://127.0.0.1:${REACT_PORT}"
+REPORT_PATH="${ARTIFACT_DIR}/synthetic-core-browser-smoke.json"
+VIEWPORT="${PYTC_BROWSER_SMOKE_VIEWPORT:-1280x720}"
+
+mkdir -p "${ARTIFACT_DIR}" "$(dirname "${PROJECT_ROOT}")"
+rm -f \
+ "${ARTIFACT_DIR}/api.pid" \
+ "${ARTIFACT_DIR}/react.pid" \
+ "${ARTIFACT_DIR}/synthetic-core.db" \
+ "${ARTIFACT_DIR}/synthetic-core.db-shm" \
+ "${ARTIFACT_DIR}/synthetic-core.db-wal" \
+ "${REPORT_PATH}"
+
+export PYTC_INITIAL_PROJECT_ROOT="${PROJECT_ROOT}"
+export PYTC_INITIAL_PROJECT_KIND="synthetic"
+export PYTC_INITIAL_PROJECT_TITLE="Synthetic Segmentation Core Loop"
+export PYTC_INITIAL_IMAGE_PATH="${PROJECT_ROOT}/data/raw/train-01_image.h5"
+export PYTC_INITIAL_LABEL_PATH="${PROJECT_ROOT}/data/seg/ground_truth/train-01_ground_truth.h5"
+export PYTC_INITIAL_MASK_PATH="${PYTC_INITIAL_LABEL_PATH}"
+export PYTC_INITIAL_CONFIG_PATH="${PROJECT_ROOT}/configs/Synthetic-Core-Loop-BC.yaml"
+export PYTC_DATABASE_URL="sqlite:///${ARTIFACT_DIR}/synthetic-core.db"
+export PYTC_API_HOST="127.0.0.1"
+export PYTC_API_PORT="${API_PORT}"
+export PYTC_ALLOWED_ORIGINS="${REACT_BASE_URL}"
+export REACT_APP_INITIAL_PROJECT_ROOT="${PROJECT_ROOT}"
+export REACT_APP_SERVER_PROTOCOL="http"
+export REACT_APP_SERVER_URL="127.0.0.1:${API_PORT}"
+
+STARTED_PIDS=()
+
+terminate_process_tree() {
+ local pid="$1"
+ local child
+ while IFS= read -r child; do
+ [[ -n "${child}" ]] && terminate_process_tree "${child}"
+ done < <(pgrep -P "${pid}" 2>/dev/null || true)
+ kill "${pid}" >/dev/null 2>&1 || true
+}
+
+cleanup() {
+ local exit_code=$?
+ local pid
+ for pid in "${STARTED_PIDS[@]:-}"; do
+ if kill -0 "${pid}" >/dev/null 2>&1; then
+ terminate_process_tree "${pid}"
+ fi
+ done
+ for pid in "${STARTED_PIDS[@]:-}"; do
+ wait "${pid}" >/dev/null 2>&1 || true
+ done
+ exit "${exit_code}"
+}
+trap cleanup EXIT INT TERM
+
+wait_for_http() {
+ local name="$1"
+ local url="$2"
+ local log_path="$3"
+ local attempt
+ for attempt in $(seq 1 90); do
+ if curl --fail --silent --show-error "${url}" >/dev/null 2>&1; then
+ echo "${name} ready at ${url}"
+ return 0
+ fi
+ if ((attempt % 15 == 0)); then
+ echo "Waiting for ${name} at ${url} (${attempt}/90)"
+ fi
+ sleep 1
+ done
+ echo "${name} failed to become ready at ${url}" >&2
+ tail -n 100 "${log_path}" >&2 || true
+ return 1
+}
+
+cd "${ROOT_DIR}"
+
+uv run python scripts/create_synthetic_project.py \
+ --output-dir "${PROJECT_ROOT}" \
+ --reset | tee "${ARTIFACT_DIR}/fixture.log"
+
+uv run python -m server_api.main >"${ARTIFACT_DIR}/api.log" 2>&1 &
+API_PID=$!
+STARTED_PIDS+=("${API_PID}")
+echo "${API_PID}" >"${ARTIFACT_DIR}/api.pid"
+
+(
+ cd client
+ exec env \
+ BROWSER=none \
+ CI=false \
+ HOST=127.0.0.1 \
+ PORT="${REACT_PORT}" \
+ npm start
+) >"${ARTIFACT_DIR}/react.log" 2>&1 &
+REACT_PID=$!
+STARTED_PIDS+=("${REACT_PID}")
+echo "${REACT_PID}" >"${ARTIFACT_DIR}/react.pid"
+
+wait_for_http "FastAPI" "${API_BASE_URL}/health" "${ARTIFACT_DIR}/api.log"
+
+MOUNT_PAYLOAD="$(uv run python -c \
+ 'import json, os; print(json.dumps({"directory_path": os.environ["PYTC_INITIAL_PROJECT_ROOT"], "destination_path": "root", "mount_name": os.environ["PYTC_INITIAL_PROJECT_TITLE"]}))')"
+curl --fail --silent --show-error \
+ -X POST \
+ -H "Content-Type: application/json" \
+ --data-binary "${MOUNT_PAYLOAD}" \
+ "${API_BASE_URL}/files/mount" >"${ARTIFACT_DIR}/mount.json"
+echo "Synthetic project mounted."
+
+wait_for_http "React" "${REACT_BASE_URL}" "${ARTIFACT_DIR}/react.log"
+
+uv run python scripts/browser_synthetic_core_smoke.py \
+ --base-url "${REACT_BASE_URL}" \
+ --api-base-url "${API_BASE_URL}" \
+ --timeout-ms 45000 \
+ --viewport "${VIEWPORT}" \
+ --screenshot "${ARTIFACT_DIR}/synthetic-core-browser-smoke.png" \
+ --report "${REPORT_PATH}"
diff --git a/scripts/run_dbos_operation_spike.py b/scripts/run_dbos_operation_spike.py
new file mode 100644
index 00000000..80057c93
--- /dev/null
+++ b/scripts/run_dbos_operation_spike.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+import pathlib
+import sys
+
+REPO_ROOT = pathlib.Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+ sys.path.insert(0, str(REPO_ROOT))
+
+from spikes.dbos_operation.runner import main
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/start.sh b/scripts/start.sh
index 164dffbc..37f55d91 100755
--- a/scripts/start.sh
+++ b/scripts/start.sh
@@ -44,6 +44,36 @@ fi
mkdir -p "${LOG_DIR}"
+configure_synthetic_project() {
+ if [[ "${PYTC_SYNTHETIC_PROJECT:-1}" != "1" || -n "${PYTC_INITIAL_PROJECT_ROOT:-}" ]]; then
+ return 0
+ fi
+
+ local project_root="${PYTC_SYNTHETIC_PROJECT_ROOT:-${ROOT_DIR}/.pytc/synthetic-core-project}"
+ local generator_args=(
+ python scripts/create_synthetic_project.py
+ --output-dir "${project_root}"
+ )
+ if [[ "${PYTC_SYNTHETIC_PROJECT_RESET:-0}" == "1" ]]; then
+ generator_args+=(--reset)
+ fi
+
+ echo "Preparing deterministic synthetic project..."
+ "${UV_BIN}" run --directory "${ROOT_DIR}" "${generator_args[@]}"
+ export PYTC_INITIAL_PROJECT_ROOT="${project_root}"
+ export PYTC_INITIAL_PROJECT_KIND="synthetic"
+ export PYTC_INITIAL_PROJECT_TITLE="Synthetic Segmentation Core Loop"
+ export PYTC_INITIAL_IMAGE_PATH="${project_root}/data/raw/train-01_image.h5"
+ export PYTC_INITIAL_LABEL_PATH="${project_root}/data/seg/ground_truth/train-01_ground_truth.h5"
+ export PYTC_INITIAL_MASK_PATH="${project_root}/data/seg/ground_truth/train-01_ground_truth.h5"
+ export PYTC_INITIAL_CONFIG_PATH="${project_root}/configs/Synthetic-Core-Loop-BC.yaml"
+ export PYTC_DATABASE_URL="${PYTC_DATABASE_URL:-sqlite:///${ROOT_DIR}/.pytc/synthetic-core.db}"
+ export PYTC_AUTOMOUNT_INITIAL_PROJECT="${PYTC_AUTOMOUNT_INITIAL_PROJECT:-1}"
+ export REACT_APP_INITIAL_PROJECT_ROOT="${project_root}"
+}
+
+configure_synthetic_project
+
STARTED_PIDS=()
relative_log_path() {
@@ -137,6 +167,24 @@ start_service \
"${LOG_DIR}/api-server.log" \
env PYTHONDONTWRITEBYTECODE=1 "${UV_BIN}" run --directory "${ROOT_DIR}" python -m server_api.main
+initialize_local_project() {
+ if [[ "${PYTC_AUTOMOUNT_INITIAL_PROJECT:-0}" != "1" || -z "${PYTC_INITIAL_PROJECT_ROOT:-}" ]]; then
+ return 0
+ fi
+
+ local payload
+ payload="$(PYTC_MOUNT_ROOT="${PYTC_INITIAL_PROJECT_ROOT}" "${UV_BIN}" run --directory "${ROOT_DIR}" python -c 'import json, os; print(json.dumps({"directory_path": os.environ["PYTC_MOUNT_ROOT"], "destination_path": "root"}))')"
+ curl -sf "http://localhost:4242/api/workflows/current" >/dev/null
+ curl -sf \
+ -X POST \
+ -H "Content-Type: application/json" \
+ --data-binary "${payload}" \
+ "http://localhost:4242/files/mount" >/dev/null
+ echo "Initial project mounted: ${PYTC_INITIAL_PROJECT_ROOT}"
+}
+
+initialize_local_project
+
start_service \
"PyTC server" \
4243 \
diff --git a/server_api/auth/database.py b/server_api/auth/database.py
index 7533c849..80137595 100644
--- a/server_api/auth/database.py
+++ b/server_api/auth/database.py
@@ -1,9 +1,22 @@
+import os
+from pathlib import Path
+
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
+DATABASE_URL = os.getenv("PYTC_DATABASE_URL", "sqlite:///server_api/auth/sql_app.db")
+
+if DATABASE_URL.startswith("sqlite:///"):
+ database_path = DATABASE_URL.removeprefix("sqlite:///")
+ if database_path and database_path != ":memory:":
+ Path(database_path).expanduser().parent.mkdir(parents=True, exist_ok=True)
+
engine = create_engine(
- "sqlite:///server_api/auth/sql_app.db", connect_args={"check_same_thread": False}
+ DATABASE_URL,
+ connect_args={"check_same_thread": False}
+ if DATABASE_URL.startswith("sqlite:")
+ else {},
)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
Base = declarative_base()
diff --git a/server_api/auth/models.py b/server_api/auth/models.py
index f45ee28c..3d6e2566 100644
--- a/server_api/auth/models.py
+++ b/server_api/auth/models.py
@@ -203,6 +203,14 @@ class Config:
from_attributes = True
+class FilePageResponse(BaseModel):
+ items: List[FileResponse]
+ total: int
+ offset: int
+ limit: int
+ has_more: bool
+
+
class Token(BaseModel):
access_token: str
token_type: str
diff --git a/server_api/auth/router.py b/server_api/auth/router.py
index a3b271c0..5543751b 100644
--- a/server_api/auth/router.py
+++ b/server_api/auth/router.py
@@ -1,12 +1,23 @@
-from fastapi import APIRouter, Depends, HTTPException, status, UploadFile, File, Form
+from fastapi import (
+ APIRouter,
+ Depends,
+ File,
+ Form,
+ HTTPException,
+ Query,
+ UploadFile,
+ status,
+)
from fastapi.responses import Response
from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
+from sqlalchemy import func, or_
from sqlalchemy.orm import Session
from . import models, utils, database
from jose import JWTError, jwt
-from typing import Any, List, Optional
+from typing import Any, List, Optional, Union
from datetime import datetime, timezone
from urllib.parse import parse_qs, unquote, urlparse
+import anyio
import json
import shutil
import os
@@ -15,6 +26,7 @@
import re
import math
from server_api.utils.utils import resolve_existing_path
+from server_api.workflows.volume_io import open_volume_store
try: # Optional preview dependency
import cv2
@@ -37,6 +49,7 @@
IGNORED_SYSTEM_FILENAMES = {
".ds_store",
"thumbs.db",
+ "workflow_preference.json",
".pytc_proofreading.json",
".pytc_instance_labels.tif",
".pytc_project_context.json",
@@ -48,6 +61,11 @@
PROJECT_AUDIT_MAX_SAMPLE_VALUES = 250_000
PROJECT_AUDIT_MAX_FINDINGS = 24
PROJECT_CONTEXT_FILENAME = ".pytc_project_context.json"
+FILE_LIST_DEFAULT_LIMIT = 100
+FILE_LIST_MAX_LIMIT = 500
+FILE_LIST_MAINTENANCE_LIMIT = 500
+FILE_PREVIEW_MAX_SOURCE_DIM = 1024
+FILE_PREVIEW_LIMITER = anyio.CapacityLimiter(2)
VOLUME_EXTENSIONS = {
".h5",
@@ -68,6 +86,14 @@
CONFIG_EXTENSIONS = {".yaml", ".yml", ".json", ".toml"}
CHECKPOINT_EXTENSIONS = {".pt", ".pth", ".pth.tar", ".ckpt", ".onnx"}
TEXT_SIGNAL_EXTENSIONS = CONFIG_EXTENSIONS | {".md", ".txt", ".csv", ".tsv"}
+FILE_PICKER_VOLUME_EXTENSIONS = VOLUME_EXTENSIONS | {
+ ".png",
+ ".jpg",
+ ".jpeg",
+ ".gif",
+ ".bmp",
+ ".webp",
+}
def _format_size(size_bytes: int) -> str:
@@ -90,7 +116,8 @@ def _is_ignored_system_file(name: Optional[str]) -> bool:
def _project_suggestion_candidates() -> List[dict]:
repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
workspace_root = os.path.abspath(os.path.join(repo_root, ".."))
- return [
+ initial_project_root = os.getenv("PYTC_INITIAL_PROJECT_ROOT", "").strip()
+ candidates = [
{
"id": "yixiao-tapereader-xri-case-study",
"name": "yixiao_tapereader_xri_case_study",
@@ -169,6 +196,31 @@ def _project_suggestion_candidates() -> List[dict]:
"recommended": False,
},
]
+ if initial_project_root:
+ candidates.insert(
+ 0,
+ {
+ "id": "initial-project",
+ "name": os.path.basename(initial_project_root.rstrip(os.sep)),
+ "directory_path": initial_project_root,
+ "description": "Active local development project.",
+ "recommended": True,
+ },
+ )
+ for candidate in candidates[1:]:
+ candidate["recommended"] = False
+
+ unique_candidates = []
+ seen_paths = set()
+ for candidate in candidates:
+ normalized_path = os.path.abspath(
+ os.path.expanduser(candidate["directory_path"])
+ )
+ if normalized_path in seen_paths:
+ continue
+ seen_paths.add(normalized_path)
+ unique_candidates.append(candidate)
+ return unique_candidates
def _lower_path_parts(path: str) -> List[str]:
@@ -1998,16 +2050,24 @@ def _is_managed_upload_path(user_id: int, physical_path: Optional[str]) -> bool:
return False
-def _repair_stale_mounted_entries(db: Session, user_id: int) -> None:
- candidates = (
- db.query(models.File)
- .filter(
- models.File.user_id == user_id,
- models.File.physical_path.isnot(None),
- models.File.path != "root",
- )
- .all()
+def _repair_stale_mounted_entries(
+ db: Session,
+ user_id: int,
+ *,
+ parent: Optional[str] = None,
+ max_candidates: Optional[int] = None,
+) -> None:
+ query = db.query(models.File).filter(
+ models.File.user_id == user_id,
+ models.File.physical_path.isnot(None),
+ models.File.path != "root",
)
+ if parent is not None:
+ query = query.filter(models.File.path == parent)
+ query = query.order_by(models.File.id.asc())
+ if max_candidates is not None:
+ query = query.limit(max_candidates)
+ candidates = query.all()
changed = False
for entry in candidates:
@@ -2040,16 +2100,24 @@ def _repair_stale_mounted_entries(db: Session, user_id: int) -> None:
db.commit()
-def _prune_missing_managed_upload_entries(db: Session, user_id: int) -> None:
- candidates = (
- db.query(models.File)
- .filter(
- models.File.user_id == user_id,
- models.File.is_folder.is_(False),
- models.File.physical_path.isnot(None),
- )
- .all()
+def _prune_missing_managed_upload_entries(
+ db: Session,
+ user_id: int,
+ *,
+ parent: Optional[str] = None,
+ max_candidates: Optional[int] = None,
+) -> None:
+ query = db.query(models.File).filter(
+ models.File.user_id == user_id,
+ models.File.is_folder.is_(False),
+ models.File.physical_path.isnot(None),
)
+ if parent is not None:
+ query = query.filter(models.File.path == parent)
+ query = query.order_by(models.File.id.asc())
+ if max_candidates is not None:
+ query = query.limit(max_candidates)
+ candidates = query.all()
removed = False
for entry in candidates:
@@ -2223,14 +2291,53 @@ def list_current_user_projects(
# File Management Endpoints
-@router.get("/files", response_model=List[models.FileResponse])
+def _visible_files_query(query):
+ lower_name = func.lower(models.File.name)
+ return query.filter(
+ ~lower_name.in_(sorted(IGNORED_SYSTEM_FILENAMES)),
+ ~lower_name.like(".__pytc_runtime_%"),
+ )
+
+
+def _volume_picker_files_query(query):
+ lower_name = func.lower(models.File.name)
+ return query.filter(
+ or_(
+ models.File.is_folder.is_(True),
+ *(
+ lower_name.like(f"%{extension}")
+ for extension in sorted(FILE_PICKER_VOLUME_EXTENSIONS)
+ ),
+ )
+ )
+
+
+@router.get(
+ "/files",
+ response_model=Union[List[models.FileResponse], models.FilePageResponse],
+)
def get_files(
parent: Optional[str] = None,
+ offset: Optional[int] = Query(default=None, ge=0),
+ limit: Optional[int] = Query(default=None, ge=1, le=FILE_LIST_MAX_LIMIT),
+ volume_only: bool = False,
current_user: models.User = Depends(get_current_user),
db: Session = Depends(database.get_db),
):
- _repair_stale_mounted_entries(db, current_user.id)
- _prune_missing_managed_upload_entries(db, current_user.id)
+ paged = offset is not None or limit is not None
+ maintenance_limit = FILE_LIST_MAINTENANCE_LIMIT if paged else None
+ _repair_stale_mounted_entries(
+ db,
+ current_user.id,
+ parent=parent,
+ max_candidates=maintenance_limit,
+ )
+ _prune_missing_managed_upload_entries(
+ db,
+ current_user.id,
+ parent=parent,
+ max_candidates=maintenance_limit,
+ )
query = db.query(models.File).filter(models.File.user_id == current_user.id)
if parent is not None:
if parent != "root":
@@ -2254,24 +2361,33 @@ def get_files(
)
query = query.filter(models.File.path == parent)
- return [
- file
- for file in query.order_by(
+ query = _visible_files_query(query)
+ ordered_query = query.order_by(models.File.is_folder.desc(), models.File.name.asc())
+ if not paged:
+ return ordered_query.all()
+
+ page_offset = offset or 0
+ page_limit = limit or FILE_LIST_DEFAULT_LIMIT
+ if volume_only:
+ ordered_query = _volume_picker_files_query(query).order_by(
models.File.is_folder.desc(), models.File.name.asc()
- ).all()
- if not _is_ignored_system_file(file.name)
- ]
+ )
+ total = ordered_query.order_by(None).count()
+ items = ordered_query.offset(page_offset).limit(page_limit).all()
+ return models.FilePageResponse(
+ items=items,
+ total=total,
+ offset=page_offset,
+ limit=page_limit,
+ has_more=page_offset + len(items) < total,
+ )
-@router.get("/files/preview/{file_id}")
-def file_preview(
+def _get_preview_file(
file_id: int,
current_user: models.User = Depends(get_current_user),
db: Session = Depends(database.get_db),
-):
- if cv2 is None or np is None:
- raise HTTPException(status_code=500, detail="Preview dependencies missing")
-
+) -> models.File:
file = (
db.query(models.File)
.filter(models.File.id == file_id, models.File.user_id == current_user.id)
@@ -2281,51 +2397,81 @@ def file_preview(
raise HTTPException(status_code=404, detail="File not found")
if not file.physical_path or not os.path.exists(file.physical_path):
raise HTTPException(status_code=404, detail="File not found on disk")
+ return file
- def to_uint8(arr):
- if arr is None:
- return None
- if arr.dtype == np.uint8:
- return arr
- arr = arr.astype(np.float32)
- min_val = np.nanmin(arr)
- max_val = np.nanmax(arr)
- if max_val <= min_val:
- return np.zeros_like(arr, dtype=np.uint8)
- scaled = (arr - min_val) / (max_val - min_val)
- return np.clip(scaled * 255.0, 0, 255).astype(np.uint8)
-
- def load_image(path: str) -> Optional["np.ndarray"]:
- ext = os.path.splitext(path)[1].lower()
- if ext in {".tif", ".tiff"} and tifffile is not None:
- try:
- img = tifffile.imread(path)
- except Exception:
- img = None
- else:
- img = cv2.imread(path, cv2.IMREAD_UNCHANGED)
- if img is None:
+
+def _to_preview_uint8(arr):
+ if arr is None:
+ return None
+ if arr.dtype == np.uint8:
+ return arr
+ arr = arr.astype(np.float32)
+ min_val = np.nanmin(arr)
+ max_val = np.nanmax(arr)
+ if max_val <= min_val:
+ return np.zeros_like(arr, dtype=np.uint8)
+ scaled = (arr - min_val) / (max_val - min_val)
+ return np.clip(scaled * 255.0, 0, 255).astype(np.uint8)
+
+
+def _volume_preview_crop(shape: tuple[int, ...]) -> tuple[tuple[slice, ...], int]:
+ if len(shape) < 2:
+ raise ValueError("Preview volume must have at least two dimensions")
+
+ has_color_axis = len(shape) >= 3 and shape[-1] in (3, 4)
+ retained_dimensions = 3 if has_color_axis else 2
+ leading_dimensions = len(shape) - retained_dimensions
+ height_axis = leading_dimensions
+ width_axis = leading_dimensions + 1
+ source_max = max(shape[height_axis], shape[width_axis])
+ stride = max(1, math.ceil(source_max / FILE_PREVIEW_MAX_SOURCE_DIM))
+ crop = [
+ slice(dimension // 2, dimension // 2 + 1)
+ for dimension in shape[:leading_dimensions]
+ ]
+ crop.extend(
+ [
+ slice(None, None, stride),
+ slice(None, None, stride),
+ ]
+ )
+ if has_color_axis:
+ crop.append(slice(None))
+ return tuple(crop), leading_dimensions
+
+
+def _load_preview_image(path: str) -> Optional["np.ndarray"]:
+ extension = _project_extension(path)
+ if extension in VOLUME_EXTENSIONS:
+ try:
+ with open_volume_store(path) as store:
+ crop, leading_dimensions = _volume_preview_crop(store.metadata.shape)
+ img = store.read(crop)
+ img = np.asarray(img)
+ for _ in range(leading_dimensions):
+ img = np.take(img, 0, axis=0)
+ except Exception:
return None
+ else:
+ img = cv2.imread(path, cv2.IMREAD_UNCHANGED)
- img = np.asarray(img)
- if img.ndim == 2:
- return to_uint8(img)
- if img.ndim == 3:
- if img.shape[2] in (3, 4):
- if img.shape[2] == 4:
- img = img[:, :, :3]
- return to_uint8(img)
- mid = img.shape[0] // 2
- return to_uint8(img[mid])
- if img.ndim == 4:
- mid = img.shape[0] // 2
- img = img[mid]
- if img.ndim == 3 and img.shape[2] == 4:
- img = img[:, :, :3]
- return to_uint8(img)
+ if img is None:
return None
+ img = np.asarray(img)
+ if img.ndim == 2:
+ return _to_preview_uint8(img)
+ if img.ndim == 3 and img.shape[2] in (3, 4):
+ if img.shape[2] == 4:
+ img = img[:, :, :3]
+ return _to_preview_uint8(img)
+ return None
+
+
+def _render_file_preview(path: str) -> bytes:
+ if cv2 is None or np is None:
+ raise HTTPException(status_code=500, detail="Preview dependencies missing")
- image = load_image(file.physical_path)
+ image = _load_preview_image(path)
if image is None:
raise HTTPException(status_code=415, detail="Unsupported image format")
@@ -2339,7 +2485,19 @@ def load_image(path: str) -> Optional["np.ndarray"]:
success, buffer = cv2.imencode(".png", image)
if not success:
raise HTTPException(status_code=500, detail="Failed to encode preview")
- return Response(content=buffer.tobytes(), media_type="image/png")
+ return buffer.tobytes()
+
+
+@router.get("/files/preview/{file_id}")
+async def file_preview(
+ file: models.File = Depends(_get_preview_file),
+):
+ content = await anyio.to_thread.run_sync(
+ _render_file_preview,
+ file.physical_path,
+ limiter=FILE_PREVIEW_LIMITER,
+ )
+ return Response(content=content, media_type="image/png")
@router.post("/files/upload", response_model=models.FileResponse)
diff --git a/server_api/chatbot/logging_utils.py b/server_api/chatbot/logging_utils.py
index 41dcdfe9..906831a7 100644
--- a/server_api/chatbot/logging_utils.py
+++ b/server_api/chatbot/logging_utils.py
@@ -1,13 +1,26 @@
import logging
+import re
import time
import uuid
from typing import Any, Optional
logger = logging.getLogger(__name__)
+_REQUEST_ID_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
def request_id_from_request(request: Any) -> str:
- return request.headers.get("x-request-id") or str(uuid.uuid4())
+ existing = getattr(getattr(request, "state", None), "request_id", None)
+ if existing:
+ return existing
+ supplied_request_id = request.headers.get("x-request-id")
+ request_id = (
+ supplied_request_id
+ if supplied_request_id and _REQUEST_ID_PATTERN.fullmatch(supplied_request_id)
+ else str(uuid.uuid4())
+ )
+ if getattr(request, "state", None) is not None:
+ request.state.request_id = request_id
+ return request_id
def log_request_summary(
diff --git a/server_api/ehtool/data_manager.py b/server_api/ehtool/data_manager.py
index 45b7b328..3e4d42fe 100644
--- a/server_api/ehtool/data_manager.py
+++ b/server_api/ehtool/data_manager.py
@@ -1631,11 +1631,7 @@ def _load_volume(self, path: str) -> Dict[str, Any]:
# Single file
if path_obj.is_file():
- lower_name = path_obj.name.lower()
- if lower_name.endswith((".tif", ".tiff")):
- volume = tifffile.imread(file_path)
- else:
- volume = load_volume(path)
+ volume = load_volume(path)
if volume.ndim == 2:
return {
diff --git a/server_api/ehtool/router.py b/server_api/ehtool/router.py
index 3f4bbdfd..ff9714c0 100644
--- a/server_api/ehtool/router.py
+++ b/server_api/ehtool/router.py
@@ -182,17 +182,20 @@ async def load_detection_dataset(
_data_managers[db_session.id] = data_manager
if workflow:
+ workflow_patch = {
+ "stage": "proofreading",
+ "title": request.project_name or workflow.title,
+ "image_path": request.dataset_path,
+ "label_path": request.mask_path,
+ "mask_path": request.mask_path,
+ "proofreading_session_id": db_session.id,
+ }
+ if not workflow.dataset_path:
+ workflow_patch["dataset_path"] = request.dataset_path
update_workflow_fields(
db,
workflow,
- {
- "stage": "proofreading",
- "title": request.project_name or workflow.title,
- "dataset_path": request.dataset_path,
- "image_path": request.dataset_path,
- "mask_path": request.mask_path,
- "proofreading_session_id": db_session.id,
- },
+ workflow_patch,
commit=True,
)
append_workflow_event(
diff --git a/server_api/errors.py b/server_api/errors.py
new file mode 100644
index 00000000..f8e74d8d
--- /dev/null
+++ b/server_api/errors.py
@@ -0,0 +1,258 @@
+"""Shared API error responses and exception handlers."""
+
+import logging
+from typing import Any, Dict, List, Optional
+
+from fastapi import FastAPI, Request
+from fastapi.encoders import jsonable_encoder
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+from starlette.exceptions import HTTPException as StarletteHTTPException
+
+from server_api.chatbot.logging_utils import request_id_from_request
+
+logger = logging.getLogger(__name__)
+ERROR_SCHEMA_VERSION = 1
+PROBLEM_JSON_MEDIA_TYPE = "application/problem+json"
+PROBLEM_TYPE_BASE = "https://seg.bio/problems"
+
+
+def _message_from_detail(detail: Any, fallback: str) -> str:
+ if isinstance(detail, str) and detail.strip():
+ return detail.strip()
+ if isinstance(detail, dict):
+ for key in ("user_message", "message", "detail", "reason"):
+ value = detail.get(key)
+ if isinstance(value, str) and value.strip():
+ return value.strip()
+ return fallback
+
+
+def _error_metadata(status_code: int) -> Dict[str, Any]:
+ if status_code == 400:
+ return {
+ "code": "invalid_request",
+ "category": "validation",
+ "title": "Request could not be completed",
+ "retryable": False,
+ "recovery_actions": ["review_input"],
+ }
+ if status_code == 401:
+ return {
+ "code": "authentication_required",
+ "category": "authentication",
+ "title": "Authentication required",
+ "retryable": False,
+ "recovery_actions": ["sign_in"],
+ }
+ if status_code == 403:
+ return {
+ "code": "permission_denied",
+ "category": "authorization",
+ "title": "Permission denied",
+ "retryable": False,
+ "recovery_actions": ["contact_admin"],
+ }
+ if status_code == 404:
+ return {
+ "code": "not_found",
+ "category": "resource",
+ "title": "Resource not found",
+ "retryable": False,
+ "recovery_actions": ["go_back"],
+ }
+ if status_code == 409:
+ return {
+ "code": "conflict",
+ "category": "state",
+ "title": "State conflict",
+ "retryable": False,
+ "recovery_actions": ["refresh"],
+ }
+ if status_code == 413:
+ return {
+ "code": "payload_too_large",
+ "category": "resource",
+ "title": "Request is too large",
+ "retryable": False,
+ "recovery_actions": ["reduce_request"],
+ }
+ if status_code == 422:
+ return {
+ "code": "validation_failed",
+ "category": "validation",
+ "title": "Some request values are invalid",
+ "retryable": False,
+ "recovery_actions": ["review_input"],
+ }
+ if status_code == 429:
+ return {
+ "code": "rate_limited",
+ "category": "availability",
+ "title": "Too many requests",
+ "retryable": True,
+ "recovery_actions": ["retry_later"],
+ }
+ if status_code in {502, 503, 504}:
+ return {
+ "code": "service_unavailable",
+ "category": "availability",
+ "title": "Service temporarily unavailable",
+ "retryable": True,
+ "recovery_actions": ["retry"],
+ }
+ if status_code >= 500:
+ return {
+ "code": "internal_error",
+ "category": "internal",
+ "title": "Unexpected server error",
+ "retryable": True,
+ "recovery_actions": ["retry", "view_logs"],
+ }
+ return {
+ "code": "request_failed",
+ "category": "request",
+ "title": "Request failed",
+ "retryable": False,
+ "recovery_actions": ["go_back"],
+ }
+
+
+def _accepts_problem_json(request: Request) -> bool:
+ """Require an explicit media type during the compatibility transition."""
+ for media_range in request.headers.get("accept", "").split(","):
+ parts = [part.strip() for part in media_range.split(";")]
+ if not parts or parts[0].lower() != PROBLEM_JSON_MEDIA_TYPE:
+ continue
+ quality = next(
+ (
+ part.split("=", 1)[1].strip()
+ for part in parts[1:]
+ if part.lower().startswith("q=")
+ ),
+ "1",
+ )
+ try:
+ if float(quality) > 0:
+ return True
+ except ValueError:
+ continue
+ return False
+
+
+def _append_vary_header(headers: Dict[str, str], field: str) -> None:
+ existing_key = next((key for key in headers if key.lower() == "vary"), "Vary")
+ existing_fields = {
+ item.strip().lower()
+ for item in headers.get(existing_key, "").split(",")
+ if item.strip()
+ }
+ if field.lower() not in existing_fields:
+ headers[existing_key] = ", ".join(
+ item for item in (headers.get(existing_key, "").strip(), field) if item
+ )
+
+
+def build_error_response(
+ *,
+ status_code: int,
+ request_id: str,
+ detail: Any,
+ message_fallback: str,
+ instance: str,
+ use_problem_json: bool = False,
+ headers: Optional[Dict[str, str]] = None,
+ validation_errors: Optional[List[Dict[str, Any]]] = None,
+) -> JSONResponse:
+ metadata = _error_metadata(status_code)
+ error = {
+ "schema_version": ERROR_SCHEMA_VERSION,
+ **metadata,
+ "message": _message_from_detail(detail, message_fallback),
+ "request_id": request_id,
+ }
+ if validation_errors is not None:
+ error["validation_errors"] = validation_errors
+
+ problem = {
+ "type": f"{PROBLEM_TYPE_BASE}/{metadata['code'].replace('_', '-')}",
+ "title": metadata["title"],
+ "status": status_code,
+ "instance": f"{instance}#request-{request_id}",
+ }
+ if use_problem_json:
+ problem["detail"] = error["message"]
+ problem["error"] = error
+ if detail != error["message"]:
+ problem["legacy_detail"] = detail
+ if validation_errors is not None:
+ problem["validation_errors"] = validation_errors
+ else:
+ # Preserve FastAPI's historical detail value for existing consumers.
+ problem["detail"] = detail
+ problem["error"] = error
+
+ response_headers = dict(headers or {})
+ response_headers.setdefault("x-request-id", request_id)
+ response_headers.setdefault("cache-control", "no-store")
+ _append_vary_header(response_headers, "Accept")
+ return JSONResponse(
+ status_code=status_code,
+ content=jsonable_encoder(problem),
+ headers=response_headers,
+ media_type=PROBLEM_JSON_MEDIA_TYPE if use_problem_json else None,
+ )
+
+
+def install_error_handlers(app: FastAPI) -> None:
+ @app.exception_handler(StarletteHTTPException)
+ async def http_exception_handler(
+ request: Request, exc: StarletteHTTPException
+ ) -> JSONResponse:
+ request_id = request_id_from_request(request)
+ return build_error_response(
+ status_code=exc.status_code,
+ request_id=request_id,
+ detail=exc.detail,
+ message_fallback="The request could not be completed.",
+ instance=request.url.path,
+ use_problem_json=_accepts_problem_json(request),
+ headers=exc.headers,
+ )
+
+ @app.exception_handler(RequestValidationError)
+ async def validation_exception_handler(
+ request: Request, exc: RequestValidationError
+ ) -> JSONResponse:
+ request_id = request_id_from_request(request)
+ errors = exc.errors()
+ return build_error_response(
+ status_code=422,
+ request_id=request_id,
+ detail=errors,
+ message_fallback="Review the highlighted request values and try again.",
+ instance=request.url.path,
+ use_problem_json=_accepts_problem_json(request),
+ validation_errors=errors,
+ )
+
+ @app.exception_handler(Exception)
+ async def unexpected_exception_handler(
+ request: Request, exc: Exception
+ ) -> JSONResponse:
+ request_id = request_id_from_request(request)
+ logger.exception(
+ "Unhandled API error request_id=%s method=%s path=%s",
+ request_id,
+ request.method,
+ request.url.path,
+ exc_info=exc,
+ )
+ return build_error_response(
+ status_code=500,
+ request_id=request_id,
+ detail="An unexpected server error occurred.",
+ message_fallback="An unexpected server error occurred.",
+ instance=request.url.path,
+ use_problem_json=_accepts_problem_json(request),
+ )
diff --git a/server_api/main.py b/server_api/main.py
index 4d5fd0a3..564d86a6 100644
--- a/server_api/main.py
+++ b/server_api/main.py
@@ -19,6 +19,7 @@
import requests
import uvicorn
+import numpy as np
from fastapi import Depends, FastAPI, HTTPException, Request, UploadFile
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
@@ -39,12 +40,23 @@
from server_api.auth.database import get_db
from server_api.auth.router import get_current_user
from server_api.ehtool import router as ehtool_router
+from server_api.errors import install_error_handlers
from server_api.chatbot.logging_utils import (
log_request_summary,
request_id_from_request,
)
from server_api.workflows import router as workflow_router
-from server_api.workflows.db_models import WorkflowCommand, WorkflowEvent
+from server_api.workflows import operation_router as workflow_operation_router
+from server_api.workflows.db_models import (
+ WorkflowCommand,
+ WorkflowEvent,
+ WorkflowOperation,
+)
+from server_api.workflows.operation_service import (
+ create_workflow_operation,
+ operation_to_dict,
+ transition_workflow_operation,
+)
from server_api.workflows.service import (
append_event_for_workflow_if_present,
command_to_dict,
@@ -56,7 +68,7 @@
update_workflow_fields,
)
from server_api.project_manager import router as pm_router
-from server_api.workflows.volume_io import load_volume
+from server_api.workflows.volume_io import VolumeStore, open_volume_store
from server_api.workflows.volume_pairs import (
_is_chunked_volume_directory,
_is_neuroglancer_volume_file,
@@ -251,6 +263,7 @@ def _ensure_sqlite_column(table_name: str, column_name: str, ddl: str) -> None:
)
app = FastAPI()
+install_error_handlers(app)
# Ensure uploads directory exists
os.makedirs("uploads", exist_ok=True)
@@ -259,6 +272,11 @@ def _ensure_sqlite_column(table_name: str, column_name: str, ddl: str) -> None:
app.include_router(auth_router.router)
app.include_router(ehtool_router.router, prefix="/eh", tags=["ehtool"])
app.include_router(workflow_router.router, prefix="/api/workflows", tags=["workflows"])
+app.include_router(
+ workflow_operation_router.router,
+ prefix="/api/workflows",
+ tags=["workflow-operations"],
+)
app.include_router(pm_router.router, prefix="/api/pm", tags=["project-manager"])
app.add_middleware(
@@ -267,6 +285,7 @@ def _ensure_sqlite_column(table_name: str, column_name: str, ddl: str) -> None:
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
+ expose_headers=["x-request-id"],
)
logger = logging.getLogger(__name__)
@@ -384,6 +403,79 @@ def _build_neuroglancer_layer(
return _with_neuroglancer_image_shader(neuroglancer_module, source, shader)
+class _NeuroglancerSegmentationStore:
+ """Validate and convert segmentation labels one requested chunk at a time."""
+
+ def __init__(self, store: VolumeStore):
+ if store.ndim != 3:
+ raise ValueError(
+ "Storage-backed segmentation viewing requires a 3D label volume. "
+ "Convert multi-channel predictions to a 3D label volume first."
+ )
+ self._store = store
+ self.shape = store.shape
+ source_dtype = np.dtype(store.dtype)
+ if np.issubdtype(source_dtype, np.bool_):
+ self.dtype = np.dtype(np.uint8)
+ elif np.issubdtype(source_dtype, np.unsignedinteger):
+ self.dtype = source_dtype
+ elif np.issubdtype(source_dtype, np.integer) or np.issubdtype(
+ source_dtype, np.floating
+ ):
+ self.dtype = np.dtype(np.uint64)
+ else:
+ raise ValueError(
+ f"Segmentation volume dtype {source_dtype} is not supported."
+ )
+
+ def __getitem__(self, key):
+ chunk = np.asarray(self._store[key])
+ if chunk.size == 0:
+ return chunk.astype(self.dtype, copy=False)
+ if np.issubdtype(chunk.dtype, np.floating):
+ if not np.all(np.isfinite(chunk)):
+ raise ValueError(
+ "Segmentation volumes must not contain NaN or infinite values."
+ )
+ rounded = np.rint(chunk)
+ if not np.allclose(chunk, rounded):
+ raise ValueError("Segmentation volumes must use integer-valued labels.")
+ chunk = rounded
+ if np.issubdtype(chunk.dtype, np.signedinteger) or np.issubdtype(
+ chunk.dtype, np.floating
+ ):
+ if int(chunk.min()) < 0:
+ raise ValueError(
+ "Segmentation volumes must contain non-negative label ids."
+ )
+ return chunk.astype(self.dtype, copy=False)
+
+
+def _open_neuroglancer_volume_sources(
+ image_path: pathlib.Path,
+ label_path: Optional[pathlib.Path],
+):
+ resources: List[VolumeStore] = []
+ try:
+ image_store = open_volume_store(str(image_path))
+ resources.append(image_store)
+ if image_store.ndim != 3:
+ raise ValueError(
+ f"Image volume must be 3D for Neuroglancer, got {image_store.shape}."
+ )
+
+ label_source = None
+ if label_path is not None:
+ label_store = open_volume_store(str(label_path))
+ resources.append(label_store)
+ label_source = _NeuroglancerSegmentationStore(label_store)
+ return image_store, label_source, resources
+ except Exception:
+ for resource in resources:
+ resource.close()
+ raise
+
+
class ClientAppLogEvent(BaseModel):
event: str
level: str = "INFO"
@@ -726,6 +818,54 @@ def _build_training_body_from_command(
}
+def _workflow_command_run_response(
+ workflow,
+ command: WorkflowCommand,
+ operation: WorkflowOperation,
+) -> dict[str, Any]:
+ result = decode_json(operation.result_json)
+ return {
+ "workflow_id": workflow.id,
+ "command": command_to_dict(command),
+ "operation": operation_to_dict(operation),
+ "worker": result.get("worker", {}),
+ "run_id": result.get("run_id"),
+ "started_event_id": result.get("started_event_id"),
+ }
+
+
+def _fail_command_operation(
+ db: Session,
+ *,
+ command: WorkflowCommand,
+ operation: WorkflowOperation,
+ error_payload: dict[str, Any],
+ retryable: bool,
+) -> None:
+ fail_workflow_command(
+ db,
+ command,
+ error_payload=error_payload,
+ retryable=retryable,
+ commit=False,
+ )
+ if operation.status in {"queued", "running"}:
+ transition_workflow_operation(
+ db,
+ operation,
+ status="failed",
+ expected_status=operation.status,
+ error_payload=error_payload,
+ lease_owner=(
+ "server_api.training_runner" if operation.status == "running" else None
+ ),
+ commit=False,
+ )
+ db.commit()
+ db.refresh(command)
+ db.refresh(operation)
+
+
@app.on_event("startup")
async def configure_app_event_logging():
log_path = configure_process_logging("server_api")
@@ -911,6 +1051,17 @@ def _neuroglancer_token_from_url(viewer_url: str) -> Optional[str]:
return None
+def _close_neuroglancer_entry_resources(entry: dict[str, Any]) -> None:
+ for resource in entry.get("resources") or []:
+ try:
+ resource.close()
+ except Exception:
+ logger.warning(
+ "Failed to close an evicted Neuroglancer backing resource.",
+ exc_info=True,
+ )
+
+
def _cleanup_retained_neuroglancer_viewers(now: Optional[float] = None):
if now is None:
now = time.time()
@@ -928,11 +1079,14 @@ def _cleanup_retained_neuroglancer_viewers(now: Optional[float] = None):
"mode": entry.get("mode"),
}
)
- _retained_neuroglancer_viewers.pop(token, None)
+ removed = _retained_neuroglancer_viewers.pop(token, None)
+ if removed is not None:
+ _close_neuroglancer_entry_resources(removed)
max_viewers = max(PYTC_NEUROGLANCER_MAX_VIEWERS, 0)
while max_viewers and len(_retained_neuroglancer_viewers) > max_viewers:
token, entry = _retained_neuroglancer_viewers.popitem(last=False)
+ _close_neuroglancer_entry_resources(entry)
evicted.append(
{
"token": token,
@@ -954,6 +1108,7 @@ def _retain_neuroglancer_viewer(
workflow_id: Optional[int] = None,
image_path: Optional[str] = None,
label_path: Optional[str] = None,
+ resources: Optional[List[VolumeStore]] = None,
) -> Optional[str]:
token = getattr(viewer, "token", None) or _neuroglancer_token_from_url(
internal_viewer_url
@@ -974,6 +1129,8 @@ def _retain_neuroglancer_viewer(
now = time.time()
with _retained_neuroglancer_viewers_lock:
if PYTC_NEUROGLANCER_MAX_VIEWERS <= 0:
+ for entry in _retained_neuroglancer_viewers.values():
+ _close_neuroglancer_entry_resources(entry)
_retained_neuroglancer_viewers.clear()
append_app_event(
component="server_api",
@@ -988,6 +1145,9 @@ def _retain_neuroglancer_viewer(
return token
evicted = _cleanup_retained_neuroglancer_viewers(now)
+ replaced = _retained_neuroglancer_viewers.pop(token, None)
+ if replaced is not None:
+ _close_neuroglancer_entry_resources(replaced)
_retained_neuroglancer_viewers[token] = {
"viewer": viewer,
"public_url": public_url,
@@ -996,6 +1156,7 @@ def _retain_neuroglancer_viewer(
"workflow_id": workflow_id,
"image_path": image_path,
"label_path": label_path,
+ "resources": list(resources or []),
"created_at": now,
}
_retained_neuroglancer_viewers.move_to_end(token)
@@ -1779,23 +1940,15 @@ async def neuroglancer(
names=["z", "y", "x"], units=["nm", "nm", "nm"], scales=scales
)
try:
- im = load_volume(str(resolved_image_path), label="image")
- except Exception as e:
- raise HTTPException(
- status_code=400, detail=f"Failed to read image volume: {str(e)}"
+ im, gt, volume_resources = _open_neuroglancer_volume_sources(
+ resolved_image_path,
+ resolved_label_path,
)
- try:
- gt = (
- load_volume(str(resolved_label_path), label="label")
- if resolved_label_path
- else None
- )
- if gt is not None:
- gt = _normalize_segmentation_volume_for_neuroglancer(gt)
except Exception as e:
raise HTTPException(
- status_code=400, detail=f"Failed to prepare label volume: {str(e)}"
- )
+ status_code=400,
+ detail=f"Failed to prepare storage-backed volume layers: {str(e)}",
+ ) from e
def ngLayer(
data,
@@ -1844,6 +1997,7 @@ def ngLayer(
workflow_id=workflow_id,
image_path=str(resolved_image_path),
label_path=str(resolved_label_path) if resolved_label_path else None,
+ resources=volume_resources,
)
append_app_event(
component="server_api",
@@ -2051,23 +2205,16 @@ async def neuroglancer_proofread(
raise HTTPException(status_code=400, detail=str(exc)) from exc
try:
- im = load_volume(str(resolved_image_path), label="image")
+ im, gt, volume_resources = _open_neuroglancer_volume_sources(
+ resolved_image_path,
+ resolved_label_path,
+ )
except Exception as exc:
raise HTTPException(
- status_code=400, detail=f"Failed to read image volume: {str(exc)}"
+ status_code=400,
+ detail=f"Failed to prepare storage-backed proofreading layers: {str(exc)}",
) from exc
- gt = None
- if resolved_label_path:
- try:
- gt = load_volume(str(resolved_label_path), label="label")
- gt = _normalize_segmentation_volume_for_neuroglancer(gt)
- except Exception as exc:
- raise HTTPException(
- status_code=400,
- detail=f"Failed to prepare label volume for proofreading: {str(exc)}",
- ) from exc
-
neuroglancer.set_server_bind_address(
PYTC_NEUROGLANCER_BIND_HOST, PYTC_NEUROGLANCER_PORT
)
@@ -2281,6 +2428,7 @@ def handle_save_review(_action_state):
workflow_id=workflow_id,
image_path=str(resolved_image_path),
label_path=str(resolved_label_path) if resolved_label_path else None,
+ resources=volume_resources,
)
response_payload = {
"url": public_url,
@@ -2450,6 +2598,70 @@ async def run_workflow_command(
detail=f"Unsupported workflow command type: {command.command_type}",
)
+ if command.status == "submitted":
+ completed_operation = (
+ db.query(WorkflowOperation)
+ .filter(
+ WorkflowOperation.workflow_id == workflow.id,
+ WorkflowOperation.command_id == command.id,
+ WorkflowOperation.status == "succeeded",
+ )
+ .order_by(WorkflowOperation.id.desc())
+ .first()
+ )
+ if completed_operation is not None:
+ return _workflow_command_run_response(
+ workflow,
+ command,
+ completed_operation,
+ )
+ raise HTTPException(
+ status_code=409, detail="Workflow command was already submitted."
+ )
+
+ operation_query = db.query(WorkflowOperation).filter(
+ WorkflowOperation.workflow_id == workflow.id,
+ WorkflowOperation.command_id == command.id,
+ )
+ latest_operation = operation_query.order_by(WorkflowOperation.id.desc()).first()
+ if latest_operation is not None and latest_operation.status in {
+ "queued",
+ "running",
+ "succeeded",
+ }:
+ operation = latest_operation
+ else:
+ operation = create_workflow_operation(
+ db,
+ workflow_id=workflow.id,
+ operation_type="start_training",
+ idempotency_key=(
+ f"workflow-command:{command.id}:attempt:{operation_query.count() + 1}"
+ ),
+ actor=command.actor,
+ command_id=command.id,
+ input_payload=decode_json(command.input_json),
+ metadata={
+ "command_type": command.command_type,
+ "execution_scope": "worker_submission",
+ },
+ commit=True,
+ )
+ if operation.status == "succeeded":
+ if command.status != "submitted":
+ command = submit_workflow_command(
+ db,
+ command,
+ result_payload=decode_json(operation.result_json),
+ commit=True,
+ )
+ return _workflow_command_run_response(workflow, command, operation)
+ if operation.status != "queued":
+ raise HTTPException(
+ status_code=409,
+ detail=f"Workflow command operation is already {operation.status}.",
+ )
+
try:
body = _build_training_body_from_command(command, workflow)
body = _runtime_body_with_workflow_fallbacks(body, workflow, mode="training")
@@ -2457,8 +2669,20 @@ async def run_workflow_command(
db,
command,
lease_owner="server_api.training_runner",
- commit=True,
+ commit=False,
+ )
+ operation = transition_workflow_operation(
+ db,
+ operation,
+ status="running",
+ expected_status="queued",
+ lease_owner="server_api.training_runner",
+ metadata={"run_id": body.get("run_id")},
+ commit=False,
)
+ db.commit()
+ db.refresh(command)
+ db.refresh(operation)
update_workflow_fields(
db,
workflow,
@@ -2494,36 +2718,43 @@ async def run_workflow_command(
json_body=body,
timeout=30,
)
- command = submit_workflow_command(
- db,
- command,
- result_payload={
- "worker": worker_data,
- "run_id": body.get("run_id"),
- "started_event_id": started_event.id if started_event else None,
- "submitted": True,
- },
- commit=True,
- )
- return {
- "workflow_id": workflow.id,
- "command": command_to_dict(command),
+ operation_result = {
"worker": worker_data,
"run_id": body.get("run_id"),
"started_event_id": started_event.id if started_event else None,
+ "submitted": True,
}
+ command = submit_workflow_command(
+ db,
+ command,
+ result_payload=operation_result,
+ commit=False,
+ )
+ operation = transition_workflow_operation(
+ db,
+ operation,
+ status="succeeded",
+ expected_status="running",
+ result_payload=operation_result,
+ lease_owner="server_api.training_runner",
+ commit=False,
+ )
+ db.commit()
+ db.refresh(command)
+ db.refresh(operation)
+ return _workflow_command_run_response(workflow, command, operation)
except HTTPException as exc:
error_payload = {
"error": "HTTPException",
"status_code": exc.status_code,
"detail": exc.detail,
}
- fail_workflow_command(
+ _fail_command_operation(
db,
- command,
+ command=command,
+ operation=operation,
error_payload=error_payload,
retryable=exc.status_code in {503, 504},
- commit=True,
)
append_event_for_workflow_if_present(
db,
@@ -2545,12 +2776,12 @@ async def run_workflow_command(
"error": type(exc).__name__,
"detail": str(exc),
}
- fail_workflow_command(
+ _fail_command_operation(
db,
- command,
+ command=command,
+ operation=operation,
error_payload=error_payload,
retryable=False,
- commit=True,
)
append_event_for_workflow_if_present(
db,
diff --git a/server_api/synthetic_project.py b/server_api/synthetic_project.py
new file mode 100644
index 00000000..e6ee2cf5
--- /dev/null
+++ b/server_api/synthetic_project.py
@@ -0,0 +1,361 @@
+"""Deterministic local project fixture for the core segmentation workflow."""
+
+from __future__ import annotations
+
+import json
+import shutil
+from pathlib import Path
+from typing import Any
+
+import h5py
+import numpy as np
+
+GENERATOR_VERSION = "pytc-synthetic-core/v1"
+VOLUME_SHAPE = (12, 160, 160)
+VOLUME_CHUNKS = (4, 64, 64)
+VOXEL_SIZE_NM = (40.0, 8.0, 8.0)
+
+
+def _instance_labels(offset: tuple[int, int, int] = (0, 0, 0)) -> np.ndarray:
+ z, y, x = np.indices(VOLUME_SHAPE)
+ labels = np.zeros(VOLUME_SHAPE, dtype=np.uint16)
+ objects = (
+ (1, (4, 43, 48), (3, 18, 24)),
+ (2, (7, 102, 61), (3, 24, 17)),
+ (3, (5, 82, 118), (2, 16, 28)),
+ (4, (8, 125, 126), (2, 18, 20)),
+ )
+ for label_id, center, radius in objects:
+ shifted_center = tuple(center[index] + offset[index] for index in range(3))
+ distance = sum(
+ ((axis - shifted_center[index]) / radius[index]) ** 2
+ for index, axis in enumerate((z, y, x))
+ )
+ labels[distance <= 1.0] = label_id
+ return labels
+
+
+def _image_from_labels(labels: np.ndarray, seed: int) -> np.ndarray:
+ rng = np.random.default_rng(seed)
+ z, y, x = np.indices(labels.shape)
+ image = (
+ 112.0
+ + 18.0 * np.sin(x / 9.0)
+ + 13.0 * np.cos(y / 13.0)
+ + 7.0 * np.sin((x + y + z * 5) / 17.0)
+ + rng.normal(0.0, 10.0, labels.shape)
+ )
+ image += np.where(labels > 0, 36.0 + (labels % 3) * 9.0, 0.0)
+ boundary = np.zeros(labels.shape, dtype=bool)
+ for axis in range(3):
+ boundary |= labels != np.roll(labels, 1, axis=axis)
+ image[boundary] -= 45.0
+ return np.clip(image, 0, 255).astype(np.uint8)
+
+
+def _draft_labels(labels: np.ndarray) -> np.ndarray:
+ draft = labels.copy()
+ draft[draft == 3] = 0
+ draft[:, 92:105, 55:72][draft[:, 92:105, 55:72] == 2] = 0
+ draft[3:7, 24:37, 112:130] = 9
+ return draft
+
+
+def _write_h5(path: Path, data: np.ndarray, *, role: str) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ with h5py.File(path, "w") as handle:
+ dataset = handle.create_dataset(
+ "data",
+ data=data,
+ chunks=VOLUME_CHUNKS,
+ compression="gzip",
+ compression_opts=4,
+ shuffle=True,
+ )
+ dataset.attrs["axes"] = "zyx"
+ dataset.attrs["role"] = role
+ dataset.attrs["voxel_size_nm"] = VOXEL_SIZE_NM
+ handle.attrs["generator"] = GENERATOR_VERSION
+ handle.attrs["synthetic"] = True
+
+
+def _config_text(root: Path) -> str:
+ train_image = root / "data/raw/train-01_image.h5"
+ train_label = root / "data/seg/ground_truth/train-01_ground_truth.h5"
+ target_image = root / "data/raw/target-01_image.h5"
+ output_path = root / "runtime/training"
+ inference_path = root / "runtime/inference"
+ return f"""# Deterministic local workflow fixture. Not a benchmark configuration.
+SYSTEM:
+ NUM_GPUS: 1
+ NUM_CPUS: 0
+ PARALLEL: single
+
+MODEL:
+ ARCHITECTURE: unet_plus_3d
+ BLOCK_TYPE: residual_se
+ INPUT_SIZE: [9, 65, 65]
+ OUTPUT_SIZE: [9, 65, 65]
+ IN_PLANES: 1
+ OUT_PLANES: 2
+ NORM_MODE: gn
+ FILTERS: [8, 12, 16, 24, 32]
+ TARGET_OPT: ["0", "4-1-1"]
+ LOSS_OPTION:
+ - [WeightedBCEWithLogitsLoss, DiceLoss]
+ - [WeightedBCEWithLogitsLoss, DiceLoss]
+ LOSS_WEIGHT: [[1.0, 0.5], [1.0, 0.5]]
+ WEIGHT_OPT: [["1", "0"], ["1", "0"]]
+ OUTPUT_ACT: [["none", "sigmoid"], ["none", "sigmoid"]]
+
+DATASET:
+ INPUT_PATH: ""
+ IMAGE_NAME: {train_image}
+ LABEL_NAME: {train_label}
+ OUTPUT_PATH: {output_path}
+ PAD_SIZE: [4, 16, 16]
+ PAD_MODE: reflect
+ IMAGE_SCALE: [1.0, 1.0, 1.0]
+ LABEL_SCALE: [1.0, 1.0, 1.0]
+ DATA_SCALE: [1.0, 1.0, 1.0]
+ NORMALIZE_RANGE: true
+ MEAN: 0.5
+ STD: 0.5
+ MATCH_ACT: none
+ IS_ABSOLUTE_PATH: true
+ DISTRIBUTED: false
+ DO_2D: false
+ LOAD_2D: false
+ DO_CHUNK_TITLE: 0
+ DROP_CHANNEL: false
+ ENSURE_MIN_SIZE: false
+ LABEL_BINARY: false
+ LABEL_MAG: 0
+ LABEL_VAST: false
+ REDUCE_LABEL: true
+ VALID_RATIO: 0.25
+ VALID_MASK_NAME: null
+ VAL_IMAGE_NAME: null
+ VAL_LABEL_NAME: null
+ VAL_VALID_MASK_NAME: null
+ VAL_PAD_SIZE: [0, 0, 0]
+ REJECT_SAMPLING:
+ SIZE_THRES: -1
+ DIVERSITY: -1
+ NUM_TRIAL: 10
+ P: 0.95
+
+AUGMENTOR:
+ ENABLED: false
+
+SOLVER:
+ NAME: AdamW
+ LR_SCHEDULER_NAME: WarmupCosineLR
+ BASE_LR: 0.0003
+ BETAS: [0.9, 0.999]
+ WEIGHT_DECAY: 0.0001
+ ITERATION_STEP: 1
+ ITERATION_SAVE: 5
+ ITERATION_VAL: 1000000
+ ITERATION_TOTAL: 10
+ SAMPLES_PER_BATCH: 1
+ ITERATION_RESTART: false
+ WARMUP_FACTOR: 0.001
+ WARMUP_ITERS: 2
+ WARMUP_METHOD: linear
+
+MONITOR:
+ ITERATION_NUM: [2, 5]
+ LOG_OPT: [1, 1, 0]
+ VIS_OPT: [0, 8]
+
+INFERENCE:
+ INPUT_SIZE: [9, 65, 65]
+ OUTPUT_SIZE: [9, 65, 65]
+ IMAGE_NAME: {target_image}
+ OUTPUT_PATH: {inference_path}
+ OUTPUT_NAME: target-01_prediction.h5
+ OUTPUT_ACT: ["sigmoid", "sigmoid"]
+ PAD_SIZE: [4, 16, 16]
+ AUG_MODE: mean
+ AUG_NUM: 1
+ STRIDE: [8, 64, 64]
+ SAMPLES_PER_BATCH: 1
+ DO_EVAL: false
+ UNPAD: true
+"""
+
+
+def _manifest() -> dict[str, Any]:
+ return {
+ "schema_version": 1,
+ "generator": GENERATOR_VERSION,
+ "synthetic": True,
+ "title": "Synthetic Segmentation Core Loop",
+ "description": "Fixed local fixture for end-to-end workflow and UI testing.",
+ "imaging_modality": "Synthetic volumetric microscopy",
+ "target_structure": "synthetic organelles",
+ "task_family": "3D instance segmentation",
+ "task": "segmentation, proofreading, retraining, and evaluation",
+ "voxel_size": {"zyx_nm": list(VOXEL_SIZE_NM)},
+ "active_paths": {
+ "image_root": "data/raw",
+ "label_root": "data/seg",
+ "config": "configs/Synthetic-Core-Loop-BC.yaml",
+ },
+ "volumes": [
+ {
+ "id": "train-01",
+ "split": "train",
+ "image": "data/raw/train-01_image.h5",
+ "segmentation": "data/seg/ground_truth/train-01_ground_truth.h5",
+ "mask_state": "ground_truth",
+ },
+ {
+ "id": "train-02",
+ "split": "train",
+ "image": "data/raw/train-02_image.h5",
+ "segmentation": "data/seg/ground_truth/train-02_ground_truth.h5",
+ "mask_state": "ground_truth",
+ },
+ {
+ "id": "review-01",
+ "split": "proofreading",
+ "image": "data/raw/review-01_image.h5",
+ "segmentation": "data/seg/draft/review-01_seg.h5",
+ "mask_state": "needs_proofreading",
+ },
+ {
+ "id": "target-01",
+ "split": "inference",
+ "image": "data/raw/target-01_image.h5",
+ "segmentation": None,
+ "mask_state": "missing_segmentation",
+ },
+ ],
+ "initial_progress_summary": {
+ "total": 4,
+ "ground_truth": 2,
+ "needs_proofreading": 1,
+ "missing_segmentation": 1,
+ },
+ "workflow_split": {
+ "ground_truth_training_volumes": ["train-01", "train-02"],
+ "proofreading_volumes": ["review-01"],
+ "image_only_inference_targets": ["target-01"],
+ },
+ }
+
+
+def _is_current(root: Path) -> bool:
+ try:
+ manifest = json.loads((root / "project_manifest.json").read_text())
+ except (OSError, json.JSONDecodeError):
+ return False
+ required = (
+ "data/raw/train-01_image.h5",
+ "data/raw/train-02_image.h5",
+ "data/raw/review-01_image.h5",
+ "data/raw/target-01_image.h5",
+ "data/seg/ground_truth/train-01_ground_truth.h5",
+ "data/seg/ground_truth/train-02_ground_truth.h5",
+ "data/seg/draft/review-01_seg.h5",
+ "outputs/predictions/baseline_review-01.h5",
+ "outputs/predictions/candidate_review-01.h5",
+ "configs/Synthetic-Core-Loop-BC.yaml",
+ "notes/README.md",
+ )
+ return manifest.get("generator") == GENERATOR_VERSION and all(
+ (root / relative_path).is_file() for relative_path in required
+ )
+
+
+def _has_generator_marker(root: Path) -> bool:
+ try:
+ manifest = json.loads((root / "project_manifest.json").read_text())
+ except (OSError, json.JSONDecodeError):
+ return False
+ return str(manifest.get("generator") or "").startswith("pytc-synthetic-core/")
+
+
+def create_synthetic_project(output_dir: str | Path, *, reset: bool = False) -> dict:
+ """Create or restore the fixed project and return its resolved contract."""
+ root = Path(output_dir).expanduser().resolve()
+ if root.exists() and any(root.iterdir()) and not _has_generator_marker(root):
+ raise ValueError(
+ f"Refusing to modify non-synthetic directory without a generator marker: {root}"
+ )
+ if reset and root.exists():
+ shutil.rmtree(root)
+ if _is_current(root):
+ return {"created": False, "generator": GENERATOR_VERSION, "root": str(root)}
+
+ root.mkdir(parents=True, exist_ok=True)
+ cases = {
+ "train-01": (_instance_labels(), 101),
+ "train-02": (_instance_labels((0, 7, -5)), 202),
+ "review-01": (_instance_labels((0, -6, 8)), 303),
+ "target-01": (_instance_labels((0, 4, 5)), 404),
+ }
+ for name, (labels, seed) in cases.items():
+ _write_h5(
+ root / f"data/raw/{name}_image.h5",
+ _image_from_labels(labels, seed),
+ role="image",
+ )
+
+ for name in ("train-01", "train-02"):
+ _write_h5(
+ root / f"data/seg/ground_truth/{name}_ground_truth.h5",
+ cases[name][0],
+ role="ground_truth",
+ )
+
+ review_labels = cases["review-01"][0]
+ draft = _draft_labels(review_labels)
+ (root / "data/seg/draft/review-01_draft_seg.h5").unlink(missing_ok=True)
+ _write_h5(
+ root / "data/seg/draft/review-01_seg.h5",
+ draft,
+ role="draft_segmentation",
+ )
+ _write_h5(
+ root / "outputs/predictions/baseline_review-01.h5",
+ draft,
+ role="baseline_prediction",
+ )
+ _write_h5(
+ root / "outputs/predictions/candidate_review-01.h5",
+ review_labels,
+ role="candidate_prediction",
+ )
+
+ config_path = root / "configs/Synthetic-Core-Loop-BC.yaml"
+ config_path.parent.mkdir(parents=True, exist_ok=True)
+ config_path.write_text(_config_text(root), encoding="utf-8")
+ (root / "runtime/training").mkdir(parents=True, exist_ok=True)
+ (root / "runtime/inference").mkdir(parents=True, exist_ok=True)
+
+ notes_path = root / "notes/README.md"
+ notes_path.parent.mkdir(parents=True, exist_ok=True)
+ notes_path.write_text(
+ """# Synthetic Segmentation Core Loop
+
+This deterministic project tests the application workflow, not biological validity or model quality.
+
+- `train-01` and `train-02` have confirmed ground-truth instance labels.
+- `review-01` has a deliberately incomplete draft with one missing object and one false positive.
+- `target-01` is image-only and should be offered as an inference target.
+- Baseline and candidate predictions make comparison and evaluation views available immediately.
+
+Expected initial progress: 4 volumes, 2 ground truth, 1 needs proofreading, and 1 missing segmentation.
+The HDF5 dataset key is `data`; volumes are chunked and compressed to exercise storage-backed reads.
+Use the repository reset command before a canonical test run. Generated results under `runtime/` are disposable.
+""",
+ encoding="utf-8",
+ )
+ (root / "project_manifest.json").write_text(
+ json.dumps(_manifest(), indent=2, sort_keys=True) + "\n",
+ encoding="utf-8",
+ )
+ return {"created": True, "generator": GENERATOR_VERSION, "root": str(root)}
diff --git a/server_api/workflows/agent_action_execution.py b/server_api/workflows/agent_action_execution.py
new file mode 100644
index 00000000..82ce85d8
--- /dev/null
+++ b/server_api/workflows/agent_action_execution.py
@@ -0,0 +1,421 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from typing import Any, Dict, Optional
+
+from fastapi import HTTPException
+from sqlalchemy.orm import Session
+
+from .agent_actions import (
+ AgentActionReceipt,
+ ComputeEvaluationAction,
+ ComputeEvaluationPayload,
+ canonical_action_policy,
+ load_action_envelope_json,
+ load_action_receipt_json,
+ validate_action_for_execution,
+ validate_receipt_for_action,
+)
+from .db_models import WorkflowEvent, WorkflowOperation
+from .evaluation_service import create_computed_evaluation_result
+from .operation_service import (
+ create_workflow_operation,
+ get_workflow_operation_or_404,
+ transition_workflow_operation,
+)
+from .service import decode_json
+
+_EXECUTOR_LEASE = "agent-action:compute-evaluation"
+
+
+def _as_utc(value: Optional[datetime]) -> Optional[datetime]:
+ if value is None:
+ return None
+ if value.tzinfo is None:
+ return value.replace(tzinfo=timezone.utc)
+ return value.astimezone(timezone.utc)
+
+
+def _now() -> datetime:
+ return datetime.now(timezone.utc)
+
+
+def validate_persisted_action_approval(
+ db: Session, envelope: ComputeEvaluationAction
+) -> WorkflowEvent:
+ """Bind an approved envelope to the matching persisted agent proposal."""
+
+ decision = (
+ db.query(WorkflowEvent)
+ .filter(
+ WorkflowEvent.id == envelope.approval.event_id,
+ WorkflowEvent.workflow_id == envelope.workflow_id,
+ WorkflowEvent.event_type == "agent.proposal_approved",
+ WorkflowEvent.actor == "user",
+ )
+ .first()
+ )
+ decision_payload = decode_json(decision.payload_json) if decision else {}
+ proposal_id = decision_payload.get("proposal_event_id")
+ proposal = (
+ db.query(WorkflowEvent)
+ .filter(
+ WorkflowEvent.id == proposal_id,
+ WorkflowEvent.workflow_id == envelope.workflow_id,
+ WorkflowEvent.event_type == "agent.proposal_created",
+ )
+ .first()
+ )
+ if decision is None or proposal is None or proposal.approval_status != "approved":
+ raise HTTPException(
+ status_code=409,
+ detail=(
+ "compute_evaluation requires a matching persisted and approved "
+ "agent proposal"
+ ),
+ )
+
+ proposal_payload = decode_json(proposal.payload_json)
+ registry = proposal_payload.get("registry")
+ params = proposal_payload.get("params")
+ client_effects = params.get("client_effects") if isinstance(params, dict) else None
+ workflow_action = (
+ client_effects.get("workflow_action")
+ if isinstance(client_effects, dict)
+ else None
+ )
+ if (
+ not isinstance(registry, dict)
+ or registry.get("action_type") != envelope.kind
+ or not isinstance(workflow_action, dict)
+ or workflow_action.get("kind") != envelope.kind
+ ):
+ raise HTTPException(
+ status_code=409,
+ detail="Approval proposal does not match the compute_evaluation envelope",
+ )
+ proposal_action = ComputeEvaluationPayload.model_validate(
+ workflow_action
+ ).model_dump(exclude_none=True)
+ action_field_names = set(ComputeEvaluationPayload.model_fields)
+ envelope_action = envelope.model_dump(
+ include=action_field_names,
+ exclude_none=True,
+ )
+ if not envelope_action.get("metadata"):
+ envelope_action.pop("metadata", None)
+ if proposal_action != envelope_action:
+ raise HTTPException(
+ status_code=409,
+ detail="Approved proposal parameters do not match the action envelope",
+ )
+ return decision
+
+
+def stage_and_execute_compute_evaluation_proposal(
+ db: Session,
+ *,
+ workflow_id: int,
+ proposal: WorkflowEvent,
+ approval_event: WorkflowEvent,
+ workflow_action: Dict[str, Any],
+ user_id: int,
+ correlation_id: Optional[str] = None,
+) -> tuple[WorkflowOperation, AgentActionReceipt]:
+ action_parameters = ComputeEvaluationPayload.model_validate(
+ workflow_action
+ ).model_dump(exclude={"kind"}, exclude_none=True)
+ input_artifacts = [
+ {
+ "logical_name": name,
+ "artifact_type": "segmentation_volume",
+ "role": role,
+ "path": path,
+ }
+ for name, role, path in (
+ (
+ "baseline-prediction",
+ "baseline_prediction",
+ action_parameters.get("baseline_prediction_path"),
+ ),
+ (
+ "candidate-prediction",
+ "candidate_prediction",
+ action_parameters.get("candidate_prediction_path"),
+ ),
+ (
+ "ground-truth",
+ "ground_truth",
+ action_parameters.get("ground_truth_path"),
+ ),
+ )
+ if path
+ ]
+ expected_output_artifacts = []
+ if action_parameters.get("report_path"):
+ expected_output_artifacts.append(
+ {
+ "logical_name": "evaluation-report",
+ "artifact_type": "evaluation_report",
+ "role": "case_study_evidence",
+ "path": action_parameters["report_path"],
+ }
+ )
+ resolved_correlation_id = correlation_id or f"agent-proposal:{proposal.id}"
+ envelope = validate_action_for_execution(
+ {
+ "action_id": f"proposal:{proposal.id}:compute_evaluation",
+ "kind": "compute_evaluation",
+ "workflow_id": workflow_id,
+ "requested_by": "agent",
+ "idempotency_key": f"agent-proposal:{proposal.id}:compute_evaluation",
+ "correlation_id": resolved_correlation_id,
+ "execution_owner": "server_workflow",
+ "policy": canonical_action_policy("compute_evaluation").model_dump(),
+ "approval": {
+ "status": "approved",
+ "event_id": approval_event.id,
+ "decided_by": f"user:{user_id}",
+ },
+ "input_artifacts": input_artifacts,
+ "expected_output_artifacts": expected_output_artifacts,
+ **action_parameters,
+ }
+ )
+ if not isinstance(envelope, ComputeEvaluationAction): # pragma: no cover
+ raise TypeError("compute evaluation proposal resolved to another action type")
+ validate_persisted_action_approval(db, envelope)
+ operation = create_workflow_operation(
+ db,
+ workflow_id=workflow_id,
+ operation_type="agent_action:compute_evaluation",
+ idempotency_key=envelope.idempotency_key,
+ correlation_id=envelope.correlation_id,
+ actor=envelope.requested_by,
+ input_payload=envelope.model_dump(mode="json", exclude_none=True),
+ metadata={
+ "action_id": envelope.action_id,
+ "action_schema_version": envelope.schema_version,
+ "approval_event_id": approval_event.id,
+ "proposal_event_id": proposal.id,
+ "execution_owner": envelope.execution_owner,
+ "risk_level": envelope.policy.risk_level,
+ },
+ commit=True,
+ )
+ receipt = execute_compute_evaluation_operation(db, operation)
+ operation = get_workflow_operation_or_404(
+ db,
+ workflow_id=workflow_id,
+ operation_id=operation.id,
+ )
+ return operation, receipt
+
+
+def _flatten_receipt_metrics(metrics: Dict[str, Any]) -> Dict[str, float]:
+ flattened: Dict[str, float] = {}
+ for section in ("baseline", "candidate", "delta"):
+ values = metrics.get(section)
+ if not isinstance(values, dict):
+ continue
+ for name, value in values.items():
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
+ continue
+ flattened[f"{section}_{name}"] = float(value)
+ return flattened
+
+
+def _stored_receipt(operation: WorkflowOperation) -> Optional[AgentActionReceipt]:
+ payload = decode_json(operation.result_json).get("receipt")
+ if not isinstance(payload, dict):
+ return None
+ action = load_action_envelope_json(operation.input_json or "{}")
+ receipt = load_action_receipt_json(json.dumps(payload))
+ return validate_receipt_for_action(action, receipt)
+
+
+def _failure_details(exc: Exception) -> Dict[str, Any]:
+ if isinstance(exc, (FileNotFoundError, ValueError)):
+ return {
+ "code": "evaluation_input_invalid",
+ "retryable": False,
+ }
+ if isinstance(exc, RuntimeError):
+ return {
+ "code": "evaluation_runtime_failed",
+ "retryable": True,
+ }
+ return {
+ "code": "evaluation_handler_failed",
+ "retryable": False,
+ }
+
+
+def execute_compute_evaluation_operation(
+ db: Session, operation: WorkflowOperation
+) -> AgentActionReceipt:
+ envelope = load_action_envelope_json(operation.input_json or "{}")
+ if envelope.policy.requires_approval and envelope.approval.status != "approved":
+ raise HTTPException(
+ status_code=409,
+ detail=f"{envelope.kind} cannot execute without an approved action envelope",
+ )
+ if not isinstance(envelope, ComputeEvaluationAction):
+ raise HTTPException(
+ status_code=409,
+ detail="Only compute_evaluation operations are executable on this route",
+ )
+ if operation.operation_type != "agent_action:compute_evaluation":
+ raise HTTPException(
+ status_code=409,
+ detail="Workflow operation type does not match compute_evaluation",
+ )
+ validate_persisted_action_approval(db, envelope)
+
+ if operation.status in {"succeeded", "failed", "cancelled"}:
+ receipt = _stored_receipt(operation)
+ if receipt is None:
+ raise HTTPException(
+ status_code=409,
+ detail="Terminal operation does not contain a typed action receipt",
+ )
+ return receipt
+ if operation.status != "queued":
+ raise HTTPException(
+ status_code=409,
+ detail=f"Operation cannot execute from {operation.status}",
+ )
+
+ operation = transition_workflow_operation(
+ db,
+ operation,
+ status="running",
+ expected_status="queued",
+ progress=0.05,
+ lease_owner=_EXECUTOR_LEASE,
+ commit=True,
+ )
+ started_at = _as_utc(operation.started_at) or _now()
+
+ try:
+ required_paths = {
+ "baseline_prediction_path": envelope.baseline_prediction_path,
+ "candidate_prediction_path": envelope.candidate_prediction_path,
+ "ground_truth_path": envelope.ground_truth_path,
+ }
+ missing = [name for name, value in required_paths.items() if not value]
+ if missing:
+ raise ValueError(
+ "compute_evaluation is missing required fields: " + ", ".join(missing)
+ )
+
+ evaluation = create_computed_evaluation_result(
+ db,
+ workflow_id=envelope.workflow_id,
+ baseline_prediction_path=envelope.baseline_prediction_path,
+ candidate_prediction_path=envelope.candidate_prediction_path,
+ ground_truth_path=envelope.ground_truth_path,
+ baseline_dataset=envelope.baseline_dataset,
+ candidate_dataset=envelope.candidate_dataset,
+ ground_truth_dataset=envelope.ground_truth_dataset,
+ crop=envelope.crop,
+ baseline_channel=envelope.baseline_channel,
+ candidate_channel=envelope.candidate_channel,
+ ground_truth_channel=envelope.ground_truth_channel,
+ name=envelope.name,
+ baseline_run_id=envelope.baseline_run_id,
+ candidate_run_id=envelope.candidate_run_id,
+ model_version_id=envelope.model_version_id,
+ report_path=envelope.report_path,
+ metadata={
+ **envelope.metadata,
+ "action_id": envelope.action_id,
+ "correlation_id": envelope.correlation_id,
+ "operation_id": operation.id,
+ },
+ commit=False,
+ )
+ metrics = decode_json(evaluation.metrics_json)
+ produced_artifacts = []
+ if evaluation.report_artifact_id:
+ produced_artifacts.append(
+ {
+ "artifact_id": evaluation.report_artifact_id,
+ "artifact_type": "evaluation_report",
+ "role": "case_study_evidence",
+ "path": evaluation.report_path,
+ }
+ )
+ receipt_payload = {
+ "receipt_id": f"operation:{operation.id}:receipt",
+ "action_id": envelope.action_id,
+ "kind": envelope.kind,
+ "workflow_id": envelope.workflow_id,
+ "idempotency_key": envelope.idempotency_key,
+ "correlation_id": envelope.correlation_id,
+ "status": "succeeded",
+ "operation_id": operation.id,
+ "produced_artifacts": produced_artifacts,
+ "evaluation_result_id": evaluation.id,
+ "metrics": _flatten_receipt_metrics(metrics),
+ "started_at": started_at,
+ "completed_at": _now(),
+ }
+ receipt = validate_receipt_for_action(envelope, receipt_payload)
+ operation = transition_workflow_operation(
+ db,
+ operation,
+ status="succeeded",
+ expected_status="running",
+ lease_owner=_EXECUTOR_LEASE,
+ result_payload={
+ "receipt": receipt.model_dump(mode="json"),
+ "evidence": {
+ "evaluation_result_id": evaluation.id,
+ "report_artifact_id": evaluation.report_artifact_id,
+ "report_path": evaluation.report_path,
+ "metrics": metrics,
+ },
+ },
+ commit=True,
+ )
+ return receipt
+ except Exception as exc:
+ db.rollback()
+ operation = get_workflow_operation_or_404(
+ db,
+ workflow_id=envelope.workflow_id,
+ operation_id=operation.id,
+ )
+ failure = _failure_details(exc)
+ message = str(exc).strip() or "Evaluation handler failed"
+ receipt_payload = {
+ "receipt_id": f"operation:{operation.id}:receipt",
+ "action_id": envelope.action_id,
+ "kind": envelope.kind,
+ "workflow_id": envelope.workflow_id,
+ "idempotency_key": envelope.idempotency_key,
+ "correlation_id": envelope.correlation_id,
+ "status": "failed",
+ "operation_id": operation.id,
+ "error": {
+ **failure,
+ "message": message[:500],
+ "details": {"exception_type": type(exc).__name__},
+ },
+ "started_at": started_at,
+ "completed_at": _now(),
+ }
+ receipt = validate_receipt_for_action(envelope, receipt_payload)
+ operation = transition_workflow_operation(
+ db,
+ operation,
+ status="failed",
+ expected_status="running",
+ lease_owner=_EXECUTOR_LEASE,
+ result_payload={"receipt": receipt.model_dump(mode="json")},
+ error_payload=receipt.error.model_dump(mode="json"),
+ commit=True,
+ )
+ return receipt
diff --git a/server_api/workflows/agent_actions.py b/server_api/workflows/agent_actions.py
new file mode 100644
index 00000000..ee5a94ea
--- /dev/null
+++ b/server_api/workflows/agent_actions.py
@@ -0,0 +1,830 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Annotated, Any, Dict, List, Literal, Optional, Union
+
+from pydantic import (
+ BaseModel,
+ ConfigDict,
+ Field,
+ StringConstraints,
+ TypeAdapter,
+ model_validator,
+)
+
+RiskLevel = Literal[
+ "read_only",
+ "prefills_form",
+ "loads_editor",
+ "writes_workflow_record",
+ "modifies_workspace",
+ "runs_job",
+ "controls_job",
+ "exports_evidence",
+]
+
+
+class _StrictActionPayload(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+
+class ChooseProjectDataPayload(_StrictActionPayload):
+ kind: Literal["choose_project_data"]
+
+
+class LoadVisualizationPayload(_StrictActionPayload):
+ kind: Literal["load_visualization"]
+
+
+class StartInferencePayload(_StrictActionPayload):
+ kind: Literal["start_inference"]
+
+
+class StopInferencePayload(_StrictActionPayload):
+ kind: Literal["stop_inference"]
+
+
+class StartProofreadingPayload(_StrictActionPayload):
+ kind: Literal["start_proofreading"]
+
+
+class StartTrainingPayload(_StrictActionPayload):
+ kind: Literal["start_training"]
+ autopick_parameters: Optional[bool] = None
+ parameter_mode: Optional[str] = None
+ volume_subset: Optional[Dict[str, Any]] = None
+
+
+class StopTrainingPayload(_StrictActionPayload):
+ kind: Literal["stop_training"]
+
+
+RuntimeActionPayload = Annotated[
+ Union[
+ ChooseProjectDataPayload,
+ LoadVisualizationPayload,
+ StartInferencePayload,
+ StopInferencePayload,
+ StartProofreadingPayload,
+ StartTrainingPayload,
+ StopTrainingPayload,
+ ],
+ Field(discriminator="kind"),
+]
+
+
+class ComputeEvaluationPayload(_StrictActionPayload):
+ kind: Literal["compute_evaluation"]
+ name: Optional[str] = None
+ baseline_prediction_path: Optional[str] = None
+ candidate_prediction_path: Optional[str] = None
+ ground_truth_path: Optional[str] = None
+ baseline_dataset: Optional[str] = None
+ candidate_dataset: Optional[str] = None
+ ground_truth_dataset: Optional[str] = None
+ crop: Optional[str] = None
+ baseline_channel: Optional[int] = Field(default=None, ge=0)
+ candidate_channel: Optional[int] = Field(default=None, ge=0)
+ ground_truth_channel: Optional[int] = Field(default=None, ge=0)
+ baseline_run_id: Optional[int] = None
+ candidate_run_id: Optional[int] = None
+ model_version_id: Optional[int] = None
+ report_path: Optional[str] = None
+ metadata: Optional[Dict[str, Any]] = None
+
+
+class ExportBundlePayload(_StrictActionPayload):
+ kind: Literal["export_bundle"]
+
+
+class ProposeRetrainingStagePayload(_StrictActionPayload):
+ kind: Literal["propose_retraining_stage"]
+ corrected_mask_path: Optional[str] = None
+
+
+WorkflowActionPayload = Annotated[
+ Union[
+ ComputeEvaluationPayload,
+ ExportBundlePayload,
+ ProposeRetrainingStagePayload,
+ ],
+ Field(discriminator="kind"),
+]
+
+
+class MountProjectPayload(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ directory_path: str
+ mount_name: Optional[str] = None
+ destination_path: Optional[str] = None
+ workflow_patch: Optional[Dict[str, Any]] = None
+
+
+class ClientEffectsPayload(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ navigate_to: Optional[str] = None
+ runtime_action: Optional[RuntimeActionPayload] = None
+ workflow_action: Optional[WorkflowActionPayload] = None
+ mount_project: Optional[MountProjectPayload] = None
+ reset_workspace: Optional[bool] = None
+ start_new_workflow: Optional[Any] = None
+ show_workflow_context: Optional[bool] = None
+ refresh_insights: Optional[bool] = None
+ refresh_project_progress: Optional[bool] = None
+ training_volume_subset: Optional[Any] = None
+ set_training_config_preset: Optional[str] = None
+ set_training_image_path: Optional[str] = None
+ set_training_label_path: Optional[str] = None
+ set_training_log_path: Optional[str] = None
+ set_training_output_path: Optional[str] = None
+ set_inference_checkpoint_path: Optional[str] = None
+ set_inference_config_preset: Optional[str] = None
+ set_inference_image_path: Optional[str] = None
+ set_inference_label_path: Optional[str] = None
+ set_inference_output_path: Optional[str] = None
+ set_visualization_image_path: Optional[str] = None
+ set_visualization_label_path: Optional[str] = None
+ set_visualization_scales: Optional[Any] = None
+ set_proofreading_dataset_path: Optional[str] = None
+ set_proofreading_image_path: Optional[str] = None
+ set_proofreading_label_path: Optional[str] = None
+ set_proofreading_mask_path: Optional[str] = None
+ set_proofreading_project_name: Optional[str] = None
+
+
+class StageRetrainingPayload(BaseModel):
+ model_config = ConfigDict(extra="allow")
+
+ corrected_mask_path: Optional[str] = None
+
+
+class StartTrainingRunPayload(BaseModel):
+ model_config = ConfigDict(extra="allow")
+
+ client_effects: Optional[ClientEffectsPayload] = None
+
+
+class RunClientEffectsPayload(BaseModel):
+ model_config = ConfigDict(extra="allow")
+
+ client_effects: ClientEffectsPayload
+
+
+@dataclass(frozen=True)
+class AgentActionDefinition:
+ action_type: str
+ risk_level: RiskLevel
+ requires_approval: bool
+ execution_owner: Literal["browser_navigation", "server_workflow", "server_runtime"]
+ specialist_agent_type: str
+
+
+RUNTIME_ACTIONS: Dict[str, AgentActionDefinition] = {
+ "choose_project_data": AgentActionDefinition(
+ "choose_project_data",
+ "prefills_form",
+ False,
+ "browser_navigation",
+ "data_agent",
+ ),
+ "load_visualization": AgentActionDefinition(
+ "load_visualization",
+ "read_only",
+ False,
+ "browser_navigation",
+ "visualization_agent",
+ ),
+ "start_inference": AgentActionDefinition(
+ "start_inference", "runs_job", True, "server_runtime", "inference_agent"
+ ),
+ "stop_inference": AgentActionDefinition(
+ "stop_inference", "controls_job", True, "server_runtime", "inference_agent"
+ ),
+ "start_proofreading": AgentActionDefinition(
+ "start_proofreading",
+ "loads_editor",
+ True,
+ "server_workflow",
+ "proofreading_agent",
+ ),
+ "start_training": AgentActionDefinition(
+ "start_training", "runs_job", True, "server_runtime", "training_agent"
+ ),
+ "stop_training": AgentActionDefinition(
+ "stop_training", "controls_job", True, "server_runtime", "training_agent"
+ ),
+}
+
+
+WORKFLOW_ACTIONS: Dict[str, AgentActionDefinition] = {
+ "compute_evaluation": AgentActionDefinition(
+ "compute_evaluation",
+ "writes_workflow_record",
+ True,
+ "server_workflow",
+ "evaluation_agent",
+ ),
+ "export_bundle": AgentActionDefinition(
+ "export_bundle",
+ "exports_evidence",
+ True,
+ "server_workflow",
+ "evidence_agent",
+ ),
+ "propose_retraining_stage": AgentActionDefinition(
+ "propose_retraining_stage",
+ "writes_workflow_record",
+ True,
+ "server_workflow",
+ "training_agent",
+ ),
+}
+
+
+NAVIGATION_SPECIALISTS = {
+ "files": "data_agent",
+ "visualization": "visualization_agent",
+ "mask-proofreading": "proofreading_agent",
+ "training": "training_agent",
+ "inference": "inference_agent",
+ "project-progress": "project_manager",
+}
+
+
+NonEmptyString = Annotated[
+ str,
+ StringConstraints(strip_whitespace=True, min_length=1, max_length=500),
+]
+IdempotencyKey = Annotated[
+ str,
+ StringConstraints(strip_whitespace=True, min_length=1, max_length=255),
+]
+CorrelationId = Annotated[
+ str,
+ StringConstraints(strip_whitespace=True, min_length=1, max_length=255),
+]
+ApprovalStatus = Literal["not_required", "pending", "approved", "rejected"]
+ExecutionOwner = Literal["browser_navigation", "server_workflow", "server_runtime"]
+OperationStatus = Literal["queued", "running", "succeeded", "failed", "cancelled"]
+ReceiptStatus = Literal["accepted", "running", "succeeded", "failed", "cancelled"]
+
+
+class _StrictBoundaryModel(BaseModel):
+ model_config = ConfigDict(extra="forbid", strict=True)
+
+
+class ArtifactReference(_StrictBoundaryModel):
+ """Stable artifact identity used at the action execution boundary."""
+
+ artifact_id: Optional[int] = Field(default=None, gt=0)
+ logical_name: Optional[NonEmptyString] = None
+ artifact_type: Optional[NonEmptyString] = None
+ role: Optional[NonEmptyString] = None
+ path: Optional[NonEmptyString] = None
+ uri: Optional[NonEmptyString] = None
+ checksum: Optional[NonEmptyString] = None
+ media_type: Optional[NonEmptyString] = None
+ immutable: bool = True
+
+ @model_validator(mode="after")
+ def require_identity(self) -> "ArtifactReference":
+ if not any(
+ (
+ self.artifact_id,
+ self.logical_name,
+ self.path,
+ self.uri,
+ self.checksum,
+ )
+ ):
+ raise ValueError(
+ "artifact reference requires artifact_id, logical_name, path, uri, "
+ "or checksum"
+ )
+ return self
+
+
+class WorkflowStagePrecondition(_StrictBoundaryModel):
+ kind: Literal["workflow_stage"]
+ allowed_stages: List[NonEmptyString] = Field(min_length=1)
+
+
+class ArtifactAvailablePrecondition(_StrictBoundaryModel):
+ kind: Literal["artifact_available"]
+ artifact: ArtifactReference
+
+
+class WorkflowFieldPrecondition(_StrictBoundaryModel):
+ kind: Literal["workflow_field_present"]
+ field_name: NonEmptyString
+
+
+class OperationStatusPrecondition(_StrictBoundaryModel):
+ kind: Literal["operation_status"]
+ operation_id: int = Field(gt=0)
+ allowed_statuses: List[OperationStatus] = Field(min_length=1)
+
+
+ActionPrecondition = Annotated[
+ Union[
+ WorkflowStagePrecondition,
+ ArtifactAvailablePrecondition,
+ WorkflowFieldPrecondition,
+ OperationStatusPrecondition,
+ ],
+ Field(discriminator="kind"),
+]
+
+
+class ActionPolicy(_StrictBoundaryModel):
+ risk_level: RiskLevel
+ requires_approval: bool
+ approval_reason: Optional[NonEmptyString] = None
+
+
+class ActionApproval(_StrictBoundaryModel):
+ status: ApprovalStatus
+ event_id: Optional[int] = Field(default=None, gt=0)
+ decided_by: Optional[NonEmptyString] = None
+ decided_at: Optional[datetime] = None
+
+ @model_validator(mode="after")
+ def require_decision_evidence(self) -> "ActionApproval":
+ if self.status in {"approved", "rejected"}:
+ if self.event_id is None or self.decided_by is None:
+ raise ValueError(
+ "approved or rejected actions require event_id and decided_by"
+ )
+ elif any((self.event_id, self.decided_by, self.decided_at)):
+ raise ValueError(
+ "approval decision evidence is only valid for approved or rejected "
+ "actions"
+ )
+ return self
+
+
+ALL_ACTION_DEFINITIONS: Dict[str, AgentActionDefinition] = {
+ **RUNTIME_ACTIONS,
+ **WORKFLOW_ACTIONS,
+}
+
+
+class _ActionEnvelopeBase(_StrictBoundaryModel):
+ schema_version: Literal["workflow.action/v1"] = "workflow.action/v1"
+ action_id: NonEmptyString
+ kind: str
+ workflow_id: int = Field(gt=0)
+ requested_by: Literal["user", "agent", "system"]
+ idempotency_key: IdempotencyKey
+ correlation_id: CorrelationId
+ execution_owner: ExecutionOwner
+ policy: ActionPolicy
+ approval: ActionApproval
+ input_artifacts: List[ArtifactReference] = Field(default_factory=list)
+ expected_output_artifacts: List[ArtifactReference] = Field(default_factory=list)
+ preconditions: List[ActionPrecondition] = Field(default_factory=list)
+ created_at: Optional[datetime] = None
+
+ @model_validator(mode="after")
+ def enforce_registry_policy(self) -> "_ActionEnvelopeBase":
+ definition = ALL_ACTION_DEFINITIONS.get(self.kind)
+ if definition is None:
+ raise ValueError(f"Unsupported action envelope kind: {self.kind}")
+ if self.policy.risk_level != definition.risk_level:
+ raise ValueError(
+ f"risk_level for {self.kind} must be {definition.risk_level}"
+ )
+ if self.policy.requires_approval != definition.requires_approval:
+ raise ValueError(
+ f"requires_approval for {self.kind} must be "
+ f"{definition.requires_approval}"
+ )
+ if self.execution_owner != definition.execution_owner:
+ raise ValueError(
+ f"execution_owner for {self.kind} must be "
+ f"{definition.execution_owner}"
+ )
+ if definition.requires_approval:
+ if self.approval.status == "not_required":
+ raise ValueError(f"{self.kind} requires an approval status")
+ elif self.approval.status != "not_required":
+ raise ValueError(f"{self.kind} does not accept an approval decision")
+ return self
+
+
+class ChooseProjectDataAction(_ActionEnvelopeBase):
+ kind: Literal["choose_project_data"]
+
+
+class LoadVisualizationAction(_ActionEnvelopeBase):
+ kind: Literal["load_visualization"]
+
+
+class StartInferenceAction(_ActionEnvelopeBase):
+ kind: Literal["start_inference"]
+
+
+class StopInferenceAction(_ActionEnvelopeBase):
+ kind: Literal["stop_inference"]
+ operation_id: Optional[int] = Field(default=None, gt=0)
+
+
+class StartProofreadingAction(_ActionEnvelopeBase):
+ kind: Literal["start_proofreading"]
+
+
+class TrainingVolumeSubset(_StrictBoundaryModel):
+ selection_basis: Optional[NonEmptyString] = None
+ training_statuses: List[NonEmptyString] = Field(default_factory=list)
+ train_volume_count: Optional[int] = Field(default=None, ge=0)
+ target_volume_count: Optional[int] = Field(default=None, ge=0)
+ review_volume_count: Optional[int] = Field(default=None, ge=0)
+ manifest_path: Optional[NonEmptyString] = None
+
+
+class StartTrainingAction(_ActionEnvelopeBase):
+ kind: Literal["start_training"]
+ autopick_parameters: Optional[bool] = None
+ parameter_mode: Optional[NonEmptyString] = None
+ volume_subset: Optional[TrainingVolumeSubset] = None
+
+
+class StopTrainingAction(_ActionEnvelopeBase):
+ kind: Literal["stop_training"]
+ operation_id: Optional[int] = Field(default=None, gt=0)
+
+
+class ComputeEvaluationAction(_ActionEnvelopeBase):
+ kind: Literal["compute_evaluation"]
+ name: Optional[NonEmptyString] = None
+ baseline_prediction_path: Optional[NonEmptyString] = None
+ candidate_prediction_path: Optional[NonEmptyString] = None
+ ground_truth_path: Optional[NonEmptyString] = None
+ baseline_dataset: Optional[NonEmptyString] = None
+ candidate_dataset: Optional[NonEmptyString] = None
+ ground_truth_dataset: Optional[NonEmptyString] = None
+ crop: Optional[NonEmptyString] = None
+ baseline_channel: Optional[int] = Field(default=None, ge=0)
+ candidate_channel: Optional[int] = Field(default=None, ge=0)
+ ground_truth_channel: Optional[int] = Field(default=None, ge=0)
+ baseline_run_id: Optional[int] = Field(default=None, gt=0)
+ candidate_run_id: Optional[int] = Field(default=None, gt=0)
+ model_version_id: Optional[int] = Field(default=None, gt=0)
+ report_path: Optional[NonEmptyString] = None
+ metadata: Dict[str, Any] = Field(default_factory=dict)
+
+
+class ExportBundleAction(_ActionEnvelopeBase):
+ kind: Literal["export_bundle"]
+
+
+class ProposeRetrainingStageAction(_ActionEnvelopeBase):
+ kind: Literal["propose_retraining_stage"]
+ corrected_mask_path: Optional[NonEmptyString] = None
+
+
+AgentActionEnvelope = Annotated[
+ Union[
+ ChooseProjectDataAction,
+ LoadVisualizationAction,
+ StartInferenceAction,
+ StopInferenceAction,
+ StartProofreadingAction,
+ StartTrainingAction,
+ StopTrainingAction,
+ ComputeEvaluationAction,
+ ExportBundleAction,
+ ProposeRetrainingStageAction,
+ ],
+ Field(discriminator="kind"),
+]
+
+
+class AgentActionError(_StrictBoundaryModel):
+ code: NonEmptyString
+ message: NonEmptyString
+ retryable: bool = False
+ details: Dict[str, Any] = Field(default_factory=dict)
+
+
+class _ActionReceiptBase(_StrictBoundaryModel):
+ schema_version: Literal["workflow.action-receipt/v1"] = "workflow.action-receipt/v1"
+ receipt_id: NonEmptyString
+ action_id: NonEmptyString
+ kind: str
+ workflow_id: int = Field(gt=0)
+ idempotency_key: IdempotencyKey
+ correlation_id: CorrelationId
+ status: ReceiptStatus
+ operation_id: Optional[int] = Field(default=None, gt=0)
+ produced_artifacts: List[ArtifactReference] = Field(default_factory=list)
+ error: Optional[AgentActionError] = None
+ started_at: Optional[datetime] = None
+ completed_at: Optional[datetime] = None
+
+ @model_validator(mode="after")
+ def validate_receipt_state(self) -> "_ActionReceiptBase":
+ if self.status == "failed" and self.error is None:
+ raise ValueError("failed action receipts require an error")
+ if self.status != "failed" and self.error is not None:
+ raise ValueError("error is only valid for failed action receipts")
+ if self.status in {"succeeded", "failed", "cancelled"}:
+ if self.completed_at is None:
+ raise ValueError("terminal action receipts require completed_at")
+ elif self.completed_at is not None:
+ raise ValueError("non-terminal action receipts cannot have completed_at")
+ if (
+ self.started_at is not None
+ and self.completed_at is not None
+ and self.completed_at < self.started_at
+ ):
+ raise ValueError("completed_at cannot precede started_at")
+
+ success_fields = {
+ "choose_project_data": "selected_artifacts",
+ "load_visualization": "viewer_url",
+ "start_inference": "run_id",
+ "stop_inference": "stopped",
+ "start_proofreading": "proofreading_session_id",
+ "start_training": "run_id",
+ "stop_training": "stopped",
+ "compute_evaluation": "evaluation_result_id",
+ "export_bundle": "bundle_artifact",
+ "propose_retraining_stage": "workflow_event_id",
+ }
+ success_field = success_fields.get(self.kind)
+ if self.status == "succeeded" and success_field:
+ if not getattr(self, success_field, None):
+ raise ValueError(
+ f"succeeded {self.kind} receipts require {success_field}"
+ )
+ return self
+
+
+class ChooseProjectDataReceipt(_ActionReceiptBase):
+ kind: Literal["choose_project_data"]
+ selected_artifacts: List[ArtifactReference] = Field(default_factory=list)
+
+
+class LoadVisualizationReceipt(_ActionReceiptBase):
+ kind: Literal["load_visualization"]
+ viewer_url: Optional[NonEmptyString] = None
+
+
+class StartInferenceReceipt(_ActionReceiptBase):
+ kind: Literal["start_inference"]
+ run_id: Optional[NonEmptyString] = None
+
+
+class StopInferenceReceipt(_ActionReceiptBase):
+ kind: Literal["stop_inference"]
+ stopped: Optional[bool] = None
+
+
+class StartProofreadingReceipt(_ActionReceiptBase):
+ kind: Literal["start_proofreading"]
+ proofreading_session_id: Optional[int] = Field(default=None, gt=0)
+ viewer_url: Optional[NonEmptyString] = None
+
+
+class StartTrainingReceipt(_ActionReceiptBase):
+ kind: Literal["start_training"]
+ run_id: Optional[NonEmptyString] = None
+
+
+class StopTrainingReceipt(_ActionReceiptBase):
+ kind: Literal["stop_training"]
+ stopped: Optional[bool] = None
+
+
+class ComputeEvaluationReceipt(_ActionReceiptBase):
+ kind: Literal["compute_evaluation"]
+ evaluation_result_id: Optional[int] = Field(default=None, gt=0)
+ metrics: Dict[str, float] = Field(default_factory=dict)
+
+
+class ExportBundleReceipt(_ActionReceiptBase):
+ kind: Literal["export_bundle"]
+ bundle_artifact: Optional[ArtifactReference] = None
+
+
+class ProposeRetrainingStageReceipt(_ActionReceiptBase):
+ kind: Literal["propose_retraining_stage"]
+ workflow_event_id: Optional[int] = Field(default=None, gt=0)
+ staged_workflow_stage: Literal["retraining_staged"] = "retraining_staged"
+
+
+AgentActionReceipt = Annotated[
+ Union[
+ ChooseProjectDataReceipt,
+ LoadVisualizationReceipt,
+ StartInferenceReceipt,
+ StopInferenceReceipt,
+ StartProofreadingReceipt,
+ StartTrainingReceipt,
+ StopTrainingReceipt,
+ ComputeEvaluationReceipt,
+ ExportBundleReceipt,
+ ProposeRetrainingStageReceipt,
+ ],
+ Field(discriminator="kind"),
+]
+
+
+_ACTION_ENVELOPE_ADAPTER = TypeAdapter(AgentActionEnvelope)
+_ACTION_RECEIPT_ADAPTER = TypeAdapter(AgentActionReceipt)
+
+
+def canonical_action_policy(kind: str) -> ActionPolicy:
+ definition = ALL_ACTION_DEFINITIONS.get(kind)
+ if definition is None:
+ raise ValueError(f"Unsupported action envelope kind: {kind}")
+ return ActionPolicy(
+ risk_level=definition.risk_level,
+ requires_approval=definition.requires_approval,
+ )
+
+
+def validate_action_envelope(payload: Any) -> AgentActionEnvelope:
+ return _ACTION_ENVELOPE_ADAPTER.validate_python(payload)
+
+
+def validate_action_for_execution(payload: Any) -> AgentActionEnvelope:
+ envelope = validate_action_envelope(payload)
+ if envelope.policy.requires_approval and envelope.approval.status != "approved":
+ raise ValueError(
+ f"{envelope.kind} cannot execute without an approved action envelope"
+ )
+ return envelope
+
+
+def validate_action_receipt(payload: Any) -> AgentActionReceipt:
+ return _ACTION_RECEIPT_ADAPTER.validate_python(payload)
+
+
+def validate_receipt_for_action(
+ action_payload: Any, receipt_payload: Any
+) -> AgentActionReceipt:
+ action = validate_action_envelope(action_payload)
+ receipt = validate_action_receipt(receipt_payload)
+ matching_fields = (
+ "action_id",
+ "kind",
+ "workflow_id",
+ "idempotency_key",
+ "correlation_id",
+ )
+ mismatches = [
+ field
+ for field in matching_fields
+ if getattr(action, field) != getattr(receipt, field)
+ ]
+ if mismatches:
+ raise ValueError(
+ "action receipt does not match its envelope: " + ", ".join(mismatches)
+ )
+ return receipt
+
+
+def dump_action_envelope_json(payload: Any) -> bytes:
+ return _ACTION_ENVELOPE_ADAPTER.dump_json(validate_action_envelope(payload))
+
+
+def load_action_envelope_json(payload: Union[str, bytes]) -> AgentActionEnvelope:
+ return _ACTION_ENVELOPE_ADAPTER.validate_json(payload)
+
+
+def dump_action_receipt_json(payload: Any) -> bytes:
+ return _ACTION_RECEIPT_ADAPTER.dump_json(validate_action_receipt(payload))
+
+
+def load_action_receipt_json(payload: Union[str, bytes]) -> AgentActionReceipt:
+ return _ACTION_RECEIPT_ADAPTER.validate_json(payload)
+
+
+def action_envelope_json_schema() -> Dict[str, Any]:
+ return _ACTION_ENVELOPE_ADAPTER.json_schema()
+
+
+def action_receipt_json_schema() -> Dict[str, Any]:
+ return _ACTION_RECEIPT_ADAPTER.json_schema()
+
+
+def _validate_effects(client_effects: Dict[str, Any]) -> None:
+ validator = getattr(ClientEffectsPayload, "model_validate", None)
+ if validator is not None:
+ validator(client_effects)
+ else: # pragma: no cover - Pydantic v1 compatibility
+ ClientEffectsPayload.parse_obj(client_effects)
+
+
+def resolve_agent_action(
+ action_id: str, client_effects: Optional[Dict[str, Any]]
+) -> AgentActionDefinition:
+ effects = client_effects or {}
+ _validate_effects(effects)
+
+ runtime_action = effects.get("runtime_action") or {}
+ runtime_kind = runtime_action.get("kind")
+ if runtime_kind:
+ return RUNTIME_ACTIONS[str(runtime_kind)]
+
+ workflow_action = effects.get("workflow_action") or {}
+ workflow_kind = workflow_action.get("kind")
+ if workflow_kind:
+ return WORKFLOW_ACTIONS[str(workflow_kind)]
+
+ if effects.get("mount_project"):
+ return AgentActionDefinition(
+ "mount_project", "modifies_workspace", True, "server_workflow", "data_agent"
+ )
+ if effects.get("reset_workspace"):
+ return AgentActionDefinition(
+ "reset_workspace",
+ "modifies_workspace",
+ True,
+ "server_workflow",
+ "data_agent",
+ )
+ if effects.get("start_new_workflow"):
+ return AgentActionDefinition(
+ "start_new_workflow",
+ "writes_workflow_record",
+ True,
+ "server_workflow",
+ "project_manager",
+ )
+
+ if any(key.startswith("set_") for key in effects) or effects.get(
+ "training_volume_subset"
+ ):
+ navigate_to = str(effects.get("navigate_to") or "")
+ return AgentActionDefinition(
+ action_id,
+ "prefills_form",
+ False,
+ "browser_navigation",
+ NAVIGATION_SPECIALISTS.get(navigate_to, "project_manager"),
+ )
+
+ if effects.get("show_workflow_context") or effects.get("refresh_insights"):
+ return AgentActionDefinition(
+ (
+ "show_workflow_context"
+ if effects.get("show_workflow_context")
+ else "refresh_context"
+ ),
+ "read_only",
+ False,
+ "browser_navigation",
+ "project_manager",
+ )
+
+ if effects.get("navigate_to"):
+ destination = str(effects["navigate_to"])
+ return AgentActionDefinition(
+ f"open_{destination}",
+ "read_only",
+ False,
+ "browser_navigation",
+ NAVIGATION_SPECIALISTS.get(destination, "project_manager"),
+ )
+
+ return AgentActionDefinition(
+ action_id,
+ "read_only",
+ False,
+ "browser_navigation",
+ "project_manager",
+ )
+
+
+def validate_agent_proposal(
+ action: str, payload: Optional[Dict[str, Any]]
+) -> AgentActionDefinition:
+ params = payload or {}
+ if action == "stage_retraining_from_corrections":
+ StageRetrainingPayload.model_validate(params)
+ return AgentActionDefinition(
+ action,
+ "writes_workflow_record",
+ True,
+ "server_workflow",
+ "training_agent",
+ )
+ if action == "start_training_run":
+ validated = StartTrainingRunPayload.model_validate(params)
+ if validated.client_effects is not None:
+ return resolve_agent_action(
+ action, validated.client_effects.model_dump(exclude_none=True)
+ )
+ return RUNTIME_ACTIONS["start_training"]
+ if action == "run_client_effects":
+ validated = RunClientEffectsPayload.model_validate(params)
+ return resolve_agent_action(
+ action, validated.client_effects.model_dump(exclude_none=True)
+ )
+ raise ValueError(f"Unsupported agent proposal action: {action}")
diff --git a/server_api/workflows/db_models.py b/server_api/workflows/db_models.py
index 67f773a8..ebd638ee 100644
--- a/server_api/workflows/db_models.py
+++ b/server_api/workflows/db_models.py
@@ -1,5 +1,6 @@
from sqlalchemy import (
Boolean,
+ CheckConstraint,
Column,
DateTime,
Float,
@@ -99,6 +100,12 @@ class WorkflowSession(Base):
cascade="all, delete-orphan",
order_by="WorkflowVolumeState.volume_id",
)
+ operations = relationship(
+ "WorkflowOperation",
+ back_populates="workflow",
+ cascade="all, delete-orphan",
+ order_by="WorkflowOperation.created_at",
+ )
class WorkflowEvent(Base):
@@ -166,6 +173,67 @@ class WorkflowCommand(Base):
approval_event = relationship("WorkflowEvent", foreign_keys=[approval_event_id])
+class WorkflowOperation(Base):
+ """Durable lifecycle record for asynchronous workflow work.
+
+ Commands capture an approved intent. Operations capture execution state and can
+ therefore outlive the API process that requested or started the work.
+ """
+
+ __tablename__ = "workflow_operations"
+ __table_args__ = (
+ UniqueConstraint(
+ "workflow_id",
+ "idempotency_key",
+ name="uq_workflow_operations_workflow_id_idempotency_key",
+ ),
+ CheckConstraint(
+ "status IN ('queued', 'running', 'succeeded', 'failed', 'cancelled')",
+ name="ck_workflow_operations_status",
+ ),
+ CheckConstraint(
+ "progress IS NULL OR (progress >= 0 AND progress <= 1)",
+ name="ck_workflow_operations_progress",
+ ),
+ )
+
+ id = Column(Integer, primary_key=True, index=True)
+ workflow_id = Column(
+ Integer, ForeignKey("workflow_sessions.id"), nullable=False, index=True
+ )
+ operation_type = Column(String, nullable=False, index=True)
+ status = Column(String, default="queued", nullable=False, index=True)
+ idempotency_key = Column(String, nullable=False, index=True)
+ correlation_id = Column(String, nullable=False, index=True)
+ actor = Column(String, default="system", nullable=False, index=True)
+ command_id = Column(
+ Integer, ForeignKey("workflow_commands.id"), nullable=True, index=True
+ )
+ model_run_id = Column(
+ Integer, ForeignKey("workflow_model_runs.id"), nullable=True, index=True
+ )
+ input_json = Column(Text, nullable=True)
+ result_json = Column(Text, nullable=True)
+ error_json = Column(Text, nullable=True)
+ metadata_json = Column(Text, nullable=True)
+ progress = Column(Float, nullable=True)
+ attempt_count = Column(Integer, default=0, nullable=False)
+ lease_owner = Column(String, nullable=True, index=True)
+ lease_expires_at = Column(DateTime(timezone=True), nullable=True)
+ heartbeat_at = Column(DateTime(timezone=True), nullable=True)
+ cancellation_requested_at = Column(DateTime(timezone=True), nullable=True)
+ started_at = Column(DateTime(timezone=True), nullable=True)
+ completed_at = Column(DateTime(timezone=True), nullable=True)
+ created_at = Column(DateTime(timezone=True), server_default=func.now())
+ updated_at = Column(
+ DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
+ )
+
+ workflow = relationship("WorkflowSession", back_populates="operations")
+ command = relationship("WorkflowCommand")
+ model_run = relationship("WorkflowModelRun")
+
+
class WorkflowArtifact(Base):
__tablename__ = "workflow_artifacts"
__table_args__ = (
diff --git a/server_api/workflows/evaluation_service.py b/server_api/workflows/evaluation_service.py
new file mode 100644
index 00000000..939d85c7
--- /dev/null
+++ b/server_api/workflows/evaluation_service.py
@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+from typing import Any, Dict, Optional
+
+from sqlalchemy.orm import Session
+
+from .db_models import WorkflowEvaluationResult
+from .evaluation import compute_before_after_evaluation, write_evaluation_report
+from .service import create_workflow_artifact, encode_json
+
+
+def create_computed_evaluation_result(
+ db: Session,
+ *,
+ workflow_id: int,
+ baseline_prediction_path: str,
+ candidate_prediction_path: str,
+ ground_truth_path: str,
+ baseline_dataset: Optional[str] = None,
+ candidate_dataset: Optional[str] = None,
+ ground_truth_dataset: Optional[str] = None,
+ crop: Optional[str] = None,
+ baseline_channel: Optional[int] = None,
+ candidate_channel: Optional[int] = None,
+ ground_truth_channel: Optional[int] = None,
+ name: Optional[str] = None,
+ baseline_run_id: Optional[int] = None,
+ candidate_run_id: Optional[int] = None,
+ model_version_id: Optional[int] = None,
+ report_path: Optional[str] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ commit: bool = True,
+) -> WorkflowEvaluationResult:
+ """Compute and persist the canonical before/after evaluation record."""
+
+ metrics = compute_before_after_evaluation(
+ baseline_prediction_path=baseline_prediction_path,
+ candidate_prediction_path=candidate_prediction_path,
+ ground_truth_path=ground_truth_path,
+ baseline_dataset=baseline_dataset,
+ candidate_dataset=candidate_dataset,
+ ground_truth_dataset=ground_truth_dataset,
+ crop=crop,
+ baseline_channel=baseline_channel,
+ candidate_channel=candidate_channel,
+ ground_truth_channel=ground_truth_channel,
+ )
+ evaluation_metadata = {
+ **(metadata or {}),
+ "baseline_prediction_path": baseline_prediction_path,
+ "candidate_prediction_path": candidate_prediction_path,
+ "ground_truth_path": ground_truth_path,
+ "baseline_dataset": baseline_dataset,
+ "candidate_dataset": candidate_dataset,
+ "ground_truth_dataset": ground_truth_dataset,
+ "crop": crop,
+ "baseline_channel": baseline_channel,
+ "candidate_channel": candidate_channel,
+ "ground_truth_channel": ground_truth_channel,
+ }
+ summary = (
+ "Before/after evaluation computed. "
+ f"Dice delta: {metrics.get('summary', {}).get('dice_delta')}."
+ )
+
+ persisted_report_path = report_path
+ if persisted_report_path:
+ report_payload = {
+ "workflow_id": workflow_id,
+ "name": name,
+ "summary": summary,
+ "metrics": metrics,
+ "metadata": evaluation_metadata,
+ }
+ persisted_report_path = write_evaluation_report(
+ persisted_report_path, report_payload
+ )
+
+ report_artifact = None
+ if persisted_report_path:
+ report_artifact = create_workflow_artifact(
+ db,
+ workflow_id=workflow_id,
+ artifact_type="evaluation_report",
+ role="case_study_evidence",
+ path=persisted_report_path,
+ metadata={"source": "computed_evaluation_result"},
+ commit=False,
+ )
+
+ result = WorkflowEvaluationResult(
+ workflow_id=workflow_id,
+ name=name or "before-after-evaluation",
+ baseline_run_id=baseline_run_id,
+ candidate_run_id=candidate_run_id,
+ model_version_id=model_version_id,
+ report_artifact_id=report_artifact.id if report_artifact else None,
+ report_path=persisted_report_path,
+ summary=summary,
+ metrics_json=encode_json(metrics),
+ metadata_json=encode_json(evaluation_metadata),
+ )
+ db.add(result)
+ db.flush()
+ if commit:
+ db.commit()
+ db.refresh(result)
+ return result
diff --git a/server_api/workflows/operation_router.py b/server_api/workflows/operation_router.py
new file mode 100644
index 00000000..0233a283
--- /dev/null
+++ b/server_api/workflows/operation_router.py
@@ -0,0 +1,365 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from pydantic import BaseModel, Field
+from sqlalchemy.orm import Session
+
+from server_api.auth import models as auth_models
+from server_api.auth.database import get_db
+from server_api.auth.router import get_current_user
+
+from .db_models import WorkflowOperation
+from .agent_actions import (
+ AgentActionEnvelope,
+ AgentActionReceipt,
+ ComputeEvaluationAction,
+ action_envelope_json_schema,
+ validate_action_for_execution,
+)
+from .agent_action_execution import (
+ execute_compute_evaluation_operation,
+ validate_persisted_action_approval,
+)
+from .operation_service import (
+ OPERATION_STATUSES,
+ create_workflow_operation,
+ get_workflow_operation_or_404,
+ heartbeat_workflow_operation,
+ operation_to_dict,
+ request_workflow_operation_cancellation,
+ transition_workflow_operation,
+)
+from .service import decode_json, get_user_workflow_or_404
+
+router = APIRouter()
+
+
+class WorkflowOperationCreateRequest(BaseModel):
+ operation_type: str = Field(min_length=1, max_length=120)
+ idempotency_key: str = Field(min_length=1, max_length=255)
+ correlation_id: Optional[str] = Field(default=None, max_length=255)
+ actor: str = "system"
+ command_id: Optional[int] = None
+ model_run_id: Optional[int] = None
+ input: Dict[str, Any] = Field(default_factory=dict)
+ metadata: Dict[str, Any] = Field(default_factory=dict)
+
+
+class WorkflowOperationTransitionRequest(BaseModel):
+ status: str
+ expected_status: Optional[str] = None
+ result: Optional[Dict[str, Any]] = None
+ error: Optional[Dict[str, Any]] = None
+ metadata: Optional[Dict[str, Any]] = None
+ progress: Optional[float] = None
+ lease_owner: Optional[str] = None
+ lease_expires_at: Optional[datetime] = None
+
+
+class WorkflowOperationHeartbeatRequest(BaseModel):
+ progress: Optional[float] = None
+ metadata: Optional[Dict[str, Any]] = None
+ lease_owner: Optional[str] = None
+ lease_expires_at: Optional[datetime] = None
+
+
+class WorkflowOperationCancellationRequest(BaseModel):
+ reason: Optional[str] = Field(default=None, max_length=1000)
+
+
+class WorkflowOperationResponse(BaseModel):
+ id: int
+ workflow_id: int
+ operation_type: str
+ status: str
+ idempotency_key: str
+ correlation_id: str
+ actor: str
+ command_id: Optional[int] = None
+ model_run_id: Optional[int] = None
+ input: Dict[str, Any] = Field(default_factory=dict)
+ result: Dict[str, Any] = Field(default_factory=dict)
+ error: Dict[str, Any] = Field(default_factory=dict)
+ metadata: Dict[str, Any] = Field(default_factory=dict)
+ progress: Optional[float] = None
+ attempt_count: int = 0
+ lease_owner: Optional[str] = None
+ lease_expires_at: Any = None
+ heartbeat_at: Any = None
+ cancellation_requested_at: Any = None
+ started_at: Any = None
+ completed_at: Any = None
+ created_at: Any
+ updated_at: Any
+
+
+def _response(operation: WorkflowOperation) -> WorkflowOperationResponse:
+ return WorkflowOperationResponse(**operation_to_dict(operation))
+
+
+def _owned_operation(
+ db: Session,
+ *,
+ workflow_id: int,
+ operation_id: int,
+ user_id: int,
+) -> WorkflowOperation:
+ get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user_id)
+ return get_workflow_operation_or_404(
+ db,
+ workflow_id=workflow_id,
+ operation_id=operation_id,
+ )
+
+
+@router.get("/action-envelopes/schema")
+def get_action_envelope_schema():
+ return action_envelope_json_schema()
+
+
+@router.post(
+ "/{workflow_id}/action-operations",
+ response_model=WorkflowOperationResponse,
+ status_code=202,
+)
+def stage_action_operation(
+ workflow_id: int,
+ body: AgentActionEnvelope,
+ user: auth_models.User = Depends(get_current_user),
+ db: Session = Depends(get_db),
+):
+ workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id)
+ if body.workflow_id != workflow.id:
+ raise HTTPException(
+ status_code=409,
+ detail="Action envelope workflow_id does not match the request path",
+ )
+ try:
+ envelope = validate_action_for_execution(body)
+ except ValueError as exc:
+ raise HTTPException(status_code=409, detail=str(exc)) from exc
+ if envelope.execution_owner == "browser_navigation":
+ raise HTTPException(
+ status_code=409,
+ detail="Browser-owned actions cannot be staged as server operations",
+ )
+ approval_metadata: Dict[str, Any] = {}
+ if isinstance(envelope, ComputeEvaluationAction):
+ approval_event = validate_persisted_action_approval(db, envelope)
+ approval_metadata = {
+ "approval_event_id": approval_event.id,
+ "proposal_event_id": decode_json(approval_event.payload_json).get(
+ "proposal_event_id"
+ ),
+ }
+
+ input_payload = envelope.model_dump(mode="json", exclude_none=True)
+ operation = create_workflow_operation(
+ db,
+ workflow_id=workflow.id,
+ operation_type=f"agent_action:{envelope.kind}",
+ idempotency_key=envelope.idempotency_key,
+ correlation_id=envelope.correlation_id,
+ actor=envelope.requested_by,
+ input_payload=input_payload,
+ metadata={
+ "action_id": envelope.action_id,
+ "action_schema_version": envelope.schema_version,
+ "execution_owner": envelope.execution_owner,
+ "risk_level": envelope.policy.risk_level,
+ **approval_metadata,
+ },
+ commit=True,
+ )
+ return _response(operation)
+
+
+@router.post(
+ "/{workflow_id}/action-operations/{operation_id}/execute",
+ response_model=AgentActionReceipt,
+)
+def execute_action_operation(
+ workflow_id: int,
+ operation_id: int,
+ user: auth_models.User = Depends(get_current_user),
+ db: Session = Depends(get_db),
+):
+ operation = _owned_operation(
+ db,
+ workflow_id=workflow_id,
+ operation_id=operation_id,
+ user_id=user.id,
+ )
+ return execute_compute_evaluation_operation(db, operation)
+
+
+@router.post(
+ "/{workflow_id}/operations",
+ response_model=WorkflowOperationResponse,
+)
+def create_operation(
+ workflow_id: int,
+ body: WorkflowOperationCreateRequest,
+ user: auth_models.User = Depends(get_current_user),
+ db: Session = Depends(get_db),
+):
+ workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id)
+ operation = create_workflow_operation(
+ db,
+ workflow_id=workflow.id,
+ operation_type=body.operation_type,
+ idempotency_key=body.idempotency_key,
+ correlation_id=body.correlation_id,
+ actor=body.actor,
+ command_id=body.command_id,
+ model_run_id=body.model_run_id,
+ input_payload=body.input,
+ metadata=body.metadata,
+ commit=True,
+ )
+ return _response(operation)
+
+
+@router.get(
+ "/{workflow_id}/operations",
+ response_model=List[WorkflowOperationResponse],
+)
+def list_operations(
+ workflow_id: int,
+ status: Optional[str] = None,
+ operation_type: Optional[str] = None,
+ limit: int = Query(default=100, ge=1, le=500),
+ user: auth_models.User = Depends(get_current_user),
+ db: Session = Depends(get_db),
+):
+ get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id)
+ if status is not None and status not in OPERATION_STATUSES:
+ raise HTTPException(
+ status_code=400,
+ detail=(
+ "operation status must be one of: "
+ f"{', '.join(sorted(OPERATION_STATUSES))}"
+ ),
+ )
+ query = db.query(WorkflowOperation).filter(
+ WorkflowOperation.workflow_id == workflow_id
+ )
+ if status is not None:
+ query = query.filter(WorkflowOperation.status == status)
+ if operation_type is not None:
+ query = query.filter(WorkflowOperation.operation_type == operation_type)
+ operations = query.order_by(
+ WorkflowOperation.created_at.desc(), WorkflowOperation.id.desc()
+ ).limit(limit)
+ return [_response(operation) for operation in operations]
+
+
+@router.get(
+ "/{workflow_id}/operations/{operation_id}",
+ response_model=WorkflowOperationResponse,
+)
+def get_operation(
+ workflow_id: int,
+ operation_id: int,
+ user: auth_models.User = Depends(get_current_user),
+ db: Session = Depends(get_db),
+):
+ return _response(
+ _owned_operation(
+ db,
+ workflow_id=workflow_id,
+ operation_id=operation_id,
+ user_id=user.id,
+ )
+ )
+
+
+@router.post(
+ "/{workflow_id}/operations/{operation_id}/transitions",
+ response_model=WorkflowOperationResponse,
+)
+def transition_operation(
+ workflow_id: int,
+ operation_id: int,
+ body: WorkflowOperationTransitionRequest,
+ user: auth_models.User = Depends(get_current_user),
+ db: Session = Depends(get_db),
+):
+ operation = _owned_operation(
+ db,
+ workflow_id=workflow_id,
+ operation_id=operation_id,
+ user_id=user.id,
+ )
+ operation = transition_workflow_operation(
+ db,
+ operation,
+ status=body.status,
+ expected_status=body.expected_status,
+ result_payload=body.result,
+ error_payload=body.error,
+ metadata=body.metadata,
+ progress=body.progress,
+ lease_owner=body.lease_owner,
+ lease_expires_at=body.lease_expires_at,
+ commit=True,
+ )
+ return _response(operation)
+
+
+@router.post(
+ "/{workflow_id}/operations/{operation_id}/heartbeat",
+ response_model=WorkflowOperationResponse,
+)
+def heartbeat_operation(
+ workflow_id: int,
+ operation_id: int,
+ body: WorkflowOperationHeartbeatRequest,
+ user: auth_models.User = Depends(get_current_user),
+ db: Session = Depends(get_db),
+):
+ operation = _owned_operation(
+ db,
+ workflow_id=workflow_id,
+ operation_id=operation_id,
+ user_id=user.id,
+ )
+ operation = heartbeat_workflow_operation(
+ db,
+ operation,
+ progress=body.progress,
+ metadata=body.metadata,
+ lease_owner=body.lease_owner,
+ lease_expires_at=body.lease_expires_at,
+ commit=True,
+ )
+ return _response(operation)
+
+
+@router.post(
+ "/{workflow_id}/operations/{operation_id}/cancel",
+ response_model=WorkflowOperationResponse,
+)
+def cancel_operation(
+ workflow_id: int,
+ operation_id: int,
+ body: Optional[WorkflowOperationCancellationRequest] = None,
+ user: auth_models.User = Depends(get_current_user),
+ db: Session = Depends(get_db),
+):
+ operation = _owned_operation(
+ db,
+ workflow_id=workflow_id,
+ operation_id=operation_id,
+ user_id=user.id,
+ )
+ operation = request_workflow_operation_cancellation(
+ db,
+ operation,
+ reason=body.reason if body else None,
+ commit=True,
+ )
+ return _response(operation)
diff --git a/server_api/workflows/operation_service.py b/server_api/workflows/operation_service.py
new file mode 100644
index 00000000..38193437
--- /dev/null
+++ b/server_api/workflows/operation_service.py
@@ -0,0 +1,400 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any, Dict, Optional
+from uuid import uuid4
+
+from fastapi import HTTPException
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+
+from .db_models import (
+ WorkflowCommand,
+ WorkflowModelRun,
+ WorkflowOperation,
+)
+from .service import decode_json, encode_json, validate_actor
+
+OPERATION_STATUSES = {"queued", "running", "succeeded", "failed", "cancelled"}
+TERMINAL_OPERATION_STATUSES = {"succeeded", "failed", "cancelled"}
+OPERATION_TRANSITIONS = {
+ "queued": {"running", "failed", "cancelled"},
+ "running": {"succeeded", "failed", "cancelled"},
+ "succeeded": set(),
+ "failed": set(),
+ "cancelled": set(),
+}
+
+
+def _now() -> datetime:
+ return datetime.now(timezone.utc)
+
+
+def _merge_dicts(
+ base: Optional[Dict[str, Any]], patch: Optional[Dict[str, Any]]
+) -> Dict[str, Any]:
+ merged = dict(base or {})
+ for key, value in (patch or {}).items():
+ if isinstance(value, dict) and isinstance(merged.get(key), dict):
+ merged[key] = _merge_dicts(merged[key], value)
+ else:
+ merged[key] = value
+ return merged
+
+
+def _validate_progress(progress: Optional[float]) -> Optional[float]:
+ if progress is None:
+ return None
+ value = float(progress)
+ if value < 0 or value > 1:
+ raise HTTPException(status_code=400, detail="progress must be between 0 and 1")
+ return value
+
+
+def operation_to_dict(operation: WorkflowOperation) -> Dict[str, Any]:
+ return {
+ "id": operation.id,
+ "workflow_id": operation.workflow_id,
+ "operation_type": operation.operation_type,
+ "status": operation.status,
+ "idempotency_key": operation.idempotency_key,
+ "correlation_id": operation.correlation_id,
+ "actor": operation.actor,
+ "command_id": operation.command_id,
+ "model_run_id": operation.model_run_id,
+ "input": decode_json(operation.input_json),
+ "result": decode_json(operation.result_json),
+ "error": decode_json(operation.error_json),
+ "metadata": decode_json(operation.metadata_json),
+ "progress": operation.progress,
+ "attempt_count": operation.attempt_count,
+ "lease_owner": operation.lease_owner,
+ "lease_expires_at": operation.lease_expires_at,
+ "heartbeat_at": operation.heartbeat_at,
+ "cancellation_requested_at": operation.cancellation_requested_at,
+ "started_at": operation.started_at,
+ "completed_at": operation.completed_at,
+ "created_at": operation.created_at,
+ "updated_at": operation.updated_at,
+ }
+
+
+def get_workflow_operation_or_404(
+ db: Session, *, workflow_id: int, operation_id: int
+) -> WorkflowOperation:
+ operation = (
+ db.query(WorkflowOperation)
+ .filter(
+ WorkflowOperation.id == operation_id,
+ WorkflowOperation.workflow_id == workflow_id,
+ )
+ .first()
+ )
+ if operation is None:
+ raise HTTPException(status_code=404, detail="Workflow operation not found")
+ return operation
+
+
+def _validate_linked_record(
+ db: Session,
+ *,
+ model: Any,
+ record_id: Optional[int],
+ workflow_id: int,
+ label: str,
+) -> None:
+ if record_id is None:
+ return
+ record = db.query(model).filter(model.id == record_id).first()
+ if record is None or record.workflow_id != workflow_id:
+ raise HTTPException(
+ status_code=400,
+ detail=f"{label} must belong to the workflow",
+ )
+
+
+def _assert_idempotent_match(
+ operation: WorkflowOperation,
+ *,
+ operation_type: str,
+ actor: str,
+ command_id: Optional[int],
+ model_run_id: Optional[int],
+ input_payload: Dict[str, Any],
+ metadata: Dict[str, Any],
+) -> WorkflowOperation:
+ expected = {
+ "operation_type": operation_type,
+ "actor": actor,
+ "command_id": command_id,
+ "model_run_id": model_run_id,
+ "input": input_payload,
+ "metadata": metadata,
+ }
+ actual = {
+ "operation_type": operation.operation_type,
+ "actor": operation.actor,
+ "command_id": operation.command_id,
+ "model_run_id": operation.model_run_id,
+ "input": decode_json(operation.input_json),
+ "metadata": decode_json(operation.metadata_json),
+ }
+ if actual != expected:
+ raise HTTPException(
+ status_code=409,
+ detail="idempotency_key is already used by a different operation request",
+ )
+ return operation
+
+
+def create_workflow_operation(
+ db: Session,
+ *,
+ workflow_id: int,
+ operation_type: str,
+ idempotency_key: str,
+ correlation_id: Optional[str] = None,
+ actor: str = "system",
+ command_id: Optional[int] = None,
+ model_run_id: Optional[int] = None,
+ input_payload: Optional[Dict[str, Any]] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ commit: bool = False,
+) -> WorkflowOperation:
+ operation_type = operation_type.strip()
+ idempotency_key = idempotency_key.strip()
+ if not operation_type:
+ raise HTTPException(status_code=400, detail="operation_type is required")
+ if not idempotency_key:
+ raise HTTPException(status_code=400, detail="idempotency_key is required")
+ correlation_id = (correlation_id or str(uuid4())).strip()
+ if not correlation_id:
+ raise HTTPException(status_code=400, detail="correlation_id cannot be blank")
+ actor = validate_actor(actor)
+ input_payload = input_payload or {}
+ metadata = metadata or {}
+ _validate_linked_record(
+ db,
+ model=WorkflowCommand,
+ record_id=command_id,
+ workflow_id=workflow_id,
+ label="command_id",
+ )
+ _validate_linked_record(
+ db,
+ model=WorkflowModelRun,
+ record_id=model_run_id,
+ workflow_id=workflow_id,
+ label="model_run_id",
+ )
+
+ existing = (
+ db.query(WorkflowOperation)
+ .filter(
+ WorkflowOperation.workflow_id == workflow_id,
+ WorkflowOperation.idempotency_key == idempotency_key,
+ )
+ .first()
+ )
+ if existing is not None:
+ return _assert_idempotent_match(
+ existing,
+ operation_type=operation_type,
+ actor=actor,
+ command_id=command_id,
+ model_run_id=model_run_id,
+ input_payload=input_payload,
+ metadata=metadata,
+ )
+
+ operation = WorkflowOperation(
+ workflow_id=workflow_id,
+ operation_type=operation_type,
+ status="queued",
+ idempotency_key=idempotency_key,
+ correlation_id=correlation_id,
+ actor=actor,
+ command_id=command_id,
+ model_run_id=model_run_id,
+ input_json=encode_json(input_payload),
+ metadata_json=encode_json(metadata),
+ )
+ try:
+ with db.begin_nested():
+ db.add(operation)
+ db.flush()
+ except IntegrityError:
+ existing = (
+ db.query(WorkflowOperation)
+ .filter(
+ WorkflowOperation.workflow_id == workflow_id,
+ WorkflowOperation.idempotency_key == idempotency_key,
+ )
+ .first()
+ )
+ if existing is None:
+ raise
+ operation = _assert_idempotent_match(
+ existing,
+ operation_type=operation_type,
+ actor=actor,
+ command_id=command_id,
+ model_run_id=model_run_id,
+ input_payload=input_payload,
+ metadata=metadata,
+ )
+ if commit:
+ db.commit()
+ db.refresh(operation)
+ return operation
+
+
+def transition_workflow_operation(
+ db: Session,
+ operation: WorkflowOperation,
+ *,
+ status: str,
+ expected_status: Optional[str] = None,
+ result_payload: Optional[Dict[str, Any]] = None,
+ error_payload: Optional[Dict[str, Any]] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ progress: Optional[float] = None,
+ lease_owner: Optional[str] = None,
+ lease_expires_at: Optional[datetime] = None,
+ commit: bool = False,
+) -> WorkflowOperation:
+ if status not in OPERATION_STATUSES:
+ raise HTTPException(
+ status_code=400,
+ detail=(
+ "operation status must be one of: "
+ f"{', '.join(sorted(OPERATION_STATUSES))}"
+ ),
+ )
+ if expected_status is not None and operation.status != expected_status:
+ raise HTTPException(
+ status_code=409,
+ detail=(
+ f"Operation status changed: expected {expected_status}, "
+ f"found {operation.status}"
+ ),
+ )
+ if (
+ operation.status == "running"
+ and operation.lease_owner
+ and status in TERMINAL_OPERATION_STATUSES
+ and lease_owner != operation.lease_owner
+ ):
+ raise HTTPException(
+ status_code=409, detail="Operation is leased by another worker"
+ )
+ if operation.status == status and status in TERMINAL_OPERATION_STATUSES:
+ return operation
+ if status not in OPERATION_TRANSITIONS[operation.status]:
+ raise HTTPException(
+ status_code=409,
+ detail=f"Operation cannot transition from {operation.status} to {status}",
+ )
+
+ now = _now()
+ operation.status = status
+ if metadata is not None:
+ operation.metadata_json = encode_json(
+ _merge_dicts(decode_json(operation.metadata_json), metadata)
+ )
+ if progress is not None:
+ operation.progress = _validate_progress(progress)
+
+ if status == "running":
+ operation.attempt_count = int(operation.attempt_count or 0) + 1
+ operation.started_at = operation.started_at or now
+ operation.completed_at = None
+ operation.heartbeat_at = now
+ operation.lease_owner = lease_owner
+ operation.lease_expires_at = lease_expires_at
+ operation.error_json = None
+ else:
+ operation.completed_at = now
+ operation.lease_owner = None
+ operation.lease_expires_at = None
+ if status == "succeeded":
+ operation.progress = 1.0
+ operation.result_json = encode_json(result_payload or {})
+ operation.error_json = None
+ elif status == "failed":
+ if result_payload is not None:
+ operation.result_json = encode_json(result_payload)
+ operation.error_json = encode_json(error_payload or {})
+ elif status == "cancelled" and error_payload is not None:
+ operation.error_json = encode_json(error_payload)
+
+ db.flush()
+ if commit:
+ db.commit()
+ db.refresh(operation)
+ return operation
+
+
+def heartbeat_workflow_operation(
+ db: Session,
+ operation: WorkflowOperation,
+ *,
+ progress: Optional[float] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ lease_owner: Optional[str] = None,
+ lease_expires_at: Optional[datetime] = None,
+ commit: bool = False,
+) -> WorkflowOperation:
+ if operation.status != "running":
+ raise HTTPException(
+ status_code=409,
+ detail="Only running operations can receive heartbeats",
+ )
+ if lease_owner and operation.lease_owner and lease_owner != operation.lease_owner:
+ raise HTTPException(
+ status_code=409, detail="Operation is leased by another worker"
+ )
+ operation.heartbeat_at = _now()
+ if progress is not None:
+ operation.progress = _validate_progress(progress)
+ if metadata is not None:
+ operation.metadata_json = encode_json(
+ _merge_dicts(decode_json(operation.metadata_json), metadata)
+ )
+ if lease_owner is not None:
+ operation.lease_owner = lease_owner
+ if lease_expires_at is not None:
+ operation.lease_expires_at = lease_expires_at
+ db.flush()
+ if commit:
+ db.commit()
+ db.refresh(operation)
+ return operation
+
+
+def request_workflow_operation_cancellation(
+ db: Session,
+ operation: WorkflowOperation,
+ *,
+ reason: Optional[str] = None,
+ commit: bool = False,
+) -> WorkflowOperation:
+ if operation.status in TERMINAL_OPERATION_STATUSES:
+ return operation
+ now = _now()
+ operation.cancellation_requested_at = operation.cancellation_requested_at or now
+ if reason:
+ operation.metadata_json = encode_json(
+ _merge_dicts(
+ decode_json(operation.metadata_json),
+ {"cancellation": {"reason": reason}},
+ )
+ )
+ if operation.status == "queued":
+ operation.status = "cancelled"
+ operation.completed_at = now
+ db.flush()
+ if commit:
+ db.commit()
+ db.refresh(operation)
+ return operation
diff --git a/server_api/workflows/router.py b/server_api/workflows/router.py
index 15aeb8f1..e8a6c228 100644
--- a/server_api/workflows/router.py
+++ b/server_api/workflows/router.py
@@ -10,7 +10,7 @@
import requests
from fastapi import APIRouter, Depends, HTTPException
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, ValidationError
from sqlalchemy.orm import Session
from app_event_logger import append_app_event
@@ -34,6 +34,7 @@
WorkflowEvent,
WorkflowModelRun,
WorkflowModelVersion,
+ WorkflowOperation,
WorkflowRegionHotspot,
WorkflowSession,
WorkflowVolumeState,
@@ -64,7 +65,17 @@
)
from .bundle_export import build_export_bundle, write_export_bundle_directory
from .agent_plan import build_case_study_plan_graph
-from .evaluation import compute_before_after_evaluation, write_evaluation_report
+from .agent_actions import (
+ AgentActionReceipt,
+ resolve_agent_action,
+ validate_agent_proposal,
+)
+from .agent_action_execution import (
+ execute_compute_evaluation_operation,
+ stage_and_execute_compute_evaluation_proposal,
+)
+from .operation_service import operation_to_dict
+from .evaluation_service import create_computed_evaluation_result
from .metrics import compute_workflow_metrics
from .volume_pairs import discover_neuroglancer_volume_pairs
@@ -298,6 +309,8 @@ class AgentActionResult(BaseModel):
events: List[WorkflowEventResponse]
client_effects: Dict[str, Any] = Field(default_factory=dict)
commands: List[WorkflowCommandResponse] = Field(default_factory=list)
+ operation: Optional[Dict[str, Any]] = None
+ receipt: Optional[AgentActionReceipt] = None
class WorkflowHotspotItem(BaseModel):
@@ -885,6 +898,32 @@ def _get_pending_proposal_or_404(
return event
+def _related_proposal_event(
+ db: Session,
+ *,
+ workflow_id: int,
+ proposal_id: int,
+ event_type: str,
+) -> Optional[WorkflowEvent]:
+ candidates = (
+ db.query(WorkflowEvent)
+ .filter(
+ WorkflowEvent.workflow_id == workflow_id,
+ WorkflowEvent.event_type == event_type,
+ )
+ .order_by(WorkflowEvent.id.desc())
+ .all()
+ )
+ return next(
+ (
+ event
+ for event in candidates
+ if decode_json(event.payload_json).get("proposal_event_id") == proposal_id
+ ),
+ None,
+ )
+
+
def _get_agent_plan_or_404(
db: Session, *, workflow_id: int, plan_id: int
) -> WorkflowAgentPlan:
@@ -1390,44 +1429,11 @@ def _client_effects_to_command(client_effects: Dict[str, Any]) -> str:
def _infer_action_risk(client_effects: Optional[Dict[str, Any]]) -> str:
- effects = client_effects or {}
- runtime_kind = (effects.get("runtime_action") or {}).get("kind")
- workflow_action_kind = (effects.get("workflow_action") or {}).get("kind")
- if runtime_kind in {"start_inference", "start_training"}:
- return "runs_job"
- if runtime_kind in {"stop_inference", "stop_training"}:
- return "controls_job"
- if runtime_kind == "start_proofreading":
- return "loads_editor"
- if runtime_kind == "choose_project_data":
- return "prefills_form"
- if effects.get("mount_project") or effects.get("reset_workspace"):
- return "modifies_workspace"
- if effects.get("start_new_workflow"):
- return "writes_workflow_record"
- if workflow_action_kind == "export_bundle":
- return "exports_evidence"
- if workflow_action_kind in {"compute_evaluation", "propose_retraining_stage"}:
- return "writes_workflow_record"
- if any(key.startswith("set_") for key in effects):
- return "prefills_form"
- if effects.get("navigate_to") or effects.get("show_workflow_context"):
- return "read_only"
- if effects.get("refresh_insights"):
- return "read_only"
- return "read_only"
+ return resolve_agent_action("workflow_action", client_effects).risk_level
def _requires_action_approval(client_effects: Optional[Dict[str, Any]]) -> bool:
- risk = _infer_action_risk(client_effects)
- return risk in {
- "runs_job",
- "controls_job",
- "loads_editor",
- "exports_evidence",
- "writes_workflow_record",
- "modifies_workspace",
- }
+ return resolve_agent_action("workflow_action", client_effects).requires_approval
def _action_risk_tier(risk_level: str) -> str:
@@ -1540,58 +1546,15 @@ def _agent_trace_kwargs(agent: Dict[str, Any]) -> Dict[str, str]:
def _specialist_agent_for_action(
action_type: str, client_effects: Dict[str, Any]
) -> Dict[str, Any]:
- navigate_to = str((client_effects or {}).get("navigate_to") or "")
- if action_type in {"start_training", "open_training"} or "training" in navigate_to:
- return _agent_descriptor("training_agent")
- if (
- action_type in {"start_inference", "open_inference"}
- or "inference" in navigate_to
- ):
- return _agent_descriptor("inference_agent")
- if action_type in {"start_proofreading"} or "proofreading" in navigate_to:
- return _agent_descriptor("proofreading_agent")
- if "visualization" in navigate_to or action_type.startswith("open_visualization"):
- return _agent_descriptor("visualization_agent")
- if action_type in {"export_bundle"}:
- return _agent_descriptor("evidence_agent")
- if action_type in {"compute_evaluation"}:
- return _agent_descriptor("evaluation_agent")
- if (
- action_type in {"mount_project", "choose_project_data"}
- or navigate_to == "files"
- ):
- return _agent_descriptor("data_agent")
- if action_type in {
- "show_workflow_context",
- "refresh_context",
- "open_project-progress",
- }:
- return _agent_descriptor("project_manager")
- return _agent_descriptor("project_manager")
+ definition = resolve_agent_action(action_type, client_effects)
+ return _agent_descriptor(definition.specialist_agent_type)
def _action_type_from_effects(
action_id: str,
client_effects: Optional[Dict[str, Any]],
) -> str:
- effects = client_effects or {}
- runtime_kind = (effects.get("runtime_action") or {}).get("kind")
- workflow_kind = (effects.get("workflow_action") or {}).get("kind")
- if runtime_kind:
- return str(runtime_kind)
- if workflow_kind:
- return str(workflow_kind)
- if effects.get("mount_project"):
- return "mount_project"
- if effects.get("start_new_workflow"):
- return "start_new_workflow"
- if effects.get("show_workflow_context"):
- return "show_workflow_context"
- if effects.get("refresh_insights"):
- return "refresh_context"
- if effects.get("navigate_to"):
- return f"open_{effects.get('navigate_to')}"
- return action_id
+ return resolve_agent_action(action_id, client_effects).action_type
def _action_target_from_effects(
@@ -1829,7 +1792,8 @@ def _build_action_card_payload(
requires_approval: bool,
disabled_reason: Optional[str],
) -> Dict[str, Any]:
- action_type = _action_type_from_effects(action_id, client_effects)
+ definition = resolve_agent_action(action_id, client_effects)
+ action_type = definition.action_type
specialist_agent = _specialist_agent_for_action(action_type, client_effects)
blockers = [disabled_reason] if disabled_reason else []
return {
@@ -1855,6 +1819,12 @@ def _build_action_card_payload(
"summary_fields": _summary_fields_from_effects(client_effects),
"expected_effects": _expected_effects_from_client_effects(client_effects),
"executor": "bounded_app_routine",
+ "execution_owner": definition.execution_owner,
+ "registry_policy": {
+ "risk_level": definition.risk_level,
+ "requires_approval": definition.requires_approval,
+ "specialist_agent_type": definition.specialist_agent_type,
+ },
}
@@ -4996,7 +4966,13 @@ def _project_progress_volume_candidates(
return []
def is_candidate(child: pathlib.Path) -> bool:
- if any(part.startswith(".") or part == "__pycache__" for part in child.parts):
+ try:
+ relative_parts = child.relative_to(absolute).parts
+ except ValueError:
+ relative_parts = child.parts
+ if any(
+ part.startswith(".") or part == "__pycache__" for part in relative_parts
+ ):
return False
if not child.is_file() and not (
child.is_dir() and _project_extension(child.name) in {".zarr", ".n5"}
@@ -6078,6 +6054,14 @@ def _workflow_overview_actions(
)
)
if needs_proofreading > 0:
+ draft_volume = next(
+ (
+ volume
+ for volume in progress.get("volumes") or []
+ if volume.get("status") == "needs_proofreading"
+ ),
+ {},
+ )
actions.append(
WorkflowOverviewAction(
id="proofread-draft-masks",
@@ -6085,7 +6069,13 @@ def _workflow_overview_actions(
detail=f"Review {needs_proofreading} volume(s) before treating them as ground truth.",
target_view="mask-proofreading",
priority="high" if ground_truth <= 0 else "normal",
- client_effects={"navigate_to": "mask-proofreading"},
+ client_effects={
+ "navigate_to": "mask-proofreading",
+ "set_proofreading_dataset_path": draft_volume.get("image_path"),
+ "set_proofreading_mask_path": draft_volume.get("segmentation_path"),
+ "set_proofreading_project_name": workflow.title,
+ "runtime_action": {"kind": "start_proofreading"},
+ },
)
)
if ground_truth > 0 and missing_segmentation > 0:
@@ -8190,7 +8180,9 @@ def compute_evaluation_result(
):
workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id)
try:
- metrics = compute_before_after_evaluation(
+ result = create_computed_evaluation_result(
+ db,
+ workflow_id=workflow.id,
baseline_prediction_path=body.baseline_prediction_path,
candidate_prediction_path=body.candidate_prediction_path,
ground_truth_path=body.ground_truth_path,
@@ -8201,6 +8193,13 @@ def compute_evaluation_result(
baseline_channel=body.baseline_channel,
candidate_channel=body.candidate_channel,
ground_truth_channel=body.ground_truth_channel,
+ name=body.name,
+ baseline_run_id=body.baseline_run_id,
+ candidate_run_id=body.candidate_run_id,
+ model_version_id=body.model_version_id,
+ report_path=body.report_path,
+ metadata=body.metadata,
+ commit=True,
)
except FileNotFoundError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@@ -8208,62 +8207,6 @@ def compute_evaluation_result(
raise HTTPException(status_code=400, detail=str(exc)) from exc
except RuntimeError as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
-
- metadata = {
- **body.metadata,
- "baseline_prediction_path": body.baseline_prediction_path,
- "candidate_prediction_path": body.candidate_prediction_path,
- "ground_truth_path": body.ground_truth_path,
- "baseline_dataset": body.baseline_dataset,
- "candidate_dataset": body.candidate_dataset,
- "ground_truth_dataset": body.ground_truth_dataset,
- "crop": body.crop,
- "baseline_channel": body.baseline_channel,
- "candidate_channel": body.candidate_channel,
- "ground_truth_channel": body.ground_truth_channel,
- }
- summary = (
- "Before/after evaluation computed. "
- f"Dice delta: {metrics.get('summary', {}).get('dice_delta')}."
- )
- report_path = body.report_path
- if report_path:
- report_payload = {
- "workflow_id": workflow.id,
- "name": body.name,
- "summary": summary,
- "metrics": metrics,
- "metadata": metadata,
- }
- report_path = write_evaluation_report(report_path, report_payload)
-
- report_artifact = None
- if report_path:
- report_artifact = create_workflow_artifact(
- db,
- workflow_id=workflow.id,
- artifact_type="evaluation_report",
- role="case_study_evidence",
- path=report_path,
- metadata={"source": "computed_evaluation_result"},
- commit=False,
- )
-
- result = WorkflowEvaluationResult(
- workflow_id=workflow.id,
- name=body.name or "before-after-evaluation",
- baseline_run_id=body.baseline_run_id,
- candidate_run_id=body.candidate_run_id,
- model_version_id=body.model_version_id,
- report_artifact_id=report_artifact.id if report_artifact else None,
- report_path=report_path,
- summary=summary,
- metrics_json=encode_json(metrics),
- metadata_json=encode_json(metadata),
- )
- db.add(result)
- db.commit()
- db.refresh(result)
return _evaluation_response(result)
@@ -8947,6 +8890,10 @@ async def create_agent_action(
db: Session = Depends(get_db),
):
workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id)
+ try:
+ definition = validate_agent_proposal(body.action, body.payload)
+ except (ValidationError, ValueError) as exc:
+ raise HTTPException(status_code=422, detail=str(exc)) from exc
summary = body.summary or f"Agent proposed: {body.action}"
event = append_workflow_event(
db,
@@ -8955,7 +8902,17 @@ async def create_agent_action(
event_type="agent.proposal_created",
stage=workflow.stage,
summary=summary,
- payload={"action": body.action, "params": body.payload},
+ payload={
+ "action": body.action,
+ "params": body.payload,
+ "registry": {
+ "action_type": definition.action_type,
+ "risk_level": definition.risk_level,
+ "requires_approval": definition.requires_approval,
+ "execution_owner": definition.execution_owner,
+ "specialist_agent_type": definition.specialist_agent_type,
+ },
+ },
approval_status="pending",
commit=True,
)
@@ -8974,14 +8931,95 @@ async def approve_agent_action(
db: Session = Depends(get_db),
):
workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id)
- proposal = _get_pending_proposal_or_404(
- db, workflow_id=workflow.id, event_id=event_id
+ proposal = (
+ db.query(WorkflowEvent)
+ .filter(
+ WorkflowEvent.id == event_id,
+ WorkflowEvent.workflow_id == workflow.id,
+ WorkflowEvent.event_type == "agent.proposal_created",
+ )
+ .first()
)
+ if proposal is None:
+ raise HTTPException(status_code=404, detail="Agent proposal not found")
action_payload = _proposal_action_payload(proposal)
action = action_payload.get("action")
params = action_payload.get("params", {})
if not isinstance(params, dict):
params = {}
+ if proposal.approval_status == "approved":
+ client_effects = params.get("client_effects")
+ workflow_action = (
+ client_effects.get("workflow_action")
+ if isinstance(client_effects, dict)
+ else None
+ )
+ if (
+ action != "run_client_effects"
+ or not isinstance(workflow_action, dict)
+ or workflow_action.get("kind") != "compute_evaluation"
+ ):
+ raise HTTPException(status_code=400, detail="Agent proposal is not pending")
+ approved = _related_proposal_event(
+ db,
+ workflow_id=workflow.id,
+ proposal_id=proposal.id,
+ event_type="agent.proposal_approved",
+ )
+ staged = _related_proposal_event(
+ db,
+ workflow_id=workflow.id,
+ proposal_id=proposal.id,
+ event_type="evaluation.agent_action_approved",
+ )
+ if approved is None:
+ raise HTTPException(
+ status_code=409,
+ detail="Approved proposal is missing its decision event",
+ )
+ operation = (
+ db.query(WorkflowOperation)
+ .filter(
+ WorkflowOperation.workflow_id == workflow.id,
+ WorkflowOperation.idempotency_key
+ == f"agent-proposal:{proposal.id}:compute_evaluation",
+ )
+ .first()
+ )
+ if operation is None:
+ requested_correlation_id = params.get("correlation_id")
+ operation, receipt = stage_and_execute_compute_evaluation_proposal(
+ db,
+ workflow_id=workflow.id,
+ proposal=proposal,
+ approval_event=approved,
+ workflow_action=workflow_action,
+ user_id=user.id,
+ correlation_id=(
+ requested_correlation_id
+ if isinstance(requested_correlation_id, str)
+ else None
+ ),
+ )
+ else:
+ receipt = execute_compute_evaluation_operation(db, operation)
+ replay_effects = dict(client_effects)
+ replay_effects.pop("workflow_action", None)
+ return AgentActionResult(
+ workflow=_workflow_response(workflow),
+ proposal=_event_response(proposal),
+ events=[
+ _event_response(event)
+ for event in (approved, staged)
+ if event is not None
+ ],
+ client_effects={**replay_effects, "workflow_stage": workflow.stage},
+ commands=[],
+ operation=operation_to_dict(operation),
+ receipt=receipt,
+ )
+ if proposal.approval_status != "pending":
+ raise HTTPException(status_code=400, detail="Agent proposal is not pending")
params, applied_overrides = _apply_agent_action_overrides(
str(action or ""),
params,
@@ -8997,6 +9035,7 @@ async def approve_agent_action(
if action == "start_training_run":
client_effects = _training_run_effects_from_proposal(workflow, params)
+ resolve_agent_action(str(action), client_effects)
corrected_mask_path = (
client_effects.get("set_training_label_path")
or params.get("label_path")
@@ -9080,6 +9119,10 @@ async def approve_agent_action(
status_code=400,
detail="Approved client-effect action is missing client_effects.",
)
+ try:
+ resolve_agent_action(str(action), client_effects)
+ except ValidationError as exc:
+ raise HTTPException(status_code=422, detail=str(exc)) from exc
proposal.approval_status = "approved"
db.commit()
db.refresh(proposal)
@@ -9098,13 +9141,26 @@ async def approve_agent_action(
),
commit=True,
)
+ workflow_action = client_effects.get("workflow_action")
+ server_executes_evaluation = (
+ isinstance(workflow_action, dict)
+ and workflow_action.get("kind") == "compute_evaluation"
+ )
staged = append_workflow_event(
db,
workflow_id=workflow.id,
actor="system",
- event_type="agent.client_effects_approved",
+ event_type=(
+ "evaluation.agent_action_approved"
+ if server_executes_evaluation
+ else "agent.client_effects_approved"
+ ),
stage=workflow.stage,
- summary="Approved in-app assistant action for client execution.",
+ summary=(
+ "Approved agent evaluation action for server execution."
+ if server_executes_evaluation
+ else "Approved in-app assistant action for client execution."
+ ),
payload={
"proposal_event_id": proposal.id,
"item_id": params.get("item_id"),
@@ -9116,12 +9172,37 @@ async def approve_agent_action(
},
commit=True,
)
+ operation_payload = None
+ receipt = None
+ approved_client_effects = dict(client_effects)
+ if server_executes_evaluation:
+ requested_correlation_id = params.get("correlation_id")
+ operation, receipt = stage_and_execute_compute_evaluation_proposal(
+ db,
+ workflow_id=workflow.id,
+ proposal=proposal,
+ approval_event=approved,
+ workflow_action=workflow_action,
+ user_id=user.id,
+ correlation_id=(
+ requested_correlation_id
+ if isinstance(requested_correlation_id, str)
+ else None
+ ),
+ )
+ operation_payload = operation_to_dict(operation)
+ approved_client_effects.pop("workflow_action", None)
return AgentActionResult(
workflow=_workflow_response(workflow),
proposal=_event_response(proposal),
events=[_event_response(approved), _event_response(staged)],
- client_effects={**client_effects, "workflow_stage": workflow.stage},
+ client_effects={
+ **approved_client_effects,
+ "workflow_stage": workflow.stage,
+ },
commands=[],
+ operation=operation_payload,
+ receipt=receipt,
)
corrected_mask_path = (
diff --git a/server_api/workflows/service.py b/server_api/workflows/service.py
index 6745f0c2..0bd16910 100644
--- a/server_api/workflows/service.py
+++ b/server_api/workflows/service.py
@@ -51,6 +51,56 @@ def _initial_project_defaults() -> Dict[str, Any]:
if not INITIAL_PROJECT_ROOT:
return {}
normalized_root = INITIAL_PROJECT_ROOT.lower()
+ if (
+ os.getenv("PYTC_INITIAL_PROJECT_KIND", "").lower() == "synthetic"
+ or "synthetic-core-project" in normalized_root
+ ):
+ image_path = os.getenv(
+ "PYTC_INITIAL_IMAGE_PATH",
+ os.path.join(INITIAL_PROJECT_ROOT, "data/raw/train-01_image.h5"),
+ )
+ label_path = os.getenv(
+ "PYTC_INITIAL_LABEL_PATH",
+ os.path.join(
+ INITIAL_PROJECT_ROOT,
+ "data/seg/ground_truth/train-01_ground_truth.h5",
+ ),
+ )
+ return {
+ "title": os.getenv(
+ "PYTC_INITIAL_PROJECT_TITLE",
+ "Synthetic Segmentation Core Loop",
+ ),
+ "dataset_path": INITIAL_PROJECT_ROOT,
+ "image_path": image_path,
+ "label_path": label_path,
+ "mask_path": os.getenv("PYTC_INITIAL_MASK_PATH", label_path),
+ "config_path": os.getenv(
+ "PYTC_INITIAL_CONFIG_PATH",
+ os.path.join(
+ INITIAL_PROJECT_ROOT,
+ "configs/Synthetic-Core-Loop-BC.yaml",
+ ),
+ ),
+ "metadata": {
+ "created_from": "initial_project_default",
+ "synthetic": True,
+ "project_context": {
+ "imaging_modality": "Synthetic volumetric microscopy",
+ "target_structure": "synthetic organelles",
+ "task_family": "3D instance segmentation",
+ "task_goal": "segmentation, proofreading, retraining, and evaluation",
+ "optimization_priority": "workflow contract and recovery behavior",
+ "mask_status": "mixed: 2 ground-truth masks, 1 draft mask, 1 image-only target",
+ "training_policy": "train only on confirmed ground-truth masks",
+ "image_only_strategy": "run inference on image-only volumes after training",
+ "voxel_size_nm": [40, 8, 8],
+ "voxel_size_source": "project_manifest.json",
+ },
+ "visualization_scales": [40, 8, 8],
+ "visualization_scales_source": "project_manifest.json",
+ },
+ }
if (
"yixiao" in normalized_root
or "tapereader" in normalized_root
diff --git a/server_api/workflows/volume_io.py b/server_api/workflows/volume_io.py
index c2e4b1ba..79fdd1f6 100644
--- a/server_api/workflows/volume_io.py
+++ b/server_api/workflows/volume_io.py
@@ -1,7 +1,9 @@
from __future__ import annotations
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
from pathlib import Path
-from typing import Any, List, Optional, Sequence, Tuple, Union
+from typing import Any, Callable, List, Optional, Sequence, Tuple, Union
import numpy as np
@@ -34,6 +36,128 @@
)
+@dataclass(frozen=True)
+class VolumeMetadata:
+ """Storage-level metadata available without materializing voxel data."""
+
+ path: str
+ format: str
+ shape: Tuple[int, ...]
+ dtype: np.dtype
+ dataset_key: Optional[str] = None
+ chunks: Optional[Tuple[int, ...]] = None
+
+ @property
+ def ndim(self) -> int:
+ return len(self.shape)
+
+
+class VolumeStore(ABC):
+ """A bounded region-reader for a single array inside a volume artifact."""
+
+ @property
+ @abstractmethod
+ def metadata(self) -> VolumeMetadata:
+ raise NotImplementedError
+
+ @property
+ def shape(self) -> Tuple[int, ...]:
+ return self.metadata.shape
+
+ @property
+ def dtype(self) -> np.dtype:
+ return self.metadata.dtype
+
+ @property
+ def ndim(self) -> int:
+ return self.metadata.ndim
+
+ @abstractmethod
+ def read(
+ self,
+ crop: CropSpec = None,
+ *,
+ channel: Optional[int] = None,
+ reference_ndim: Optional[int] = None,
+ label: str = "volume",
+ ) -> np.ndarray:
+ raise NotImplementedError
+
+ def close(self) -> None:
+ """Release resources held by the backing artifact."""
+
+ def __enter__(self) -> "VolumeStore":
+ return self
+
+ def __exit__(self, *_exc_info: Any) -> None:
+ self.close()
+
+
+class ArrayVolumeStore(VolumeStore):
+ """VolumeStore adapter for array-like objects supporting basic indexing."""
+
+ def __init__(
+ self,
+ data: Any,
+ *,
+ path: Path,
+ format: str,
+ dataset_key: Optional[str] = None,
+ close: Optional[Callable[[], None]] = None,
+ ) -> None:
+ self._data = data
+ self._close = close
+ self._closed = False
+ chunks = getattr(data, "chunks", None)
+ self._metadata = VolumeMetadata(
+ path=str(path),
+ format=format,
+ shape=tuple(int(value) for value in data.shape),
+ dtype=np.dtype(data.dtype),
+ dataset_key=dataset_key,
+ chunks=(
+ tuple(int(value) for value in chunks)
+ if chunks is not None and all(value is not None for value in chunks)
+ else None
+ ),
+ )
+
+ @property
+ def metadata(self) -> VolumeMetadata:
+ return self._metadata
+
+ def read(
+ self,
+ crop: CropSpec = None,
+ *,
+ channel: Optional[int] = None,
+ reference_ndim: Optional[int] = None,
+ label: str = "volume",
+ ) -> np.ndarray:
+ if self._closed:
+ raise RuntimeError("Volume store is closed")
+ return _as_array(
+ self._data,
+ parse_crop(crop),
+ channel=channel,
+ reference_ndim=reference_ndim,
+ label=label,
+ )
+
+ def __getitem__(self, key: Any) -> Any:
+ """Expose storage-backed slicing to consumers such as Neuroglancer."""
+ if self._closed:
+ raise RuntimeError("Volume store is closed")
+ return self._data[key]
+
+ def close(self) -> None:
+ if self._closed:
+ return
+ self._closed = True
+ if self._close is not None:
+ self._close()
+
+
def split_dataset_ref(path: str) -> Tuple[str, Optional[str]]:
if "::" not in path:
return path, None
@@ -216,28 +340,6 @@ def _select_h5_dataset(handle: Any, dataset_key: Optional[str]) -> Any:
return handle[datasets[0]]
-def _read_hdf5(
- path: Path,
- dataset_key: Optional[str],
- crop: Optional[Tuple[slice, ...]],
- *,
- channel: Optional[int] = None,
- reference_ndim: Optional[int] = None,
- label: str = "volume",
-) -> np.ndarray:
- import h5py
-
- with h5py.File(path, "r") as handle:
- dataset = _select_h5_dataset(handle, dataset_key)
- return _as_array(
- dataset,
- crop,
- channel=channel,
- reference_ndim=reference_ndim,
- label=label,
- )
-
-
def _is_zarr_array(value: Any) -> bool:
return (
hasattr(value, "shape")
@@ -287,197 +389,194 @@ def _select_zarr_array(store: Any, dataset_key: Optional[str]) -> Any:
return store[arrays[0]]
-def _read_zarr(
- path: Path,
- dataset_key: Optional[str],
- crop: Optional[Tuple[slice, ...]],
- *,
- channel: Optional[int] = None,
- reference_ndim: Optional[int] = None,
- label: str = "volume",
-) -> np.ndarray:
- import zarr
-
- store = zarr.open(str(path), mode="r")
- array = _select_zarr_array(store, dataset_key)
- return _as_array(
- array,
- crop,
- channel=channel,
- reference_ndim=reference_ndim,
- label=label,
- )
-
-
-def _read_npz(
- path: Path,
- dataset_key: Optional[str],
- crop: Optional[Tuple[slice, ...]],
- *,
- channel: Optional[int] = None,
- reference_ndim: Optional[int] = None,
- label: str = "volume",
-) -> np.ndarray:
- with np.load(path) as loaded:
- key = dataset_key
- if not key:
- for candidate in COMMON_DATASET_NAMES:
- if candidate in loaded:
- key = candidate
- break
- if not key:
- keys = list(loaded.keys())
- if not keys:
- raise ValueError("NPZ file does not contain any arrays")
- key = keys[0]
- return _as_array(
- loaded[key],
- crop,
- channel=channel,
- reference_ndim=reference_ndim,
- label=label,
- )
-
-
-def _read_nifti(
- path: Path,
- crop: Optional[Tuple[slice, ...]],
- *,
- channel: Optional[int] = None,
- reference_ndim: Optional[int] = None,
- label: str = "volume",
-) -> np.ndarray:
- try:
- import nibabel as nib
- except Exception as exc: # pragma: no cover - optional dependency
- raise RuntimeError("nibabel is required to read NIfTI volumes") from exc
-
- image = nib.load(str(path))
- data = image.dataobj
- return _as_array(
- data,
- crop,
- channel=channel,
- reference_ndim=reference_ndim,
- label=label,
- )
-
-
-def _read_mrc(
- path: Path,
- crop: Optional[Tuple[slice, ...]],
- *,
- channel: Optional[int] = None,
- reference_ndim: Optional[int] = None,
- label: str = "volume",
-) -> np.ndarray:
- try:
- import mrcfile
- except Exception as exc: # pragma: no cover - optional dependency
- raise RuntimeError("mrcfile is required to read MRC/MAP volumes") from exc
-
- with mrcfile.open(str(path), permissive=True) as handle:
- return _as_array(
- handle.data,
- crop,
- channel=channel,
- reference_ndim=reference_ndim,
- label=label,
- )
-
-
-def load_volume(
+def _close_all(*resources: Any) -> Callable[[], None]:
+ def close() -> None:
+ first_error: Optional[Exception] = None
+ for resource in resources:
+ close_resource = getattr(resource, "close", None)
+ if not callable(close_resource):
+ continue
+ try:
+ close_resource()
+ except Exception as exc: # pragma: no cover - defensive cleanup
+ first_error = first_error or exc
+ if first_error is not None:
+ raise first_error
+
+ return close
+
+
+def _select_npz_array(loaded: Any, dataset_key: Optional[str]) -> Tuple[str, Any]:
+ key = dataset_key
+ if not key:
+ for candidate in COMMON_DATASET_NAMES:
+ if candidate in loaded:
+ key = candidate
+ break
+ if not key:
+ keys = list(loaded.keys())
+ if not keys:
+ raise ValueError("NPZ file does not contain any arrays")
+ key = keys[0]
+ if key not in loaded:
+ raise ValueError(f"NPZ array {key!r} not found")
+ return key, loaded[key]
+
+
+def open_volume_store(
path: str,
*,
dataset_key: Optional[str] = None,
- crop: CropSpec = None,
- channel: Optional[int] = None,
- reference_ndim: Optional[int] = None,
- label: str = "volume",
-) -> np.ndarray:
+) -> VolumeStore:
+ """Open a volume for metadata inspection and bounded region reads.
+
+ Callers should use this as a context manager. HDF5, NPY, Zarr/N5, NIfTI,
+ MRC, and TIFF expose their backing array directly; indexing therefore occurs
+ before NumPy materialization. Formats without random-access support retain an
+ eager compatibility fallback.
+ """
+
file_path, inline_dataset_key = split_dataset_ref(str(path))
dataset_key = dataset_key or inline_dataset_key
target = Path(file_path).expanduser()
if not target.exists():
raise FileNotFoundError(f"Volume artifact does not exist: {target}")
- crop_slices = parse_crop(crop)
lower_name = target.name.lower()
lower_path = str(target).lower()
if lower_name.endswith((".h5", ".hdf5", ".hdf")):
- return _read_hdf5(
- target,
- dataset_key,
- crop_slices,
- channel=channel,
- reference_ndim=reference_ndim,
- label=label,
- )
+ import h5py
+
+ handle = h5py.File(target, "r")
+ try:
+ data = _select_h5_dataset(handle, dataset_key)
+ selected_key = data.name.lstrip("/")
+ return ArrayVolumeStore(
+ data,
+ path=target,
+ format="hdf5",
+ dataset_key=selected_key,
+ close=handle.close,
+ )
+ except Exception:
+ handle.close()
+ raise
+
if lower_name.endswith((".tif", ".tiff", ".ome.tif", ".ome.tiff")):
import tifffile
+ import zarr
+
+ handle = tifffile.TiffFile(str(target))
+ try:
+ tiff_store = handle.series[0].aszarr()
+ data = zarr.open(tiff_store, mode="r")
+ return ArrayVolumeStore(
+ data,
+ path=target,
+ format="ome-tiff" if ".ome.tif" in lower_name else "tiff",
+ close=_close_all(tiff_store, handle),
+ )
+ except Exception:
+ handle.close()
+ return ArrayVolumeStore(
+ tifffile.imread(str(target)),
+ path=target,
+ format="ome-tiff" if ".ome.tif" in lower_name else "tiff",
+ )
- return _as_array(
- tifffile.imread(str(target)),
- crop_slices,
- channel=channel,
- reference_ndim=reference_ndim,
- label=label,
- )
if lower_name.endswith(".npy"):
- return _as_array(
- np.load(target, mmap_mode="r"),
- crop_slices,
- channel=channel,
- reference_ndim=reference_ndim,
- label=label,
+ data = np.load(target, mmap_mode="r")
+ mmap = getattr(data, "_mmap", None)
+ return ArrayVolumeStore(
+ data,
+ path=target,
+ format="npy",
+ close=getattr(mmap, "close", None),
)
+
if lower_name.endswith(".npz"):
- return _read_npz(
- target,
- dataset_key,
- crop_slices,
- channel=channel,
- reference_ndim=reference_ndim,
- label=label,
- )
+ loaded = np.load(target)
+ try:
+ selected_key, data = _select_npz_array(loaded, dataset_key)
+ return ArrayVolumeStore(
+ data,
+ path=target,
+ format="npz",
+ dataset_key=selected_key,
+ close=loaded.close,
+ )
+ except Exception:
+ loaded.close()
+ raise
+
if target.is_dir() or lower_name.endswith((".zarr", ".n5")):
- return _read_zarr(
- target,
- dataset_key,
- crop_slices,
- channel=channel,
- reference_ndim=reference_ndim,
- label=label,
+ import zarr
+
+ root = zarr.open(str(target), mode="r")
+ data = _select_zarr_array(root, dataset_key)
+ selected_key = dataset_key or getattr(data, "path", None) or None
+ return ArrayVolumeStore(
+ data,
+ path=target,
+ format="n5" if lower_name.endswith(".n5") else "zarr",
+ dataset_key=selected_key,
)
+
if lower_path.endswith((".nii", ".nii.gz")):
- return _read_nifti(
- target,
- crop_slices,
- channel=channel,
- reference_ndim=reference_ndim,
- label=label,
+ try:
+ import nibabel as nib
+ except Exception as exc: # pragma: no cover - optional dependency
+ raise RuntimeError("nibabel is required to read NIfTI volumes") from exc
+ image = nib.load(str(target))
+ return ArrayVolumeStore(
+ image.dataobj,
+ path=target,
+ format="nifti",
+ close=getattr(image, "uncache", None),
)
+
if lower_name.endswith((".mrc", ".map", ".rec")):
- return _read_mrc(
- target,
- crop_slices,
- channel=channel,
- reference_ndim=reference_ndim,
- label=label,
+ try:
+ import mrcfile
+ except Exception as exc: # pragma: no cover - optional dependency
+ raise RuntimeError("mrcfile is required to read MRC/MAP volumes") from exc
+ handle = mrcfile.open(str(target), permissive=True)
+ return ArrayVolumeStore(
+ handle.data,
+ path=target,
+ format="mrc",
+ close=handle.close,
)
+
if lower_name.endswith((".png", ".jpg", ".jpeg", ".bmp")):
import imageio.v3 as iio
- return _as_array(
+ return ArrayVolumeStore(
iio.imread(target),
- crop_slices,
- channel=channel,
- reference_ndim=reference_ndim,
- label=label,
+ path=target,
+ format=target.suffix.lower().lstrip("."),
)
raise ValueError(
f"Unsupported volume format for {target}. Supported formats: "
+ "; ".join(SUPPORTED_VOLUME_FORMATS)
)
+
+
+def load_volume(
+ path: str,
+ *,
+ dataset_key: Optional[str] = None,
+ crop: CropSpec = None,
+ channel: Optional[int] = None,
+ reference_ndim: Optional[int] = None,
+ label: str = "volume",
+) -> np.ndarray:
+ with open_volume_store(path, dataset_key=dataset_key) as store:
+ return store.read(
+ crop,
+ channel=channel,
+ reference_ndim=reference_ndim,
+ label=label,
+ )
diff --git a/spikes/__init__.py b/spikes/__init__.py
new file mode 100644
index 00000000..9d527b6f
--- /dev/null
+++ b/spikes/__init__.py
@@ -0,0 +1 @@
+"""Isolated engineering spikes that are not imported by production services."""
diff --git a/spikes/dbos_operation/__init__.py b/spikes/dbos_operation/__init__.py
new file mode 100644
index 00000000..a3f0413f
--- /dev/null
+++ b/spikes/dbos_operation/__init__.py
@@ -0,0 +1 @@
+"""DBOS durable-operation evaluation spike."""
diff --git a/spikes/dbos_operation/runner.py b/spikes/dbos_operation/runner.py
new file mode 100644
index 00000000..b96f1b22
--- /dev/null
+++ b/spikes/dbos_operation/runner.py
@@ -0,0 +1,234 @@
+from __future__ import annotations
+
+import argparse
+import json
+import pathlib
+from typing import Any, Dict, Optional, Sequence
+
+from dbos import DBOS, DBOSClient, SetWorkflowID
+
+from .runtime import (
+ APP_NAME,
+ APP_VERSION,
+ PROGRESS_EVENT,
+ QUEUE_NAME,
+ synthetic_operation,
+)
+
+RESULT_PREFIX = "DBOS_SPIKE_RESULT="
+
+
+def sqlite_database_url(database_path: str) -> str:
+ path = pathlib.Path(database_path).expanduser().resolve()
+ path.parent.mkdir(parents=True, exist_ok=True)
+ return f"sqlite:///{path}"
+
+
+def _workflow_status(database_url: str, workflow_id: str) -> Optional[Dict[str, Any]]:
+ client = DBOSClient(system_database_url=database_url)
+ try:
+ rows = client.list_workflows(workflow_ids=[workflow_id], limit=1)
+ if not rows:
+ return None
+ row = rows[0]
+ return {
+ "workflow_id": row.workflow_id,
+ "status": row.status,
+ "name": row.name,
+ "queue_name": row.queue_name,
+ "executor_id": row.executor_id,
+ "app_version": row.app_version,
+ "recovery_attempts": row.recovery_attempts,
+ "output": row.output,
+ "error": str(row.error) if row.error else None,
+ }
+ finally:
+ client.destroy()
+
+
+def _progress(database_url: str, workflow_id: str) -> Optional[Dict[str, Any]]:
+ client = DBOSClient(system_database_url=database_url)
+ try:
+ value = client.get_event(workflow_id, PROGRESS_EVENT, timeout_seconds=0)
+ return value if isinstance(value, dict) else None
+ finally:
+ client.destroy()
+
+
+def _launch(database_url: str, *, register_queue: bool) -> None:
+ DBOS(
+ config={
+ "name": APP_NAME,
+ "application_version": APP_VERSION,
+ # A single-server replacement process must reuse the executor identity
+ # so startup recovery can claim that executor's interrupted workflows.
+ "executor_id": "pytc-dbos-operation-spike",
+ "system_database_url": database_url,
+ }
+ )
+ DBOS.launch()
+ if register_queue:
+ DBOS.register_queue(
+ QUEUE_NAME,
+ concurrency=1,
+ worker_concurrency=1,
+ polling_interval_sec=0.05,
+ )
+
+
+def _wait_for_result(handle: Any, database_url: str) -> Dict[str, Any]:
+ try:
+ result = handle.get_result()
+ return {"result": result, "cancelled": False}
+ except Exception:
+ status = _workflow_status(database_url, handle.workflow_id)
+ if status and status["status"] == "CANCELLED":
+ return {"result": None, "cancelled": True}
+ raise
+
+
+def execute(args: argparse.Namespace) -> Dict[str, Any]:
+ database_url = sqlite_database_url(args.database)
+ pathlib.Path(args.workspace).expanduser().resolve().mkdir(
+ parents=True, exist_ok=True
+ )
+ _launch(database_url, register_queue=args.register_queue)
+ queue_name = QUEUE_NAME if args.register_queue else f"{QUEUE_NAME}-paused"
+ try:
+ with SetWorkflowID(args.workflow_id):
+ first = DBOS.enqueue_workflow(
+ queue_name,
+ synthetic_operation,
+ str(pathlib.Path(args.workspace).expanduser().resolve()),
+ args.workflow_id,
+ args.correlation_id,
+ args.steps,
+ args.step_runtime,
+ args.inter_step_delay,
+ )
+ duplicate_workflow_id = None
+ if args.duplicate_submission:
+ with SetWorkflowID(args.workflow_id):
+ duplicate = DBOS.enqueue_workflow(
+ queue_name,
+ synthetic_operation,
+ str(pathlib.Path(args.workspace).expanduser().resolve()),
+ args.workflow_id,
+ args.correlation_id,
+ args.steps,
+ args.step_runtime,
+ args.inter_step_delay,
+ )
+ duplicate_workflow_id = duplicate.workflow_id
+
+ wait_result = (
+ _wait_for_result(first, database_url) if args.wait else {"result": None}
+ )
+ return {
+ "workflow_id": first.workflow_id,
+ "duplicate_workflow_id": duplicate_workflow_id,
+ "database_url": database_url,
+ "progress": _progress(database_url, args.workflow_id),
+ "workflow": _workflow_status(database_url, args.workflow_id),
+ **wait_result,
+ }
+ finally:
+ DBOS.destroy(workflow_completion_timeout_sec=1)
+
+
+def recover(args: argparse.Namespace) -> Dict[str, Any]:
+ database_url = sqlite_database_url(args.database)
+ _launch(database_url, register_queue=True)
+ try:
+ handle = DBOS.retrieve_workflow(args.workflow_id)
+ wait_result = _wait_for_result(handle, database_url)
+ return {
+ "workflow_id": handle.workflow_id,
+ "database_url": database_url,
+ "progress": _progress(database_url, args.workflow_id),
+ "workflow": _workflow_status(database_url, args.workflow_id),
+ **wait_result,
+ }
+ finally:
+ DBOS.destroy(workflow_completion_timeout_sec=1)
+
+
+def status(args: argparse.Namespace) -> Dict[str, Any]:
+ database_url = sqlite_database_url(args.database)
+ return {
+ "workflow_id": args.workflow_id,
+ "database_url": database_url,
+ "progress": _progress(database_url, args.workflow_id),
+ "workflow": _workflow_status(database_url, args.workflow_id),
+ }
+
+
+def cancel(args: argparse.Namespace) -> Dict[str, Any]:
+ database_url = sqlite_database_url(args.database)
+ client = DBOSClient(system_database_url=database_url)
+ try:
+ client.cancel_workflow(args.workflow_id)
+ finally:
+ client.destroy()
+ return status(args)
+
+
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(
+ description="Run the isolated DBOS durable-operation spike."
+ )
+ subparsers = parser.add_subparsers(dest="command", required=True)
+
+ execute_parser = subparsers.add_parser(
+ "execute", help="Idempotently enqueue a synthetic operation."
+ )
+ execute_parser.add_argument("--database", required=True)
+ execute_parser.add_argument("--workspace", required=True)
+ execute_parser.add_argument("--workflow-id", required=True)
+ execute_parser.add_argument("--correlation-id", default="dbos-spike")
+ execute_parser.add_argument("--steps", type=int, default=3)
+ execute_parser.add_argument("--step-runtime", type=float, default=0.01)
+ execute_parser.add_argument("--inter-step-delay", type=float, default=0.05)
+ execute_parser.add_argument(
+ "--register-queue", action=argparse.BooleanOptionalAction, default=True
+ )
+ execute_parser.add_argument(
+ "--duplicate-submission", action="store_true", default=False
+ )
+ execute_parser.add_argument(
+ "--wait", action=argparse.BooleanOptionalAction, default=True
+ )
+ execute_parser.set_defaults(handler=execute)
+
+ recover_parser = subparsers.add_parser(
+ "recover", help="Launch a new executor and wait for recovered work."
+ )
+ recover_parser.add_argument("--database", required=True)
+ recover_parser.add_argument("--workflow-id", required=True)
+ recover_parser.set_defaults(handler=recover)
+
+ status_parser = subparsers.add_parser(
+ "status", help="Read durable status and progress without launching a worker."
+ )
+ status_parser.add_argument("--database", required=True)
+ status_parser.add_argument("--workflow-id", required=True)
+ status_parser.set_defaults(handler=status)
+
+ cancel_parser = subparsers.add_parser(
+ "cancel", help="Cancel queued or running work through DBOSClient."
+ )
+ cancel_parser.add_argument("--database", required=True)
+ cancel_parser.add_argument("--workflow-id", required=True)
+ cancel_parser.set_defaults(handler=cancel)
+ return parser
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+ args = build_parser().parse_args(argv)
+ result = args.handler(args)
+ print(RESULT_PREFIX + json.dumps(result, sort_keys=True), flush=True)
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/spikes/dbos_operation/runtime.py b/spikes/dbos_operation/runtime.py
new file mode 100644
index 00000000..42130c6d
--- /dev/null
+++ b/spikes/dbos_operation/runtime.py
@@ -0,0 +1,124 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import pathlib
+import time
+from typing import Any, Dict
+
+from dbos import DBOS
+
+APP_NAME = "pytc-dbos-operation-spike"
+APP_VERSION = "pytc-dbos-operation-spike-v1"
+QUEUE_NAME = "synthetic-operations"
+PROGRESS_EVENT = "operation_progress"
+
+
+def workflow_storage_key(workflow_id: str) -> str:
+ return hashlib.sha256(workflow_id.encode("utf-8")).hexdigest()[:20]
+
+
+def marker_directory(workspace: str, workflow_id: str) -> pathlib.Path:
+ return pathlib.Path(workspace) / "markers" / workflow_storage_key(workflow_id)
+
+
+def _append_json_line(path: pathlib.Path, payload: Dict[str, Any]) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ encoded = json.dumps(payload, sort_keys=True) + "\n"
+ with path.open("a", encoding="utf-8") as stream:
+ stream.write(encoded)
+ stream.flush()
+ os.fsync(stream.fileno())
+
+
+@DBOS.step(name="pytc.synthetic_operation_step.v1")
+def execute_synthetic_step(
+ workspace: str,
+ workflow_id: str,
+ step_index: int,
+ step_runtime_seconds: float,
+) -> Dict[str, Any]:
+ """Perform an idempotent external effect representative of a compute chunk."""
+
+ markers = marker_directory(workspace, workflow_id)
+ markers.mkdir(parents=True, exist_ok=True)
+ marker = markers / f"step-{step_index:04d}.json"
+ attempt = {
+ "workflow_id": workflow_id,
+ "step_index": step_index,
+ "pid": os.getpid(),
+ "attempted_at": time.time(),
+ }
+ try:
+ with marker.open("x", encoding="utf-8") as stream:
+ stream.write(json.dumps(attempt, sort_keys=True))
+ stream.flush()
+ os.fsync(stream.fileno())
+ effect = "created"
+ except FileExistsError:
+ effect = "already_exists"
+ _append_json_line(markers / "duplicate-attempts.jsonl", attempt)
+
+ if step_runtime_seconds > 0:
+ time.sleep(step_runtime_seconds)
+ return {
+ "step_index": step_index,
+ "effect": effect,
+ "marker": str(marker),
+ }
+
+
+@DBOS.workflow(name="pytc.synthetic_operation.v1")
+def synthetic_operation(
+ workspace: str,
+ workflow_id: str,
+ correlation_id: str,
+ total_steps: int,
+ step_runtime_seconds: float,
+ inter_step_delay_seconds: float,
+) -> Dict[str, Any]:
+ if total_steps < 1:
+ raise ValueError("total_steps must be positive")
+
+ progress: Dict[str, Any] = {
+ "status": "running",
+ "workflow_id": workflow_id,
+ "correlation_id": correlation_id,
+ "completed_steps": 0,
+ "total_steps": total_steps,
+ "progress": 0.0,
+ }
+ DBOS.set_event(PROGRESS_EVENT, progress)
+
+ step_results = []
+ for step_index in range(total_steps):
+ step_results.append(
+ execute_synthetic_step(
+ workspace,
+ workflow_id,
+ step_index,
+ step_runtime_seconds,
+ )
+ )
+ completed_steps = step_index + 1
+ progress = {
+ **progress,
+ "completed_steps": completed_steps,
+ "progress": completed_steps / total_steps,
+ }
+ DBOS.set_event(PROGRESS_EVENT, progress)
+ if completed_steps < total_steps and inter_step_delay_seconds > 0:
+ DBOS.sleep(inter_step_delay_seconds)
+
+ result = {
+ "status": "succeeded",
+ "workflow_id": workflow_id,
+ "correlation_id": correlation_id,
+ "completed_steps": total_steps,
+ "total_steps": total_steps,
+ "progress": 1.0,
+ "step_results": step_results,
+ }
+ DBOS.set_event(PROGRESS_EVENT, result)
+ return result
diff --git a/tests/test_agent_action_envelopes.py b/tests/test_agent_action_envelopes.py
new file mode 100644
index 00000000..81eb008b
--- /dev/null
+++ b/tests/test_agent_action_envelopes.py
@@ -0,0 +1,307 @@
+from datetime import datetime, timezone
+
+import pytest
+from pydantic import ValidationError
+
+from server_api.workflows.agent_actions import (
+ ALL_ACTION_DEFINITIONS,
+ action_envelope_json_schema,
+ action_receipt_json_schema,
+ canonical_action_policy,
+ dump_action_envelope_json,
+ dump_action_receipt_json,
+ load_action_envelope_json,
+ load_action_receipt_json,
+ validate_action_for_execution,
+ validate_action_envelope,
+ validate_action_receipt,
+ validate_receipt_for_action,
+)
+
+
+def _envelope(kind="start_training", **overrides):
+ definition = ALL_ACTION_DEFINITIONS[kind]
+ payload = {
+ "action_id": f"action-{kind}",
+ "kind": kind,
+ "workflow_id": 17,
+ "requested_by": "agent",
+ "idempotency_key": f"workflow:17:{kind}:v1",
+ "correlation_id": "request-4c0d",
+ "execution_owner": definition.execution_owner,
+ "policy": {
+ "risk_level": definition.risk_level,
+ "requires_approval": definition.requires_approval,
+ },
+ "approval": {
+ "status": "pending" if definition.requires_approval else "not_required"
+ },
+ "input_artifacts": [
+ {
+ "artifact_id": 9,
+ "artifact_type": "volume",
+ "role": "training_image",
+ "uri": "zarr:///datasets/mito25/image",
+ "checksum": "sha256:image-v1",
+ }
+ ],
+ "expected_output_artifacts": [
+ {
+ "logical_name": "candidate-checkpoint",
+ "artifact_type": "model_checkpoint",
+ "role": "candidate",
+ }
+ ],
+ "preconditions": [
+ {
+ "kind": "workflow_stage",
+ "allowed_stages": ["proofreading", "retraining_staged"],
+ },
+ {
+ "kind": "artifact_available",
+ "artifact": {"artifact_id": 9},
+ },
+ ],
+ **overrides,
+ }
+ return payload
+
+
+def _receipt(kind="start_training", **overrides):
+ action_fields = {
+ "choose_project_data": {
+ "selected_artifacts": [{"artifact_id": 2}],
+ },
+ "load_visualization": {"viewer_url": "https://viewer.test/v/1"},
+ "start_inference": {"run_id": "inference-1"},
+ "stop_inference": {"stopped": True},
+ "start_proofreading": {"proofreading_session_id": 8},
+ "start_training": {"run_id": "training-1"},
+ "stop_training": {"stopped": True},
+ "compute_evaluation": {"evaluation_result_id": 11},
+ "export_bundle": {
+ "bundle_artifact": {
+ "artifact_id": 12,
+ "artifact_type": "evidence_bundle",
+ }
+ },
+ "propose_retraining_stage": {"workflow_event_id": 13},
+ }
+ return {
+ "receipt_id": f"receipt-{kind}",
+ "action_id": f"action-{kind}",
+ "kind": kind,
+ "workflow_id": 17,
+ "idempotency_key": f"workflow:17:{kind}:v1",
+ "correlation_id": "request-4c0d",
+ "status": "succeeded",
+ "operation_id": 31,
+ "produced_artifacts": [{"logical_name": "output"}],
+ "started_at": datetime(2026, 7, 21, 13, 0, tzinfo=timezone.utc),
+ "completed_at": datetime(2026, 7, 21, 13, 5, tzinfo=timezone.utc),
+ **action_fields[kind],
+ **overrides,
+ }
+
+
+@pytest.mark.parametrize("kind", sorted(ALL_ACTION_DEFINITIONS))
+def test_every_registered_core_action_has_a_discriminated_envelope(kind):
+ envelope = validate_action_envelope(_envelope(kind))
+
+ assert envelope.kind == kind
+ assert envelope.policy == canonical_action_policy(kind)
+ assert envelope.execution_owner == ALL_ACTION_DEFINITIONS[kind].execution_owner
+
+
+def test_action_envelope_schema_exposes_kind_discriminator_and_core_mappings():
+ schema = action_envelope_json_schema()
+
+ assert schema["discriminator"]["propertyName"] == "kind"
+ assert set(schema["discriminator"]["mapping"]) == set(ALL_ACTION_DEFINITIONS)
+ assert len(schema["oneOf"]) == len(ALL_ACTION_DEFINITIONS)
+
+
+def test_action_receipt_schema_matches_action_discriminator():
+ schema = action_receipt_json_schema()
+
+ assert schema["discriminator"]["propertyName"] == "kind"
+ assert set(schema["discriminator"]["mapping"]) == set(ALL_ACTION_DEFINITIONS)
+
+
+def test_action_envelope_json_round_trip_preserves_typed_boundary_fields():
+ payload = _envelope(
+ "start_training",
+ approval={
+ "status": "approved",
+ "event_id": 21,
+ "decided_by": "user:3",
+ "decided_at": datetime(2026, 7, 21, 12, 59, tzinfo=timezone.utc),
+ },
+ autopick_parameters=True,
+ parameter_mode="agent_default",
+ volume_subset={
+ "selection_basis": "project_progress",
+ "training_statuses": ["ground_truth"],
+ "train_volume_count": 2,
+ "target_volume_count": 1,
+ "review_volume_count": 0,
+ "manifest_path": "/datasets/mito25/volume_subset_manifest.json",
+ },
+ )
+
+ original = validate_action_envelope(payload)
+ restored = load_action_envelope_json(dump_action_envelope_json(original))
+
+ assert type(restored) is type(original)
+ assert restored.model_dump() == original.model_dump()
+ assert restored.input_artifacts[0].artifact_id == 9
+ assert restored.preconditions[1].kind == "artifact_available"
+
+
+def test_action_receipt_json_round_trip_preserves_typed_result():
+ original = validate_action_receipt(
+ _receipt(
+ "compute_evaluation",
+ metrics={"iou": 0.84, "dice": 0.91},
+ )
+ )
+ restored = load_action_receipt_json(dump_action_receipt_json(original))
+
+ assert type(restored) is type(original)
+ assert restored.model_dump() == original.model_dump()
+ assert restored.evaluation_result_id == 11
+ assert restored.metrics["dice"] == pytest.approx(0.91)
+
+
+def test_envelope_rejects_policy_or_execution_owner_spoofing():
+ with pytest.raises(ValidationError, match="risk_level for start_training"):
+ validate_action_envelope(
+ _envelope(
+ policy={
+ "risk_level": "read_only",
+ "requires_approval": True,
+ }
+ )
+ )
+
+ with pytest.raises(ValidationError, match="execution_owner for start_training"):
+ validate_action_envelope(_envelope(execution_owner="browser_navigation"))
+
+
+def test_envelope_rejects_invalid_approval_evidence_and_status():
+ with pytest.raises(ValidationError, match="require event_id and decided_by"):
+ validate_action_envelope(
+ _envelope(approval={"status": "approved", "event_id": 9})
+ )
+
+ with pytest.raises(ValidationError, match="does not accept an approval decision"):
+ validate_action_envelope(
+ _envelope("choose_project_data", approval={"status": "pending"})
+ )
+
+
+def test_execution_gate_requires_approved_envelope_for_risky_actions():
+ with pytest.raises(ValueError, match="cannot execute without an approved"):
+ validate_action_for_execution(_envelope("start_training"))
+
+ approved = _envelope(
+ "start_training",
+ approval={
+ "status": "approved",
+ "event_id": 23,
+ "decided_by": "user:4",
+ },
+ )
+ assert validate_action_for_execution(approved).kind == "start_training"
+ assert validate_action_for_execution(_envelope("choose_project_data")).kind == (
+ "choose_project_data"
+ )
+
+
+def test_envelope_rejects_untyped_artifacts_preconditions_and_action_fields():
+ with pytest.raises(ValidationError, match="artifact reference requires"):
+ validate_action_envelope(_envelope(input_artifacts=[{"role": "image"}]))
+
+ with pytest.raises(ValidationError, match="precondition"):
+ validate_action_envelope(
+ _envelope(preconditions=[{"kind": "shell_exit_code", "value": 0}])
+ )
+
+ with pytest.raises(ValidationError):
+ validate_action_envelope(_envelope(shell_command="rm -rf /"))
+
+ with pytest.raises(ValidationError):
+ validate_action_envelope(_envelope(volume_subset={"shell_command": "rm -rf /"}))
+
+
+def test_failed_receipt_requires_typed_error_and_terminal_timestamp():
+ failed_payload = _receipt(
+ "start_inference",
+ status="failed",
+ run_id=None,
+ error={
+ "code": "worker_unavailable",
+ "message": "The inference worker did not respond.",
+ "retryable": True,
+ "details": {"status_code": 503},
+ },
+ )
+ receipt = validate_action_receipt(failed_payload)
+
+ assert receipt.error.code == "worker_unavailable"
+ assert receipt.error.retryable is True
+
+ with pytest.raises(ValidationError, match="require an error"):
+ validate_action_receipt({**failed_payload, "error": None})
+
+ with pytest.raises(ValidationError, match="require completed_at"):
+ validate_action_receipt({**failed_payload, "completed_at": None})
+
+ with pytest.raises(ValidationError, match="cannot precede started_at"):
+ validate_action_receipt(
+ {
+ **failed_payload,
+ "completed_at": datetime(2026, 7, 21, 12, 0, tzinfo=timezone.utc),
+ }
+ )
+
+
+def test_receipt_must_match_action_identity_and_correlation():
+ action = _envelope(
+ "start_training",
+ approval={
+ "status": "approved",
+ "event_id": 23,
+ "decided_by": "user:4",
+ },
+ )
+ receipt = _receipt("start_training")
+
+ assert validate_receipt_for_action(action, receipt).run_id == "training-1"
+ with pytest.raises(ValueError, match="correlation_id"):
+ validate_receipt_for_action(
+ action,
+ {**receipt, "correlation_id": "different-request"},
+ )
+
+
+@pytest.mark.parametrize("kind", sorted(ALL_ACTION_DEFINITIONS))
+def test_succeeded_receipts_require_action_specific_result(kind):
+ receipt = validate_action_receipt(_receipt(kind))
+ assert receipt.kind == kind
+
+ result_field = {
+ "choose_project_data": "selected_artifacts",
+ "load_visualization": "viewer_url",
+ "start_inference": "run_id",
+ "stop_inference": "stopped",
+ "start_proofreading": "proofreading_session_id",
+ "start_training": "run_id",
+ "stop_training": "stopped",
+ "compute_evaluation": "evaluation_result_id",
+ "export_bundle": "bundle_artifact",
+ "propose_retraining_stage": "workflow_event_id",
+ }[kind]
+ missing_value = [] if result_field == "selected_artifacts" else None
+ with pytest.raises(ValidationError, match=f"require {result_field}"):
+ validate_action_receipt(_receipt(kind, **{result_field: missing_value}))
diff --git a/tests/test_agent_action_registry.py b/tests/test_agent_action_registry.py
new file mode 100644
index 00000000..c48077a0
--- /dev/null
+++ b/tests/test_agent_action_registry.py
@@ -0,0 +1,99 @@
+import pytest
+from pydantic import ValidationError
+
+from server_api.workflows.agent_actions import (
+ resolve_agent_action,
+ validate_agent_proposal,
+)
+
+
+def test_runtime_action_registry_owns_risk_and_approval_policy():
+ action = resolve_agent_action(
+ "launch-model",
+ {
+ "navigate_to": "training",
+ "runtime_action": {"kind": "start_training"},
+ },
+ )
+
+ assert action.action_type == "start_training"
+ assert action.risk_level == "runs_job"
+ assert action.requires_approval is True
+ assert action.execution_owner == "server_runtime"
+ assert action.specialist_agent_type == "training_agent"
+
+
+def test_navigation_action_remains_browser_owned_and_read_only():
+ action = resolve_agent_action("show-files", {"navigate_to": "files"})
+
+ assert action.action_type == "open_files"
+ assert action.risk_level == "read_only"
+ assert action.requires_approval is False
+ assert action.execution_owner == "browser_navigation"
+ assert action.specialist_agent_type == "data_agent"
+
+
+def test_prefill_action_is_typed_without_becoming_domain_execution():
+ action = resolve_agent_action(
+ "prepare-training",
+ {
+ "navigate_to": "training",
+ "set_training_image_path": "/data/image.zarr",
+ "set_training_label_path": "/data/labels.zarr",
+ },
+ )
+
+ assert action.risk_level == "prefills_form"
+ assert action.requires_approval is False
+ assert action.execution_owner == "browser_navigation"
+
+
+def test_unknown_client_effect_cannot_bypass_registry_validation():
+ with pytest.raises(ValidationError):
+ resolve_agent_action("mystery", {"silently_delete_project": True})
+
+
+def test_registered_runtime_kind_rejects_unregistered_parameters():
+ with pytest.raises(ValidationError):
+ resolve_agent_action(
+ "launch-model",
+ {
+ "runtime_action": {
+ "kind": "start_inference",
+ "shell_command": "rm -rf /",
+ }
+ },
+ )
+
+
+def test_registered_workflow_kind_rejects_unregistered_parameters():
+ with pytest.raises(ValidationError):
+ resolve_agent_action(
+ "export",
+ {
+ "workflow_action": {
+ "kind": "export_bundle",
+ "destination": "/unapproved/path",
+ }
+ },
+ )
+
+
+def test_persisted_client_effect_proposal_is_validated_before_approval():
+ action = validate_agent_proposal(
+ "run_client_effects",
+ {
+ "client_effects": {
+ "navigate_to": "inference",
+ "runtime_action": {"kind": "start_inference"},
+ }
+ },
+ )
+
+ assert action.action_type == "start_inference"
+ assert action.execution_owner == "server_runtime"
+
+
+def test_unknown_persisted_proposal_type_is_rejected():
+ with pytest.raises(ValueError, match="Unsupported agent proposal action"):
+ validate_agent_proposal("invented_action", {})
diff --git a/tests/test_agent_evaluation_execution.py b/tests/test_agent_evaluation_execution.py
new file mode 100644
index 00000000..3f89d419
--- /dev/null
+++ b/tests/test_agent_evaluation_execution.py
@@ -0,0 +1,261 @@
+import pathlib
+import tempfile
+import unittest
+
+import numpy as np
+import pytest
+
+pytest.importorskip("sqlalchemy")
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+pytest.importorskip("fastapi")
+pytest.importorskip("tifffile")
+import tifffile
+from fastapi.testclient import TestClient
+
+from server_api.auth import database as auth_database
+from server_api.auth import models
+from server_api.main import app as server_api_app
+
+
+class AgentEvaluationExecutionTests(unittest.TestCase):
+ def setUp(self):
+ self.temp_dir = tempfile.TemporaryDirectory()
+ self.root = pathlib.Path(self.temp_dir.name)
+ self.db_path = self.root / "agent-evaluation.db"
+ self.engine = create_engine(
+ f"sqlite:///{self.db_path}", connect_args={"check_same_thread": False}
+ )
+ self.SessionLocal = sessionmaker(
+ autocommit=False, autoflush=False, bind=self.engine
+ )
+ models.Base.metadata.create_all(bind=self.engine)
+
+ def override_get_db():
+ db = self.SessionLocal()
+ try:
+ yield db
+ finally:
+ db.close()
+
+ server_api_app.dependency_overrides[auth_database.get_db] = override_get_db
+ self.client = TestClient(server_api_app)
+ response = self.client.get("/api/workflows/current")
+ self.assertEqual(response.status_code, 200)
+ self.workflow_id = response.json()["workflow"]["id"]
+
+ def tearDown(self):
+ server_api_app.dependency_overrides.clear()
+ self.engine.dispose()
+ self.temp_dir.cleanup()
+
+ def _volume_paths(self):
+ ground_truth_path = self.root / "ground-truth.tif"
+ baseline_path = self.root / "baseline.tif"
+ candidate_path = self.root / "candidate.tif"
+ ground_truth = np.zeros((1, 4, 4), dtype=np.uint8)
+ ground_truth[:, 1:3, 1:3] = 1
+ baseline = np.zeros_like(ground_truth)
+ baseline[:, 1:2, 1:2] = 1
+ tifffile.imwrite(str(ground_truth_path), ground_truth)
+ tifffile.imwrite(str(baseline_path), baseline)
+ tifffile.imwrite(str(candidate_path), ground_truth)
+ return baseline_path, candidate_path, ground_truth_path
+
+ def _workflow_action(self, **overrides):
+ baseline, candidate, ground_truth = self._volume_paths()
+ action = {
+ "kind": "compute_evaluation",
+ "name": "agent-before-after",
+ "baseline_prediction_path": str(baseline),
+ "candidate_prediction_path": str(candidate),
+ "ground_truth_path": str(ground_truth),
+ "report_path": str(self.root / "agent-evaluation-report.json"),
+ "metadata": {"source": "typed-agent-action-test"},
+ }
+ action.update(overrides)
+ return action
+
+ def _propose(self, workflow_action, correlation_id="proposal-correlation-7"):
+ response = self.client.post(
+ f"/api/workflows/{self.workflow_id}/agent-actions",
+ json={
+ "action": "run_client_effects",
+ "summary": "Compute approved before/after evaluation.",
+ "payload": {
+ "correlation_id": correlation_id,
+ "client_effects": {
+ "navigate_to": "project-progress",
+ "workflow_action": workflow_action,
+ },
+ },
+ },
+ )
+ self.assertEqual(response.status_code, 200, response.text)
+ return response.json()
+
+ def _envelope(self, proposal_id, workflow_action, approval):
+ return {
+ "action_id": f"proposal:{proposal_id}:compute_evaluation",
+ "kind": "compute_evaluation",
+ "workflow_id": self.workflow_id,
+ "requested_by": "agent",
+ "idempotency_key": f"agent-proposal:{proposal_id}:compute_evaluation",
+ "correlation_id": "proposal-correlation-7",
+ "execution_owner": "server_workflow",
+ "policy": {
+ "risk_level": "writes_workflow_record",
+ "requires_approval": True,
+ },
+ "approval": approval,
+ **{key: value for key, value in workflow_action.items() if key != "kind"},
+ }
+
+ def test_approval_executes_once_and_returns_terminal_typed_evidence(self):
+ workflow_action = self._workflow_action()
+ proposal = self._propose(workflow_action)
+
+ approved = self.client.post(
+ f"/api/workflows/{self.workflow_id}/agent-actions/"
+ f"{proposal['id']}/approve"
+ )
+ self.assertEqual(approved.status_code, 200, approved.text)
+ payload = approved.json()
+
+ receipt = payload["receipt"]
+ operation = payload["operation"]
+ approval_event = payload["events"][0]
+ self.assertEqual(receipt["status"], "succeeded")
+ self.assertEqual(receipt["kind"], "compute_evaluation")
+ self.assertEqual(receipt["correlation_id"], "proposal-correlation-7")
+ self.assertEqual(receipt["operation_id"], operation["id"])
+ self.assertEqual(operation["status"], "succeeded")
+ self.assertEqual(operation["correlation_id"], receipt["correlation_id"])
+ self.assertEqual(
+ operation["result"]["receipt"]["evaluation_result_id"],
+ receipt["evaluation_result_id"],
+ )
+ self.assertEqual(approval_event["event_type"], "agent.proposal_approved")
+ self.assertEqual(
+ payload["events"][1]["event_type"],
+ "evaluation.agent_action_approved",
+ )
+ self.assertIn("server execution", payload["events"][1]["summary"])
+ self.assertEqual(
+ operation["input"]["approval"]["event_id"], approval_event["id"]
+ )
+ self.assertNotEqual(approval_event["id"], proposal["id"])
+ self.assertNotIn("workflow_action", payload["client_effects"])
+ self.assertTrue(pathlib.Path(workflow_action["report_path"]).exists())
+
+ approval_retry = self.client.post(
+ f"/api/workflows/{self.workflow_id}/agent-actions/"
+ f"{proposal['id']}/approve"
+ )
+ self.assertEqual(approval_retry.status_code, 200, approval_retry.text)
+ self.assertEqual(approval_retry.json()["operation"]["id"], operation["id"])
+ self.assertEqual(approval_retry.json()["receipt"], receipt)
+ self.assertNotIn("workflow_action", approval_retry.json()["client_effects"])
+
+ stage_url = f"/api/workflows/{self.workflow_id}/action-operations"
+ duplicate_stage = self.client.post(stage_url, json=operation["input"])
+ self.assertEqual(duplicate_stage.status_code, 202, duplicate_stage.text)
+ self.assertEqual(duplicate_stage.json()["id"], operation["id"])
+ tampered_stage = self.client.post(
+ stage_url,
+ json={
+ **operation["input"],
+ "baseline_prediction_path": str(self.root / "different-baseline.tif"),
+ },
+ )
+ self.assertEqual(tampered_stage.status_code, 409)
+ self.assertIn("parameters do not match", tampered_stage.json()["detail"])
+
+ execute_url = (
+ f"/api/workflows/{self.workflow_id}/action-operations/"
+ f"{operation['id']}/execute"
+ )
+ first_retry = self.client.post(execute_url)
+ second_retry = self.client.post(execute_url)
+ self.assertEqual(first_retry.status_code, 200, first_retry.text)
+ self.assertEqual(second_retry.status_code, 200, second_retry.text)
+ self.assertEqual(first_retry.json(), receipt)
+ self.assertEqual(second_retry.json(), receipt)
+
+ evaluations = self.client.get(
+ f"/api/workflows/{self.workflow_id}/evaluation-results"
+ )
+ self.assertEqual(evaluations.status_code, 200)
+ self.assertEqual(len(evaluations.json()), 1)
+
+ def test_pending_or_missing_approval_cannot_stage_evaluation(self):
+ workflow_action = self._workflow_action()
+ proposal = self._propose(workflow_action)
+ stage_url = f"/api/workflows/{self.workflow_id}/action-operations"
+
+ pending = self.client.post(
+ stage_url,
+ json=self._envelope(proposal["id"], workflow_action, {"status": "pending"}),
+ )
+ self.assertEqual(pending.status_code, 409)
+ self.assertIn("approved", pending.json()["detail"])
+
+ fabricated = self.client.post(
+ stage_url,
+ json=self._envelope(
+ proposal["id"],
+ workflow_action,
+ {
+ "status": "approved",
+ "event_id": proposal["id"],
+ "decided_by": "user:1",
+ },
+ ),
+ )
+ self.assertEqual(fabricated.status_code, 409)
+ self.assertIn("persisted and approved", fabricated.json()["detail"])
+
+ def test_handler_failure_is_terminal_correlated_and_idempotent(self):
+ workflow_action = self._workflow_action(
+ baseline_prediction_path=str(self.root / "missing-baseline.tif"),
+ report_path=None,
+ )
+ proposal = self._propose(
+ workflow_action, correlation_id="proposal-correlation-failure"
+ )
+
+ approved = self.client.post(
+ f"/api/workflows/{self.workflow_id}/agent-actions/"
+ f"{proposal['id']}/approve"
+ )
+ self.assertEqual(approved.status_code, 200, approved.text)
+ payload = approved.json()
+ receipt = payload["receipt"]
+ operation = payload["operation"]
+
+ self.assertEqual(receipt["status"], "failed")
+ self.assertEqual(receipt["error"]["code"], "evaluation_input_invalid")
+ self.assertEqual(receipt["correlation_id"], "proposal-correlation-failure")
+ self.assertEqual(operation["status"], "failed")
+ self.assertEqual(operation["result"]["receipt"], receipt)
+ self.assertEqual(operation["error"]["code"], receipt["error"]["code"])
+ self.assertNotIn("workflow_action", payload["client_effects"])
+
+ execute_url = (
+ f"/api/workflows/{self.workflow_id}/action-operations/"
+ f"{operation['id']}/execute"
+ )
+ duplicate = self.client.post(execute_url)
+ self.assertEqual(duplicate.status_code, 200, duplicate.text)
+ self.assertEqual(duplicate.json(), receipt)
+
+ evaluations = self.client.get(
+ f"/api/workflows/{self.workflow_id}/evaluation-results"
+ )
+ self.assertEqual(evaluations.status_code, 200)
+ self.assertEqual(evaluations.json(), [])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_browser_synthetic_core_smoke.py b/tests/test_browser_synthetic_core_smoke.py
new file mode 100644
index 00000000..051d6482
--- /dev/null
+++ b/tests/test_browser_synthetic_core_smoke.py
@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import json
+
+from scripts import browser_synthetic_core_smoke as smoke
+
+
+def test_parser_defaults_target_local_synthetic_app() -> None:
+ args = smoke._build_arg_parser().parse_args([])
+
+ assert args.base_url == smoke.DEFAULT_BASE_URL
+ assert args.report == smoke.DEFAULT_REPORT
+ assert args.viewport == "1280x900"
+ assert not args.skip_reload
+
+
+def test_main_records_a_successful_smoke(monkeypatch, tmp_path) -> None:
+ monkeypatch.setattr(
+ smoke,
+ "run_smoke",
+ lambda **kwargs: {
+ "passed": True,
+ "base_url": kwargs["base_url"],
+ "checks": ["bounded file picker navigated"],
+ },
+ )
+ report_path = tmp_path / "report.json"
+
+ result = smoke.main(["--skip-reload", "--report", str(report_path)])
+
+ assert result == 0
+ report = json.loads(report_path.read_text(encoding="utf-8"))
+ assert report["passed"] is True
+ assert report["checks"] == ["bounded file picker navigated"]
+
+
+def test_main_records_browser_failures(monkeypatch, tmp_path) -> None:
+ monkeypatch.setattr(
+ smoke,
+ "run_smoke",
+ lambda **_kwargs: (_ for _ in ()).throw(AssertionError("picker failed")),
+ )
+ report_path = tmp_path / "failed.json"
+
+ result = smoke.main(["--report", str(report_path)])
+
+ assert result == 1
+ report = json.loads(report_path.read_text(encoding="utf-8"))
+ assert report["passed"] is False
+ assert report["error"] == "picker failed"
diff --git a/tests/test_dbos_operation_spike.py b/tests/test_dbos_operation_spike.py
new file mode 100644
index 00000000..4ef0bf04
--- /dev/null
+++ b/tests/test_dbos_operation_spike.py
@@ -0,0 +1,268 @@
+import hashlib
+import json
+import pathlib
+import subprocess
+import sys
+import time
+
+import pytest
+
+dbos = pytest.importorskip("dbos")
+from dbos import DBOSClient
+
+RESULT_PREFIX = "DBOS_SPIKE_RESULT="
+RUNNER_MODULE = "spikes.dbos_operation.runner"
+PROGRESS_EVENT = "operation_progress"
+
+
+def _command(*args):
+ return [sys.executable, "-m", RUNNER_MODULE, *map(str, args)]
+
+
+def _parse_result(output):
+ for line in reversed(output.splitlines()):
+ if line.startswith(RESULT_PREFIX):
+ return json.loads(line[len(RESULT_PREFIX) :])
+ raise AssertionError(f"Spike runner did not emit a result: {output[-2000:]}")
+
+
+def _run(*args, timeout=30):
+ completed = subprocess.run(
+ _command(*args),
+ check=True,
+ capture_output=True,
+ text=True,
+ timeout=timeout,
+ )
+ return _parse_result(completed.stdout)
+
+
+def _database_url(path):
+ return f"sqlite:///{path.resolve()}"
+
+
+def _client_state(database, workflow_id):
+ client = DBOSClient(system_database_url=_database_url(database))
+ try:
+ rows = client.list_workflows(workflow_ids=[workflow_id], limit=1)
+ status = rows[0].status if rows else None
+ progress = client.get_event(
+ workflow_id,
+ PROGRESS_EVENT,
+ timeout_seconds=0,
+ )
+ return status, progress
+ finally:
+ client.destroy()
+
+
+def _wait_for(database, workflow_id, predicate, timeout=15):
+ deadline = time.monotonic() + timeout
+ last_state = (None, None)
+ while time.monotonic() < deadline:
+ try:
+ last_state = _client_state(database, workflow_id)
+ except Exception:
+ time.sleep(0.05)
+ continue
+ if predicate(*last_state):
+ return last_state
+ time.sleep(0.05)
+ raise AssertionError(f"Timed out waiting for DBOS state; last state={last_state}")
+
+
+def _markers(workspace, workflow_id):
+ storage_key = hashlib.sha256(workflow_id.encode("utf-8")).hexdigest()[:20]
+ return workspace / "markers" / storage_key
+
+
+def test_idempotent_submission_executes_external_effects_once(tmp_path):
+ database = tmp_path / "dbos.sqlite"
+ workspace = tmp_path / "workspace"
+ workflow_id = "operation-idempotency-1"
+
+ result = _run(
+ "execute",
+ "--database",
+ database,
+ "--workspace",
+ workspace,
+ "--workflow-id",
+ workflow_id,
+ "--correlation-id",
+ "request-idempotency-1",
+ "--steps",
+ 3,
+ "--duplicate-submission",
+ )
+
+ assert result["workflow_id"] == workflow_id
+ assert result["duplicate_workflow_id"] == workflow_id
+ assert result["workflow"]["status"] == "SUCCESS"
+ assert result["progress"]["progress"] == 1.0
+ marker_dir = _markers(workspace, workflow_id)
+ assert len(list(marker_dir.glob("step-*.json"))) == 3
+ assert not (marker_dir / "duplicate-attempts.jsonl").exists()
+
+
+def test_queued_cancellation_prevents_execution(tmp_path):
+ database = tmp_path / "dbos.sqlite"
+ workspace = tmp_path / "workspace"
+ workflow_id = "operation-cancel-queued"
+
+ submitted = _run(
+ "execute",
+ "--database",
+ database,
+ "--workspace",
+ workspace,
+ "--workflow-id",
+ workflow_id,
+ "--steps",
+ 3,
+ "--no-register-queue",
+ "--no-wait",
+ )
+ assert submitted["workflow"]["status"] == "ENQUEUED"
+
+ cancelled = _run(
+ "cancel",
+ "--database",
+ database,
+ "--workflow-id",
+ workflow_id,
+ )
+ assert cancelled["workflow"]["status"] == "CANCELLED"
+ assert not _markers(workspace, workflow_id).exists()
+
+
+def test_running_cancellation_stops_at_durable_boundary(tmp_path):
+ database = tmp_path / "dbos.sqlite"
+ workspace = tmp_path / "workspace"
+ workflow_id = "operation-cancel-running"
+ process = subprocess.Popen(
+ _command(
+ "execute",
+ "--database",
+ database,
+ "--workspace",
+ workspace,
+ "--workflow-id",
+ workflow_id,
+ "--steps",
+ 20,
+ "--step-runtime",
+ 0.01,
+ "--inter-step-delay",
+ 0.5,
+ ),
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ text=True,
+ )
+ try:
+ _wait_for(
+ database,
+ workflow_id,
+ lambda status, progress: (
+ status == "PENDING"
+ and isinstance(progress, dict)
+ and progress.get("completed_steps", 0) >= 1
+ ),
+ )
+ client = DBOSClient(system_database_url=_database_url(database))
+ try:
+ client.cancel_workflow(workflow_id)
+ finally:
+ client.destroy()
+ _wait_for(
+ database,
+ workflow_id,
+ lambda status, _progress: status == "CANCELLED",
+ )
+ output, _ = process.communicate(timeout=15)
+ result = _parse_result(output)
+ finally:
+ if process.poll() is None:
+ process.kill()
+ process.wait(timeout=5)
+
+ assert result["cancelled"] is True
+ marker_dir = _markers(workspace, workflow_id)
+ completed_markers = list(marker_dir.glob("step-*.json"))
+ assert 1 <= len(completed_markers) < 20
+ assert not (marker_dir / "duplicate-attempts.jsonl").exists()
+
+
+def test_process_kill_recovers_from_last_completed_step(tmp_path):
+ database = tmp_path / "dbos.sqlite"
+ workspace = tmp_path / "workspace"
+ workflow_id = "operation-restart-recovery"
+ process = subprocess.Popen(
+ _command(
+ "execute",
+ "--database",
+ database,
+ "--workspace",
+ workspace,
+ "--workflow-id",
+ workflow_id,
+ "--correlation-id",
+ "request-recovery-1",
+ "--steps",
+ 3,
+ "--step-runtime",
+ 0.01,
+ "--inter-step-delay",
+ 3,
+ ),
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ )
+ try:
+ status_before, progress_before = _wait_for(
+ database,
+ workflow_id,
+ lambda status, progress: (
+ status == "PENDING"
+ and isinstance(progress, dict)
+ and progress.get("completed_steps") == 1
+ ),
+ )
+ assert status_before == "PENDING"
+ assert progress_before["correlation_id"] == "request-recovery-1"
+ process.kill()
+ process.wait(timeout=5)
+ finally:
+ if process.poll() is None:
+ process.kill()
+ process.wait(timeout=5)
+
+ status_after_kill, persisted_progress = _wait_for(
+ database,
+ workflow_id,
+ lambda status, progress: (
+ status == "PENDING"
+ and isinstance(progress, dict)
+ and progress.get("completed_steps") == 1
+ ),
+ )
+ assert status_after_kill == "PENDING"
+ assert persisted_progress["progress"] == pytest.approx(1 / 3)
+
+ recovered = _run(
+ "recover",
+ "--database",
+ database,
+ "--workflow-id",
+ workflow_id,
+ timeout=20,
+ )
+ assert recovered["workflow"]["status"] == "SUCCESS"
+ assert recovered["workflow"]["recovery_attempts"] >= 2
+ assert recovered["progress"]["completed_steps"] == 3
+ assert recovered["progress"]["correlation_id"] == "request-recovery-1"
+
+ marker_dir = _markers(workspace, workflow_id)
+ assert len(list(marker_dir.glob("step-*.json"))) == 3
+ assert not (marker_dir / "duplicate-attempts.jsonl").exists()
diff --git a/tests/test_error_contract.py b/tests/test_error_contract.py
new file mode 100644
index 00000000..3e080282
--- /dev/null
+++ b/tests/test_error_contract.py
@@ -0,0 +1,172 @@
+import unittest
+
+from fastapi import FastAPI, HTTPException
+from fastapi.testclient import TestClient
+from pydantic import BaseModel
+
+from server_api.errors import PROBLEM_JSON_MEDIA_TYPE, install_error_handlers
+
+
+class ExamplePayload(BaseModel):
+ count: int
+
+
+def build_test_app() -> FastAPI:
+ app = FastAPI()
+ install_error_handlers(app)
+
+ @app.get("/missing")
+ def missing():
+ raise HTTPException(status_code=404, detail="Dataset was not found")
+
+ @app.post("/validate")
+ def validate(payload: ExamplePayload):
+ return payload
+
+ @app.get("/upstream")
+ def upstream():
+ raise HTTPException(
+ status_code=503,
+ detail={
+ "user_message": "The model worker is offline",
+ "error": "ConnectionError",
+ },
+ )
+
+ @app.get("/failure")
+ def failure():
+ raise RuntimeError("private implementation detail")
+
+ return app
+
+
+class ErrorContractTests(unittest.TestCase):
+ def setUp(self):
+ self.client = TestClient(build_test_app(), raise_server_exceptions=False)
+
+ def test_http_error_preserves_detail_and_adds_recovery_contract(self):
+ response = self.client.get("/missing", headers={"x-request-id": "req-123"})
+
+ self.assertEqual(response.status_code, 404)
+ self.assertEqual(response.headers["x-request-id"], "req-123")
+ self.assertEqual(response.headers["content-type"], "application/json")
+ self.assertIn("Accept", response.headers["vary"])
+ self.assertEqual(response.json()["detail"], "Dataset was not found")
+ self.assertEqual(response.json()["status"], 404)
+ self.assertEqual(response.json()["type"], "https://seg.bio/problems/not-found")
+ self.assertEqual(response.json()["title"], "Resource not found")
+ self.assertEqual(response.json()["instance"], "/missing#request-req-123")
+ self.assertEqual(
+ response.json()["error"],
+ {
+ "schema_version": 1,
+ "code": "not_found",
+ "category": "resource",
+ "title": "Resource not found",
+ "retryable": False,
+ "recovery_actions": ["go_back"],
+ "message": "Dataset was not found",
+ "request_id": "req-123",
+ },
+ )
+
+ def test_validation_error_includes_field_diagnostics(self):
+ response = self.client.post("/validate", json={"count": "invalid"})
+
+ self.assertEqual(response.status_code, 422)
+ body = response.json()
+ self.assertEqual(body["error"]["code"], "validation_failed")
+ self.assertEqual(body["detail"], body["error"]["validation_errors"])
+ self.assertTrue(body["error"]["request_id"])
+
+ def test_problem_json_representation_is_rfc_9457_compatible(self):
+ response = self.client.get(
+ "/upstream",
+ headers={
+ "accept": PROBLEM_JSON_MEDIA_TYPE,
+ "x-request-id": "problem-503",
+ },
+ )
+
+ self.assertEqual(response.status_code, 503)
+ self.assertEqual(response.headers["content-type"], PROBLEM_JSON_MEDIA_TYPE)
+ body = response.json()
+ self.assertEqual(
+ {
+ "type": body["type"],
+ "title": body["title"],
+ "status": body["status"],
+ "detail": body["detail"],
+ "instance": body["instance"],
+ },
+ {
+ "type": "https://seg.bio/problems/service-unavailable",
+ "title": "Service temporarily unavailable",
+ "status": 503,
+ "detail": "The model worker is offline",
+ "instance": "/upstream#request-problem-503",
+ },
+ )
+ self.assertEqual(
+ body["legacy_detail"],
+ {
+ "user_message": "The model worker is offline",
+ "error": "ConnectionError",
+ },
+ )
+ self.assertEqual(body["error"]["code"], "service_unavailable")
+
+ def test_problem_json_validation_detail_is_string_with_extensions(self):
+ response = self.client.post(
+ "/validate",
+ json={"count": "invalid"},
+ headers={"accept": PROBLEM_JSON_MEDIA_TYPE},
+ )
+
+ self.assertEqual(response.status_code, 422)
+ body = response.json()
+ self.assertIsInstance(body["detail"], str)
+ self.assertEqual(body["legacy_detail"], body["validation_errors"])
+ self.assertEqual(body["validation_errors"], body["error"]["validation_errors"])
+
+ def test_default_representation_preserves_non_string_detail(self):
+ response = self.client.get("/upstream")
+
+ self.assertEqual(
+ response.json()["detail"],
+ {
+ "user_message": "The model worker is offline",
+ "error": "ConnectionError",
+ },
+ )
+
+ def test_problem_json_must_be_explicitly_acceptable(self):
+ response = self.client.get(
+ "/missing", headers={"accept": "application/problem+json;q=0, */*"}
+ )
+
+ self.assertEqual(response.headers["content-type"], "application/json")
+
+ def test_invalid_caller_request_id_is_not_reflected(self):
+ response = self.client.get(
+ "/missing", headers={"x-request-id": "invalid request id"}
+ )
+
+ self.assertEqual(response.status_code, 404)
+ self.assertNotEqual(response.headers["x-request-id"], "invalid request id")
+ self.assertEqual(
+ response.headers["x-request-id"], response.json()["error"]["request_id"]
+ )
+
+ def test_unexpected_error_does_not_expose_exception_text(self):
+ response = self.client.get("/failure")
+
+ self.assertEqual(response.status_code, 500)
+ body = response.json()
+ self.assertEqual(body["error"]["code"], "internal_error")
+ self.assertNotIn("private implementation detail", response.text)
+ self.assertEqual(response.headers["x-request-id"], body["error"]["request_id"])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_file_workspace_routes.py b/tests/test_file_workspace_routes.py
index 26c7510b..97c1b937 100644
--- a/tests/test_file_workspace_routes.py
+++ b/tests/test_file_workspace_routes.py
@@ -3,6 +3,8 @@
import tempfile
import unittest
import json
+from types import SimpleNamespace
+from unittest import mock
from fastapi.testclient import TestClient
from sqlalchemy import create_engine
@@ -133,6 +135,112 @@ def test_missing_parent_file_listing_returns_404(self):
self.assertEqual(response.status_code, 404)
self.assertIn("no longer mounted", response.json()["detail"])
+ def test_file_listing_pagination_is_opt_in_and_filters_before_count(self):
+ self._create_file(name="nested", is_folder=True)
+ self._create_file(name="a-image.h5")
+ self._create_file(name="b-mask.tif")
+ self._create_file(name="notes.txt")
+ self._create_file(name=".DS_Store")
+ self._create_file(name="workflow_preference.json")
+
+ legacy_response = self.client.get("/files")
+ self.assertEqual(legacy_response.status_code, 200)
+ self.assertIsInstance(legacy_response.json(), list)
+ self.assertEqual(
+ [item["name"] for item in legacy_response.json()],
+ ["nested", "a-image.h5", "b-mask.tif", "notes.txt"],
+ )
+
+ first_page = self.client.get(
+ "/files",
+ params={"offset": 0, "limit": 2, "volume_only": True},
+ )
+ self.assertEqual(first_page.status_code, 200)
+ payload = first_page.json()
+ self.assertEqual(
+ [item["name"] for item in payload["items"]], ["nested", "a-image.h5"]
+ )
+ self.assertEqual(payload["total"], 3)
+ self.assertEqual(payload["offset"], 0)
+ self.assertEqual(payload["limit"], 2)
+ self.assertTrue(payload["has_more"])
+
+ second_page = self.client.get(
+ "/files",
+ params={"offset": 2, "limit": 2, "volume_only": True},
+ ).json()
+ self.assertEqual(
+ [item["name"] for item in second_page["items"]], ["b-mask.tif"]
+ )
+ self.assertFalse(second_page["has_more"])
+
+ def test_file_preview_reads_one_hdf5_volume_slice(self):
+ try:
+ import h5py
+ import numpy as np
+ except Exception:
+ self.skipTest("HDF5 preview dependencies are not installed")
+
+ volume_path = pathlib.Path(self.temp_dir.name) / "preview.h5"
+ with h5py.File(volume_path, "w") as handle:
+ handle.create_dataset(
+ "data",
+ data=np.arange(5 * 20 * 24, dtype=np.uint16).reshape(5, 20, 24),
+ chunks=(1, 20, 24),
+ )
+ file_id = self._create_file(
+ name="preview.h5",
+ physical_path=str(volume_path),
+ file_type="application/x-hdf5",
+ )
+
+ response = self.client.get(f"/files/preview/{file_id}")
+
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(response.headers["content-type"], "image/png")
+ self.assertTrue(response.content.startswith(b"\x89PNG\r\n\x1a\n"))
+
+ def test_file_preview_requests_bounded_middle_volume_region(self):
+ try:
+ import numpy as np
+ except Exception:
+ self.skipTest("NumPy preview dependency is not installed")
+
+ volume_path = pathlib.Path(self.temp_dir.name) / "bounded.tif"
+ volume_path.write_bytes(b"placeholder")
+ file_id = self._create_file(
+ name="bounded.tif",
+ physical_path=str(volume_path),
+ file_type="image/tiff",
+ )
+ requested_crops = []
+
+ class FakeStore:
+ metadata = SimpleNamespace(shape=(20, 2000, 3000))
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_exc_info):
+ return None
+
+ def read(self, crop):
+ requested_crops.append(crop)
+ return np.zeros((1, 667, 1000), dtype=np.uint16)
+
+ with mock.patch(
+ "server_api.auth.router.open_volume_store",
+ return_value=FakeStore(),
+ ):
+ response = self.client.get(f"/files/preview/{file_id}")
+
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(requested_crops), 1)
+ crop = requested_crops[0]
+ self.assertEqual(crop[0], slice(10, 11))
+ self.assertEqual(crop[1], slice(None, None, 3))
+ self.assertEqual(crop[2], slice(None, None, 3))
+
def test_project_context_profile_is_saved_in_hidden_project_file(self):
project_root = pathlib.Path(self.temp_dir.name) / "profile-project"
project_root.mkdir()
diff --git a/tests/test_neuroglancer_storage_sources.py b/tests/test_neuroglancer_storage_sources.py
new file mode 100644
index 00000000..c63f311e
--- /dev/null
+++ b/tests/test_neuroglancer_storage_sources.py
@@ -0,0 +1,114 @@
+import numpy as np
+import pytest
+
+from server_api.main import (
+ _NeuroglancerSegmentationStore,
+ _build_neuroglancer_local_volume_source,
+ _open_neuroglancer_volume_sources,
+)
+from server_api.workflows.volume_io import ArrayVolumeStore
+
+
+class RecordingLabelArray:
+ shape = (8, 16, 24)
+ dtype = np.dtype(np.int32)
+ chunks = (2, 4, 6)
+
+ def __init__(self, fill_value=7):
+ self.fill_value = fill_value
+ self.requested_keys = []
+
+ def __array__(self, *_args, **_kwargs):
+ raise AssertionError("full label volume was materialized")
+
+ def __getitem__(self, key):
+ self.requested_keys.append(key)
+ return np.full((2, 4, 6), self.fill_value, dtype=self.dtype)
+
+
+def test_segmentation_source_validates_and_converts_only_requested_chunk(tmp_path):
+ backing = RecordingLabelArray()
+ store = ArrayVolumeStore(
+ backing,
+ path=tmp_path / "labels.zarr",
+ format="zarr",
+ )
+ source = _NeuroglancerSegmentationStore(store)
+
+ chunk = source[1:3, 4:8, 6:12]
+
+ assert source.shape == backing.shape
+ assert source.dtype == np.dtype(np.uint64)
+ assert chunk.dtype == np.uint64
+ assert backing.requested_keys == [(slice(1, 3), slice(4, 8), slice(6, 12))]
+
+
+def test_segmentation_source_rejects_negative_labels_per_chunk(tmp_path):
+ store = ArrayVolumeStore(
+ RecordingLabelArray(fill_value=-1),
+ path=tmp_path / "negative-labels.zarr",
+ format="zarr",
+ )
+ source = _NeuroglancerSegmentationStore(store)
+
+ with pytest.raises(ValueError, match="non-negative"):
+ source[0:2, 0:4, 0:6]
+
+
+def test_segmentation_source_requires_preprocessed_3d_labels(tmp_path):
+ store = ArrayVolumeStore(
+ np.zeros((2, 8, 16, 24), dtype=np.float32),
+ path=tmp_path / "prediction.zarr",
+ format="zarr",
+ )
+
+ with pytest.raises(ValueError, match="requires a 3D label volume"):
+ _NeuroglancerSegmentationStore(store)
+
+
+def test_neuroglancer_local_volume_reads_hdf5_subvolumes_on_demand(tmp_path):
+ h5py = pytest.importorskip("h5py")
+ neuroglancer = pytest.importorskip("neuroglancer")
+ image_path = tmp_path / "image.h5"
+ label_path = tmp_path / "labels.h5"
+ with h5py.File(image_path, "w") as handle:
+ handle.create_dataset(
+ "data",
+ data=np.arange(8 * 16 * 24, dtype=np.uint8).reshape(8, 16, 24),
+ chunks=(2, 4, 6),
+ )
+ with h5py.File(label_path, "w") as handle:
+ handle.create_dataset(
+ "data",
+ data=np.ones((8, 16, 24), dtype=np.int32),
+ chunks=(2, 4, 6),
+ )
+
+ image, labels, resources = _open_neuroglancer_volume_sources(image_path, label_path)
+ try:
+ dimensions = neuroglancer.CoordinateSpace(
+ names=["z", "y", "x"], units=["nm", "nm", "nm"], scales=[1, 1, 1]
+ )
+ image_volume = _build_neuroglancer_local_volume_source(
+ neuroglancer, image, dimensions, volume_type="image"
+ )
+ label_volume = _build_neuroglancer_local_volume_source(
+ neuroglancer, labels, dimensions, volume_type="segmentation"
+ )
+ start = np.array([1, 2, 3], dtype=np.int64)
+ end = np.array([3, 6, 9], dtype=np.int64)
+
+ image_payload, image_content_type = image_volume.get_encoded_subvolume(
+ "raw", start, end, "1,1,1"
+ )
+ label_payload, label_content_type = label_volume.get_encoded_subvolume(
+ "raw", start, end, "1,1,1"
+ )
+
+ assert image_payload
+ assert label_payload
+ assert image_content_type == "application/octet-stream"
+ assert label_content_type == "application/octet-stream"
+ finally:
+ for resource in resources:
+ resource.close()
diff --git a/tests/test_neuroglancer_url_contract.py b/tests/test_neuroglancer_url_contract.py
index e051e31b..b95a8385 100644
--- a/tests/test_neuroglancer_url_contract.py
+++ b/tests/test_neuroglancer_url_contract.py
@@ -12,6 +12,14 @@ def __init__(self, token):
self.token = token
+class DummyResource:
+ def __init__(self):
+ self.closed = False
+
+ def close(self):
+ self.closed = True
+
+
def make_request(*, host="localhost:4242", scheme="http", extra_headers=None):
headers = [(b"host", host.encode("utf-8"))]
for key, value in (extra_headers or {}).items():
@@ -125,6 +133,29 @@ def test_retain_neuroglancer_viewer_evicts_oldest_over_capacity(self):
finally:
server_main.PYTC_NEUROGLANCER_MAX_VIEWERS = previous_limit
+ def test_evicted_viewer_closes_storage_backing_resources(self):
+ previous_limit = server_main.PYTC_NEUROGLANCER_MAX_VIEWERS
+ server_main.PYTC_NEUROGLANCER_MAX_VIEWERS = 1
+ old_resource = DummyResource()
+ try:
+ server_main._retain_neuroglancer_viewer(
+ DummyViewer("old"),
+ public_url="https://viewer.example.com/ng/v/old/",
+ internal_viewer_url="http://127.0.0.1:4244/v/old/",
+ mode="visualization",
+ resources=[old_resource],
+ )
+ server_main._retain_neuroglancer_viewer(
+ DummyViewer("new"),
+ public_url="https://viewer.example.com/ng/v/new/",
+ internal_viewer_url="http://127.0.0.1:4244/v/new/",
+ mode="visualization",
+ )
+
+ self.assertTrue(old_resource.closed)
+ finally:
+ server_main.PYTC_NEUROGLANCER_MAX_VIEWERS = previous_limit
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/test_pytc_runtime_routes.py b/tests/test_pytc_runtime_routes.py
index aee1fc9f..ce53075a 100644
--- a/tests/test_pytc_runtime_routes.py
+++ b/tests/test_pytc_runtime_routes.py
@@ -23,6 +23,8 @@
_resolve_raw_image_shader,
)
from server_api.main import _coerce_neuroglancer_scales
+from server_api.workflows.db_models import WorkflowCommand
+from server_api.workflows.service import encode_json
from server_pytc.main import app as server_pytc_app
from server_pytc.services import model as model_service
@@ -521,6 +523,7 @@ def test_durable_training_command_runner_launches_worker_and_records_state(self)
captured = {}
def fake_worker(method, endpoint, json_body=None, **_kwargs):
+ captured["calls"] = captured.get("calls", 0) + 1
captured["method"] = method
captured["endpoint"] = endpoint
captured["json_body"] = json_body
@@ -530,11 +533,28 @@ def fake_worker(method, endpoint, json_body=None, **_kwargs):
run_response = self.client.post(
f"/api/workflows/{workflow_id}/commands/{command['id']}/run"
)
+ duplicate_run_response = self.client.post(
+ f"/api/workflows/{workflow_id}/commands/{command['id']}/run"
+ )
self.assertEqual(run_response.status_code, 200)
+ self.assertEqual(duplicate_run_response.status_code, 200)
payload = run_response.json()
self.assertEqual(payload["command"]["status"], "submitted")
self.assertEqual(payload["command"]["attempt_count"], 1)
+ self.assertEqual(payload["operation"]["status"], "succeeded")
+ self.assertEqual(payload["operation"]["operation_type"], "start_training")
+ self.assertEqual(payload["operation"]["command_id"], command["id"])
+ self.assertEqual(
+ payload["operation"]["idempotency_key"],
+ f"workflow-command:{command['id']}:attempt:1",
+ )
+ self.assertEqual(payload["operation"]["result"]["worker"]["pid"], 4242)
+ self.assertEqual(
+ duplicate_run_response.json()["operation"]["id"],
+ payload["operation"]["id"],
+ )
+ self.assertEqual(captured["calls"], 1)
self.assertEqual(payload["worker"]["pid"], 4242)
self.assertEqual(captured["method"], "post")
self.assertEqual(captured["endpoint"], "/start_model_training")
@@ -556,6 +576,13 @@ def fake_worker(method, endpoint, json_body=None, **_kwargs):
self.assertEqual(commands_response.status_code, 200)
self.assertEqual(commands_response.json()[0]["status"], "submitted")
+ operations_response = self.client.get(
+ f"/api/workflows/{workflow_id}/operations"
+ )
+ self.assertEqual(operations_response.status_code, 200)
+ self.assertEqual(len(operations_response.json()), 1)
+ self.assertEqual(operations_response.json()[0]["status"], "succeeded")
+
events_response = self.client.get(f"/api/workflows/{workflow_id}/events")
self.assertEqual(events_response.status_code, 200)
started_events = [
@@ -570,6 +597,83 @@ def fake_worker(method, endpoint, json_body=None, **_kwargs):
f"workflow-command-{command['id']}",
)
+ def test_durable_training_command_failure_records_retryable_operation(self):
+ workflow_id = self._workflow_id()
+ project_root = pathlib.Path(self.temp_dir.name) / "failed-command-project"
+ image_path = project_root / "image.h5"
+ label_path = project_root / "label.tif"
+ output_path = project_root / "output"
+ project_root.mkdir(parents=True)
+ output_path.mkdir()
+ image_path.write_text("image", encoding="utf-8")
+ label_path.write_text("label", encoding="utf-8")
+
+ db = self.SessionLocal()
+ try:
+ command = WorkflowCommand(
+ workflow_id=workflow_id,
+ command_type="start_training",
+ status="queued",
+ idempotency_key="test:failed-training-command",
+ actor="user",
+ input_json=encode_json(
+ {
+ "trainingConfig": "DATASET: {}\n",
+ "outputPath": str(output_path),
+ "logPath": str(output_path),
+ "inputImagePath": str(image_path),
+ "inputLabelPath": str(label_path),
+ }
+ ),
+ )
+ db.add(command)
+ db.commit()
+ db.refresh(command)
+ command_id = command.id
+ finally:
+ db.close()
+
+ with patch(
+ "server_api.main._proxy_to_worker",
+ side_effect=HTTPException(status_code=503, detail="worker unavailable"),
+ ):
+ failed_response = self.client.post(
+ f"/api/workflows/{workflow_id}/commands/{command_id}/run"
+ )
+
+ self.assertEqual(failed_response.status_code, 503)
+ commands_response = self.client.get(f"/api/workflows/{workflow_id}/commands")
+ failed_command = next(
+ item for item in commands_response.json() if item["id"] == command_id
+ )
+ self.assertEqual(failed_command["status"], "retry_pending")
+ self.assertEqual(failed_command["attempt_count"], 1)
+
+ operations_response = self.client.get(
+ f"/api/workflows/{workflow_id}/operations"
+ )
+ self.assertEqual(operations_response.status_code, 200)
+ operations = operations_response.json()
+ self.assertEqual(len(operations), 1)
+ self.assertEqual(operations[0]["status"], "failed")
+ self.assertEqual(operations[0]["command_id"], command_id)
+ self.assertEqual(operations[0]["error"]["status_code"], 503)
+
+ with patch(
+ "server_api.main._proxy_to_worker",
+ return_value={"status": "started", "pid": 4243},
+ ):
+ retry_response = self.client.post(
+ f"/api/workflows/{workflow_id}/commands/{command_id}/run"
+ )
+
+ self.assertEqual(retry_response.status_code, 200)
+ self.assertEqual(retry_response.json()["operation"]["status"], "succeeded")
+ self.assertEqual(
+ retry_response.json()["operation"]["idempotency_key"],
+ f"workflow-command:{command_id}:attempt:2",
+ )
+
class ModelServiceTests(unittest.TestCase):
def tearDown(self):
@@ -652,12 +756,14 @@ def test_runtime_log_lines_are_exported_to_app_event_log(self):
self.assertEqual(records[1]["stream"], "stdout")
def test_detect_chunk_tile_mismatch_for_direct_h5_volume(self):
- diagnostic = model_service._detect_chunk_tile_mismatch("""
+ diagnostic = model_service._detect_chunk_tile_mismatch(
+ """
DATASET:
DO_CHUNK_TITLE: 1
IMAGE_NAME: /tmp/train-volume.h5
LABEL_NAME: /tmp/train-label.h5
-""")
+"""
+ )
self.assertIsNotNone(diagnostic)
self.assertEqual(diagnostic["code"], "tile_dataset_direct_volume_mismatch")
diff --git a/tests/test_synthetic_project.py b/tests/test_synthetic_project.py
new file mode 100644
index 00000000..bebf86ef
--- /dev/null
+++ b/tests/test_synthetic_project.py
@@ -0,0 +1,211 @@
+import json
+import pathlib
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import h5py
+import numpy as np
+from fastapi.testclient import TestClient
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from server_api.auth import database as auth_database
+from server_api.auth import models
+from server_api.auth.router import (
+ _project_suggestion_candidates,
+ _scan_project_profile,
+)
+from server_api.main import app as server_api_app
+from server_api.synthetic_project import (
+ GENERATOR_VERSION,
+ VOLUME_CHUNKS,
+ VOLUME_SHAPE,
+ create_synthetic_project,
+)
+from server_api.workflows import service as workflow_service
+
+
+class SyntheticProjectTests(unittest.TestCase):
+ def setUp(self):
+ self.temp_dir = tempfile.TemporaryDirectory()
+ self.project_root = (
+ pathlib.Path(self.temp_dir.name) / ".pytc/synthetic-core-project"
+ )
+
+ def tearDown(self):
+ self.temp_dir.cleanup()
+
+ def test_generator_is_deterministic_idempotent_and_resettable(self):
+ first = create_synthetic_project(self.project_root)
+ self.assertTrue(first["created"])
+ self.assertEqual(first["generator"], GENERATOR_VERSION)
+
+ with h5py.File(self.project_root / "data/raw/train-01_image.h5") as handle:
+ first_image = handle["data"][:]
+ self.assertEqual(handle["data"].shape, VOLUME_SHAPE)
+ self.assertEqual(handle["data"].chunks, VOLUME_CHUNKS)
+ self.assertEqual(handle["data"].compression, "gzip")
+ self.assertTrue(handle.attrs["synthetic"])
+
+ notes_path = self.project_root / "notes/README.md"
+ notes_path.write_text("local testing note\n", encoding="utf-8")
+ second = create_synthetic_project(self.project_root)
+ self.assertFalse(second["created"])
+ self.assertEqual(notes_path.read_text(encoding="utf-8"), "local testing note\n")
+
+ restored = create_synthetic_project(self.project_root, reset=True)
+ self.assertTrue(restored["created"])
+ self.assertIn(
+ "Expected initial progress", notes_path.read_text(encoding="utf-8")
+ )
+ with h5py.File(self.project_root / "data/raw/train-01_image.h5") as handle:
+ np.testing.assert_array_equal(handle["data"][:], first_image)
+
+ def test_fixture_encodes_expected_project_and_prediction_states(self):
+ create_synthetic_project(self.project_root)
+ manifest = json.loads(
+ (self.project_root / "project_manifest.json").read_text(encoding="utf-8")
+ )
+ self.assertEqual(
+ manifest["initial_progress_summary"],
+ {
+ "total": 4,
+ "ground_truth": 2,
+ "needs_proofreading": 1,
+ "missing_segmentation": 1,
+ },
+ )
+
+ with (
+ h5py.File(
+ self.project_root / "outputs/predictions/baseline_review-01.h5"
+ ) as baseline_handle,
+ h5py.File(
+ self.project_root / "outputs/predictions/candidate_review-01.h5"
+ ) as candidate_handle,
+ ):
+ baseline = baseline_handle["data"][:]
+ candidate = candidate_handle["data"][:]
+ self.assertGreater(np.count_nonzero(baseline != candidate), 0)
+ self.assertIn(9, np.unique(baseline))
+ self.assertNotIn(9, np.unique(candidate))
+
+ profile = _scan_project_profile(str(self.project_root), audit_detail="summary")
+ self.assertEqual(profile["volume_sets"][0]["image_count"], 4)
+ self.assertEqual(profile["volume_sets"][0]["label_count"], 3)
+ self.assertEqual(profile["volume_sets"][0]["pair_count"], 3)
+ self.assertEqual(
+ profile["context_hints"]["imaging_modality"],
+ "Synthetic volumetric microscopy",
+ )
+ self.assertEqual(profile["context_hints"]["voxel_size_nm"], [40.0, 8.0, 8.0])
+
+ def test_generator_refuses_to_overwrite_an_unmarked_directory(self):
+ self.project_root.mkdir(parents=True)
+ (self.project_root / "important.txt").write_text("keep", encoding="utf-8")
+
+ with self.assertRaisesRegex(ValueError, "non-synthetic directory"):
+ create_synthetic_project(self.project_root, reset=True)
+
+ self.assertEqual(
+ (self.project_root / "important.txt").read_text(encoding="utf-8"),
+ "keep",
+ )
+
+ def test_initial_project_candidate_and_workflow_defaults_use_fixture(self):
+ create_synthetic_project(self.project_root)
+ with (
+ patch.dict(
+ "os.environ",
+ {
+ "PYTC_INITIAL_PROJECT_ROOT": str(self.project_root),
+ "PYTC_INITIAL_PROJECT_KIND": "synthetic",
+ },
+ clear=False,
+ ),
+ patch.object(
+ workflow_service,
+ "INITIAL_PROJECT_ROOT",
+ str(self.project_root),
+ ),
+ ):
+ candidates = _project_suggestion_candidates()
+ defaults = workflow_service._initial_project_defaults()
+
+ self.assertEqual(candidates[0]["id"], "initial-project")
+ self.assertTrue(candidates[0]["recommended"])
+ self.assertFalse(any(item["recommended"] for item in candidates[1:]))
+ self.assertEqual(defaults["title"], "Synthetic Segmentation Core Loop")
+ self.assertEqual(
+ defaults["image_path"],
+ str(self.project_root / "data/raw/train-01_image.h5"),
+ )
+ self.assertTrue(defaults["metadata"]["synthetic"])
+ self.assertEqual(
+ defaults["metadata"]["project_context"]["training_policy"],
+ "train only on confirmed ground-truth masks",
+ )
+
+
+class SyntheticProjectProgressTests(unittest.TestCase):
+ def setUp(self):
+ self.temp_dir = tempfile.TemporaryDirectory()
+ self.project_root = (
+ pathlib.Path(self.temp_dir.name) / ".pytc/synthetic-core-project"
+ )
+ create_synthetic_project(self.project_root)
+
+ self.engine = create_engine(
+ f"sqlite:///{pathlib.Path(self.temp_dir.name) / 'test.db'}",
+ connect_args={"check_same_thread": False},
+ )
+ self.SessionLocal = sessionmaker(
+ autocommit=False,
+ autoflush=False,
+ bind=self.engine,
+ )
+ models.Base.metadata.create_all(bind=self.engine)
+
+ def override_get_db():
+ with self.SessionLocal() as db:
+ yield db
+
+ server_api_app.dependency_overrides[auth_database.get_db] = override_get_db
+ self.client = TestClient(server_api_app)
+
+ def tearDown(self):
+ server_api_app.dependency_overrides.clear()
+ self.engine.dispose()
+ self.temp_dir.cleanup()
+
+ def test_real_progress_endpoint_reports_fixture_contract(self):
+ current_response = self.client.get("/api/workflows/current")
+ self.assertEqual(current_response.status_code, 200)
+ workflow_id = current_response.json()["workflow"]["id"]
+ patch_response = self.client.patch(
+ f"/api/workflows/{workflow_id}",
+ json={
+ "title": "Synthetic Segmentation Core Loop",
+ "dataset_path": str(self.project_root),
+ "image_path": str(self.project_root / "data/raw"),
+ "label_path": str(self.project_root / "data/seg"),
+ "config_path": str(
+ self.project_root / "configs/Synthetic-Core-Loop-BC.yaml"
+ ),
+ },
+ )
+ self.assertEqual(patch_response.status_code, 200)
+
+ progress_response = self.client.get(
+ f"/api/workflows/{workflow_id}/project-progress"
+ )
+ self.assertEqual(progress_response.status_code, 200)
+ payload = progress_response.json()
+ self.assertEqual(payload["summary"]["total"], 4)
+ self.assertEqual(payload["summary"]["ground_truth"], 2)
+ self.assertEqual(payload["summary"]["needs_proofreading"], 1)
+ self.assertEqual(payload["summary"]["missing_segmentation"], 1)
+ states = {row["name"]: row["status"] for row in payload["volumes"]}
+ self.assertEqual(states["review-01_image.h5"], "needs_proofreading")
+ self.assertEqual(states["target-01_image.h5"], "missing_segmentation")
diff --git a/tests/test_volume_io.py b/tests/test_volume_io.py
index 9e6e36d4..b8dd6dc4 100644
--- a/tests/test_volume_io.py
+++ b/tests/test_volume_io.py
@@ -4,7 +4,12 @@
h5py = pytest.importorskip("h5py")
pytest.importorskip("tifffile")
-from server_api.workflows.volume_io import load_volume, parse_crop
+from server_api.workflows.volume_io import (
+ ArrayVolumeStore,
+ load_volume,
+ open_volume_store,
+ parse_crop,
+)
def test_parse_crop_accepts_voxel_slice_strings():
@@ -27,6 +32,102 @@ def test_load_volume_reads_hdf5_dataset_with_crop(tmp_path):
np.testing.assert_array_equal(loaded, volume[1:3, 2:5, 1:4])
+def test_open_volume_store_exposes_metadata_without_reading_data(tmp_path):
+ path = tmp_path / "volume.h5"
+ with h5py.File(path, "w") as handle:
+ handle.create_dataset(
+ "data",
+ shape=(64, 128, 256),
+ chunks=(8, 32, 32),
+ dtype=np.uint16,
+ )
+
+ with open_volume_store(str(path), dataset_key="data") as store:
+ assert store.metadata.shape == (64, 128, 256)
+ assert store.metadata.chunks == (8, 32, 32)
+ assert store.metadata.dtype == np.dtype(np.uint16)
+ assert store.metadata.dataset_key == "data"
+ assert store.metadata.format == "hdf5"
+
+
+def test_volume_store_indexes_region_before_numpy_materialization(tmp_path):
+ class RecordingArray:
+ shape = (100, 200, 300)
+ dtype = np.dtype(np.uint16)
+ chunks = (10, 20, 30)
+
+ def __init__(self):
+ self.requested_keys = []
+
+ def __array__(self, *_args, **_kwargs):
+ raise AssertionError("full backing array was materialized")
+
+ def __getitem__(self, key):
+ self.requested_keys.append(key)
+ return np.zeros((2, 3, 4), dtype=self.dtype)
+
+ backing = RecordingArray()
+ store = ArrayVolumeStore(
+ backing,
+ path=tmp_path / "recording.zarr",
+ format="zarr",
+ )
+
+ result = store.read("1:3,10:13,20:24")
+
+ assert result.shape == (2, 3, 4)
+ assert backing.requested_keys == [(slice(1, 3), slice(10, 13), slice(20, 24))]
+
+
+def test_volume_store_supports_storage_backed_array_protocol(tmp_path):
+ class RecordingArray:
+ shape = (100, 200, 300)
+ dtype = np.dtype(np.uint16)
+ chunks = (10, 20, 30)
+
+ def __init__(self):
+ self.requested_keys = []
+
+ def __array__(self, *_args, **_kwargs):
+ raise AssertionError("full backing array was materialized")
+
+ def __getitem__(self, key):
+ self.requested_keys.append(key)
+ return np.ones((2, 3, 4), dtype=self.dtype)
+
+ backing = RecordingArray()
+ store = ArrayVolumeStore(
+ backing,
+ path=tmp_path / "viewer.zarr",
+ format="zarr",
+ )
+
+ region = store[1:3, 10:13, 20:24]
+
+ assert store.shape == (100, 200, 300)
+ assert store.ndim == 3
+ assert store.dtype == np.dtype(np.uint16)
+ assert region.shape == (2, 3, 4)
+ assert backing.requested_keys == [(slice(1, 3), slice(10, 13), slice(20, 24))]
+
+
+def test_volume_store_context_closes_backing_resource(tmp_path):
+ closed = []
+ store = ArrayVolumeStore(
+ np.zeros((2, 3, 4), dtype=np.uint8),
+ path=tmp_path / "volume.npy",
+ format="npy",
+ close=lambda: closed.append(True),
+ )
+
+ with store:
+ assert store.read("0:1").shape == (1, 3, 4)
+
+ assert closed == [True]
+ with pytest.raises(RuntimeError, match="closed"):
+ store.read("0:1")
+
+
def test_load_volume_reads_inline_hdf5_dataset_reference(tmp_path):
path = tmp_path / "volume.h5"
volume = np.arange(2 * 3 * 4, dtype=np.uint8).reshape(2, 3, 4)
@@ -70,3 +171,35 @@ def test_load_volume_rejects_channel_for_already_3d_volume(tmp_path):
reference_ndim=3,
label="mask",
)
+
+
+def test_load_volume_reads_compressed_tiff_crop(tmp_path):
+ tifffile = pytest.importorskip("tifffile")
+ path = tmp_path / "volume.tif"
+ volume = np.arange(8 * 16 * 24, dtype=np.uint16).reshape(8, 16, 24)
+ tifffile.imwrite(
+ path,
+ volume,
+ compression="zlib",
+ metadata={"axes": "ZYX"},
+ )
+
+ loaded = load_volume(str(path), crop="2:5,4:9,6:12")
+
+ np.testing.assert_array_equal(loaded, volume[2:5, 4:9, 6:12])
+
+
+def test_open_volume_store_reads_zarr_region(tmp_path):
+ zarr = pytest.importorskip("zarr")
+ path = tmp_path / "volume.zarr"
+ volume = np.arange(6 * 12 * 18, dtype=np.uint16).reshape(6, 12, 18)
+ root = zarr.open(str(path), mode="w")
+ create_array = getattr(root, "create_array", root.create_dataset)
+ create_array("raw", data=volume, chunks=(2, 4, 6))
+
+ with open_volume_store(f"{path}::raw") as store:
+ loaded = store.read("1:4,3:8,5:11")
+ assert store.metadata.shape == volume.shape
+ assert store.metadata.chunks == (2, 4, 6)
+
+ np.testing.assert_array_equal(loaded, volume[1:4, 3:8, 5:11])
diff --git a/tests/test_workflow_operations.py b/tests/test_workflow_operations.py
new file mode 100644
index 00000000..4993af52
--- /dev/null
+++ b/tests/test_workflow_operations.py
@@ -0,0 +1,338 @@
+import pathlib
+import tempfile
+import unittest
+
+import pytest
+
+pytest.importorskip("sqlalchemy")
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+pytest.importorskip("fastapi")
+from fastapi.testclient import TestClient
+
+from server_api.auth import database as auth_database
+from server_api.auth import models
+from server_api.main import app as server_api_app
+
+
+class WorkflowOperationRouteTests(unittest.TestCase):
+ def setUp(self):
+ self.temp_dir = tempfile.TemporaryDirectory()
+ self.db_path = pathlib.Path(self.temp_dir.name) / "operation-test.db"
+ self.engine = create_engine(
+ f"sqlite:///{self.db_path}", connect_args={"check_same_thread": False}
+ )
+ self.SessionLocal = sessionmaker(
+ autocommit=False, autoflush=False, bind=self.engine
+ )
+ models.Base.metadata.create_all(bind=self.engine)
+
+ def override_get_db():
+ db = self.SessionLocal()
+ try:
+ yield db
+ finally:
+ db.close()
+
+ server_api_app.dependency_overrides[auth_database.get_db] = override_get_db
+ self.client = TestClient(server_api_app)
+ response = self.client.get("/api/workflows/current")
+ self.assertEqual(response.status_code, 200)
+ self.workflow_id = response.json()["workflow"]["id"]
+
+ def tearDown(self):
+ server_api_app.dependency_overrides.clear()
+ self.engine.dispose()
+ self.temp_dir.cleanup()
+
+ def _create_operation(self, key="train:dataset-a:v1", **overrides):
+ body = {
+ "operation_type": "training",
+ "idempotency_key": key,
+ "actor": "user",
+ "input": {"config_path": "/tmp/config.yaml"},
+ "metadata": {"source": "test"},
+ **overrides,
+ }
+ return self.client.post(
+ f"/api/workflows/{self.workflow_id}/operations",
+ json=body,
+ )
+
+ def _training_action(self, **overrides):
+ body = {
+ "action_id": "action-training-1",
+ "kind": "start_training",
+ "workflow_id": self.workflow_id,
+ "requested_by": "agent",
+ "idempotency_key": "action:start-training:1",
+ "correlation_id": "conversation-1",
+ "execution_owner": "server_runtime",
+ "policy": {
+ "risk_level": "runs_job",
+ "requires_approval": True,
+ "approval_reason": "Training consumes compute resources.",
+ },
+ "approval": {
+ "status": "approved",
+ "event_id": 1,
+ "decided_by": "test-user",
+ },
+ "input_artifacts": [
+ {
+ "logical_name": "synthetic-training-config",
+ "role": "config",
+ "path": "configs/Synthetic-Core-Loop-BC.yaml",
+ }
+ ],
+ "preconditions": [
+ {
+ "kind": "workflow_stage",
+ "allowed_stages": ["training_ready"],
+ }
+ ],
+ "autopick_parameters": True,
+ }
+ body.update(overrides)
+ return body
+
+ def test_approved_action_envelope_stages_one_idempotent_operation(self):
+ url = f"/api/workflows/{self.workflow_id}/action-operations"
+ body = self._training_action()
+
+ first_response = self.client.post(url, json=body)
+ second_response = self.client.post(url, json=body)
+
+ self.assertEqual(first_response.status_code, 202)
+ self.assertEqual(second_response.status_code, 202)
+ first = first_response.json()
+ self.assertEqual(second_response.json()["id"], first["id"])
+ self.assertEqual(first["status"], "queued")
+ self.assertEqual(first["operation_type"], "agent_action:start_training")
+ self.assertEqual(first["input"]["kind"], "start_training")
+ self.assertEqual(first["metadata"]["action_id"], "action-training-1")
+ self.assertEqual(first["correlation_id"], "conversation-1")
+
+ def test_action_operation_rejects_unapproved_and_wrong_workflow_envelopes(self):
+ url = f"/api/workflows/{self.workflow_id}/action-operations"
+ unapproved = self._training_action(
+ approval={"status": "pending"},
+ idempotency_key="action:start-training:pending",
+ )
+ unapproved_response = self.client.post(url, json=unapproved)
+ self.assertEqual(unapproved_response.status_code, 409)
+ self.assertIn("approved", unapproved_response.json()["detail"])
+
+ wrong_workflow = self._training_action(
+ workflow_id=self.workflow_id + 1,
+ idempotency_key="action:start-training:wrong-workflow",
+ )
+ wrong_response = self.client.post(url, json=wrong_workflow)
+ self.assertEqual(wrong_response.status_code, 409)
+ self.assertIn("workflow_id", wrong_response.json()["detail"])
+
+ def test_action_operation_rejects_browser_owned_actions(self):
+ url = f"/api/workflows/{self.workflow_id}/action-operations"
+ body = {
+ **self._training_action(),
+ "action_id": "action-choose-project-1",
+ "kind": "choose_project_data",
+ "idempotency_key": "action:choose-project:1",
+ "execution_owner": "browser_navigation",
+ "policy": {
+ "risk_level": "prefills_form",
+ "requires_approval": False,
+ },
+ "approval": {"status": "not_required"},
+ }
+ body.pop("autopick_parameters")
+ response = self.client.post(url, json=body)
+ self.assertEqual(response.status_code, 409)
+ self.assertIn("Browser-owned", response.json()["detail"])
+
+ def test_action_envelope_schema_is_exposed_with_a_discriminator(self):
+ response = self.client.get("/api/workflows/action-envelopes/schema")
+ self.assertEqual(response.status_code, 200)
+ schema = response.json()
+ self.assertEqual(schema["discriminator"]["propertyName"], "kind")
+ self.assertGreaterEqual(len(schema["oneOf"]), 10)
+
+ def test_operation_lifecycle_is_persisted_and_queryable(self):
+ created_response = self._create_operation()
+ self.assertEqual(created_response.status_code, 200)
+ created = created_response.json()
+ self.assertEqual(created["status"], "queued")
+ self.assertEqual(created["attempt_count"], 0)
+ self.assertTrue(created["correlation_id"])
+
+ running_response = self.client.post(
+ (
+ f"/api/workflows/{self.workflow_id}/operations/"
+ f"{created['id']}/transitions"
+ ),
+ json={
+ "status": "running",
+ "expected_status": "queued",
+ "lease_owner": "worker-1",
+ "progress": 0.1,
+ },
+ )
+ self.assertEqual(running_response.status_code, 200)
+ running = running_response.json()
+ self.assertEqual(running["status"], "running")
+ self.assertEqual(running["attempt_count"], 1)
+ self.assertEqual(running["lease_owner"], "worker-1")
+ self.assertIsNotNone(running["started_at"])
+ self.assertIsNotNone(running["heartbeat_at"])
+
+ heartbeat_response = self.client.post(
+ (
+ f"/api/workflows/{self.workflow_id}/operations/"
+ f"{created['id']}/heartbeat"
+ ),
+ json={
+ "lease_owner": "worker-1",
+ "progress": 0.55,
+ "metadata": {"runtime": {"pid": 123}},
+ },
+ )
+ self.assertEqual(heartbeat_response.status_code, 200)
+ heartbeat = heartbeat_response.json()
+ self.assertEqual(heartbeat["progress"], 0.55)
+ self.assertEqual(heartbeat["metadata"]["runtime"]["pid"], 123)
+
+ succeeded_response = self.client.post(
+ (
+ f"/api/workflows/{self.workflow_id}/operations/"
+ f"{created['id']}/transitions"
+ ),
+ json={
+ "status": "succeeded",
+ "expected_status": "running",
+ "lease_owner": "worker-1",
+ "result": {"checkpoint_path": "/tmp/model.pth"},
+ },
+ )
+ self.assertEqual(succeeded_response.status_code, 200)
+ succeeded = succeeded_response.json()
+ self.assertEqual(succeeded["status"], "succeeded")
+ self.assertEqual(succeeded["progress"], 1.0)
+ self.assertEqual(succeeded["result"]["checkpoint_path"], "/tmp/model.pth")
+ self.assertIsNone(succeeded["lease_owner"])
+ self.assertIsNotNone(succeeded["completed_at"])
+
+ get_response = self.client.get(
+ f"/api/workflows/{self.workflow_id}/operations/{created['id']}"
+ )
+ self.assertEqual(get_response.status_code, 200)
+ self.assertEqual(get_response.json()["status"], "succeeded")
+
+ list_response = self.client.get(
+ f"/api/workflows/{self.workflow_id}/operations",
+ params={"status": "succeeded", "operation_type": "training"},
+ )
+ self.assertEqual(list_response.status_code, 200)
+ self.assertEqual([item["id"] for item in list_response.json()], [created["id"]])
+
+ def test_creation_is_idempotent_and_rejects_key_reuse(self):
+ first = self._create_operation().json()
+ second_response = self._create_operation()
+ self.assertEqual(second_response.status_code, 200)
+ self.assertEqual(second_response.json()["id"], first["id"])
+ self.assertEqual(
+ second_response.json()["correlation_id"], first["correlation_id"]
+ )
+
+ conflict_response = self._create_operation(
+ input={"config_path": "/tmp/different.yaml"}
+ )
+ self.assertEqual(conflict_response.status_code, 409)
+ self.assertIn("idempotency_key", conflict_response.json()["detail"])
+
+ def test_running_cancellation_is_requested_before_worker_acknowledgement(self):
+ operation = self._create_operation(key="inference:dataset-b:v1").json()
+ transition_url = (
+ f"/api/workflows/{self.workflow_id}/operations/"
+ f"{operation['id']}/transitions"
+ )
+ start_response = self.client.post(
+ transition_url,
+ json={"status": "running", "expected_status": "queued"},
+ )
+ self.assertEqual(start_response.status_code, 200)
+
+ cancel_response = self.client.post(
+ (
+ f"/api/workflows/{self.workflow_id}/operations/"
+ f"{operation['id']}/cancel"
+ ),
+ json={"reason": "User selected another checkpoint."},
+ )
+ self.assertEqual(cancel_response.status_code, 200)
+ cancellation_requested = cancel_response.json()
+ self.assertEqual(cancellation_requested["status"], "running")
+ self.assertIsNotNone(cancellation_requested["cancellation_requested_at"])
+ self.assertEqual(
+ cancellation_requested["metadata"]["cancellation"]["reason"],
+ "User selected another checkpoint.",
+ )
+
+ acknowledged_response = self.client.post(
+ transition_url,
+ json={"status": "cancelled", "expected_status": "running"},
+ )
+ self.assertEqual(acknowledged_response.status_code, 200)
+ self.assertEqual(acknowledged_response.json()["status"], "cancelled")
+
+ invalid_response = self.client.post(
+ transition_url,
+ json={"status": "succeeded"},
+ )
+ self.assertEqual(invalid_response.status_code, 409)
+
+ def test_queued_cancellation_is_immediately_terminal(self):
+ operation = self._create_operation(key="evaluation:cancel-before-start").json()
+ response = self.client.post(
+ (
+ f"/api/workflows/{self.workflow_id}/operations/"
+ f"{operation['id']}/cancel"
+ )
+ )
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(response.json()["status"], "cancelled")
+ self.assertIsNotNone(response.json()["completed_at"])
+
+ def test_expected_status_and_lease_owner_prevent_stale_updates(self):
+ operation = self._create_operation(key="export:evidence:v1").json()
+ transition_url = (
+ f"/api/workflows/{self.workflow_id}/operations/"
+ f"{operation['id']}/transitions"
+ )
+ stale_response = self.client.post(
+ transition_url,
+ json={"status": "running", "expected_status": "running"},
+ )
+ self.assertEqual(stale_response.status_code, 409)
+
+ self.client.post(
+ transition_url,
+ json={"status": "running", "lease_owner": "worker-1"},
+ )
+ heartbeat_response = self.client.post(
+ (
+ f"/api/workflows/{self.workflow_id}/operations/"
+ f"{operation['id']}/heartbeat"
+ ),
+ json={"lease_owner": "worker-2", "progress": 0.5},
+ )
+ self.assertEqual(heartbeat_response.status_code, 409)
+
+ bad_progress_response = self.client.post(
+ (
+ f"/api/workflows/{self.workflow_id}/operations/"
+ f"{operation['id']}/heartbeat"
+ ),
+ json={"lease_owner": "worker-1", "progress": 1.5},
+ )
+ self.assertEqual(bad_progress_response.status_code, 400)
diff --git a/tests/test_workflow_routes.py b/tests/test_workflow_routes.py
index a2161633..e01a08ee 100644
--- a/tests/test_workflow_routes.py
+++ b/tests/test_workflow_routes.py
@@ -1288,6 +1288,23 @@ def test_workflow_overview_summarizes_phase_progress_and_next_actions(self):
action_ids = [action["id"] for action in payload["recommended_next_actions"]]
self.assertIn("proofread-draft-masks", action_ids)
self.assertIn("train-from-ground-truth", action_ids)
+ proofread_action = next(
+ action
+ for action in payload["recommended_next_actions"]
+ if action["id"] == "proofread-draft-masks"
+ )
+ self.assertEqual(
+ proofread_action["client_effects"]["set_proofreading_dataset_path"],
+ str((image_dir / "vol_b_im.h5").resolve()),
+ )
+ self.assertEqual(
+ proofread_action["client_effects"]["set_proofreading_mask_path"],
+ str((label_dir / "vol_b_seg.h5").resolve()),
+ )
+ self.assertEqual(
+ proofread_action["client_effects"]["runtime_action"]["kind"],
+ "start_proofreading",
+ )
blocker_ids = [blocker["id"] for blocker in payload["blockers"]]
self.assertIn("draft_masks_need_review", blocker_ids)
@@ -2849,6 +2866,10 @@ def test_ehtool_load_classify_save_and_export_append_workflow_events(self):
mask[1, 4:6, 4:6] = 3
tifffile.imwrite(str(image_path), image)
tifffile.imwrite(str(mask_path), mask)
+ self.client.patch(
+ f"/api/workflows/{workflow_id}",
+ json={"dataset_path": str(data_root)},
+ )
load_response = self.client.post(
"/eh/detection/load",
@@ -2866,6 +2887,9 @@ def test_ehtool_load_classify_save_and_export_append_workflow_events(self):
updated_workflow = workflow_response.json()["workflow"]
self.assertEqual(updated_workflow["stage"], "proofreading")
self.assertEqual(updated_workflow["proofreading_session_id"], session_id)
+ self.assertEqual(updated_workflow["dataset_path"], str(data_root))
+ self.assertEqual(updated_workflow["image_path"], str(image_path))
+ self.assertEqual(updated_workflow["label_path"], str(mask_path))
instances_response = self.client.get(
"/eh/detection/instances", params={"session_id": session_id}
diff --git a/uv.lock b/uv.lock
index b6546c54..aec60937 100644
--- a/uv.lock
+++ b/uv.lock
@@ -93,6 +93,24 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" },
]
+[[package]]
+name = "aiosqlite"
+version = "0.22.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/4e/8a/64761f4005f17809769d23e518d915db74e6310474e733e3593cfc854ef1/aiosqlite-0.22.1.tar.gz", hash = "sha256:043e0bd78d32888c0a9ca90fc788b38796843360c855a7262a532813133a0650", size = 14821, upload-time = "2025-12-23T19:25:43.997Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/00/b7/e3bf5133d697a08128598c8d0abc5e16377b51465a33756de24fa7dee953/aiosqlite-0.22.1-py3-none-any.whl", hash = "sha256:21c002eb13823fad740196c5a2e9d8e62f6243bd9e7e4a1f87fb5e44ecb4fceb", size = 17405, upload-time = "2025-12-23T19:25:42.139Z" },
+]
+
+[[package]]
+name = "annotated-doc"
+version = "0.0.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" },
+]
+
[[package]]
name = "annotated-types"
version = "0.7.0"
@@ -685,6 +703,28 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686, upload-time = "2024-06-09T16:20:16.715Z" },
]
+[[package]]
+name = "dbos"
+version = "2.28.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "psycopg", extra = ["binary"] },
+ { name = "python-dateutil" },
+ { name = "pyyaml" },
+ { name = "sqlalchemy", extra = ["asyncio"] },
+ { name = "typer-slim" },
+ { name = "websockets" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c6/b1/8a6c8ace5764e19850f2fd3f010251c3135daf1d682f43566c18dd0d432f/dbos-2.28.0.tar.gz", hash = "sha256:e738d62baf3953639d3e70988912b533ffd9de047d8b90d12755c10c03cac6fe", size = 592187, upload-time = "2026-07-21T15:24:08.551Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/25/d4/17f83581209153dff0bb7f6369c83d98d06303f8674746a7d47875add155/dbos-2.28.0-py3-none-any.whl", hash = "sha256:ba1bec5fcb2505b1bdd99251056cecf79669337699a81d5ef2b7598f701f31b6", size = 236036, upload-time = "2026-07-21T15:24:06.354Z" },
+]
+
+[package.optional-dependencies]
+aiosqlite = [
+ { name = "aiosqlite" },
+]
+
[[package]]
name = "debugpy"
version = "1.8.20"
@@ -1826,6 +1866,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/59/1b/6ef961f543593969d25b2afe57a3564200280528caa9bd1082eecdd7b3bc/markdown-3.10.1-py3-none-any.whl", hash = "sha256:867d788939fe33e4b736426f5b9f651ad0c0ae0ecf89df0ca5d1176c70812fe3", size = 107684, upload-time = "2026-01-21T18:09:27.203Z" },
]
+[[package]]
+name = "markdown-it-py"
+version = "4.2.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "mdurl" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" },
+]
+
[[package]]
name = "markupsafe"
version = "3.0.3"
@@ -1919,6 +1971,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl", hash = "sha256:d56ce5156ba6085e00a9d54fead6ed29a9c47e215cd1bba2e976ef39f5710a76", size = 9516, upload-time = "2025-10-23T09:00:20.675Z" },
]
+[[package]]
+name = "mdurl"
+version = "0.1.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" },
+]
+
[[package]]
name = "mistune"
version = "3.2.0"
@@ -2578,6 +2639,25 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/cb/28/3bfe2fa5a7b9c46fe7e13c97bda14c895fb10fa2ebf1d0abb90e0cea7ee1/platformdirs-4.5.1-py3-none-any.whl", hash = "sha256:d03afa3963c806a9bed9d5125c8f4cb2fdaf74a55ab60e5d59b3fde758104d31", size = 18731, upload-time = "2025-12-05T13:52:56.823Z" },
]
+[[package]]
+name = "playwright"
+version = "1.61.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "greenlet" },
+ { name = "pyee" },
+]
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/44/ee/31e4e0db36588b817a10b299a0285082545fde7d36543c2abe498bb3d61a/playwright-1.61.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:ff138c3a604f69911e9d42fd036e55c2a171e5616edf04c1e7f60a2a285540b0", size = 43421877, upload-time = "2026-06-29T10:32:48.428Z" },
+ { url = "https://files.pythonhosted.org/packages/42/35/71395dd3ecc798965be4a3ef8c443217d4abca168e7cb34536304f9489e6/playwright-1.61.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:009588c2a7e499bc5a8b425b61fa65490968bbda9cd69e0cf2cff10f8304659a", size = 42205016, upload-time = "2026-06-29T10:32:52.104Z" },
+ { url = "https://files.pythonhosted.org/packages/f4/44/323164cf5cd1647bdefce76ffce27651aadb959d089b48f53ea40918276e/playwright-1.61.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:9f7de4536088d12037c13a52b7ea34b59270b78926bb56935070597ffac6b1af", size = 43421884, upload-time = "2026-06-29T10:32:55.773Z" },
+ { url = "https://files.pythonhosted.org/packages/ab/f8/a35bf179e4ba2522c1893635094a64e407572547bd61528820fc0abc87fe/playwright-1.61.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:54f3b39f6eab832e33458c1dd7da0b5682aedab3b09ae731b5c59fa12fd2024e", size = 47421381, upload-time = "2026-06-29T10:32:59.903Z" },
+ { url = "https://files.pythonhosted.org/packages/b7/eb/e3f922348ec17c315f98c463f72faa1181a1c3de0bfe31a8d2edf6561723/playwright-1.61.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93454322ade8c11d5d6c211bfd91bdfb9ffb4810e3e026371bcbc4bec1b7ee4c", size = 47120545, upload-time = "2026-06-29T10:33:03.574Z" },
+ { url = "https://files.pythonhosted.org/packages/c2/a6/5be4e52b40a9c0c8a073e7c5b0785c05cf5a9ea8f8a7b5b260e32d970342/playwright-1.61.0-py3-none-win32.whl", hash = "sha256:372d55a6f1248fa1dd47599686980cb8fb5bbe6fcda59eab793eb657c11d8a9b", size = 37844841, upload-time = "2026-06-29T10:33:07.361Z" },
+ { url = "https://files.pythonhosted.org/packages/6c/fd/2b78036e5fbe9d5f5645bbe08a1eac7160c51243c0093963edbcf67c35d9/playwright-1.61.0-py3-none-win_amd64.whl", hash = "sha256:35c6cc4589a5d00964a59d7b3e59641e0aac0c02f15479a7af77d20f6bc79597", size = 37844846, upload-time = "2026-06-29T10:33:10.637Z" },
+ { url = "https://files.pythonhosted.org/packages/27/0d/1b0f3c4ee4eb0514bc805b5c2f9a223e5b6de4f11a926f5235d51d0fc81b/playwright-1.61.0-py3-none-win_arm64.whl", hash = "sha256:e9fcbffcf557a8620fdedd92491eb59a32d18e23d6f3b4f6214b952be324fe51", size = 33955127, upload-time = "2026-06-29T10:33:14.008Z" },
+]
+
[[package]]
name = "pluggy"
version = "1.6.0"
@@ -2678,6 +2758,53 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" },
]
+[[package]]
+name = "psycopg"
+version = "3.3.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "typing-extensions" },
+ { name = "tzdata", marker = "sys_platform == 'win32'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/db/2f/cb91e5502ec9de1de6f1b76cfbf69531932725361168bb06963620c77e2e/psycopg-3.3.4.tar.gz", hash = "sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc", size = 165799, upload-time = "2026-05-01T23:31:55.179Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/5c/e0/7b3dee031daae7743609ce3c746565d4a3ed7c2c186479eb48e34e838c64/psycopg-3.3.4-py3-none-any.whl", hash = "sha256:b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a", size = 213001, upload-time = "2026-05-01T23:20:50.816Z" },
+]
+
+[package.optional-dependencies]
+binary = [
+ { name = "psycopg-binary", marker = "implementation_name != 'pypy'" },
+]
+
+[[package]]
+name = "psycopg-binary"
+version = "3.3.4"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/b7/bf/70d8a60488f9955cbbcd538beae44d56bb2f1d19e673b72788f2d343ff55/psycopg_binary-3.3.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b7bfff1ca23732b488cbca3076fc11bc98d520ee122514fdb17a8e20d3338f5a", size = 4609750, upload-time = "2026-05-01T23:24:20.06Z" },
+ { url = "https://files.pythonhosted.org/packages/db/b0/29e98ba210c9dbc75a6dc91e3f99b9e06ea901a62ca95804e02a1ae13e6b/psycopg_binary-3.3.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:32a6fbf8481e3a370d0d72b860d35948a693cb01281da217f7b2f307636e591a", size = 4676700, upload-time = "2026-05-01T23:25:21.727Z" },
+ { url = "https://files.pythonhosted.org/packages/8e/ab/3df087b3c12bf74e47c08204172b2fabb5a144679110d5c7ad12d9201323/psycopg_binary-3.3.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bdef84570ebbce1d42b4e7ea952d21c414c5f118ad02fee00c5625f35e134429", size = 5496319, upload-time = "2026-05-01T23:25:28.271Z" },
+ { url = "https://files.pythonhosted.org/packages/87/9a/f088207b4cd6772f9e0d8a91807e79fa2458d4eb9eb1ae406c68415f2bec/psycopg_binary-3.3.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa1cbc10768a796c96d3243656016bf4e337c81c71097270bb7b0ad6210d9765", size = 5171906, upload-time = "2026-05-01T23:25:34.004Z" },
+ { url = "https://files.pythonhosted.org/packages/48/45/4523a857f253871d75c22e1c2e79fd47e599e736bcba1bad58d83e24be02/psycopg_binary-3.3.4-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cf7f73a4a792bc5db58a4b385d8a1467e8d468f7548702fb0ed1e9b7501b1c13", size = 6762621, upload-time = "2026-05-01T23:25:41.392Z" },
+ { url = "https://files.pythonhosted.org/packages/7c/d1/925bf776503345bef428e6c45fb017d0139ddbe0e211814b585c4253dca8/psycopg_binary-3.3.4-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d7b4d40c153fa352ab3cca530f3a0baedf7621b2ebcbd7f084009522c21788fc", size = 5006319, upload-time = "2026-05-01T23:25:51.419Z" },
+ { url = "https://files.pythonhosted.org/packages/6f/aa/99727337206fbba357ca084bf4ea8b29dc986f61842a2685859af61416db/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9b1c2533af01cd7648378599f82b0b8ae32f293296e6eec5753a625bc97ef28", size = 4535388, upload-time = "2026-05-01T23:25:57.957Z" },
+ { url = "https://files.pythonhosted.org/packages/0b/a4/567ba2c37d19d8c2f63d836385dfd2495aa5897bbee6cfab104d9ee58624/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ad3bc94054876155549fdaedf4a46d1ec69d39a5bcee377148afe498e84c4b8e", size = 4224544, upload-time = "2026-05-01T23:26:03.832Z" },
+ { url = "https://files.pythonhosted.org/packages/b7/23/86457f5a82731685d7701de7bfaa5eb783dd1fecbf875321897d9d9ce33a/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:eb4eed2079c01a4850bf467deacfab56d356d4225040170af03dc9958321242d", size = 3956282, upload-time = "2026-05-01T23:26:09.983Z" },
+ { url = "https://files.pythonhosted.org/packages/a7/d8/249456df16d47de082abd9b73bce8ccdeb0293eb12e590f9150c7cbdb788/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f80e3f2b5331dbbf0901bcb658056c03eeb2c1ef31d774afb0d61598b242e744", size = 4261736, upload-time = "2026-05-01T23:26:16.798Z" },
+ { url = "https://files.pythonhosted.org/packages/15/6b/c4abe228acafd8a385c1fb615d4f1e3c9b8ad7a4e4f0e84118ba3ffeed9c/psycopg_binary-3.3.4-cp310-cp310-win_amd64.whl", hash = "sha256:574ea21a9651958f1535c5a1c649c7409e9168bcbffa29a3f2f961f58b322949", size = 3570620, upload-time = "2026-05-01T23:26:22.655Z" },
+ { url = "https://files.pythonhosted.org/packages/b6/82/df3312c0ca083d5b43b352f27d4dd8b1e614bd334473074715d9e0000da4/psycopg_binary-3.3.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:612a627d733f695b1de1f9b4bd511c15f999a5d8b915d444bbd7dd71cf3370da", size = 4609813, upload-time = "2026-05-01T23:26:30.612Z" },
+ { url = "https://files.pythonhosted.org/packages/1f/b5/d74d542458d3e8ac0571d8a88f57ca369999b9a82f4fa528052d0d7d3e4c/psycopg_binary-3.3.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:13a7f380824c35896dcac7fe0f61440f7ca49d6dc73f3c13a9a4471e6a3b302e", size = 4676799, upload-time = "2026-05-01T23:26:38.475Z" },
+ { url = "https://files.pythonhosted.org/packages/09/67/06bab9c60671999f4c6ceff1b334f3ac1f9fc5789eb467c714623ea21de9/psycopg_binary-3.3.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:276904e3452d6a23d474ef9a21eee19f20eed3d53ddd2576af033827e0ba0992", size = 5497050, upload-time = "2026-05-01T23:26:47.061Z" },
+ { url = "https://files.pythonhosted.org/packages/72/9b/023433e2b20f970de1e22d29132a95281277646da0b2e2879dd4ee94b8c1/psycopg_binary-3.3.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab8cca8ef8fb1ccf5b048ae5bd78ba55b9e4b5d472e3ce5ca39ff4d2a9c249e4", size = 5172428, upload-time = "2026-05-01T23:26:56.708Z" },
+ { url = "https://files.pythonhosted.org/packages/08/cd/ae16da8fde228a38b2fe9269bbc13cf89e0186173f2265600f02d6a71e64/psycopg_binary-3.3.4-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7465bfe6087d2d5b42d4c53b9b11ca9f218e477317a4a162a10e3c19e984ba8e", size = 6762746, upload-time = "2026-05-01T23:27:07.023Z" },
+ { url = "https://files.pythonhosted.org/packages/4f/81/0ba09fa5f5f88779093a2541a8e02489825721f258ab88058b11d68b3eb5/psycopg_binary-3.3.4-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:22cdbf5f91ef7bb91fe0c5757e1962d3127a8010256eefd9c61fcaf441802097", size = 5006033, upload-time = "2026-05-01T23:27:12.221Z" },
+ { url = "https://files.pythonhosted.org/packages/73/6a/629136040cc3497adb442a305710b5913f2a754d4630fc3d3717c4c0df65/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e2631da29253a98bd496e6c4813b24e09a4fe3fb2a9e88513305d6f8747cce95", size = 4534175, upload-time = "2026-05-01T23:27:18.248Z" },
+ { url = "https://files.pythonhosted.org/packages/7c/32/1027f843c6dc2d5d51960ee62cc0c2cf755a4c39455aff1371173edbef7d/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7f7668f30b9dd5163197e5cbf4e0efd54e00f0a859cc566ce56cfc31f4054839", size = 4224203, upload-time = "2026-05-01T23:27:24.3Z" },
+ { url = "https://files.pythonhosted.org/packages/0b/e1/380a724d9093c74adb14d4fce920ea8327838abb61f760b1448586b14a8e/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:cffc3408d77a27973f33e5d909b624cce683db5fc25964b02fe0aae7886c1007", size = 3954509, upload-time = "2026-05-01T23:27:30.815Z" },
+ { url = "https://files.pythonhosted.org/packages/db/cd/895893ae575a09c97ccfd5def070d88993d955ef34df45a881fd5ff506d6/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0579252a1202cd73e4da137a1426e2dae993ae44e757605344282af3a082848c", size = 4259551, upload-time = "2026-05-01T23:27:38.828Z" },
+ { url = "https://files.pythonhosted.org/packages/dd/c6/2330a20794e37a3ec609ef2fd8522919ec7a4395a1abf979a8e2d1775cd5/psycopg_binary-3.3.4-cp311-cp311-win_amd64.whl", hash = "sha256:41f2ec0fea529832982bcb6c9415de3c86264ebe562b77a467c0fbcd7efbba8d", size = 3572054, upload-time = "2026-05-01T23:27:45.455Z" },
+]
+
[[package]]
name = "ptyprocess"
version = "0.7.0"
@@ -2813,6 +2940,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl", hash = "sha256:fddb9fd99a5b18da837b29710391e945b1e30c135477f484084ee513adb93809", size = 51880, upload-time = "2025-11-10T14:25:45.546Z" },
]
+[[package]]
+name = "pyee"
+version = "13.0.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" },
+]
+
[[package]]
name = "pygments"
version = "2.19.2"
@@ -2861,6 +3000,7 @@ dependencies = [
{ name = "scikit-image", version = "0.26.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
{ name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "scipy", version = "1.17.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
+ { name = "sqlalchemy" },
{ name = "tensorboard" },
{ name = "tensorboard-data-server" },
{ name = "tifffile", version = "2025.5.10", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
@@ -2875,6 +3015,8 @@ dependencies = [
[package.dev-dependencies]
dev = [
{ name = "black" },
+ { name = "dbos", extra = ["aiosqlite"] },
+ { name = "playwright" },
{ name = "pytest" },
]
@@ -2903,6 +3045,7 @@ requires-dist = [
{ name = "requests", specifier = ">=2.31" },
{ name = "scikit-image", specifier = ">=0.25" },
{ name = "scipy", specifier = ">=1.11" },
+ { name = "sqlalchemy", specifier = ">=2.0,<3" },
{ name = "tensorboard", specifier = "==2.20.0" },
{ name = "tensorboard-data-server", specifier = "==0.7.2" },
{ name = "tifffile", specifier = ">=2025.5.10" },
@@ -2915,6 +3058,8 @@ requires-dist = [
[package.metadata.requires-dev]
dev = [
{ name = "black", specifier = "==26.5.1" },
+ { name = "dbos", extras = ["aiosqlite"], specifier = "==2.28.0" },
+ { name = "playwright", specifier = ">=1.54,<2" },
{ name = "pytest", specifier = "==9.0.2" },
]
@@ -3171,6 +3316,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/7e/71/44ce230e1b7fadd372515a97e32a83011f906ddded8d03e3c6aafbdedbb7/rfc3987_syntax-1.1.0-py3-none-any.whl", hash = "sha256:6c3d97604e4c5ce9f714898e05401a0445a641cfa276432b0a648c80856f6a3f", size = 8046, upload-time = "2025-07-18T01:05:03.843Z" },
]
+[[package]]
+name = "rich"
+version = "15.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "markdown-it-py" },
+ { name = "pygments" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" },
+]
+
[[package]]
name = "rpds-py"
version = "0.30.0"
@@ -3428,6 +3586,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/94/b8/f1f62a5e3c0ad2ff1d189590bfa4c46b4f3b6e49cef6f26c6ee4e575394d/setuptools-80.10.2-py3-none-any.whl", hash = "sha256:95b30ddfb717250edb492926c92b5221f7ef3fbcc2b07579bcd4a27da21d0173", size = 1064234, upload-time = "2026-01-25T22:38:15.216Z" },
]
+[[package]]
+name = "shellingham"
+version = "1.5.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" },
+]
+
[[package]]
name = "six"
version = "1.17.0"
@@ -3473,6 +3640,11 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/fc/a1/9c4efa03300926601c19c18582531b45aededfb961ab3c3585f1e24f120b/sqlalchemy-2.0.46-py3-none-any.whl", hash = "sha256:f9c11766e7e7c0a2767dda5acb006a118640c9fc0a4104214b96269bfb78399e", size = 1937882, upload-time = "2026-01-21T18:22:10.456Z" },
]
+[package.optional-dependencies]
+asyncio = [
+ { name = "greenlet" },
+]
+
[[package]]
name = "stack-data"
version = "0.6.3"
@@ -3752,6 +3924,33 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/e0/12/b05ba554d2c623bffa59922b94b0775673de251f468a9609bc9e45de95e9/triton-3.6.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8e323d608e3a9bfcc2d9efcc90ceefb764a82b99dea12a86d643c72539ad5d3", size = 188214640, upload-time = "2026-01-20T16:00:35.869Z" },
]
+[[package]]
+name = "typer"
+version = "0.27.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "annotated-doc" },
+ { name = "colorama", marker = "sys_platform == 'win32'" },
+ { name = "rich" },
+ { name = "shellingham" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/37/78/fda3361b56efc27944f24225f6ecd13d96d6fcfe37bd0eb34e2f4c63f9fc/typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5", size = 203430, upload-time = "2026-07-15T19:21:07.007Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/40/03/26a383c9e58c213199d1aad1c3d353cfc22d4444ec6d2c0bf8ad02523843/typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1", size = 122716, upload-time = "2026-07-15T19:21:05.553Z" },
+]
+
+[[package]]
+name = "typer-slim"
+version = "0.24.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "typer" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a7/a7/e6aecc4b4eb59598829a3b5076a93aff291b4fdaa2ded25efc4e1f4d219c/typer_slim-0.24.0.tar.gz", hash = "sha256:f0ed36127183f52ae6ced2ecb2521789995992c521a46083bfcdbb652d22ad34", size = 4776, upload-time = "2026-02-16T22:08:51.2Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/a7/24/5480c20380dfd18cf33d14784096dca45a24eae6102e91d49a718d3b6855/typer_slim-0.24.0-py3-none-any.whl", hash = "sha256:d5d7ee1ee2834d5020c7c616ed5e0d0f29b9a4b1dd283bdebae198ec09778d0e", size = 3394, upload-time = "2026-02-16T22:08:49.92Z" },
+]
+
[[package]]
name = "typing-extensions"
version = "4.15.0"
@@ -3892,6 +4091,54 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" },
]
+[[package]]
+name = "websockets"
+version = "16.1.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/21/f7/bc3a25c5ec26ce62ce487690becc2f3710bbc7b33338f005ad390db0b986/websockets-16.1.1.tar.gz", hash = "sha256:db234eda965dcce15df96bb9709f587cd87d4d52aaf0e80e2f34ec04c7670c57", size = 182204, upload-time = "2026-07-17T22:51:05.858Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/08/e7/d1671fb984f9dd844e1da5288070c7c23c9eaba3082d3871aae19c3ab8b9/websockets-16.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:49ae99bdfcae803a885c926bf14f886196e84925395bb3f568fef5c0f0979d7d", size = 179570, upload-time = "2026-07-17T22:48:24.032Z" },
+ { url = "https://files.pythonhosted.org/packages/99/f5/70df723bf571f5e0b1b845e0a4ff1c966eeb84f667599fc251caa37d15a3/websockets-16.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5bfd1ac19b1b9986a9c95a82d5e23a391ebb09e12c34d7be6094b86efcc35731", size = 177252, upload-time = "2026-07-17T22:48:25.775Z" },
+ { url = "https://files.pythonhosted.org/packages/90/72/2f14b2e167170b8bf1c8bb7f9b0d78000f470d41a2085a91f33e3917b6c9/websockets-16.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9246a0d063cfcbcc85f2359dd6876d681213f4790832272aa16641b4ed5d64d4", size = 177530, upload-time = "2026-07-17T22:48:27.337Z" },
+ { url = "https://files.pythonhosted.org/packages/f3/18/a17e2f0cde02dc10154c808deed7e1d8528afff93612f70d3f0a5b19b011/websockets-16.1.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1214e673c404684b9bf7154f5cf43b45025b1a6160fac3a9e438e9c1a97e22cb", size = 186038, upload-time = "2026-07-17T22:48:28.756Z" },
+ { url = "https://files.pythonhosted.org/packages/d5/b0/41de283899cf5929d637b72a508cdbc9aa40dc0f317c6b77613fd1000488/websockets-16.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90001d893bc368e302ef168d82130b4e4fdd27b85fa094682df9b667c2d48838", size = 187278, upload-time = "2026-07-17T22:48:30.328Z" },
+ { url = "https://files.pythonhosted.org/packages/50/61/874aab5257e027f9f61b5004cec65e592babca7942b1bc09f38e72b7f1fd/websockets-16.1.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:130937b167a52af203c8d58e78d67705874e82759862e3b9671a452fec4abc87", size = 189936, upload-time = "2026-07-17T22:48:31.896Z" },
+ { url = "https://files.pythonhosted.org/packages/a6/1a/42173913ac5519607220849ed417c864d77384e4119f06dbba964a50f096/websockets-16.1.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c9f23004a3d40e89c01a7955d186a6cc83418d93b749701944ce2de3e95a1f3", size = 187796, upload-time = "2026-07-17T22:48:33.344Z" },
+ { url = "https://files.pythonhosted.org/packages/1b/f4/37c1840bd89b529479aec41470b97b7c683b107ca90b6399ac5afb99dedf/websockets-16.1.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f55f0b01956a094c8587146d9558c91937e78789c333860ffaf35931a6e5dbc4", size = 186481, upload-time = "2026-07-17T22:48:34.843Z" },
+ { url = "https://files.pythonhosted.org/packages/9e/70/652d9b964adcfbeb056f42e0ca6bece34d108fe75534e74df20643cae199/websockets-16.1.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6aaface73b9c71974c6497366d8b9628357f6c9749e09c4ea3610176c63f2ae3", size = 184351, upload-time = "2026-07-17T22:48:36.307Z" },
+ { url = "https://files.pythonhosted.org/packages/13/f1/af3850e5d48d482921985be72ebcb169c6180b3a77b57bd612deebcee23b/websockets-16.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dc0fad4933f427acd5b1cec210f3ea6dce7089e1724e4b9ec6ef47c6c04d1b3b", size = 186791, upload-time = "2026-07-17T22:48:37.762Z" },
+ { url = "https://files.pythonhosted.org/packages/1d/40/1a4e3ed4969ec378dcad337e5f1472c5e292cb3e733bc392f0dc2e230abd/websockets-16.1.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f2769a0344a09e9ccf5b3cce538bc75a51b53eff3275d3896310c8552049195d", size = 185413, upload-time = "2026-07-17T22:48:39.127Z" },
+ { url = "https://files.pythonhosted.org/packages/aa/3e/4e3fa1afe8f1a6a780434cd9ba8eb422632b044eff3dd73f6af67523c147/websockets-16.1.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:f70541f3104339f59f830522d94ebadb1bf47426287381623443d8bb1cdbf33d", size = 187178, upload-time = "2026-07-17T22:48:40.676Z" },
+ { url = "https://files.pythonhosted.org/packages/71/ab/dd742766aa5dda7f349be0de49e4d565b84cf6f7f7fa02e07692f0f2bdd9/websockets-16.1.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:dc385593a42e31cd6fb60c19f0ecb015b386603818fc2c6c274fb42bd2bb4165", size = 185051, upload-time = "2026-07-17T22:48:42.098Z" },
+ { url = "https://files.pythonhosted.org/packages/ae/f5/76438c6560f416f1c0a7f587679fb97cc6e99ed336011d43ce2002dd27c1/websockets-16.1.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:387e8e4aa5df2f90b198fa3cad3478822a89cf905b6a6d6c97dc3664689640cc", size = 185846, upload-time = "2026-07-17T22:48:43.472Z" },
+ { url = "https://files.pythonhosted.org/packages/62/12/5c0320f2127823d27b2d56d611d31b0b284ad4edcb41364d66bf4c92b537/websockets-16.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fd46fff7eb62c24804d234f0051c7a8ea81285ad63e0337d3dcf33ca82aee58a", size = 186066, upload-time = "2026-07-17T22:48:44.884Z" },
+ { url = "https://files.pythonhosted.org/packages/a2/97/875986b857b955c3f9dd192cb8a1af81254dfb2ea22cc9590f0a1e020b8b/websockets-16.1.1-cp310-cp310-win32.whl", hash = "sha256:7883388947767080f094950b342b30d35a2a06b849cd967c422fa0db72b40ea9", size = 179940, upload-time = "2026-07-17T22:48:46.481Z" },
+ { url = "https://files.pythonhosted.org/packages/54/82/1013a5fe7ddae8e102bc3b4b39db81d8d28fd02100a324ce6ede8cd832b1/websockets-16.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:d57685547e0060cc6fd90ee6a28405d6bd395e525545f13c8d7cd99c78afd79f", size = 180239, upload-time = "2026-07-17T22:48:48.043Z" },
+ { url = "https://files.pythonhosted.org/packages/2b/03/47debfe28e9d6d354be5d777b67fd44c359b9eb299a5d103500bd7cc3e37/websockets-16.1.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d0fcf657e9f13ff4b177960ab2200237b12994232dfb6df16f1cfe1d4339f93c", size = 179566, upload-time = "2026-07-17T22:48:49.596Z" },
+ { url = "https://files.pythonhosted.org/packages/72/93/31efa1ed78c17e5cfc229fd449e3966e1b9cc15753204cd585cc8dd01f4a/websockets-16.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b852788aa51764e2d8e4cf5493d559326bcae5e38d16ba25ffa322b034df272a", size = 177250, upload-time = "2026-07-17T22:48:50.942Z" },
+ { url = "https://files.pythonhosted.org/packages/01/4a/542378ab3972b0c1cf1df3df3eff9591cea0d30c58c3aa3c4ddbc244e787/websockets-16.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1427fb4cf0d72f66333e2cacc3ff5f575bf2d7008166ce991a4a470b21d51a22", size = 177528, upload-time = "2026-07-17T22:48:52.59Z" },
+ { url = "https://files.pythonhosted.org/packages/33/d9/162321f63c7eed558e9e1798ed7a1e34a4f6dab51f35419e4ed7a4907979/websockets-16.1.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:da4ca1a9d72f9030b3146b8d7022719a9f3d478f61efe6f7dd51d243f61c51b2", size = 186859, upload-time = "2026-07-17T22:48:53.915Z" },
+ { url = "https://files.pythonhosted.org/packages/de/09/87df740f7430ce564bd52402e9c9458d4d0459cc7d2ee29e530c8204851b/websockets-16.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:86d7f0f8bdb25d2c632b72527325e4776430fd5bc61b9118de4e2b8ddb5f5b01", size = 188095, upload-time = "2026-07-17T22:48:55.384Z" },
+ { url = "https://files.pythonhosted.org/packages/d2/12/3d2703af7cc095f3c81904c92208cc1ae79affbc67376944b50ee9301f73/websockets-16.1.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7dfcad78ea1492ee3a9ec765cb7f51bbc17d477107aaf6b22abf7b2558d1c5a0", size = 191385, upload-time = "2026-07-17T22:48:56.742Z" },
+ { url = "https://files.pythonhosted.org/packages/1d/69/986aa0234a964a00f5149cfc46e136e96c8faad1c783474550f40d31aef4/websockets-16.1.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fb9a0a6dc3d1b3986cb88091b6899f0396651e0f74e2c9766ab8d6ffc3842e29", size = 188653, upload-time = "2026-07-17T22:48:58.134Z" },
+ { url = "https://files.pythonhosted.org/packages/35/6b/10f9d03e3970a69ba67bd3b46b87a929b586d0300fadbfe14f57c1f85490/websockets-16.1.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29dfa8114c4a620c69591c5973860f768eac29d3fd6904f37f34266cb219c512", size = 187426, upload-time = "2026-07-17T22:48:59.515Z" },
+ { url = "https://files.pythonhosted.org/packages/56/db/bb3aad62bf63d8bb3f0634b2eabffcfb3677a34bd19492110ff6869cf703/websockets-16.1.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6ff9417c0ada4d0f7d212f928303e5579bdf3ace4c802fa4afabb30995da58c3", size = 184882, upload-time = "2026-07-17T22:49:00.916Z" },
+ { url = "https://files.pythonhosted.org/packages/6c/4c/c09a2ea9bfbeccce52fdc383e5f28af4bc8843338aabac28c81489af6120/websockets-16.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8fe0b50da2d84535fb4f7b4bfa951280f97ce3d558a0443b541166d609e67b57", size = 187584, upload-time = "2026-07-17T22:49:02.283Z" },
+ { url = "https://files.pythonhosted.org/packages/c7/8b/31bb4eb4d9eaacf1fdd39d115772a8aeaedfc19b5dc262e57ffbc8a9d42c/websockets-16.1.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:34420aaa64440ebd51ac72ca8a45ef4626429438c9b02e633ae412ed43f925d3", size = 186174, upload-time = "2026-07-17T22:49:03.973Z" },
+ { url = "https://files.pythonhosted.org/packages/2f/e4/dc02d725610a1ad49e193ef91a548194d71bdc6cdf27da83067dd1f73995/websockets-16.1.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a6a61aff018180c9c50b7b0da33bfd29d378af3497429c95006c589a23a11648", size = 187986, upload-time = "2026-07-17T22:49:05.553Z" },
+ { url = "https://files.pythonhosted.org/packages/e0/73/30ed84c8bfd14c73d4af29d5ed9323c3073b48e0b7b23b67070f4e7fd59b/websockets-16.1.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:04fd29a0e2fe9414a95b00e92c67ae51bf900c50c0f8a4b2dafdad621f49ea1d", size = 185565, upload-time = "2026-07-17T22:49:06.959Z" },
+ { url = "https://files.pythonhosted.org/packages/7d/d3/4be8d4959f51e31b4f8fc0ece12b45bd3b6c0d15ea23b9990d9c11fc805f/websockets-16.1.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:5c31aa7e39ee3e8a358573257f1c0bb5c52430d1b637030dd9c8cc2c282926be", size = 186598, upload-time = "2026-07-17T22:49:08.293Z" },
+ { url = "https://files.pythonhosted.org/packages/26/fa/abb38597a52d84ed9cfacadc7a0c6f2db282c0ab23cdf72b58a666a21227/websockets-16.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d14bfb217eb4701e850f1525c9d29d79c44794cdf1c299ead25f39f8c78dea81", size = 186834, upload-time = "2026-07-17T22:49:09.766Z" },
+ { url = "https://files.pythonhosted.org/packages/59/80/1119ad08a228b90c4eb77fbe48df7836731a605f5f881ba701ca826a4a65/websockets-16.1.1-cp311-cp311-win32.whl", hash = "sha256:2e28e602bb13da44fbe518c1781a88e3b9d4c3d48d02c9bad83e546164336f57", size = 179940, upload-time = "2026-07-17T22:49:11.196Z" },
+ { url = "https://files.pythonhosted.org/packages/71/b2/e511c1c6f64a95c2f3fc54bffda0e14eaa7e9442be605c29270f7589b918/websockets-16.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:7421fad442de870a8cbf2287d1cad7e706ece0dbfeba5e911df132cbdc1cb56a", size = 180239, upload-time = "2026-07-17T22:49:12.519Z" },
+ { url = "https://files.pythonhosted.org/packages/e1/ed/71fea6e141590cafc40b14dc5943b0845606bee87bdb52a21b6a73eb4311/websockets-16.1.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:820fb8450edddae3812fd58cbc08e2bf22812cb248ecb5f06dbb82119a56e869", size = 177185, upload-time = "2026-07-17T22:50:56.665Z" },
+ { url = "https://files.pythonhosted.org/packages/01/ec/00e7eeca200facf9266a83e4cbbf1bed0e67fba1d4d45031d3e5b3d81b5c/websockets-16.1.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:125f22dbefaf1554fea66fc83851490edb284ce4f501d37ffed2752f418332d9", size = 177459, upload-time = "2026-07-17T22:50:58.197Z" },
+ { url = "https://files.pythonhosted.org/packages/75/fd/5774c4b33f7c0d8f0c51809c8b3a93456c48e3543579262cfa64eb5f522e/websockets-16.1.1-pp311-pypy311_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:30bbe120437b5648a77d3519b7024ea09530e0b5b18d3698c5a0ae536fe0cc2e", size = 178294, upload-time = "2026-07-17T22:50:59.641Z" },
+ { url = "https://files.pythonhosted.org/packages/37/c3/48e2c03d2bd79bb45948841c592d24156312dd5f58cdf8f549febe652fb6/websockets-16.1.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b6b9dadbef0cccd9f4c4ee96b08898afa73e26803bbe0f6aeb5bb12b0074206d", size = 179190, upload-time = "2026-07-17T22:51:01.129Z" },
+ { url = "https://files.pythonhosted.org/packages/2d/3f/73e511ecf2496ceac57dd4ed8388efe2bcf0769338a2dbf242c8366ae87e/websockets-16.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56cd5fc4f10a9ea8aa0804bddb7b42506cf9e136046f3b4c27de8fec9e2ecba5", size = 180330, upload-time = "2026-07-17T22:51:02.603Z" },
+ { url = "https://files.pythonhosted.org/packages/be/4d/2d0d67834092e354d2b0498f014a41249a89556bc406cf86f3e1557bb463/websockets-16.1.1-py3-none-any.whl", hash = "sha256:6abbd3e82c731c8e531714466acd5d87b5e88ac3243465337ba71d68e23ae7e3", size = 173814, upload-time = "2026-07-17T22:51:04.184Z" },
+]
+
[[package]]
name = "werkzeug"
version = "3.1.5"