From ff5af93e86efaef8d91558d1e43bc109157e9a29 Mon Sep 17 00:00:00 2001 From: Oleh Martsokha Date: Wed, 26 Aug 2026 05:44:01 +0200 Subject: [PATCH 1/7] Split runs into detections and redactions with reviewer edits Model a pipeline run as one immutable detection (analysis pass) that can be redacted many times, and let each redaction carry a set of reviewer edits. Data model: - Rename workspace_pipeline_runs -> workspace_detections (drop output_file_id; keep the 1:1 audit) and workspace_pipeline_run_usage -> workspace_detection_usage. - New workspace_redactions child table (1 detection -> N redactions), each owning its review audit and redacted output. - Status enum PIPELINE_RUN_STATUS -> DETECTION_STATUS (pending/executing/complete/ failed; drop the never-implemented cancelled and the redaction-era completed). - Add FILE_KIND 'review' for a redaction's post-edit audit. - Rename RunId -> DetectionId and add RedactionId; RunFilter/RunMetadata and the run repository/models become their detection equivalents; new redaction repo. API: - Routes /runs/{runId}/... -> /detections/{detectionId}/...; the findings endpoint is /analysis/. POST /detections/{id}/redactions/ accepts an optional EditSet (suppress a false positive, retag, or add a missed detection), applies it to the analysis, redacts, and persists a redaction (review audit + redacted output). - New GET /detections/{id}/redactions/ (list) and /detections/{id}/redactions/{redactionId}/review (the review audit). - Reviewer edits are validated against the analysis via the engine's EditSet::validate; an unknown target or self-contradiction maps to 400 through a From impl. The detection analysis is never mutated: edits land on a working clone persisted as the redaction's review audit. Events: pipeline.run.{started,analyzed,completed,failed} -> pipeline.detection. {started,completed,failed} plus pipeline.redaction.created; PipelineRunRef -> DetectionRef; the status subject is pipeline.detections.{id}.status. Update elide-runtime to the reworked edit API (EditSet::validate/apply take the report; operator-override edits removed upstream) and the EditError re-export. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018bKk1YEG4tZ69jzYVQvQL8 --- Cargo.lock | 16 +- crates/nvisy-postgres/src/model/mod.rs | 16 +- .../src/model/workspace_detection.rs | 94 +++ ..._usage.rs => workspace_detection_usage.rs} | 24 +- .../src/model/workspace_pipeline_run.rs | 102 --- .../src/model/workspace_redaction.rs | 47 ++ crates/nvisy-postgres/src/query/analytics.rs | 107 +-- crates/nvisy-postgres/src/query/mod.rs | 6 +- .../src/query/workspace_detection.rs | 629 ++++++++++++++ .../src/query/workspace_file.rs | 79 +- .../src/query/workspace_pipeline_run.rs | 625 -------------- .../src/query/workspace_redaction.rs | 130 +++ crates/nvisy-postgres/src/schema.rs | 107 ++- .../src/types/constraint/detections.rs | 14 + .../src/types/constraint/mod.rs | 12 +- .../src/types/constraint/pipeline_runs.rs | 14 - .../src/types/enums/activity_type.rs | 66 +- .../src/types/enums/detection_status.rs | 59 ++ .../src/types/enums/file_kind.rs | 8 +- crates/nvisy-postgres/src/types/enums/mod.rs | 6 +- .../src/types/enums/notification_event.rs | 33 +- .../src/types/enums/pipeline_run_status.rs | 74 -- .../src/types/enums/webhook_event.rs | 56 +- .../filtering/{runs.rs => detections.rs} | 30 +- .../nvisy-postgres/src/types/filtering/mod.rs | 4 +- .../src/types/json/activity_params.rs | 81 +- ..._run_metadata.rs => detection_metadata.rs} | 14 +- crates/nvisy-postgres/src/types/json/mod.rs | 16 +- .../src/types/json/notification_params.rs | 70 +- crates/nvisy-postgres/src/types/mod.rs | 30 +- .../nvisy-postgres/src/types/prefixed_id.rs | 9 +- crates/nvisy-server/src/handler/analytics.rs | 8 +- ...pipeline_audits.rs => detection_audits.rs} | 95 ++- crates/nvisy-server/src/handler/detections.rs | 793 ++++++++++++++++++ .../src/handler/error/engine_error.rs | 13 + .../src/handler/error/pg_error.rs | 2 +- .../src/handler/error/pg_pipeline.rs | 20 +- crates/nvisy-server/src/handler/mod.rs | 10 +- .../nvisy-server/src/handler/pipeline_runs.rs | 763 ----------------- crates/nvisy-server/src/handler/redactions.rs | 179 ++++ .../src/handler/request/detections.rs | 102 +++ .../nvisy-server/src/handler/request/mod.rs | 4 +- .../nvisy-server/src/handler/request/paths.rs | 21 +- .../src/handler/request/pipeline_runs.rs | 85 -- .../src/handler/response/analytics.rs | 66 +- .../src/handler/response/detections.rs | 91 ++ .../nvisy-server/src/handler/response/mod.rs | 6 +- .../src/handler/response/pipeline_runs.rs | 95 --- .../src/handler/response/redactions.rs | 57 ++ .../src/middleware/specification.rs | 9 +- .../nvisy-server/src/service/detection/job.rs | 41 +- .../nvisy-server/src/service/detection/mod.rs | 13 +- .../src/service/detection/service.rs | 42 +- .../src/service/detection/support.rs | 115 +-- .../src/service/detection/worker.rs | 184 ++-- .../nvisy-server/src/service/event/drainer.rs | 88 +- crates/nvisy-server/src/service/event/mod.rs | 4 +- .../src/service/event/workspace_event.rs | 34 +- crates/nvisy-server/src/service/mod.rs | 9 +- .../src/service/run_blob_store.rs | 282 ++++++- migrations/2025-05-27-011852_files/up.sql | 7 +- .../2026-01-19-045014_pipelines/down.sql | 7 +- migrations/2026-01-19-045014_pipelines/up.sql | 271 +++--- 63 files changed, 3478 insertions(+), 2616 deletions(-) create mode 100644 crates/nvisy-postgres/src/model/workspace_detection.rs rename crates/nvisy-postgres/src/model/{workspace_pipeline_run_usage.rs => workspace_detection_usage.rs} (71%) delete mode 100644 crates/nvisy-postgres/src/model/workspace_pipeline_run.rs create mode 100644 crates/nvisy-postgres/src/model/workspace_redaction.rs create mode 100644 crates/nvisy-postgres/src/query/workspace_detection.rs delete mode 100644 crates/nvisy-postgres/src/query/workspace_pipeline_run.rs create mode 100644 crates/nvisy-postgres/src/query/workspace_redaction.rs create mode 100644 crates/nvisy-postgres/src/types/constraint/detections.rs delete mode 100644 crates/nvisy-postgres/src/types/constraint/pipeline_runs.rs create mode 100644 crates/nvisy-postgres/src/types/enums/detection_status.rs delete mode 100644 crates/nvisy-postgres/src/types/enums/pipeline_run_status.rs rename crates/nvisy-postgres/src/types/filtering/{runs.rs => detections.rs} (74%) rename crates/nvisy-postgres/src/types/json/{pipeline_run_metadata.rs => detection_metadata.rs} (70%) rename crates/nvisy-server/src/handler/{pipeline_audits.rs => detection_audits.rs} (63%) create mode 100644 crates/nvisy-server/src/handler/detections.rs delete mode 100644 crates/nvisy-server/src/handler/pipeline_runs.rs create mode 100644 crates/nvisy-server/src/handler/redactions.rs create mode 100644 crates/nvisy-server/src/handler/request/detections.rs delete mode 100644 crates/nvisy-server/src/handler/request/pipeline_runs.rs create mode 100644 crates/nvisy-server/src/handler/response/detections.rs delete mode 100644 crates/nvisy-server/src/handler/response/pipeline_runs.rs create mode 100644 crates/nvisy-server/src/handler/response/redactions.rs diff --git a/Cargo.lock b/Cargo.lock index 554b9ae1..c82dfecd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3001,7 +3001,7 @@ dependencies = [ [[package]] name = "elide-export" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide-runtime?branch=main#cf0758d50a9697c0cc1530f856bf0472709ed80f" +source = "git+https://github.com/nvisycom/elide-runtime?branch=main#ff20eb27d6246185d7f491d24a5b9e601cda8a55" dependencies = [ "csv", "elide", @@ -3024,7 +3024,7 @@ dependencies = [ [[package]] name = "elide-governance" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide-runtime?branch=main#cf0758d50a9697c0cc1530f856bf0472709ed80f" +source = "git+https://github.com/nvisycom/elide-runtime?branch=main#ff20eb27d6246185d7f491d24a5b9e601cda8a55" dependencies = [ "elide-core", "elide-operator", @@ -3162,7 +3162,7 @@ dependencies = [ [[package]] name = "elide-pipeline" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide-runtime?branch=main#cf0758d50a9697c0cc1530f856bf0472709ed80f" +source = "git+https://github.com/nvisycom/elide-runtime?branch=main#ff20eb27d6246185d7f491d24a5b9e601cda8a55" dependencies = [ "bytes", "elide", @@ -3183,7 +3183,7 @@ dependencies = [ [[package]] name = "elide-provider" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide-runtime?branch=main#cf0758d50a9697c0cc1530f856bf0472709ed80f" +source = "git+https://github.com/nvisycom/elide-runtime?branch=main#ff20eb27d6246185d7f491d24a5b9e601cda8a55" dependencies = [ "elide", "elide-bento", @@ -3207,7 +3207,7 @@ dependencies = [ [[package]] name = "elide-review" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide-runtime?branch=main#cf0758d50a9697c0cc1530f856bf0472709ed80f" +source = "git+https://github.com/nvisycom/elide-runtime?branch=main#ff20eb27d6246185d7f491d24a5b9e601cda8a55" dependencies = [ "elide", "elide-governance", @@ -3231,7 +3231,7 @@ dependencies = [ [[package]] name = "elide-template" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide-runtime?branch=main#cf0758d50a9697c0cc1530f856bf0472709ed80f" +source = "git+https://github.com/nvisycom/elide-runtime?branch=main#ff20eb27d6246185d7f491d24a5b9e601cda8a55" dependencies = [ "elide-core", "elide-governance", @@ -6988,7 +6988,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck 0.4.1", - "itertools 0.13.0", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -7007,7 +7007,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.119", diff --git a/crates/nvisy-postgres/src/model/mod.rs b/crates/nvisy-postgres/src/model/mod.rs index 00a818cb..8c22b32a 100644 --- a/crates/nvisy-postgres/src/model/mod.rs +++ b/crates/nvisy-postgres/src/model/mod.rs @@ -15,14 +15,15 @@ mod workspace_activity; mod workspace_connection; mod workspace_connection_schedule; mod workspace_connection_sync; +mod workspace_detection; +mod workspace_detection_usage; mod workspace_file; mod workspace_file_imports; mod workspace_invite; mod workspace_member; mod workspace_pipeline; -mod workspace_pipeline_run; -mod workspace_pipeline_run_usage; mod workspace_policy; +mod workspace_redaction; mod workspace_webhook; // Account models @@ -48,15 +49,16 @@ pub use workspace_connection_schedule::{ pub use workspace_connection_sync::{ NewWorkspaceConnectionSync, UpdateWorkspaceConnectionSync, WorkspaceConnectionSync, }; +// Detection / pipeline models +pub use workspace_detection::{ + NewWorkspaceDetection, UpdateWorkspaceDetection, WorkspaceDetection, +}; +pub use workspace_detection_usage::{NewWorkspaceDetectionUsage, WorkspaceDetectionUsage}; pub use workspace_file::{NewWorkspaceFile, UpdateWorkspaceFile, WorkspaceFile}; pub use workspace_file_imports::{NewWorkspaceFileImport, WorkspaceFileImport}; pub use workspace_invite::{NewWorkspaceInvite, UpdateWorkspaceInvite, WorkspaceInvite}; pub use workspace_member::{NewWorkspaceMember, UpdateWorkspaceMember, WorkspaceMember}; pub use workspace_pipeline::{NewWorkspacePipeline, UpdateWorkspacePipeline, WorkspacePipeline}; -// Pipeline models -pub use workspace_pipeline_run::{ - NewWorkspacePipelineRun, UpdateWorkspacePipelineRun, WorkspacePipelineRun, -}; -pub use workspace_pipeline_run_usage::{NewWorkspacePipelineRunUsage, WorkspacePipelineRunUsage}; pub use workspace_policy::{NewWorkspacePolicy, UpdateWorkspacePolicy, WorkspacePolicy}; +pub use workspace_redaction::{NewWorkspaceRedaction, WorkspaceRedaction}; pub use workspace_webhook::{NewWorkspaceWebhook, UpdateWorkspaceWebhook, WorkspaceWebhook}; diff --git a/crates/nvisy-postgres/src/model/workspace_detection.rs b/crates/nvisy-postgres/src/model/workspace_detection.rs new file mode 100644 index 00000000..66b51d39 --- /dev/null +++ b/crates/nvisy-postgres/src/model/workspace_detection.rs @@ -0,0 +1,94 @@ +//! Workspace detection model for PostgreSQL database operations. + +use diesel::prelude::*; +use jiff_diesel::Timestamp; +use uuid::Uuid; + +use crate::schema::workspace_detections; +use crate::types::{DetectionMetadata, DetectionStatus, Json, PipelineTriggerType}; + +/// A detection: one analysis pass of a file through a pipeline. +/// +/// Detect creates the detection and stores the engine's `Audit` in the object +/// store, keeping its file id here; the detection then stays `Complete` and can +/// be redacted any number of times (each redaction is its own row). +#[derive(Debug, Clone, PartialEq, Queryable, Selectable)] +#[diesel(table_name = workspace_detections)] +#[diesel(check_for_backend(diesel::pg::Pg))] +pub struct WorkspaceDetection { + /// Unique detection identifier. + pub id: Uuid, + /// Pipeline whose config drove the detection. + pub pipeline_id: Uuid, + /// Account the detection is attributed to (the user who started it, or the + /// pipeline's creator for a system-initiated detection). + pub account_id: Uuid, + /// Source document the detection analyzes. + pub input_file_id: Uuid, + /// Audit file (`file_kind = audit`) holding the encrypted analysis. `None` + /// until analysis writes it. + pub audit_file_id: Option, + /// How the detection was initiated. + pub trigger_type: PipelineTriggerType, + /// Current detection status. + pub status: DetectionStatus, + /// Detect idempotency key (dedupes retries). + pub idempotency_key: Option, + /// Non-encrypted metadata for filtering/display. + pub metadata: Json, + /// When a worker last claimed this detection. Acts as a lease: a redelivered + /// job whose claim is still fresh is skipped, while a stale claim (a worker + /// that died mid-analysis) can be re-claimed. `None` until first claimed. + pub claimed_at: Option, + /// When the detection started. + pub started_at: Timestamp, + /// When the detection completed analysis. + pub completed_at: Option, +} + +/// Data for creating a new workspace detection. +#[derive(Debug, Default, Clone, Insertable)] +#[diesel(table_name = workspace_detections)] +#[diesel(check_for_backend(diesel::pg::Pg))] +pub struct NewWorkspaceDetection { + /// Pipeline ID (required). + pub pipeline_id: Uuid, + /// Account the detection is attributed to (required). + pub account_id: Uuid, + /// Source document ID (required). + pub input_file_id: Uuid, + /// Audit file holding the encrypted analysis (set once analyzed). + pub audit_file_id: Option, + /// Trigger type. + pub trigger_type: Option, + /// Initial status. + pub status: Option, + /// Detect idempotency key. + pub idempotency_key: Option, + /// Non-encrypted metadata for filtering/display. + pub metadata: Option>, +} + +/// Data for updating a workspace detection. +#[derive(Debug, Clone, Default, AsChangeset)] +#[diesel(table_name = workspace_detections)] +#[diesel(check_for_backend(diesel::pg::Pg))] +pub struct UpdateWorkspaceDetection { + /// Detection status. + pub status: Option, + /// Audit file holding the encrypted analysis. + pub audit_file_id: Option>, + /// Non-encrypted metadata for filtering/display. + pub metadata: Option>, + /// When a worker last claimed this detection (lease timestamp). + pub claimed_at: Option>, + /// When the detection completed analysis. + pub completed_at: Option>, +} + +impl WorkspaceDetection { + /// Returns whether analysis is done and the detection is ready to redact. + pub fn is_complete(&self) -> bool { + self.status.is_complete() + } +} diff --git a/crates/nvisy-postgres/src/model/workspace_pipeline_run_usage.rs b/crates/nvisy-postgres/src/model/workspace_detection_usage.rs similarity index 71% rename from crates/nvisy-postgres/src/model/workspace_pipeline_run_usage.rs rename to crates/nvisy-postgres/src/model/workspace_detection_usage.rs index 42d1dfd0..cca1b436 100644 --- a/crates/nvisy-postgres/src/model/workspace_pipeline_run_usage.rs +++ b/crates/nvisy-postgres/src/model/workspace_detection_usage.rs @@ -1,19 +1,19 @@ -//! Per-model inference usage for a pipeline run. +//! Per-model inference usage for a detection. use diesel::prelude::*; use uuid::Uuid; -use crate::schema::workspace_pipeline_run_usage; +use crate::schema::workspace_detection_usage; -/// One model's token usage within a run, as the provider reported it. +/// One model's token usage within a detection, as the provider reported it. #[derive(Debug, Clone, Queryable, Selectable)] -#[diesel(table_name = workspace_pipeline_run_usage)] +#[diesel(table_name = workspace_detection_usage)] #[diesel(check_for_backend(diesel::pg::Pg))] -pub struct WorkspacePipelineRunUsage { +pub struct WorkspaceDetectionUsage { /// Unique usage row identifier. pub id: Uuid, - /// The run this usage belongs to. - pub run_id: Uuid, + /// The detection this usage belongs to. + pub detection_id: Uuid, /// The model the recognizers used. pub model: String, /// The model version, if the provider reported one. @@ -29,13 +29,13 @@ pub struct WorkspacePipelineRunUsage { pub duration_ms: i64, } -/// Data for recording one model's usage on a run. +/// Data for recording one model's usage on a detection. #[derive(Debug, Clone, Insertable)] -#[diesel(table_name = workspace_pipeline_run_usage)] +#[diesel(table_name = workspace_detection_usage)] #[diesel(check_for_backend(diesel::pg::Pg))] -pub struct NewWorkspacePipelineRunUsage { - /// The run this usage belongs to. - pub run_id: Uuid, +pub struct NewWorkspaceDetectionUsage { + /// The detection this usage belongs to. + pub detection_id: Uuid, /// The model the recognizers used. pub model: String, /// The model version, if any. diff --git a/crates/nvisy-postgres/src/model/workspace_pipeline_run.rs b/crates/nvisy-postgres/src/model/workspace_pipeline_run.rs deleted file mode 100644 index d8d82558..00000000 --- a/crates/nvisy-postgres/src/model/workspace_pipeline_run.rs +++ /dev/null @@ -1,102 +0,0 @@ -//! Workspace pipeline run model for PostgreSQL database operations. - -use diesel::prelude::*; -use jiff_diesel::Timestamp; -use uuid::Uuid; - -use crate::schema::workspace_pipeline_runs; -use crate::types::{Json, PipelineRunStatus, PipelineTriggerType, RunMetadata}; - -/// A detect/redact run: one analysis of a file through a pipeline. -/// -/// Detect creates the run and stores the engine's `AnalyzedDocument` in the -/// object store, keeping its key here; the run then awaits reviewer -/// verification before redact fetches it back and consumes it. -#[derive(Debug, Clone, PartialEq, Queryable, Selectable)] -#[diesel(table_name = workspace_pipeline_runs)] -#[diesel(check_for_backend(diesel::pg::Pg))] -pub struct WorkspacePipelineRun { - /// Unique run identifier. - pub id: Uuid, - /// Pipeline whose config drove the run. - pub pipeline_id: Uuid, - /// Account the run is attributed to (the user who started it, or the - /// pipeline's creator for a system-initiated run). - pub account_id: Uuid, - /// Source document the run analyzes / redacts. - pub input_file_id: Uuid, - /// Audit file (`file_kind = audit`) holding the encrypted analysis between - /// detect and redact. `None` until analysis writes it. - pub audit_file_id: Option, - /// Redacted document (`file_kind = redacted`) produced by redact. `None` - /// until the run completes. - pub output_file_id: Option, - /// How the run was initiated. - pub trigger_type: PipelineTriggerType, - /// Current run status. - pub status: PipelineRunStatus, - /// Detect idempotency key (dedupes retries). - pub idempotency_key: Option, - /// Non-encrypted metadata for filtering/display. - pub metadata: Json, - /// When a worker last claimed this run for detection. Acts as a lease: a - /// redelivered job whose claim is still fresh is skipped, while a stale - /// claim (a worker that died mid-analysis) can be re-claimed. `None` until - /// first claimed. - pub claimed_at: Option, - /// When the run started. - pub started_at: Timestamp, - /// When the run completed. - pub completed_at: Option, -} - -/// Data for creating a new workspace pipeline run. -#[derive(Debug, Default, Clone, Insertable)] -#[diesel(table_name = workspace_pipeline_runs)] -#[diesel(check_for_backend(diesel::pg::Pg))] -pub struct NewWorkspacePipelineRun { - /// Pipeline ID (required). - pub pipeline_id: Uuid, - /// Account the run is attributed to (required). - pub account_id: Uuid, - /// Source document ID (required). - pub input_file_id: Uuid, - /// Audit file holding the encrypted analysis (set once analyzed). - pub audit_file_id: Option, - /// Redacted output file (set once completed). - pub output_file_id: Option, - /// Trigger type. - pub trigger_type: Option, - /// Initial status. - pub status: Option, - /// Detect idempotency key. - pub idempotency_key: Option, - /// Non-encrypted metadata for filtering/display. - pub metadata: Option>, -} - -/// Data for updating a workspace pipeline run. -#[derive(Debug, Clone, Default, AsChangeset)] -#[diesel(table_name = workspace_pipeline_runs)] -#[diesel(check_for_backend(diesel::pg::Pg))] -pub struct UpdateWorkspacePipelineRun { - /// Run status. - pub status: Option, - /// Audit file holding the encrypted analysis. - pub audit_file_id: Option>, - /// Redacted output file produced by redact. - pub output_file_id: Option>, - /// Non-encrypted metadata for filtering/display. - pub metadata: Option>, - /// When a worker last claimed this run for detection (lease timestamp). - pub claimed_at: Option>, - /// When the run completed. - pub completed_at: Option>, -} - -impl WorkspacePipelineRun { - /// Returns whether detection is done and the run awaits verification. - pub fn is_analyzed(&self) -> bool { - self.status.is_analyzed() - } -} diff --git a/crates/nvisy-postgres/src/model/workspace_redaction.rs b/crates/nvisy-postgres/src/model/workspace_redaction.rs new file mode 100644 index 00000000..1ef7c806 --- /dev/null +++ b/crates/nvisy-postgres/src/model/workspace_redaction.rs @@ -0,0 +1,47 @@ +//! Workspace redaction model for PostgreSQL database operations. + +use diesel::prelude::*; +use jiff_diesel::Timestamp; +use uuid::Uuid; + +use crate::schema::workspace_redactions; + +/// A redaction: one redact pass over a detection's analysis. +/// +/// A detection can be redacted many times — each redact request may carry a +/// different set of reviewer edits — so each redaction is its own row owning the +/// review audit it applied and the redacted document it produced. +#[derive(Debug, Clone, PartialEq, Queryable, Selectable)] +#[diesel(table_name = workspace_redactions)] +#[diesel(check_for_backend(diesel::pg::Pg))] +pub struct WorkspaceRedaction { + /// Unique redaction identifier. + pub id: Uuid, + /// Detection this redaction was produced from. + pub detection_id: Uuid, + /// Account that requested the redaction. + pub account_id: Uuid, + /// Review audit (`file_kind = review`) recording the applied edits and the + /// redaction outcome. `None` only if the file was later hard-deleted. + pub review_file_id: Option, + /// Redacted document (`file_kind = redacted`) this redaction produced. + /// `None` only if the file was later hard-deleted. + pub output_file_id: Option, + /// When the redaction was created. + pub created_at: Timestamp, +} + +/// Data for creating a new workspace redaction. +#[derive(Debug, Default, Clone, Insertable)] +#[diesel(table_name = workspace_redactions)] +#[diesel(check_for_backend(diesel::pg::Pg))] +pub struct NewWorkspaceRedaction { + /// Detection this redaction was produced from (required). + pub detection_id: Uuid, + /// Account that requested the redaction (required). + pub account_id: Uuid, + /// Review audit holding the applied edits and outcome. + pub review_file_id: Option, + /// Redacted output document this redaction produced. + pub output_file_id: Option, +} diff --git a/crates/nvisy-postgres/src/query/analytics.rs b/crates/nvisy-postgres/src/query/analytics.rs index c1333043..f07765f9 100644 --- a/crates/nvisy-postgres/src/query/analytics.rs +++ b/crates/nvisy-postgres/src/query/analytics.rs @@ -17,7 +17,7 @@ use diesel::sql_types::Timestamptz; use diesel_async::RunQueryDsl; use uuid::Uuid; -use crate::types::{FileKind, PipelineRunStatus}; +use crate::types::{DetectionStatus, FileKind}; use crate::{PgConnection, PgError, PgResult, schema}; /// Per-day run counts and durations, as loaded from the grouped run query. @@ -124,12 +124,12 @@ pub struct StorageByKind { pub total_bytes: i64, } -/// Run count for one `status` in a workspace. +/// Detection count for one `status` in a workspace. #[derive(Debug, Clone, Queryable)] pub struct RunStatusCount { - /// The run status this row aggregates. - pub status: PipelineRunStatus, - /// Number of runs in this status. + /// The detection status this row aggregates. + pub status: DetectionStatus, + /// Number of detections in this status. pub count: i64, } @@ -311,16 +311,16 @@ async fn load_runs_by_status( workspace_id: Uuid, ) -> PgResult> { use diesel::dsl::count_star; - use schema::workspace_pipeline_runs::dsl as runs; + use schema::workspace_detections::dsl as detections; use schema::workspace_pipelines::dsl as pipelines; - use schema::{workspace_pipeline_runs, workspace_pipelines}; + use schema::{workspace_detections, workspace_pipelines}; - workspace_pipeline_runs::table + workspace_detections::table .inner_join(workspace_pipelines::table) .filter(pipelines::workspace_id.eq(workspace_id)) .filter(pipelines::deleted_at.is_null()) - .group_by(runs::status) - .select((runs::status, count_star())) + .group_by(detections::status) + .select((detections::status, count_star())) .load(conn) .await .map_err(PgError::from) @@ -331,9 +331,9 @@ async fn load_runs_by_status( async fn load_run_durations(conn: &mut PgConnection, workspace_id: Uuid) -> PgResult { use diesel::dsl::sql; use diesel::sql_types::{BigInt, Nullable}; - use schema::workspace_pipeline_runs::dsl as runs; + use schema::workspace_detections::dsl as detections; use schema::workspace_pipelines::dsl as pipelines; - use schema::{workspace_pipeline_runs, workspace_pipelines}; + use schema::{workspace_detections, workspace_pipelines}; // Duration in milliseconds: the interval's epoch-seconds are scaled by 1000 // and rounded to a bigint in SQL, so the value crosses the boundary already in @@ -341,20 +341,20 @@ async fn load_run_durations(conn: &mut PgConnection, workspace_id: Uuid) -> PgRe // with no Diesel builtin) both return NULL over no rows. Columns are // table-qualified so the join can never make them ambiguous. let avg_ms = sql::>( - "round(avg(EXTRACT(EPOCH FROM (workspace_pipeline_runs.completed_at \ - - workspace_pipeline_runs.started_at)) * 1000))::bigint", + "round(avg(EXTRACT(EPOCH FROM (workspace_detections.completed_at \ + - workspace_detections.started_at)) * 1000))::bigint", ); let p95_ms = sql::>( "round(percentile_cont(0.95) WITHIN GROUP \ - (ORDER BY EXTRACT(EPOCH FROM (workspace_pipeline_runs.completed_at \ - - workspace_pipeline_runs.started_at))) * 1000)::bigint", + (ORDER BY EXTRACT(EPOCH FROM (workspace_detections.completed_at \ + - workspace_detections.started_at))) * 1000)::bigint", ); - let (avg_ms, p95_ms): (Option, Option) = workspace_pipeline_runs::table + let (avg_ms, p95_ms): (Option, Option) = workspace_detections::table .inner_join(workspace_pipelines::table) .filter(pipelines::workspace_id.eq(workspace_id)) .filter(pipelines::deleted_at.is_null()) - .filter(runs::completed_at.is_not_null()) + .filter(detections::completed_at.is_not_null()) .select((avg_ms, p95_ms)) .first(conn) .await @@ -372,14 +372,14 @@ async fn load_usage_by_model( ) -> PgResult> { use bigdecimal::ToPrimitive; use diesel::dsl::sum; - use schema::workspace_pipeline_run_usage::dsl as usage; - use schema::workspace_pipeline_runs::dsl as runs; + use schema::workspace_detection_usage::dsl as usage; + use schema::workspace_detections::dsl as detections; use schema::workspace_pipelines::dsl as pipelines; - use schema::{workspace_pipeline_run_usage, workspace_pipeline_runs, workspace_pipelines}; + use schema::{workspace_detection_usage, workspace_detections, workspace_pipelines}; - let rows: Vec = workspace_pipeline_run_usage::table - .inner_join(workspace_pipeline_runs::table.on(runs::id.eq(usage::run_id))) - .inner_join(workspace_pipelines::table.on(pipelines::id.eq(runs::pipeline_id))) + let rows: Vec = workspace_detection_usage::table + .inner_join(workspace_detections::table.on(detections::id.eq(usage::detection_id))) + .inner_join(workspace_pipelines::table.on(pipelines::id.eq(detections::pipeline_id))) .filter(pipelines::workspace_id.eq(workspace_id)) .filter(pipelines::deleted_at.is_null()) .group_by(usage::model) @@ -411,7 +411,7 @@ async fn load_usage_by_model( /// pipelines can never make it ambiguous. Returns a fresh fragment per call, as /// the builder consumes it in both `group_by` and `select`. fn run_day() -> diesel::expression::SqlLiteral { - diesel::dsl::sql::("date_trunc('day', workspace_pipeline_runs.started_at)") + diesel::dsl::sql::("date_trunc('day', workspace_detections.started_at)") } /// Per-day run counts and durations over `[from, to)`, scoped through the live @@ -425,9 +425,9 @@ async fn load_run_day_counts( ) -> PgResult> { use diesel::dsl::{case_when, count_star, sql, sum}; use diesel::sql_types::{BigInt, Nullable as SqlNullable}; - use schema::workspace_pipeline_runs::dsl as runs; + use schema::workspace_detections::dsl as detections; use schema::workspace_pipelines::dsl as pipelines; - use schema::{workspace_pipeline_runs, workspace_pipelines}; + use schema::{workspace_detections, workspace_pipelines}; // Durations in milliseconds (epoch-seconds scaled by 1000, rounded to bigint), // so the value crosses the boundary already in the API unit and type. The @@ -437,33 +437,34 @@ async fn load_run_day_counts( // aggregate FILTER is not available on `count(*)`. Columns are table-qualified // so the join to pipelines can never make them ambiguous. let avg_ms = sql::>( - "round(avg(EXTRACT(EPOCH FROM (workspace_pipeline_runs.completed_at \ - - workspace_pipeline_runs.started_at))) \ - FILTER (WHERE workspace_pipeline_runs.completed_at IS NOT NULL) * 1000)::bigint", + "round(avg(EXTRACT(EPOCH FROM (workspace_detections.completed_at \ + - workspace_detections.started_at))) \ + FILTER (WHERE workspace_detections.completed_at IS NOT NULL) * 1000)::bigint", ); let p95_ms = sql::>( "round(percentile_cont(0.95) WITHIN GROUP \ - (ORDER BY EXTRACT(EPOCH FROM (workspace_pipeline_runs.completed_at \ - - workspace_pipeline_runs.started_at))) \ - FILTER (WHERE workspace_pipeline_runs.completed_at IS NOT NULL) * 1000)::bigint", + (ORDER BY EXTRACT(EPOCH FROM (workspace_detections.completed_at \ + - workspace_detections.started_at))) \ + FILTER (WHERE workspace_detections.completed_at IS NOT NULL) * 1000)::bigint", ); - workspace_pipeline_runs::table + workspace_detections::table .inner_join(workspace_pipelines::table) .filter(pipelines::workspace_id.eq(workspace_id)) .filter(pipelines::deleted_at.is_null()) - .filter(runs::started_at.ge(from)) - .filter(runs::started_at.lt(to)) + .filter(detections::started_at.ge(from)) + .filter(detections::started_at.lt(to)) .group_by(run_day()) .select(( run_day(), count_star(), + sum(case_when::<_, _, BigInt>( + detections::status.eq_any(DetectionStatus::OUTCOMES), + 1i64, + ) + .otherwise(0i64)), sum( - case_when::<_, _, BigInt>(runs::status.eq_any(PipelineRunStatus::OUTCOMES), 1i64) - .otherwise(0i64), - ), - sum( - case_when::<_, _, BigInt>(runs::status.eq(PipelineRunStatus::Failed), 1i64) + case_when::<_, _, BigInt>(detections::status.eq(DetectionStatus::Failed), 1i64) .otherwise(0i64), ), avg_ms, @@ -486,30 +487,30 @@ async fn load_run_day_tokens( to: jiff_diesel::Timestamp, ) -> PgResult> { use diesel::dsl::sum; - use schema::workspace_pipeline_run_usage::dsl as usage; - use schema::workspace_pipeline_runs::dsl as runs; + use schema::workspace_detection_usage::dsl as usage; + use schema::workspace_detections::dsl as detections; use schema::workspace_pipelines::dsl as pipelines; - use schema::{workspace_pipeline_run_usage, workspace_pipeline_runs, workspace_pipelines}; + use schema::{workspace_detection_usage, workspace_detections, workspace_pipelines}; - let per_run_input = workspace_pipeline_run_usage::table - .filter(usage::run_id.eq(runs::id)) + let per_run_input = workspace_detection_usage::table + .filter(usage::detection_id.eq(detections::id)) .select(sum(usage::input_tokens)) .single_value(); - let per_run_output = workspace_pipeline_run_usage::table - .filter(usage::run_id.eq(runs::id)) + let per_run_output = workspace_detection_usage::table + .filter(usage::detection_id.eq(detections::id)) .select(sum(usage::output_tokens)) .single_value(); - let per_run_total = workspace_pipeline_run_usage::table - .filter(usage::run_id.eq(runs::id)) + let per_run_total = workspace_detection_usage::table + .filter(usage::detection_id.eq(detections::id)) .select(sum(usage::total_tokens)) .single_value(); - workspace_pipeline_runs::table + workspace_detections::table .inner_join(workspace_pipelines::table) .filter(pipelines::workspace_id.eq(workspace_id)) .filter(pipelines::deleted_at.is_null()) - .filter(runs::started_at.ge(from)) - .filter(runs::started_at.lt(to)) + .filter(detections::started_at.ge(from)) + .filter(detections::started_at.lt(to)) .group_by(run_day()) .select(( run_day(), diff --git a/crates/nvisy-postgres/src/query/mod.rs b/crates/nvisy-postgres/src/query/mod.rs index 4fbffaa8..98a3e94c 100644 --- a/crates/nvisy-postgres/src/query/mod.rs +++ b/crates/nvisy-postgres/src/query/mod.rs @@ -27,12 +27,13 @@ mod workspace_activity; mod workspace_connection; mod workspace_connection_schedule; mod workspace_connection_sync; +mod workspace_detection; mod workspace_file; mod workspace_invite; mod workspace_member; mod workspace_pipeline; -mod workspace_pipeline_run; mod workspace_policy; +mod workspace_redaction; mod workspace_webhook; pub use account::AccountRepository; @@ -51,10 +52,11 @@ pub use workspace_activity::{ActivityFilter, WorkspaceActivityRepository}; pub use workspace_connection::{ScheduledConnection, WorkspaceConnectionRepository}; pub use workspace_connection_schedule::WorkspaceConnectionScheduleRepository; pub use workspace_connection_sync::WorkspaceConnectionSyncRepository; +pub use workspace_detection::{DetectionFiles, DetectionListRow, WorkspaceDetectionRepository}; pub use workspace_file::{ExpiredFileRef, ImportedFileRef, WorkspaceFileRepository}; pub use workspace_invite::WorkspaceInviteRepository; pub use workspace_member::WorkspaceMemberRepository; pub use workspace_pipeline::WorkspacePipelineRepository; -pub use workspace_pipeline_run::{PipelineRunListRow, RunFiles, WorkspacePipelineRunRepository}; pub use workspace_policy::WorkspacePolicyRepository; +pub use workspace_redaction::WorkspaceRedactionRepository; pub use workspace_webhook::WorkspaceWebhookRepository; diff --git a/crates/nvisy-postgres/src/query/workspace_detection.rs b/crates/nvisy-postgres/src/query/workspace_detection.rs new file mode 100644 index 00000000..aa29453f --- /dev/null +++ b/crates/nvisy-postgres/src/query/workspace_detection.rs @@ -0,0 +1,629 @@ +//! Workspace detections repository for managing analysis instances. + +use std::future::Future; + +use diesel::prelude::*; +use diesel_async::RunQueryDsl; +use uuid::Uuid; + +use crate::model::{ + NewWorkspaceDetection, NewWorkspaceDetectionUsage, UpdateWorkspaceDetection, + WorkspaceDetection, WorkspacePipeline, +}; +use crate::types::{ + AccountRefRow, CursorPage, CursorPagination, DetectionFilter, DetectionStatus, Handle, +}; +use crate::{PgConnection, PgError, PgResult, schema}; + +/// Resolved display name of a detection's input file. +/// +/// `None` when the file has been removed (e.g. by retention). Redacted outputs +/// belong to redactions, not the detection, so they are not resolved here. +#[derive(Debug, Default, Clone)] +pub struct DetectionFiles { + /// Display name of the input document the detection analyzes. + pub input: Option, +} + +/// One row of a detection listing: the detection plus the context a response +/// needs to render it without follow-up lookups — the triggering account, the +/// owning pipeline's slug, and the input file's display name (`None` if the file +/// was removed). +#[derive(Debug, Clone)] +pub struct DetectionListRow { + /// The detection. + pub detection: WorkspaceDetection, + /// The account that triggered the detection. + pub account: AccountRefRow, + /// Slug of the detection's owning pipeline. + pub pipeline_slug: Handle, + /// Display name of the detection's input document, if still present. + pub input_file_name: Option, +} + +/// Repository for workspace detection database operations. +/// +/// Handles detection lifecycle management including creation, status updates, +/// completion tracking, and queries. +pub trait WorkspaceDetectionRepository { + /// Creates a new workspace detection record. + fn create_workspace_detection( + &mut self, + new_detection: NewWorkspaceDetection, + ) -> impl Future> + Send; + + /// Finds a detection by its opaque id, scoped to a workspace, returning the + /// detection and its owning pipeline. + /// + /// The detection is addressed by its own id (behind `/detections/{detectionId}`); + /// scoping through the owning pipeline keeps it workspace-bounded and hides + /// detections of soft-deleted pipelines. + fn find_workspace_detection_by_id( + &mut self, + workspace_id: Uuid, + detection_id: Uuid, + ) -> impl Future>> + Send; + + /// Finds a detection by its `(pipeline, idempotency key)` pair, for detect + /// replay. + fn find_detection_by_idempotency_key( + &mut self, + pipeline_id: Uuid, + idempotency_key: &str, + ) -> impl Future>> + Send; + + /// Lists a specific pipeline's detections with cursor pagination. `filter` + /// narrows by status and/or file (its `pipeline_id` is ignored — the listing + /// is already pipeline-scoped). + fn cursor_list_pipeline_detections( + &mut self, + pipeline_id: Uuid, + pagination: CursorPagination, + filter: &DetectionFilter, + ) -> impl Future>> + Send; + + /// Lists all detections across a workspace's pipelines with cursor + /// pagination. + /// + /// Detections carry no workspace reference of their own, so this joins through + /// the owning pipeline and filters on its workspace. `filter` narrows by + /// status, file, and/or owning pipeline; use [`cursor_list_pipeline_detections`] + /// for a single pipeline. + /// + /// [`cursor_list_pipeline_detections`]: Self::cursor_list_pipeline_detections + fn cursor_list_workspace_detections( + &mut self, + workspace_id: Uuid, + pagination: CursorPagination, + filter: &DetectionFilter, + ) -> impl Future>> + Send; + + /// Atomically claims a detection, transitioning it to `Executing`. + /// + /// Succeeds (returning the claimed detection) only when it is still `Pending`, + /// or is `Executing` but its previous claim has gone stale (`claimed_at` + /// older than `stale_before` — a worker that died mid-analysis). A detection + /// already executing under a fresh claim, or past the detect phase, yields + /// `None` so a redelivered job skips it instead of analyzing twice. + /// + /// The claim stamps `claimed_at = now()`, so the lease renews on each + /// (re)claim. Callers pass `stale_before = now - lease` computed against the + /// same clock the DB uses closely enough for a lease measured in minutes. + fn claim_detection( + &mut self, + detection_id: Uuid, + stale_before: jiff::Timestamp, + ) -> impl Future>> + Send; + + /// Resolves the display name of a detection's input file. + /// + /// An indexed lookup by id; a file removed (e.g. by retention) yields `None`. + /// Used to name a single detection's input file in its response without + /// threading a join through the shared detection lookup. + fn detection_file_names( + &mut self, + workspace_id: Uuid, + detection: &WorkspaceDetection, + ) -> impl Future> + Send; + + /// Updates a workspace detection with new data. + fn update_workspace_detection( + &mut self, + detection_id: Uuid, + updates: UpdateWorkspaceDetection, + ) -> impl Future> + Send; + + /// Transitions a detection to `Complete` only while the caller still holds + /// its claim — the detection is still `Executing` and its `claimed_at` matches + /// the value stamped when the caller claimed it. Returns `true` on success, + /// `false` if the claim has gone stale (another worker re-claimed the + /// detection after the lease expired), so the caller can abort without + /// stamping over the new owner's work. `updates` carries the analyze results + /// (audit file, metadata); status and the claim guard are applied here. + fn finalize_detection( + &mut self, + detection_id: Uuid, + claimed_at: jiff::Timestamp, + updates: UpdateWorkspaceDetection, + ) -> impl Future> + Send; + + /// Transitions a detection to `Failed` only while the caller still holds its + /// claim — the detection is still `Executing` and its `claimed_at` matches the + /// value stamped when the caller claimed it. Returns `true` on success, + /// `false` if the claim has gone stale (another worker re-claimed the + /// detection). Mirrors [`finalize_detection`](Self::finalize_detection) for + /// the failure path, so a worker whose lease expired mid-analysis cannot fail + /// a detection another worker now owns. `updates` carries the failure reason + /// and completion time; status and the claim guard are applied here. + fn fail_detection( + &mut self, + detection_id: Uuid, + claimed_at: jiff::Timestamp, + updates: UpdateWorkspaceDetection, + ) -> impl Future> + Send; + + /// Records a detection's per-model inference usage. A no-op for an empty slice + /// (a deterministic detection spends no tokens). Inserted once, at analyze + /// time. + fn record_detection_usage( + &mut self, + usage: &[NewWorkspaceDetectionUsage], + ) -> impl Future> + Send; +} + +impl WorkspaceDetectionRepository for PgConnection { + async fn create_workspace_detection( + &mut self, + new_detection: NewWorkspaceDetection, + ) -> PgResult { + use schema::workspace_detections; + + let detection = diesel::insert_into(workspace_detections::table) + .values(&new_detection) + .returning(WorkspaceDetection::as_returning()) + .get_result(self) + .await + .map_err(PgError::from)?; + + Ok(detection) + } + + async fn find_workspace_detection_by_id( + &mut self, + workspace_id: Uuid, + detection_id: Uuid, + ) -> PgResult> { + use schema::workspace_detections::dsl as detections; + use schema::{workspace_detections, workspace_pipelines}; + + // Detections carry no workspace column; scope through the owning pipeline + // so the id resolves only within its workspace, and only while that + // pipeline is live (a soft-deleted pipeline hides its detections). The + // pipeline is returned alongside so callers need no second lookup. + let detection = workspace_detections::table + .inner_join(workspace_pipelines::table) + .filter(detections::id.eq(detection_id)) + .filter(workspace_pipelines::workspace_id.eq(workspace_id)) + .filter(workspace_pipelines::deleted_at.is_null()) + .select(( + WorkspaceDetection::as_select(), + WorkspacePipeline::as_select(), + )) + .first(self) + .await + .optional() + .map_err(PgError::from)?; + + Ok(detection) + } + + async fn find_detection_by_idempotency_key( + &mut self, + pipeline_id: Uuid, + idempotency_key: &str, + ) -> PgResult> { + use schema::workspace_detections::{self, dsl}; + + let detection = workspace_detections::table + .filter(dsl::pipeline_id.eq(pipeline_id)) + .filter(dsl::idempotency_key.eq(idempotency_key)) + .select(WorkspaceDetection::as_select()) + .first(self) + .await + .optional() + .map_err(PgError::from)?; + + Ok(detection) + } + + async fn cursor_list_pipeline_detections( + &mut self, + pipeline_id: Uuid, + pagination: CursorPagination, + filter: &DetectionFilter, + ) -> PgResult> { + use schema::workspace_detections::dsl; + use schema::{accounts, workspace_detections, workspace_files, workspace_pipelines}; + + // Build base query with filters. The listing is already scoped to one + // pipeline, so `filter.pipeline_id` is not applied here. + let mut base_query = workspace_detections::table + .filter(dsl::pipeline_id.eq(pipeline_id)) + .into_boxed(); + + if let Some(status) = filter.status { + base_query = base_query.filter(dsl::status.eq(status)); + } + if let Some(file_id) = filter.input_file_id { + base_query = base_query.filter(dsl::input_file_id.eq(file_id)); + } + if let Some(account_id) = filter.account_id { + base_query = base_query.filter(dsl::account_id.eq(account_id)); + } + if let Some(trigger_type) = filter.trigger_type { + base_query = base_query.filter(dsl::trigger_type.eq(trigger_type)); + } + + let total = if pagination.include_count { + Some( + base_query + .count() + .get_result::(self) + .await + .map_err(PgError::from)?, + ) + } else { + None + }; + + // Rebuild query for fetching items. Join the owning pipeline (for its + // slug) and the input file (to name the detection's analyzed document) so + // a row is self-contained; a LEFT JOIN on the file tolerates one removed + // by retention, yielding a null name. + let mut query = workspace_detections::table + .inner_join(accounts::table) + .inner_join(workspace_pipelines::table) + .left_join(workspace_files::table.on(dsl::input_file_id.eq(workspace_files::id))) + .filter(dsl::pipeline_id.eq(pipeline_id)) + .into_boxed(); + + if let Some(status) = filter.status { + query = query.filter(dsl::status.eq(status)); + } + if let Some(file_id) = filter.input_file_id { + query = query.filter(dsl::input_file_id.eq(file_id)); + } + if let Some(account_id) = filter.account_id { + query = query.filter(dsl::account_id.eq(account_id)); + } + if let Some(trigger_type) = filter.trigger_type { + query = query.filter(dsl::trigger_type.eq(trigger_type)); + } + + let limit = pagination.fetch_limit(); + let selection = ( + WorkspaceDetection::as_select(), + ( + accounts::username, + accounts::display_name, + accounts::avatar_url, + ), + workspace_pipelines::slug, + workspace_files::display_name.nullable(), + ); + + let rows: Vec<(WorkspaceDetection, AccountRefRow, Handle, Option)> = + if let Some(cursor) = &pagination.after { + let cursor_time = jiff_diesel::Timestamp::from(cursor.timestamp); + + query + .filter( + dsl::started_at + .lt(&cursor_time) + .or(dsl::started_at.eq(&cursor_time).and(dsl::id.lt(cursor.id))), + ) + .select(selection) + .order((dsl::started_at.desc(), dsl::id.desc())) + .limit(limit) + .load(self) + .await + .map_err(PgError::from)? + } else { + query + .select(selection) + .order((dsl::started_at.desc(), dsl::id.desc())) + .limit(limit) + .load(self) + .await + .map_err(PgError::from)? + }; + + let items = rows + .into_iter() + .map( + |(detection, account, pipeline_slug, input_file_name)| DetectionListRow { + detection, + account, + pipeline_slug, + input_file_name, + }, + ) + .collect(); + + Ok(CursorPage::new(items, total, pagination.limit, |row| { + (row.detection.started_at.into(), row.detection.id) + })) + } + + async fn cursor_list_workspace_detections( + &mut self, + workspace_id: Uuid, + pagination: CursorPagination, + filter: &DetectionFilter, + ) -> PgResult> { + use schema::accounts::dsl as accounts; + use schema::workspace_detections::dsl as detections; + use schema::workspace_files::dsl as files; + use schema::workspace_pipelines::dsl as pipelines; + + // Detections have no workspace column; scope them through the owning + // pipeline. The owning pipeline's slug, the triggering account, and the + // input file's name are selected alongside each detection so the + // cross-pipeline response can name its pipeline, trigger, and analyzed + // document without a per-row lookup. The input file is LEFT-joined so a + // file removed by retention yields a null name rather than dropping the + // detection. + let scoped = || { + let mut query = detections::workspace_detections + .inner_join(pipelines::workspace_pipelines) + .inner_join(accounts::accounts) + .left_join(files::workspace_files.on(detections::input_file_id.eq(files::id))) + .filter(pipelines::workspace_id.eq(workspace_id)) + .into_boxed(); + if let Some(status) = filter.status { + query = query.filter(detections::status.eq(status)); + } + if let Some(file_id) = filter.input_file_id { + query = query.filter(detections::input_file_id.eq(file_id)); + } + if let Some(pipeline_id) = filter.pipeline_id { + query = query.filter(detections::pipeline_id.eq(pipeline_id)); + } + if let Some(account_id) = filter.account_id { + query = query.filter(detections::account_id.eq(account_id)); + } + if let Some(trigger_type) = filter.trigger_type { + query = query.filter(detections::trigger_type.eq(trigger_type)); + } + query + }; + + let total = if pagination.include_count { + Some( + scoped() + .count() + .get_result::(self) + .await + .map_err(PgError::from)?, + ) + } else { + None + }; + + let limit = pagination.fetch_limit(); + let selection = ( + WorkspaceDetection::as_select(), + pipelines::slug, + ( + accounts::username, + accounts::display_name, + accounts::avatar_url, + ), + files::display_name.nullable(), + ); + + let rows: Vec<(WorkspaceDetection, Handle, AccountRefRow, Option)> = + if let Some(cursor) = &pagination.after { + let cursor_time = jiff_diesel::Timestamp::from(cursor.timestamp); + + scoped() + .filter( + detections::started_at + .lt(&cursor_time) + .or(detections::started_at + .eq(&cursor_time) + .and(detections::id.lt(cursor.id))), + ) + .select(selection) + .order((detections::started_at.desc(), detections::id.desc())) + .limit(limit) + .load(self) + .await + .map_err(PgError::from)? + } else { + scoped() + .select(selection) + .order((detections::started_at.desc(), detections::id.desc())) + .limit(limit) + .load(self) + .await + .map_err(PgError::from)? + }; + + let items = rows + .into_iter() + .map( + |(detection, pipeline_slug, account, input_file_name)| DetectionListRow { + detection, + account, + pipeline_slug, + input_file_name, + }, + ) + .collect(); + + Ok(CursorPage::new(items, total, pagination.limit, |row| { + (row.detection.started_at.into(), row.detection.id) + })) + } + + async fn claim_detection( + &mut self, + detection_id: Uuid, + stale_before: jiff::Timestamp, + ) -> PgResult> { + use schema::workspace_detections::{self, dsl}; + + let stale_before = jiff_diesel::Timestamp::from(stale_before); + + // Claim only if still pending, or executing under a claim that has gone + // stale (a dead worker). The WHERE clause makes the transition atomic: + // two concurrent deliveries race on the same row and exactly one flips + // it to `executing`; the loser matches no row and gets `None`. + let claimed = diesel::update( + workspace_detections::table + .filter(dsl::id.eq(detection_id)) + .filter( + dsl::status.eq(DetectionStatus::Pending).or(dsl::status + .eq(DetectionStatus::Executing) + .and(dsl::claimed_at.lt(stale_before))), + ), + ) + .set(( + dsl::status.eq(DetectionStatus::Executing), + dsl::claimed_at.eq(diesel::dsl::now), + )) + .returning(WorkspaceDetection::as_returning()) + .get_result(self) + .await + .optional() + .map_err(PgError::from)?; + + Ok(claimed) + } + + async fn detection_file_names( + &mut self, + workspace_id: Uuid, + detection: &WorkspaceDetection, + ) -> PgResult { + use schema::workspace_files::{self, dsl}; + + // Select the display name only, scoped to the workspace and excluding + // soft-deleted files, so a file removed by retention resolves to `None`. + async fn name_of( + conn: &mut PgConnection, + workspace_id: Uuid, + file_id: Uuid, + ) -> PgResult> { + workspace_files::table + .filter(dsl::id.eq(file_id)) + .filter(dsl::workspace_id.eq(workspace_id)) + .filter(dsl::deleted_at.is_null()) + .select(dsl::display_name) + .first::(conn) + .await + .optional() + .map_err(PgError::from) + } + + let input = name_of(self, workspace_id, detection.input_file_id).await?; + + Ok(DetectionFiles { input }) + } + + async fn update_workspace_detection( + &mut self, + detection_id: Uuid, + updates: UpdateWorkspaceDetection, + ) -> PgResult { + use schema::workspace_detections::{self, dsl}; + + let detection = + diesel::update(workspace_detections::table.filter(dsl::id.eq(detection_id))) + .set(&updates) + .returning(WorkspaceDetection::as_returning()) + .get_result(self) + .await + .map_err(PgError::from)?; + + Ok(detection) + } + + async fn finalize_detection( + &mut self, + detection_id: Uuid, + claimed_at: jiff::Timestamp, + mut updates: UpdateWorkspaceDetection, + ) -> PgResult { + use schema::workspace_detections::{self, dsl}; + + // Force the terminal transition here; the guard makes it a no-op unless we + // still own the claim. + updates.status = Some(DetectionStatus::Complete); + let claimed_at = jiff_diesel::Timestamp::from(claimed_at); + + // Guard on the claim we hold: same detection, still `Executing`, and the + // exact `claimed_at` our claim stamped. A worker that re-claimed a stale + // detection renews `claimed_at`, so a lost claim matches no row and + // returns false. + let updated = diesel::update( + workspace_detections::table + .filter(dsl::id.eq(detection_id)) + .filter(dsl::status.eq(DetectionStatus::Executing)) + .filter(dsl::claimed_at.eq(claimed_at)), + ) + .set(&updates) + .execute(self) + .await + .map_err(PgError::from)?; + + Ok(updated == 1) + } + + async fn fail_detection( + &mut self, + detection_id: Uuid, + claimed_at: jiff::Timestamp, + mut updates: UpdateWorkspaceDetection, + ) -> PgResult { + use schema::workspace_detections::{self, dsl}; + + updates.status = Some(DetectionStatus::Failed); + let claimed_at = jiff_diesel::Timestamp::from(claimed_at); + + // Same claim guard as the complete finalize: only our still-live claim + // (detection `Executing`, `claimed_at` unchanged) may fail the detection. + let updated = diesel::update( + workspace_detections::table + .filter(dsl::id.eq(detection_id)) + .filter(dsl::status.eq(DetectionStatus::Executing)) + .filter(dsl::claimed_at.eq(claimed_at)), + ) + .set(&updates) + .execute(self) + .await + .map_err(PgError::from)?; + + Ok(updated == 1) + } + + async fn record_detection_usage( + &mut self, + usage: &[NewWorkspaceDetectionUsage], + ) -> PgResult<()> { + use schema::workspace_detection_usage; + + if usage.is_empty() { + return Ok(()); + } + + diesel::insert_into(workspace_detection_usage::table) + .values(usage) + .execute(self) + .await + .map_err(PgError::from)?; + + Ok(()) + } +} diff --git a/crates/nvisy-postgres/src/query/workspace_file.rs b/crates/nvisy-postgres/src/query/workspace_file.rs index c32f2247..68c935e8 100644 --- a/crates/nvisy-postgres/src/query/workspace_file.rs +++ b/crates/nvisy-postgres/src/query/workspace_file.rs @@ -11,8 +11,8 @@ use uuid::Uuid; use crate::model::{NewWorkspaceFile, NewWorkspaceFileImport, UpdateWorkspaceFile, WorkspaceFile}; use crate::query::search::ilike_contains; use crate::types::{ - AccountRefRow, CursorPage, CursorPagination, FileFilter, FileKind, FileSortBy, FileSortField, - OffsetPagination, PipelineRunStatus, SortOrder, WithAccountRef, + AccountRefRow, CursorPage, CursorPagination, DetectionStatus, FileFilter, FileKind, FileSortBy, + FileSortField, OffsetPagination, SortOrder, WithAccountRef, }; use crate::{PgConnection, PgError, PgResult, schema}; @@ -362,27 +362,28 @@ impl WorkspaceFileRepository for PgConnection { async fn files_due_for_expiry(&mut self, limit: i64) -> PgResult> { use diesel::dsl::{exists, not, now}; - use schema::workspace_pipeline_runs::dsl as runs; - use schema::{workspace_files, workspace_pipeline_runs}; - - // A run that has not finished (running or awaiting redaction) still needs - // its input document and audit blob, so those files are held back from - // expiry until the run reaches a terminal state. Otherwise an in-flight - // detect/redact could lose its source or analysis mid-flight and get - // stuck. Produced outputs are not protected — they only exist once a run - // has completed. - let active_run_holds_file = exists( - workspace_pipeline_runs::table.filter( - runs::status + use schema::workspace_detections::dsl as detections; + use schema::{workspace_detections, workspace_files}; + + // A detection that is still pending, executing, or complete needs its + // input document and audit blob: pending/executing may still analyze + // them, and a complete detection can still be redacted (redaction reads + // the audit and input). So those files are held back from expiry until the + // detection fails. Otherwise an in-flight detect/redact could lose its + // source or analysis mid-flight and get stuck. Redaction outputs are not + // protected here — they belong to redactions, not the detection. + let active_detection_holds_file = exists( + workspace_detections::table.filter( + detections::status .eq_any([ - PipelineRunStatus::Queued, - PipelineRunStatus::Analyzing, - PipelineRunStatus::Analyzed, + DetectionStatus::Pending, + DetectionStatus::Executing, + DetectionStatus::Complete, ]) .and( - runs::input_file_id + detections::input_file_id .eq(workspace_files::id) - .or(runs::audit_file_id.eq(workspace_files::id.nullable())), + .or(detections::audit_file_id.eq(workspace_files::id.nullable())), ), ), ); @@ -391,7 +392,7 @@ impl WorkspaceFileRepository for PgConnection { .filter(workspace_files::expires_at.is_not_null()) .filter(workspace_files::expires_at.lt(now)) .filter(workspace_files::deleted_at.is_null()) - .filter(not(active_run_holds_file)) + .filter(not(active_detection_holds_file)) .select(( workspace_files::id, workspace_files::storage_path, @@ -466,26 +467,38 @@ impl WorkspaceFileRepository for PgConnection { kind: FileKind, expires_at: Option, ) -> PgResult { - use schema::workspace_pipeline_runs::dsl as runs; - use schema::{workspace_files, workspace_pipeline_runs}; + use schema::workspace_detections::dsl as detections; + use schema::workspace_redactions::dsl as redactions; + use schema::{workspace_detections, workspace_files, workspace_redactions}; let expires_at = expires_at.map(jiff_diesel::Timestamp::from); - // Collect the ids of files this pipeline's runs produced for `kind`: - // redacted files are the runs' outputs, audit files their audit blobs. - // Any other kind is not pipeline-produced, so there is nothing to do. + // Collect the ids of files this pipeline produced for `kind`. Audit blobs + // belong to the pipeline's detections; redacted outputs and review blobs + // belong to the redactions of those detections (joined back to the + // pipeline through the detection). Any other kind is not pipeline-produced, + // so there is nothing to do. let file_ids: Vec = match kind { - FileKind::Redacted => workspace_pipeline_runs::table - .filter(runs::pipeline_id.eq(pipeline_id)) - .filter(runs::output_file_id.is_not_null()) - .select(runs::output_file_id.assume_not_null()) + FileKind::Audit => workspace_detections::table + .filter(detections::pipeline_id.eq(pipeline_id)) + .filter(detections::audit_file_id.is_not_null()) + .select(detections::audit_file_id.assume_not_null()) .load(self) .await .map_err(PgError::from)?, - FileKind::Audit => workspace_pipeline_runs::table - .filter(runs::pipeline_id.eq(pipeline_id)) - .filter(runs::audit_file_id.is_not_null()) - .select(runs::audit_file_id.assume_not_null()) + FileKind::Redacted => workspace_redactions::table + .inner_join(workspace_detections::table) + .filter(detections::pipeline_id.eq(pipeline_id)) + .filter(redactions::output_file_id.is_not_null()) + .select(redactions::output_file_id.assume_not_null()) + .load(self) + .await + .map_err(PgError::from)?, + FileKind::Review => workspace_redactions::table + .inner_join(workspace_detections::table) + .filter(detections::pipeline_id.eq(pipeline_id)) + .filter(redactions::review_file_id.is_not_null()) + .select(redactions::review_file_id.assume_not_null()) .load(self) .await .map_err(PgError::from)?, diff --git a/crates/nvisy-postgres/src/query/workspace_pipeline_run.rs b/crates/nvisy-postgres/src/query/workspace_pipeline_run.rs deleted file mode 100644 index 45c6595f..00000000 --- a/crates/nvisy-postgres/src/query/workspace_pipeline_run.rs +++ /dev/null @@ -1,625 +0,0 @@ -//! Workspace pipeline runs repository for managing pipeline execution instances. - -use std::future::Future; - -use diesel::prelude::*; -use diesel_async::RunQueryDsl; -use uuid::Uuid; - -use crate::model::{ - NewWorkspacePipelineRun, NewWorkspacePipelineRunUsage, UpdateWorkspacePipelineRun, - WorkspacePipeline, WorkspacePipelineRun, -}; -use crate::types::{ - AccountRefRow, CursorPage, CursorPagination, Handle, PipelineRunStatus, RunFilter, -}; -use crate::{PgConnection, PgError, PgResult, schema}; - -/// Resolved display names of a run's input and output files. -/// -/// Each is `None` when the run has no such file yet (no output before redaction) -/// or the file has been removed (e.g. by retention). -#[derive(Debug, Default, Clone)] -pub struct RunFiles { - /// Display name of the input document the run analyzes. - pub input: Option, - /// Display name of the redacted output, once the run has produced one. - pub output: Option, -} - -/// One row of a pipeline-run listing: the run plus the context a response needs -/// to render it without follow-up lookups — the triggering account, the owning -/// pipeline's slug, and the input file's display name (`None` if the file was -/// removed). -#[derive(Debug, Clone)] -pub struct PipelineRunListRow { - /// The run. - pub run: WorkspacePipelineRun, - /// The account that triggered the run. - pub account: AccountRefRow, - /// Slug of the run's owning pipeline. - pub pipeline_slug: Handle, - /// Display name of the run's input document, if still present. - pub input_file_name: Option, -} - -/// Repository for workspace pipeline run database operations. -/// -/// Handles pipeline run lifecycle management including creation, status updates, -/// completion tracking, and queries. -pub trait WorkspacePipelineRunRepository { - /// Creates a new workspace pipeline run record. - fn create_workspace_pipeline_run( - &mut self, - new_run: NewWorkspacePipelineRun, - ) -> impl Future> + Send; - - /// Finds a run by its opaque id, scoped to a workspace, returning the run - /// and its owning pipeline. - /// - /// The run is addressed by its own id (behind `/runs/{runId}`); scoping - /// through the owning pipeline keeps it workspace-bounded and hides runs of - /// soft-deleted pipelines. - fn find_workspace_run_by_id( - &mut self, - workspace_id: Uuid, - run_id: Uuid, - ) -> impl Future>> + Send; - - /// Finds a run by its `(pipeline, idempotency key)` pair, for detect replay. - fn find_pipeline_run_by_idempotency_key( - &mut self, - pipeline_id: Uuid, - idempotency_key: &str, - ) -> impl Future>> + Send; - - /// Lists a specific pipeline's runs with cursor pagination. `filter` narrows - /// by status and/or file (its `pipeline_id` is ignored — the listing is - /// already pipeline-scoped). - fn cursor_list_workspace_pipeline_runs( - &mut self, - pipeline_id: Uuid, - pagination: CursorPagination, - filter: &RunFilter, - ) -> impl Future>> + Send; - - /// Lists all runs across a workspace's pipelines with cursor pagination. - /// - /// Runs carry no workspace reference of their own, so this joins through the - /// owning pipeline and filters on its workspace. `filter` narrows by status, - /// file, and/or owning pipeline; use [`cursor_list_workspace_pipeline_runs`] - /// for a single pipeline. - /// - /// [`cursor_list_workspace_pipeline_runs`]: Self::cursor_list_workspace_pipeline_runs - fn cursor_list_workspace_runs( - &mut self, - workspace_id: Uuid, - pagination: CursorPagination, - filter: &RunFilter, - ) -> impl Future>> + Send; - - /// Atomically claims a run for detection, transitioning it to `Analyzing`. - /// - /// Succeeds (returning the claimed run) only when the run is still `Queued`, - /// or is `Analyzing` but its previous claim has gone stale (`claimed_at` - /// older than `stale_before` — a worker that died mid-analysis). A run - /// already being analyzed under a fresh claim, or past the detect phase, - /// yields `None` so a redelivered job skips it instead of analyzing twice. - /// - /// The claim stamps `claimed_at = now()`, so the lease renews on each - /// (re)claim. Callers pass `stale_before = now - lease` computed against the - /// same clock the DB uses closely enough for a lease measured in minutes. - fn claim_run_for_detection( - &mut self, - run_id: Uuid, - stale_before: jiff::Timestamp, - ) -> impl Future>> + Send; - - /// Resolves the display names of a run's input and output files. - /// - /// Two indexed lookups by id (the output only when the run has produced one); - /// a file removed (e.g. by retention) yields `None` for that name. Used to - /// name a single run's files in its response without threading a join - /// through the shared run lookup. - fn run_file_names( - &mut self, - workspace_id: Uuid, - run: &WorkspacePipelineRun, - ) -> impl Future> + Send; - - /// Updates a workspace pipeline run with new data. - fn update_workspace_pipeline_run( - &mut self, - run_id: Uuid, - updates: UpdateWorkspacePipelineRun, - ) -> impl Future> + Send; - - /// Transitions a run to `Analyzed` only while the caller still holds its - /// claim — the run is still `Analyzing` and its `claimed_at` matches the value - /// stamped when the caller claimed it. Returns `true` on success, `false` if - /// the claim has gone stale (another worker re-claimed the run after the lease - /// expired), so the caller can abort without stamping over the new owner's - /// work. `updates` carries the analyze results (audit file, metadata); status - /// and the claim guard are applied here. - fn finalize_analyzed_run( - &mut self, - run_id: Uuid, - claimed_at: jiff::Timestamp, - updates: UpdateWorkspacePipelineRun, - ) -> impl Future> + Send; - - /// Transitions a run to `Failed` only while the caller still holds its claim - /// — the run is still `Analyzing` and its `claimed_at` matches the value - /// stamped when the caller claimed it. Returns `true` on success, `false` if - /// the claim has gone stale (another worker re-claimed the run). Mirrors - /// [`finalize_analyzed_run`](Self::finalize_analyzed_run) for the failure path, - /// so a worker whose lease expired mid-analysis cannot fail a run another - /// worker now owns. `updates` carries the failure reason and completion time; - /// status and the claim guard are applied here. - fn finalize_failed_run( - &mut self, - run_id: Uuid, - claimed_at: jiff::Timestamp, - updates: UpdateWorkspacePipelineRun, - ) -> impl Future> + Send; - - /// Records a run's per-model inference usage. A no-op for an empty slice - /// (a deterministic run spends no tokens). Inserted once, at analyze time. - fn record_run_usage( - &mut self, - usage: &[NewWorkspacePipelineRunUsage], - ) -> impl Future> + Send; -} - -impl WorkspacePipelineRunRepository for PgConnection { - async fn create_workspace_pipeline_run( - &mut self, - new_run: NewWorkspacePipelineRun, - ) -> PgResult { - use schema::workspace_pipeline_runs; - - let run = diesel::insert_into(workspace_pipeline_runs::table) - .values(&new_run) - .returning(WorkspacePipelineRun::as_returning()) - .get_result(self) - .await - .map_err(PgError::from)?; - - Ok(run) - } - - async fn find_workspace_run_by_id( - &mut self, - workspace_id: Uuid, - run_id: Uuid, - ) -> PgResult> { - use schema::workspace_pipeline_runs::dsl as runs; - use schema::{workspace_pipeline_runs, workspace_pipelines}; - - // Runs carry no workspace column; scope through the owning pipeline so - // the id resolves only within its workspace, and only while that - // pipeline is live (a soft-deleted pipeline hides its runs). The - // pipeline is returned alongside so callers need no second lookup. - let run = workspace_pipeline_runs::table - .inner_join(workspace_pipelines::table) - .filter(runs::id.eq(run_id)) - .filter(workspace_pipelines::workspace_id.eq(workspace_id)) - .filter(workspace_pipelines::deleted_at.is_null()) - .select(( - WorkspacePipelineRun::as_select(), - WorkspacePipeline::as_select(), - )) - .first(self) - .await - .optional() - .map_err(PgError::from)?; - - Ok(run) - } - - async fn find_pipeline_run_by_idempotency_key( - &mut self, - pipeline_id: Uuid, - idempotency_key: &str, - ) -> PgResult> { - use schema::workspace_pipeline_runs::{self, dsl}; - - let run = workspace_pipeline_runs::table - .filter(dsl::pipeline_id.eq(pipeline_id)) - .filter(dsl::idempotency_key.eq(idempotency_key)) - .select(WorkspacePipelineRun::as_select()) - .first(self) - .await - .optional() - .map_err(PgError::from)?; - - Ok(run) - } - - async fn cursor_list_workspace_pipeline_runs( - &mut self, - pipeline_id: Uuid, - pagination: CursorPagination, - filter: &RunFilter, - ) -> PgResult> { - use schema::workspace_pipeline_runs::dsl; - use schema::{accounts, workspace_files, workspace_pipeline_runs, workspace_pipelines}; - - // Build base query with filters. The listing is already scoped to one - // pipeline, so `filter.pipeline_id` is not applied here. - let mut base_query = workspace_pipeline_runs::table - .filter(dsl::pipeline_id.eq(pipeline_id)) - .into_boxed(); - - if let Some(status) = filter.status { - base_query = base_query.filter(dsl::status.eq(status)); - } - if let Some(file_id) = filter.input_file_id { - base_query = base_query.filter(dsl::input_file_id.eq(file_id)); - } - if let Some(account_id) = filter.account_id { - base_query = base_query.filter(dsl::account_id.eq(account_id)); - } - if let Some(trigger_type) = filter.trigger_type { - base_query = base_query.filter(dsl::trigger_type.eq(trigger_type)); - } - - let total = if pagination.include_count { - Some( - base_query - .count() - .get_result::(self) - .await - .map_err(PgError::from)?, - ) - } else { - None - }; - - // Rebuild query for fetching items. Join the owning pipeline (for its - // slug) and the input file (to name the run's analyzed document) so a - // row is self-contained; a LEFT JOIN on the file tolerates one removed - // by retention, yielding a null name. - let mut query = workspace_pipeline_runs::table - .inner_join(accounts::table) - .inner_join(workspace_pipelines::table) - .left_join(workspace_files::table.on(dsl::input_file_id.eq(workspace_files::id))) - .filter(dsl::pipeline_id.eq(pipeline_id)) - .into_boxed(); - - if let Some(status) = filter.status { - query = query.filter(dsl::status.eq(status)); - } - if let Some(file_id) = filter.input_file_id { - query = query.filter(dsl::input_file_id.eq(file_id)); - } - if let Some(account_id) = filter.account_id { - query = query.filter(dsl::account_id.eq(account_id)); - } - if let Some(trigger_type) = filter.trigger_type { - query = query.filter(dsl::trigger_type.eq(trigger_type)); - } - - let limit = pagination.fetch_limit(); - let selection = ( - WorkspacePipelineRun::as_select(), - ( - accounts::username, - accounts::display_name, - accounts::avatar_url, - ), - workspace_pipelines::slug, - workspace_files::display_name.nullable(), - ); - - let rows: Vec<(WorkspacePipelineRun, AccountRefRow, Handle, Option)> = - if let Some(cursor) = &pagination.after { - let cursor_time = jiff_diesel::Timestamp::from(cursor.timestamp); - - query - .filter( - dsl::started_at - .lt(&cursor_time) - .or(dsl::started_at.eq(&cursor_time).and(dsl::id.lt(cursor.id))), - ) - .select(selection) - .order((dsl::started_at.desc(), dsl::id.desc())) - .limit(limit) - .load(self) - .await - .map_err(PgError::from)? - } else { - query - .select(selection) - .order((dsl::started_at.desc(), dsl::id.desc())) - .limit(limit) - .load(self) - .await - .map_err(PgError::from)? - }; - - let items = rows - .into_iter() - .map( - |(run, account, pipeline_slug, input_file_name)| PipelineRunListRow { - run, - account, - pipeline_slug, - input_file_name, - }, - ) - .collect(); - - Ok(CursorPage::new(items, total, pagination.limit, |row| { - (row.run.started_at.into(), row.run.id) - })) - } - - async fn cursor_list_workspace_runs( - &mut self, - workspace_id: Uuid, - pagination: CursorPagination, - filter: &RunFilter, - ) -> PgResult> { - use schema::accounts::dsl as accounts; - use schema::workspace_files::dsl as files; - use schema::workspace_pipeline_runs::dsl as runs; - use schema::workspace_pipelines::dsl as pipelines; - - // Runs have no workspace column; scope them through the owning pipeline. - // The owning pipeline's slug, the triggering account, and the input - // file's name are selected alongside each run so the cross-pipeline - // response can name its pipeline, trigger, and analyzed document without - // a per-row lookup. The input file is LEFT-joined so a file removed by - // retention yields a null name rather than dropping the run. - let scoped = || { - let mut query = runs::workspace_pipeline_runs - .inner_join(pipelines::workspace_pipelines) - .inner_join(accounts::accounts) - .left_join(files::workspace_files.on(runs::input_file_id.eq(files::id))) - .filter(pipelines::workspace_id.eq(workspace_id)) - .into_boxed(); - if let Some(status) = filter.status { - query = query.filter(runs::status.eq(status)); - } - if let Some(file_id) = filter.input_file_id { - query = query.filter(runs::input_file_id.eq(file_id)); - } - if let Some(pipeline_id) = filter.pipeline_id { - query = query.filter(runs::pipeline_id.eq(pipeline_id)); - } - if let Some(account_id) = filter.account_id { - query = query.filter(runs::account_id.eq(account_id)); - } - if let Some(trigger_type) = filter.trigger_type { - query = query.filter(runs::trigger_type.eq(trigger_type)); - } - query - }; - - let total = if pagination.include_count { - Some( - scoped() - .count() - .get_result::(self) - .await - .map_err(PgError::from)?, - ) - } else { - None - }; - - let limit = pagination.fetch_limit(); - let selection = ( - WorkspacePipelineRun::as_select(), - pipelines::slug, - ( - accounts::username, - accounts::display_name, - accounts::avatar_url, - ), - files::display_name.nullable(), - ); - - let rows: Vec<(WorkspacePipelineRun, Handle, AccountRefRow, Option)> = - if let Some(cursor) = &pagination.after { - let cursor_time = jiff_diesel::Timestamp::from(cursor.timestamp); - - scoped() - .filter( - runs::started_at.lt(&cursor_time).or(runs::started_at - .eq(&cursor_time) - .and(runs::id.lt(cursor.id))), - ) - .select(selection) - .order((runs::started_at.desc(), runs::id.desc())) - .limit(limit) - .load(self) - .await - .map_err(PgError::from)? - } else { - scoped() - .select(selection) - .order((runs::started_at.desc(), runs::id.desc())) - .limit(limit) - .load(self) - .await - .map_err(PgError::from)? - }; - - let items = rows - .into_iter() - .map( - |(run, pipeline_slug, account, input_file_name)| PipelineRunListRow { - run, - account, - pipeline_slug, - input_file_name, - }, - ) - .collect(); - - Ok(CursorPage::new(items, total, pagination.limit, |row| { - (row.run.started_at.into(), row.run.id) - })) - } - - async fn claim_run_for_detection( - &mut self, - run_id: Uuid, - stale_before: jiff::Timestamp, - ) -> PgResult> { - use schema::workspace_pipeline_runs::{self, dsl}; - - let stale_before = jiff_diesel::Timestamp::from(stale_before); - - // Claim only if still queued, or analyzing under a claim that has gone - // stale (a dead worker). The WHERE clause makes the transition atomic: - // two concurrent deliveries race on the same row and exactly one flips - // it to `analyzing`; the loser matches no row and gets `None`. - let claimed = diesel::update( - workspace_pipeline_runs::table - .filter(dsl::id.eq(run_id)) - .filter( - dsl::status.eq(PipelineRunStatus::Queued).or(dsl::status - .eq(PipelineRunStatus::Analyzing) - .and(dsl::claimed_at.lt(stale_before))), - ), - ) - .set(( - dsl::status.eq(PipelineRunStatus::Analyzing), - dsl::claimed_at.eq(diesel::dsl::now), - )) - .returning(WorkspacePipelineRun::as_returning()) - .get_result(self) - .await - .optional() - .map_err(PgError::from)?; - - Ok(claimed) - } - - async fn run_file_names( - &mut self, - workspace_id: Uuid, - run: &WorkspacePipelineRun, - ) -> PgResult { - use schema::workspace_files::{self, dsl}; - - // Select the display name only, scoped to the workspace and excluding - // soft-deleted files, so a file removed by retention resolves to `None`. - async fn name_of( - conn: &mut PgConnection, - workspace_id: Uuid, - file_id: Uuid, - ) -> PgResult> { - workspace_files::table - .filter(dsl::id.eq(file_id)) - .filter(dsl::workspace_id.eq(workspace_id)) - .filter(dsl::deleted_at.is_null()) - .select(dsl::display_name) - .first::(conn) - .await - .optional() - .map_err(PgError::from) - } - - let input = name_of(self, workspace_id, run.input_file_id).await?; - let output = match run.output_file_id { - Some(output_file_id) => name_of(self, workspace_id, output_file_id).await?, - None => None, - }; - - Ok(RunFiles { input, output }) - } - - async fn update_workspace_pipeline_run( - &mut self, - run_id: Uuid, - updates: UpdateWorkspacePipelineRun, - ) -> PgResult { - use schema::workspace_pipeline_runs::{self, dsl}; - - let run = diesel::update(workspace_pipeline_runs::table.filter(dsl::id.eq(run_id))) - .set(&updates) - .returning(WorkspacePipelineRun::as_returning()) - .get_result(self) - .await - .map_err(PgError::from)?; - - Ok(run) - } - - async fn finalize_analyzed_run( - &mut self, - run_id: Uuid, - claimed_at: jiff::Timestamp, - mut updates: UpdateWorkspacePipelineRun, - ) -> PgResult { - use schema::workspace_pipeline_runs::{self, dsl}; - - // Force the terminal transition here; the guard makes it a no-op unless we - // still own the claim. - updates.status = Some(PipelineRunStatus::Analyzed); - let claimed_at = jiff_diesel::Timestamp::from(claimed_at); - - // Guard on the claim we hold: same run, still `Analyzing`, and the exact - // `claimed_at` our claim stamped. A worker that re-claimed a stale run - // renews `claimed_at`, so a lost claim matches no row and returns false. - let updated = diesel::update( - workspace_pipeline_runs::table - .filter(dsl::id.eq(run_id)) - .filter(dsl::status.eq(PipelineRunStatus::Analyzing)) - .filter(dsl::claimed_at.eq(claimed_at)), - ) - .set(&updates) - .execute(self) - .await - .map_err(PgError::from)?; - - Ok(updated == 1) - } - - async fn finalize_failed_run( - &mut self, - run_id: Uuid, - claimed_at: jiff::Timestamp, - mut updates: UpdateWorkspacePipelineRun, - ) -> PgResult { - use schema::workspace_pipeline_runs::{self, dsl}; - - updates.status = Some(PipelineRunStatus::Failed); - let claimed_at = jiff_diesel::Timestamp::from(claimed_at); - - // Same claim guard as the analyzed finalize: only our still-live claim - // (run `Analyzing`, `claimed_at` unchanged) may fail the run. - let updated = diesel::update( - workspace_pipeline_runs::table - .filter(dsl::id.eq(run_id)) - .filter(dsl::status.eq(PipelineRunStatus::Analyzing)) - .filter(dsl::claimed_at.eq(claimed_at)), - ) - .set(&updates) - .execute(self) - .await - .map_err(PgError::from)?; - - Ok(updated == 1) - } - - async fn record_run_usage(&mut self, usage: &[NewWorkspacePipelineRunUsage]) -> PgResult<()> { - use schema::workspace_pipeline_run_usage; - - if usage.is_empty() { - return Ok(()); - } - - diesel::insert_into(workspace_pipeline_run_usage::table) - .values(usage) - .execute(self) - .await - .map_err(PgError::from)?; - - Ok(()) - } -} diff --git a/crates/nvisy-postgres/src/query/workspace_redaction.rs b/crates/nvisy-postgres/src/query/workspace_redaction.rs new file mode 100644 index 00000000..68def03f --- /dev/null +++ b/crates/nvisy-postgres/src/query/workspace_redaction.rs @@ -0,0 +1,130 @@ +//! Workspace redactions repository for managing redaction instances. + +use std::future::Future; + +use diesel::prelude::*; +use diesel_async::RunQueryDsl; +use uuid::Uuid; + +use crate::model::{NewWorkspaceRedaction, WorkspaceRedaction}; +use crate::types::{CursorPage, CursorPagination}; +use crate::{PgConnection, PgError, PgResult, schema}; + +/// Repository for workspace redaction database operations. +/// +/// A redaction is one redact pass over a detection's analysis; a detection can +/// have many. Each redaction owns the review audit it applied and the redacted +/// document it produced. +pub trait WorkspaceRedactionRepository { + /// Creates a new workspace redaction record. + fn create_redaction( + &mut self, + new_redaction: NewWorkspaceRedaction, + ) -> impl Future> + Send; + + /// Finds a redaction by its id, scoped to its owning detection. + fn find_redaction_by_id( + &mut self, + detection_id: Uuid, + redaction_id: Uuid, + ) -> impl Future>> + Send; + + /// Lists a detection's redactions with cursor pagination, newest first. + fn cursor_list_detection_redactions( + &mut self, + detection_id: Uuid, + pagination: CursorPagination, + ) -> impl Future>> + Send; +} + +impl WorkspaceRedactionRepository for PgConnection { + async fn create_redaction( + &mut self, + new_redaction: NewWorkspaceRedaction, + ) -> PgResult { + use schema::workspace_redactions; + + let redaction = diesel::insert_into(workspace_redactions::table) + .values(&new_redaction) + .returning(WorkspaceRedaction::as_returning()) + .get_result(self) + .await + .map_err(PgError::from)?; + + Ok(redaction) + } + + async fn find_redaction_by_id( + &mut self, + detection_id: Uuid, + redaction_id: Uuid, + ) -> PgResult> { + use schema::workspace_redactions::{self, dsl}; + + let redaction = workspace_redactions::table + .filter(dsl::id.eq(redaction_id)) + .filter(dsl::detection_id.eq(detection_id)) + .select(WorkspaceRedaction::as_select()) + .first(self) + .await + .optional() + .map_err(PgError::from)?; + + Ok(redaction) + } + + async fn cursor_list_detection_redactions( + &mut self, + detection_id: Uuid, + pagination: CursorPagination, + ) -> PgResult> { + use schema::workspace_redactions::{self, dsl}; + + let base_query = workspace_redactions::table.filter(dsl::detection_id.eq(detection_id)); + + let total = if pagination.include_count { + Some( + base_query + .count() + .get_result::(self) + .await + .map_err(PgError::from)?, + ) + } else { + None + }; + + let limit = pagination.fetch_limit(); + + let items: Vec = if let Some(cursor) = &pagination.after { + let cursor_time = jiff_diesel::Timestamp::from(cursor.timestamp); + + workspace_redactions::table + .filter(dsl::detection_id.eq(detection_id)) + .filter( + dsl::created_at + .lt(&cursor_time) + .or(dsl::created_at.eq(&cursor_time).and(dsl::id.lt(cursor.id))), + ) + .select(WorkspaceRedaction::as_select()) + .order((dsl::created_at.desc(), dsl::id.desc())) + .limit(limit) + .load(self) + .await + .map_err(PgError::from)? + } else { + workspace_redactions::table + .filter(dsl::detection_id.eq(detection_id)) + .select(WorkspaceRedaction::as_select()) + .order((dsl::created_at.desc(), dsl::id.desc())) + .limit(limit) + .load(self) + .await + .map_err(PgError::from)? + }; + + Ok(CursorPage::new(items, total, pagination.limit, |row| { + (row.created_at.into(), row.id) + })) + } +} diff --git a/crates/nvisy-postgres/src/schema.rs b/crates/nvisy-postgres/src/schema.rs index 32d462b3..43d26ef2 100644 --- a/crates/nvisy-postgres/src/schema.rs +++ b/crates/nvisy-postgres/src/schema.rs @@ -13,6 +13,10 @@ pub mod sql_types { #[diesel(postgres_type(name = "chat_role"))] pub struct ChatRole; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "detection_status"))] + pub struct DetectionStatus; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] #[diesel(postgres_type(name = "file_kind"))] pub struct FileKind; @@ -29,10 +33,6 @@ pub mod sql_types { #[diesel(postgres_type(name = "outbox_status"))] pub struct OutboxStatus; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "pipeline_run_status"))] - pub struct PipelineRunStatus; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] #[diesel(postgres_type(name = "pipeline_status"))] pub struct PipelineStatus; @@ -247,6 +247,42 @@ diesel::table! { } } +diesel::table! { + use diesel::sql_types::*; + + workspace_detection_usage (id) { + id -> Uuid, + detection_id -> Uuid, + model -> Text, + version -> Nullable, + input_tokens -> Nullable, + output_tokens -> Nullable, + total_tokens -> Nullable, + duration_ms -> Int8, + } +} + +diesel::table! { + use diesel::sql_types::*; + use super::sql_types::PipelineTriggerType; + use super::sql_types::DetectionStatus; + + workspace_detections (id) { + id -> Uuid, + pipeline_id -> Uuid, + account_id -> Uuid, + input_file_id -> Uuid, + audit_file_id -> Nullable, + trigger_type -> PipelineTriggerType, + status -> DetectionStatus, + idempotency_key -> Nullable, + metadata -> Jsonb, + claimed_at -> Nullable, + started_at -> Timestamptz, + completed_at -> Nullable, + } +} + diesel::table! { use diesel::sql_types::*; @@ -334,43 +370,6 @@ diesel::table! { } } -diesel::table! { - use diesel::sql_types::*; - - workspace_pipeline_run_usage (id) { - id -> Uuid, - run_id -> Uuid, - model -> Text, - version -> Nullable, - input_tokens -> Nullable, - output_tokens -> Nullable, - total_tokens -> Nullable, - duration_ms -> Int8, - } -} - -diesel::table! { - use diesel::sql_types::*; - use super::sql_types::PipelineTriggerType; - use super::sql_types::PipelineRunStatus; - - workspace_pipeline_runs (id) { - id -> Uuid, - pipeline_id -> Uuid, - account_id -> Uuid, - input_file_id -> Uuid, - audit_file_id -> Nullable, - output_file_id -> Nullable, - trigger_type -> PipelineTriggerType, - status -> PipelineRunStatus, - idempotency_key -> Nullable, - metadata -> Jsonb, - claimed_at -> Nullable, - started_at -> Timestamptz, - completed_at -> Nullable, - } -} - diesel::table! { use diesel::sql_types::*; use super::sql_types::PipelineStatus; @@ -409,6 +408,19 @@ diesel::table! { } } +diesel::table! { + use diesel::sql_types::*; + + workspace_redactions (id) { + id -> Uuid, + detection_id -> Uuid, + account_id -> Uuid, + review_file_id -> Nullable, + output_file_id -> Nullable, + created_at -> Timestamptz, + } +} + diesel::table! { use diesel::sql_types::*; use super::sql_types::WebhookEvent; @@ -465,6 +477,9 @@ diesel::joinable!(workspace_connection_syncs -> accounts (account_id)); diesel::joinable!(workspace_connection_syncs -> workspace_connection_schedule (connection_id)); diesel::joinable!(workspace_connections -> accounts (account_id)); diesel::joinable!(workspace_connections -> workspaces (workspace_id)); +diesel::joinable!(workspace_detection_usage -> workspace_detections (detection_id)); +diesel::joinable!(workspace_detections -> accounts (account_id)); +diesel::joinable!(workspace_detections -> workspace_pipelines (pipeline_id)); diesel::joinable!(workspace_file_imports -> workspace_connections (connection_id)); diesel::joinable!(workspace_file_imports -> workspace_files (file_id)); diesel::joinable!(workspace_files -> accounts (account_id)); @@ -472,13 +487,12 @@ diesel::joinable!(workspace_files -> workspaces (workspace_id)); diesel::joinable!(workspace_invites -> workspaces (workspace_id)); diesel::joinable!(workspace_members -> workspaces (workspace_id)); diesel::joinable!(workspace_pipeline_policies -> workspaces (workspace_id)); -diesel::joinable!(workspace_pipeline_run_usage -> workspace_pipeline_runs (run_id)); -diesel::joinable!(workspace_pipeline_runs -> accounts (account_id)); -diesel::joinable!(workspace_pipeline_runs -> workspace_pipelines (pipeline_id)); diesel::joinable!(workspace_pipelines -> accounts (account_id)); diesel::joinable!(workspace_pipelines -> workspaces (workspace_id)); diesel::joinable!(workspace_policies -> accounts (account_id)); diesel::joinable!(workspace_policies -> workspaces (workspace_id)); +diesel::joinable!(workspace_redactions -> accounts (account_id)); +diesel::joinable!(workspace_redactions -> workspace_detections (detection_id)); diesel::joinable!(workspace_webhooks -> accounts (created_by)); diesel::joinable!(workspace_webhooks -> workspaces (workspace_id)); diesel::joinable!(workspaces -> accounts (created_by)); @@ -494,15 +508,16 @@ diesel::allow_tables_to_appear_in_same_query!( workspace_connection_schedule, workspace_connection_syncs, workspace_connections, + workspace_detection_usage, + workspace_detections, workspace_file_imports, workspace_files, workspace_invites, workspace_members, workspace_pipeline_policies, - workspace_pipeline_run_usage, - workspace_pipeline_runs, workspace_pipelines, workspace_policies, + workspace_redactions, workspace_webhooks, workspaces, ); diff --git a/crates/nvisy-postgres/src/types/constraint/detections.rs b/crates/nvisy-postgres/src/types/constraint/detections.rs new file mode 100644 index 00000000..1e90ea74 --- /dev/null +++ b/crates/nvisy-postgres/src/types/constraint/detections.rs @@ -0,0 +1,14 @@ +//! Detections table constraint violations. + +use strum::EnumString; + +/// Detections table constraint violations. +#[derive(Debug, Clone, Copy, Eq, PartialEq, EnumString)] +pub enum WorkspaceDetectionConstraints { + #[strum(serialize = "workspace_detections_metadata_size")] + MetadataSize, + #[strum(serialize = "workspace_detections_idempotency_key_length")] + IdempotencyKeyLength, + #[strum(serialize = "workspace_detections_idempotency_idx")] + IdempotencyUnique, +} diff --git a/crates/nvisy-postgres/src/types/constraint/mod.rs b/crates/nvisy-postgres/src/types/constraint/mod.rs index dbfbed56..255bcca9 100644 --- a/crates/nvisy-postgres/src/types/constraint/mod.rs +++ b/crates/nvisy-postgres/src/types/constraint/mod.rs @@ -22,9 +22,9 @@ mod workspaces; // File-related constraint modules mod files; -// Pipeline-related constraint modules +// Detection / pipeline-related constraint modules +mod detections; mod pipeline_references; -mod pipeline_runs; mod pipelines; mod workspace_connection_syncs; @@ -36,9 +36,9 @@ pub use self::account_notifications::AccountNotificationConstraints; pub use self::accounts::AccountConstraints; pub use self::chat_messages::ChatMessageConstraints; pub use self::chat_sessions::ChatSessionConstraints; +pub use self::detections::WorkspaceDetectionConstraints; pub use self::files::WorkspaceFileConstraints; pub use self::pipeline_references::WorkspacePipelineReferenceConstraints; -pub use self::pipeline_runs::WorkspacePipelineRunConstraints; pub use self::pipelines::WorkspacePipelineConstraints; pub use self::workspace_activities::WorkspaceActivitiesConstraints; pub use self::workspace_connection_syncs::WorkspaceConnectionSyncConstraints; @@ -75,9 +75,9 @@ pub enum ConstraintViolation { // File-related constraints WorkspaceFile(WorkspaceFileConstraints), - // Pipeline-related constraints + // Detection / pipeline-related constraints WorkspacePipeline(WorkspacePipelineConstraints), - WorkspacePipelineRun(WorkspacePipelineRunConstraints), + WorkspaceDetection(WorkspaceDetectionConstraints), WorkspacePipelineReference(WorkspacePipelineReferenceConstraints), WorkspaceConnection(WorkspaceConnectionConstraints), WorkspaceConnectionSync(WorkspaceConnectionSyncConstraints), @@ -132,7 +132,7 @@ impl ConstraintViolation { WorkspaceWebhook, WorkspaceFile, WorkspacePipeline, - WorkspacePipelineRun, + WorkspaceDetection, WorkspacePipelineReference, WorkspaceConnection, WorkspaceConnectionSync, diff --git a/crates/nvisy-postgres/src/types/constraint/pipeline_runs.rs b/crates/nvisy-postgres/src/types/constraint/pipeline_runs.rs deleted file mode 100644 index 6775b56e..00000000 --- a/crates/nvisy-postgres/src/types/constraint/pipeline_runs.rs +++ /dev/null @@ -1,14 +0,0 @@ -//! Pipeline runs table constraint violations. - -use strum::EnumString; - -/// Pipeline runs table constraint violations. -#[derive(Debug, Clone, Copy, Eq, PartialEq, EnumString)] -pub enum WorkspacePipelineRunConstraints { - #[strum(serialize = "workspace_pipeline_runs_metadata_size")] - MetadataSize, - #[strum(serialize = "workspace_pipeline_runs_idempotency_key_length")] - IdempotencyKeyLength, - #[strum(serialize = "workspace_pipeline_runs_idempotency_idx")] - IdempotencyUnique, -} diff --git a/crates/nvisy-postgres/src/types/enums/activity_type.rs b/crates/nvisy-postgres/src/types/enums/activity_type.rs index a46cbc69..01079880 100644 --- a/crates/nvisy-postgres/src/types/enums/activity_type.rs +++ b/crates/nvisy-postgres/src/types/enums/activity_type.rs @@ -171,29 +171,29 @@ pub enum ActivityType { #[strum(serialize = "pipeline.deleted")] PipelineDeleted, - /// Pipeline run was started - #[db_rename = "pipeline.run.started"] - #[serde(rename = "pipeline.run.started")] - #[strum(serialize = "pipeline.run.started")] - PipelineRunStarted, - - /// Pipeline run finished detection - #[db_rename = "pipeline.run.analyzed"] - #[serde(rename = "pipeline.run.analyzed")] - #[strum(serialize = "pipeline.run.analyzed")] - PipelineRunAnalyzed, - - /// Pipeline run completed - #[db_rename = "pipeline.run.completed"] - #[serde(rename = "pipeline.run.completed")] - #[strum(serialize = "pipeline.run.completed")] - PipelineRunCompleted, - - /// Pipeline run failed - #[db_rename = "pipeline.run.failed"] - #[serde(rename = "pipeline.run.failed")] - #[strum(serialize = "pipeline.run.failed")] - PipelineRunFailed, + /// Detection was started + #[db_rename = "pipeline.detection.started"] + #[serde(rename = "pipeline.detection.started")] + #[strum(serialize = "pipeline.detection.started")] + DetectionStarted, + + /// Detection finished analysis + #[db_rename = "pipeline.detection.completed"] + #[serde(rename = "pipeline.detection.completed")] + #[strum(serialize = "pipeline.detection.completed")] + DetectionCompleted, + + /// Detection failed + #[db_rename = "pipeline.detection.failed"] + #[serde(rename = "pipeline.detection.failed")] + #[strum(serialize = "pipeline.detection.failed")] + DetectionFailed, + + /// Redaction was created + #[db_rename = "pipeline.redaction.created"] + #[serde(rename = "pipeline.redaction.created")] + #[strum(serialize = "pipeline.redaction.created")] + RedactionCreated, // Policy activities /// Policy was created @@ -217,15 +217,16 @@ pub enum ActivityType { impl ActivityType { /// The canonical dotted tag for this type, e.g. `file.created` or - /// `pipeline.run.completed` — the same string used on the wire and in the DB, - /// from the variant's `strum(serialize)`. + /// `pipeline.redaction.created` — the same string used on the wire and in the + /// DB, from the variant's `strum(serialize)`. pub fn as_tag(self) -> &'static str { self.into() } /// The object half of the tag: everything before the final segment, e.g. - /// `file` for `file.created`, `pipeline.run` for `pipeline.run.completed`, - /// `connection.sync` for `connection.sync.failed`. + /// `file` for `file.created`, `pipeline.redaction` for + /// `pipeline.redaction.created`, `connection.sync` for + /// `connection.sync.failed`. pub fn object_type(self) -> &'static str { let tag = self.as_tag(); match tag.rsplit_once('.') { @@ -235,7 +236,7 @@ impl ActivityType { } /// The action half of the tag: the final segment, e.g. `created` for - /// `file.created`, `completed` for `pipeline.run.completed`. + /// `file.created`, `created` for `pipeline.redaction.created`. pub fn action_type(self) -> &'static str { let tag = self.as_tag(); match tag.rsplit_once('.') { @@ -279,13 +280,10 @@ mod tests { assert_eq!(ActivityType::FileCreated.action_type(), "created"); // Three-part tags: object is everything before the final segment. assert_eq!( - ActivityType::PipelineRunCompleted.object_type(), - "pipeline.run" - ); - assert_eq!( - ActivityType::PipelineRunCompleted.action_type(), - "completed" + ActivityType::RedactionCreated.object_type(), + "pipeline.redaction" ); + assert_eq!(ActivityType::RedactionCreated.action_type(), "created"); assert_eq!( ActivityType::ConnectionSyncFailed.object_type(), "connection.sync" diff --git a/crates/nvisy-postgres/src/types/enums/detection_status.rs b/crates/nvisy-postgres/src/types/enums/detection_status.rs new file mode 100644 index 00000000..ee100640 --- /dev/null +++ b/crates/nvisy-postgres/src/types/enums/detection_status.rs @@ -0,0 +1,59 @@ +//! Detection status enumeration indicating the execution state of a detection. + +use diesel_derive_enum::DbEnum; +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumIter, EnumString}; + +/// The execution status of a detection (one analysis pass of a file). +/// +/// Corresponds to the `DETECTION_STATUS` PostgreSQL enum. A detection is +/// `Pending` (enqueued, no worker yet), then `Executing` (a worker is actively +/// analyzing), then settles into `Complete` (analysis done, ready to redact) or +/// `Failed`. Redaction is a separate, repeatable action over a complete +/// detection and does not change this status. +#[derive(Debug, Default, Clone, Copy, Eq, PartialEq)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Serialize, Deserialize, DbEnum, Display, EnumIter, EnumString)] +#[ExistingTypePath = "crate::schema::sql_types::DetectionStatus"] +pub enum DetectionStatus { + /// Enqueued for detection; no worker has picked it up yet. + #[db_rename = "pending"] + #[serde(rename = "pending")] + #[default] + Pending, + + /// A worker is actively analyzing the document. + #[db_rename = "executing"] + #[serde(rename = "executing")] + Executing, + + /// Analysis done; the detection is ready to redact. + #[db_rename = "complete"] + #[serde(rename = "complete")] + Complete, + + /// Detection failed with an error. + #[db_rename = "failed"] + #[serde(rename = "failed")] + Failed, +} + +impl DetectionStatus { + /// Statuses that carry a success/failure outcome: a detection reached one of + /// these iff it either finished analysis or failed. This is the correct basis + /// for an error rate (`failed / (complete + failed)`). + pub const OUTCOMES: [DetectionStatus; 2] = [DetectionStatus::Complete, DetectionStatus::Failed]; + + /// Returns whether analysis is done and the detection is ready to redact. + #[inline] + pub fn is_complete(self) -> bool { + matches!(self, DetectionStatus::Complete) + } + + /// Returns whether the detection has not finished analysis yet (pending or + /// executing). + #[inline] + pub fn is_detecting(self) -> bool { + matches!(self, DetectionStatus::Pending | DetectionStatus::Executing) + } +} diff --git a/crates/nvisy-postgres/src/types/enums/file_kind.rs b/crates/nvisy-postgres/src/types/enums/file_kind.rs index 489d3abb..0a86dd9e 100644 --- a/crates/nvisy-postgres/src/types/enums/file_kind.rs +++ b/crates/nvisy-postgres/src/types/enums/file_kind.rs @@ -26,10 +26,16 @@ pub enum FileKind { #[serde(rename = "redacted")] Redacted, - /// Engine analysis blob, not shown in file lists. + /// Engine detection-analysis blob, not shown in file lists. #[db_rename = "audit"] #[serde(rename = "audit")] Audit, + + /// Engine analysis after reviewer edits and redaction (a redaction's review + /// audit), not shown in file lists. + #[db_rename = "review"] + #[serde(rename = "review")] + Review, } impl FileKind { diff --git a/crates/nvisy-postgres/src/types/enums/mod.rs b/crates/nvisy-postgres/src/types/enums/mod.rs index 6e080aa2..1e4e7e57 100644 --- a/crates/nvisy-postgres/src/types/enums/mod.rs +++ b/crates/nvisy-postgres/src/types/enums/mod.rs @@ -29,19 +29,19 @@ pub mod workspace_role; // File-related enumerations pub mod file_kind; -// Pipeline-related enumerations -pub mod pipeline_run_status; +// Detection / pipeline-related enumerations +pub mod detection_status; pub mod pipeline_status; pub mod pipeline_trigger_type; pub use activity_type::ActivityType; pub use api_token_type::ApiTokenType; pub use chat_role::ChatRole; +pub use detection_status::DetectionStatus; pub use file_kind::FileKind; pub use invite_status::InviteStatus; pub use notification_event::NotificationEvent; pub use outbox_status::OutboxStatus; -pub use pipeline_run_status::PipelineRunStatus; pub use pipeline_status::PipelineStatus; pub use pipeline_trigger_type::PipelineTriggerType; pub use provider_type::ProviderType; diff --git a/crates/nvisy-postgres/src/types/enums/notification_event.rs b/crates/nvisy-postgres/src/types/enums/notification_event.rs index 2e3d3399..6a40a81b 100644 --- a/crates/nvisy-postgres/src/types/enums/notification_event.rs +++ b/crates/nvisy-postgres/src/types/enums/notification_event.rs @@ -7,7 +7,8 @@ use strum::{Display, EnumIter, EnumString}; /// Defines the type of notification event sent to a user. /// /// This enumeration corresponds to the `NOTIFICATION_EVENT` PostgreSQL enum and -/// is used for member, connection-sync, pipeline-run, and system notifications. +/// is used for member, connection-sync, detection, redaction, and system +/// notifications. /// The values mirror the [`WebhookEvent`](super::WebhookEvent) naming for the /// events the two channels share. #[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] @@ -37,19 +38,19 @@ pub enum NotificationEvent { #[serde(rename = "connection.sync.failed")] ConnectionSyncFailed, - // Pipeline run events - /// A pipeline run finished detection and is awaiting review - #[db_rename = "pipeline.run.analyzed"] - #[serde(rename = "pipeline.run.analyzed")] - PipelineRunAnalyzed, - - /// A pipeline run completed (redaction produced) - #[db_rename = "pipeline.run.completed"] - #[serde(rename = "pipeline.run.completed")] - PipelineRunCompleted, - - /// A pipeline run failed - #[db_rename = "pipeline.run.failed"] - #[serde(rename = "pipeline.run.failed")] - PipelineRunFailed, + // Detection / redaction events + /// A detection finished analysis and is ready to redact + #[db_rename = "pipeline.detection.completed"] + #[serde(rename = "pipeline.detection.completed")] + DetectionCompleted, + + /// A redaction was created (redacted output produced) + #[db_rename = "pipeline.redaction.created"] + #[serde(rename = "pipeline.redaction.created")] + RedactionCreated, + + /// A detection failed + #[db_rename = "pipeline.detection.failed"] + #[serde(rename = "pipeline.detection.failed")] + DetectionFailed, } diff --git a/crates/nvisy-postgres/src/types/enums/pipeline_run_status.rs b/crates/nvisy-postgres/src/types/enums/pipeline_run_status.rs deleted file mode 100644 index 2a29072e..00000000 --- a/crates/nvisy-postgres/src/types/enums/pipeline_run_status.rs +++ /dev/null @@ -1,74 +0,0 @@ -//! Pipeline run status enumeration indicating the execution state of a pipeline run. - -use diesel_derive_enum::DbEnum; -use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIter, EnumString}; - -/// Defines the execution status of a pipeline run. -/// -/// This enumeration corresponds to the `PIPELINE_RUN_STATUS` PostgreSQL enum and is used -/// to track the current state of a pipeline execution. -/// -/// The detect phase has two states: `Queued` (the run is enqueued but no worker -/// has begun) and `Analyzing` (a worker is actively analyzing). They settle into -/// `Analyzed` (detection done, awaiting review), then `Completed` after redaction. -#[derive(Debug, Default, Clone, Copy, Eq, PartialEq)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[derive(Serialize, Deserialize, DbEnum, Display, EnumIter, EnumString)] -#[ExistingTypePath = "crate::schema::sql_types::PipelineRunStatus"] -pub enum PipelineRunStatus { - /// Enqueued for detection; no worker has picked it up yet - #[db_rename = "queued"] - #[serde(rename = "queued")] - #[default] - Queued, - - /// A worker is actively analyzing the document - #[db_rename = "analyzing"] - #[serde(rename = "analyzing")] - Analyzing, - - /// Detection done; awaiting reviewer verification - #[db_rename = "analyzed"] - #[serde(rename = "analyzed")] - Analyzed, - - /// Redaction applied; run finished - #[db_rename = "completed"] - #[serde(rename = "completed")] - Completed, - - /// Run failed with error - #[db_rename = "failed"] - #[serde(rename = "failed")] - Failed, - - /// Run was cancelled by user - #[db_rename = "cancelled"] - #[serde(rename = "cancelled")] - Cancelled, -} - -impl PipelineRunStatus { - /// Statuses that carry a success/failure outcome: a run reached one of these - /// iff it either completed redaction or failed. `Cancelled` is excluded — a - /// cancelled run has no outcome — so this is the correct basis for an error - /// rate (`failed / (completed + failed)`). - pub const OUTCOMES: [PipelineRunStatus; 2] = - [PipelineRunStatus::Completed, PipelineRunStatus::Failed]; - - /// Returns whether detection is done and the run awaits verification. - #[inline] - pub fn is_analyzed(self) -> bool { - matches!(self, PipelineRunStatus::Analyzed) - } - - /// Returns whether detection is still pending (queued or analyzing). - #[inline] - pub fn is_detecting(self) -> bool { - matches!( - self, - PipelineRunStatus::Queued | PipelineRunStatus::Analyzing - ) - } -} diff --git a/crates/nvisy-postgres/src/types/enums/webhook_event.rs b/crates/nvisy-postgres/src/types/enums/webhook_event.rs index 6d092dae..c04f0a58 100644 --- a/crates/nvisy-postgres/src/types/enums/webhook_event.rs +++ b/crates/nvisy-postgres/src/types/enums/webhook_event.rs @@ -107,29 +107,29 @@ pub enum WebhookEvent { #[strum(serialize = "pipeline.deleted")] PipelineDeleted, - /// A pipeline run started - #[db_rename = "pipeline.run.started"] - #[serde(rename = "pipeline.run.started")] - #[strum(serialize = "pipeline.run.started")] - PipelineRunStarted, - - /// A pipeline run's detection finished (findings ready for review) - #[db_rename = "pipeline.run.analyzed"] - #[serde(rename = "pipeline.run.analyzed")] - #[strum(serialize = "pipeline.run.analyzed")] - PipelineRunAnalyzed, - - /// A pipeline run finished successfully - #[db_rename = "pipeline.run.completed"] - #[serde(rename = "pipeline.run.completed")] - #[strum(serialize = "pipeline.run.completed")] - PipelineRunCompleted, - - /// A pipeline run failed - #[db_rename = "pipeline.run.failed"] - #[serde(rename = "pipeline.run.failed")] - #[strum(serialize = "pipeline.run.failed")] - PipelineRunFailed, + /// A detection started + #[db_rename = "pipeline.detection.started"] + #[serde(rename = "pipeline.detection.started")] + #[strum(serialize = "pipeline.detection.started")] + DetectionStarted, + + /// A detection's analysis finished (findings ready to redact) + #[db_rename = "pipeline.detection.completed"] + #[serde(rename = "pipeline.detection.completed")] + #[strum(serialize = "pipeline.detection.completed")] + DetectionCompleted, + + /// A detection failed + #[db_rename = "pipeline.detection.failed"] + #[serde(rename = "pipeline.detection.failed")] + #[strum(serialize = "pipeline.detection.failed")] + DetectionFailed, + + /// A redaction was created + #[db_rename = "pipeline.redaction.created"] + #[serde(rename = "pipeline.redaction.created")] + #[strum(serialize = "pipeline.redaction.created")] + RedactionCreated, // Policy events /// A policy was created @@ -170,10 +170,10 @@ impl WebhookEvent { WebhookEvent::PipelineCreated | WebhookEvent::PipelineUpdated | WebhookEvent::PipelineDeleted - | WebhookEvent::PipelineRunStarted - | WebhookEvent::PipelineRunAnalyzed - | WebhookEvent::PipelineRunCompleted - | WebhookEvent::PipelineRunFailed => "pipeline", + | WebhookEvent::DetectionStarted + | WebhookEvent::DetectionCompleted + | WebhookEvent::DetectionFailed + | WebhookEvent::RedactionCreated => "pipeline", WebhookEvent::PolicyCreated | WebhookEvent::PolicyUpdated | WebhookEvent::PolicyDeleted => "policy", @@ -183,7 +183,7 @@ impl WebhookEvent { /// Returns the event as a subject string for NATS routing. /// /// The event name is already a dotted, NATS-legal subject (e.g. - /// `file.created`, `pipeline.run.completed`), so this is the event's own + /// `file.created`, `pipeline.redaction.created`), so this is the event's own /// string representation (from its `strum(serialize)`). pub fn as_subject(&self) -> &'static str { self.into() diff --git a/crates/nvisy-postgres/src/types/filtering/runs.rs b/crates/nvisy-postgres/src/types/filtering/detections.rs similarity index 74% rename from crates/nvisy-postgres/src/types/filtering/runs.rs rename to crates/nvisy-postgres/src/types/filtering/detections.rs index 99ecb196..48eea538 100644 --- a/crates/nvisy-postgres/src/types/filtering/runs.rs +++ b/crates/nvisy-postgres/src/types/filtering/detections.rs @@ -1,11 +1,11 @@ -//! Filtering options for pipeline run queries. +//! Filtering options for detection queries. use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::types::{PipelineRunStatus, PipelineTriggerType}; +use crate::types::{DetectionStatus, PipelineTriggerType}; -/// Filter options for pipeline runs. +/// Filter options for detections. /// /// Each field narrows the result when set; unset fields impose no constraint. /// The owning pipeline (single-pipeline listing) and workspace scope are applied @@ -13,40 +13,40 @@ use crate::types::{PipelineRunStatus, PipelineTriggerType}; #[derive(Debug, Default, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] -pub struct RunFilter { - /// Filter by run status. +pub struct DetectionFilter { + /// Filter by detection status. #[serde(skip_serializing_if = "Option::is_none")] - pub status: Option, - /// Filter by the source file the run analyzes. + pub status: Option, + /// Filter by the source file the detection analyzes. #[serde(skip_serializing_if = "Option::is_none")] pub input_file_id: Option, /// Filter by the owning pipeline. Ignored by the single-pipeline listing /// (already scoped to one pipeline); used by the workspace-wide listing. #[serde(skip_serializing_if = "Option::is_none")] pub pipeline_id: Option, - /// Filter by the account that triggered the run. + /// Filter by the account that triggered the detection. #[serde(skip_serializing_if = "Option::is_none")] pub account_id: Option, - /// Filter by how the run was initiated (user vs system). + /// Filter by how the detection was initiated (user vs system). #[serde(skip_serializing_if = "Option::is_none")] pub trigger_type: Option, } -impl RunFilter { +impl DetectionFilter { /// Creates a new empty filter. #[inline] pub fn new() -> Self { Self::default() } - /// Filters by run status. + /// Filters by detection status. #[inline] - pub fn with_status(mut self, status: PipelineRunStatus) -> Self { + pub fn with_status(mut self, status: DetectionStatus) -> Self { self.status = Some(status); self } - /// Filters by the source file the run analyzes. + /// Filters by the source file the detection analyzes. #[inline] pub fn with_input_file_id(mut self, input_file_id: Uuid) -> Self { self.input_file_id = Some(input_file_id); @@ -60,14 +60,14 @@ impl RunFilter { self } - /// Filters by the account that triggered the run. + /// Filters by the account that triggered the detection. #[inline] pub fn with_account_id(mut self, account_id: Uuid) -> Self { self.account_id = Some(account_id); self } - /// Filters by how the run was initiated. + /// Filters by how the detection was initiated. #[inline] pub fn with_trigger_type(mut self, trigger_type: PipelineTriggerType) -> Self { self.trigger_type = Some(trigger_type); diff --git a/crates/nvisy-postgres/src/types/filtering/mod.rs b/crates/nvisy-postgres/src/types/filtering/mod.rs index af8d668d..106ee3b8 100644 --- a/crates/nvisy-postgres/src/types/filtering/mod.rs +++ b/crates/nvisy-postgres/src/types/filtering/mod.rs @@ -1,11 +1,11 @@ //! Filtering options for database queries. +mod detections; mod files; mod invites; mod members; -mod runs; +pub use detections::DetectionFilter; pub use files::FileFilter; pub use invites::InviteFilter; pub use members::MemberFilter; -pub use runs::RunFilter; diff --git a/crates/nvisy-postgres/src/types/json/activity_params.rs b/crates/nvisy-postgres/src/types/json/activity_params.rs index cfebf954..f04472c0 100644 --- a/crates/nvisy-postgres/src/types/json/activity_params.rs +++ b/crates/nvisy-postgres/src/types/json/activity_params.rs @@ -9,7 +9,9 @@ use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::types::{ActivityType, ConnectionId, Handle, RunId, WebhookEvent, WebhookId}; +use crate::types::{ + ActivityType, ConnectionId, DetectionId, Handle, RedactionId, WebhookEvent, WebhookId, +}; /// Params of a workspace-scoped activity (`workspace.*`). #[derive(Debug, Clone, Serialize, Deserialize)] @@ -84,15 +86,26 @@ pub struct PipelineActivityParams { pub pipeline_slug: Handle, } -/// Params of a pipeline-run activity (`pipeline.run.*`). +/// Params of a detection activity (`pipeline.detection.*`). #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] -pub struct PipelineRunActivityParams { +pub struct DetectionActivityParams { /// Slug of the owning pipeline. pub pipeline_slug: Handle, - /// Id of the run. - pub run_id: RunId, + /// Id of the detection. + pub detection_id: DetectionId, +} + +/// Params of a redaction activity (`pipeline.redaction.*`). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub struct RedactionActivityParams { + /// Slug of the owning pipeline. + pub pipeline_slug: Handle, + /// Id of the redaction. + pub redaction_id: RedactionId, } /// Params of a policy activity (`policy.*`). @@ -198,18 +211,18 @@ pub enum ActivityPayload { /// A pipeline was deleted. #[serde(rename = "pipeline.deleted")] PipelineDeleted(PipelineActivityParams), - /// A pipeline run was started. - #[serde(rename = "pipeline.run.started")] - PipelineRunStarted(PipelineRunActivityParams), - /// A pipeline run finished detection. - #[serde(rename = "pipeline.run.analyzed")] - PipelineRunAnalyzed(PipelineRunActivityParams), - /// A pipeline run completed. - #[serde(rename = "pipeline.run.completed")] - PipelineRunCompleted(PipelineRunActivityParams), - /// A pipeline run failed. - #[serde(rename = "pipeline.run.failed")] - PipelineRunFailed(PipelineRunActivityParams), + /// A detection was started. + #[serde(rename = "pipeline.detection.started")] + DetectionStarted(DetectionActivityParams), + /// A detection finished analysis. + #[serde(rename = "pipeline.detection.completed")] + DetectionCompleted(DetectionActivityParams), + /// A detection failed. + #[serde(rename = "pipeline.detection.failed")] + DetectionFailed(DetectionActivityParams), + /// A redaction was created. + #[serde(rename = "pipeline.redaction.created")] + RedactionCreated(RedactionActivityParams), /// A policy was created. #[serde(rename = "policy.created")] @@ -252,10 +265,10 @@ impl ActivityPayload { ActivityPayload::PipelineCreated(_) => ActivityType::PipelineCreated, ActivityPayload::PipelineUpdated(_) => ActivityType::PipelineUpdated, ActivityPayload::PipelineDeleted(_) => ActivityType::PipelineDeleted, - ActivityPayload::PipelineRunStarted(_) => ActivityType::PipelineRunStarted, - ActivityPayload::PipelineRunAnalyzed(_) => ActivityType::PipelineRunAnalyzed, - ActivityPayload::PipelineRunCompleted(_) => ActivityType::PipelineRunCompleted, - ActivityPayload::PipelineRunFailed(_) => ActivityType::PipelineRunFailed, + ActivityPayload::DetectionStarted(_) => ActivityType::DetectionStarted, + ActivityPayload::DetectionCompleted(_) => ActivityType::DetectionCompleted, + ActivityPayload::DetectionFailed(_) => ActivityType::DetectionFailed, + ActivityPayload::RedactionCreated(_) => ActivityType::RedactionCreated, ActivityPayload::PolicyCreated(_) => ActivityType::PolicyCreated, ActivityPayload::PolicyUpdated(_) => ActivityType::PolicyUpdated, ActivityPayload::PolicyDeleted(_) => ActivityType::PolicyDeleted, @@ -294,10 +307,10 @@ impl ActivityPayload { ActivityPayload::PipelineCreated(_) => W::PipelineCreated, ActivityPayload::PipelineUpdated(_) => W::PipelineUpdated, ActivityPayload::PipelineDeleted(_) => W::PipelineDeleted, - ActivityPayload::PipelineRunStarted(_) => W::PipelineRunStarted, - ActivityPayload::PipelineRunAnalyzed(_) => W::PipelineRunAnalyzed, - ActivityPayload::PipelineRunCompleted(_) => W::PipelineRunCompleted, - ActivityPayload::PipelineRunFailed(_) => W::PipelineRunFailed, + ActivityPayload::DetectionStarted(_) => W::DetectionStarted, + ActivityPayload::DetectionCompleted(_) => W::DetectionCompleted, + ActivityPayload::DetectionFailed(_) => W::DetectionFailed, + ActivityPayload::RedactionCreated(_) => W::RedactionCreated, ActivityPayload::PolicyCreated(_) => W::PolicyCreated, ActivityPayload::PolicyUpdated(_) => W::PolicyUpdated, ActivityPayload::PolicyDeleted(_) => W::PolicyDeleted, @@ -333,10 +346,11 @@ impl ActivityPayload { | ActivityPayload::FileUpdated(p) | ActivityPayload::FileDeleted(p) => Some(p.file_id.to_string()), - ActivityPayload::PipelineRunStarted(p) - | ActivityPayload::PipelineRunAnalyzed(p) - | ActivityPayload::PipelineRunCompleted(p) - | ActivityPayload::PipelineRunFailed(p) => Some(p.run_id.to_string()), + ActivityPayload::DetectionStarted(p) + | ActivityPayload::DetectionCompleted(p) + | ActivityPayload::DetectionFailed(p) => Some(p.detection_id.to_string()), + + ActivityPayload::RedactionCreated(p) => Some(p.redaction_id.to_string()), ActivityPayload::PolicyCreated(p) | ActivityPayload::PolicyUpdated(p) @@ -380,10 +394,11 @@ impl ActivityPayload { | ActivityPayload::PipelineUpdated(p) | ActivityPayload::PipelineDeleted(p) => Some(p.pipeline_slug.to_string()), - ActivityPayload::PipelineRunStarted(p) - | ActivityPayload::PipelineRunAnalyzed(p) - | ActivityPayload::PipelineRunCompleted(p) - | ActivityPayload::PipelineRunFailed(p) => Some(p.pipeline_slug.to_string()), + ActivityPayload::DetectionStarted(p) + | ActivityPayload::DetectionCompleted(p) + | ActivityPayload::DetectionFailed(p) => Some(p.pipeline_slug.to_string()), + + ActivityPayload::RedactionCreated(p) => Some(p.pipeline_slug.to_string()), ActivityPayload::ConnectionCreated(p) | ActivityPayload::ConnectionUpdated(p) diff --git a/crates/nvisy-postgres/src/types/json/pipeline_run_metadata.rs b/crates/nvisy-postgres/src/types/json/detection_metadata.rs similarity index 70% rename from crates/nvisy-postgres/src/types/json/pipeline_run_metadata.rs rename to crates/nvisy-postgres/src/types/json/detection_metadata.rs index 747e2810..388cd2b7 100644 --- a/crates/nvisy-postgres/src/types/json/pipeline_run_metadata.rs +++ b/crates/nvisy-postgres/src/types/json/detection_metadata.rs @@ -1,4 +1,4 @@ -//! Structured metadata for the `workspace_pipeline_runs.metadata` JSONB column. +//! Structured metadata for the `workspace_detections.metadata` JSONB column. //! //! A column the server itself populates has a known shape, so it is typed rather //! than left as free-form JSON. Read with `Json::or_default` so an absent or @@ -6,21 +6,21 @@ use serde::{Deserialize, Serialize}; -/// Structured metadata for a pipeline run. +/// Structured metadata for a detection. #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase", default)] -pub struct RunMetadata { - /// Free-form labels attached to the run. +pub struct DetectionMetadata { + /// Free-form labels attached to the detection. #[serde(skip_serializing_if = "Option::is_none")] pub tags: Option>, - /// Failure reason recorded when the run failed. + /// Failure reason recorded when the detection failed. #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, /// The engine's full per-recognizer usage report (durations, per-model token /// counts), stored opaquely for drill-down. Per-model token totals for - /// aggregation live in the `workspace_pipeline_run_usage` table; this keeps - /// the detail. Absent when the run produced no usage. + /// aggregation live in the `workspace_detection_usage` table; this keeps the + /// detail. Absent when the detection produced no usage. #[serde(skip_serializing_if = "Option::is_none")] #[cfg_attr(feature = "schema", schemars(with = "Option"))] pub usage: Option, diff --git a/crates/nvisy-postgres/src/types/json/mod.rs b/crates/nvisy-postgres/src/types/json/mod.rs index fa81afb5..f4ef1128 100644 --- a/crates/nvisy-postgres/src/types/json/mod.rs +++ b/crates/nvisy-postgres/src/types/json/mod.rs @@ -5,9 +5,9 @@ //! settings / retention value types. mod activity_params; +mod detection_metadata; mod notification_params; mod pipeline_metadata; -mod pipeline_run_metadata; mod retention; mod typed_json; mod webhook_headers; @@ -15,17 +15,17 @@ mod workspace_metadata; mod workspace_settings; pub use activity_params::{ - ActivityPayload, ConnectionActivityParams, FileActivityParams, InviteActivityParams, - MemberActivityParams, PipelineActivityParams, PipelineRunActivityParams, PolicyActivityParams, - WebhookActivityParams, WorkspaceActivityParams, + ActivityPayload, ConnectionActivityParams, DetectionActivityParams, FileActivityParams, + InviteActivityParams, MemberActivityParams, PipelineActivityParams, PolicyActivityParams, + RedactionActivityParams, WebhookActivityParams, WorkspaceActivityParams, }; +pub use detection_metadata::DetectionMetadata; pub use notification_params::{ - ConnectionSyncCompletedParams, ConnectionSyncFailedParams, MemberInvitedParams, - MemberJoinedParams, NotificationPayload, PipelineRunAnalyzedParams, PipelineRunCompletedParams, - PipelineRunFailedParams, + ConnectionSyncCompletedParams, ConnectionSyncFailedParams, DetectionCompletedParams, + DetectionFailedParams, MemberInvitedParams, MemberJoinedParams, NotificationPayload, + RedactionCreatedParams, }; pub use pipeline_metadata::{PipelineMetadata, RetentionOverride}; -pub use pipeline_run_metadata::RunMetadata; pub use retention::{Retention, RetentionScope, RetentionSettings}; pub use typed_json::Json; pub use webhook_headers::{InvalidHeader, WebhookHeaders}; diff --git a/crates/nvisy-postgres/src/types/json/notification_params.rs b/crates/nvisy-postgres/src/types/json/notification_params.rs index d8f9414f..482ce3e6 100644 --- a/crates/nvisy-postgres/src/types/json/notification_params.rs +++ b/crates/nvisy-postgres/src/types/json/notification_params.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; use super::Json; -use crate::types::{ConnectionId, Handle, NotificationEvent, RunId}; +use crate::types::{ConnectionId, DetectionId, Handle, NotificationEvent, RedactionId}; /// Params of a `member.invited` notification. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -61,13 +61,13 @@ pub struct ConnectionSyncFailedParams { pub error: Option, } -/// Params of a `pipeline.run.analyzed` notification. +/// Params of a `pipeline.detection.completed` notification. #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] -pub struct PipelineRunAnalyzedParams { - /// Id of the run. - pub run_id: RunId, +pub struct DetectionCompletedParams { + /// Id of the detection. + pub detection_id: DetectionId, /// Slug of the owning pipeline. pub pipeline_slug: Handle, /// Display name of the analyzed file, if known. @@ -75,27 +75,29 @@ pub struct PipelineRunAnalyzedParams { pub input_file_name: Option, } -/// Params of a `pipeline.run.completed` notification. +/// Params of a `pipeline.redaction.created` notification. #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] -pub struct PipelineRunCompletedParams { - /// Id of the run. - pub run_id: RunId, +pub struct RedactionCreatedParams { + /// Id of the redaction. + pub redaction_id: RedactionId, + /// Id of the detection the redaction was produced from. + pub detection_id: DetectionId, /// Slug of the owning pipeline. pub pipeline_slug: Handle, - /// Display name of the analyzed file, if known. + /// Display name of the redacted file, if known. #[serde(skip_serializing_if = "Option::is_none")] pub input_file_name: Option, } -/// Params of a `pipeline.run.failed` notification. +/// Params of a `pipeline.detection.failed` notification. #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] -pub struct PipelineRunFailedParams { - /// Id of the run. - pub run_id: RunId, +pub struct DetectionFailedParams { + /// Id of the detection. + pub detection_id: DetectionId, /// Slug of the owning pipeline. pub pipeline_slug: Handle, /// Display name of the analyzed file, if known. @@ -132,17 +134,17 @@ pub enum NotificationPayload { #[serde(rename = "connection.sync.failed")] ConnectionSyncFailed(ConnectionSyncFailedParams), - /// A pipeline run finished detection and is awaiting review. - #[serde(rename = "pipeline.run.analyzed")] - PipelineRunAnalyzed(PipelineRunAnalyzedParams), + /// A detection finished analysis and is ready to redact. + #[serde(rename = "pipeline.detection.completed")] + DetectionCompleted(DetectionCompletedParams), - /// A pipeline run completed (redaction produced). - #[serde(rename = "pipeline.run.completed")] - PipelineRunCompleted(PipelineRunCompletedParams), + /// A redaction was created (redacted output produced). + #[serde(rename = "pipeline.redaction.created")] + RedactionCreated(RedactionCreatedParams), - /// A pipeline run failed. - #[serde(rename = "pipeline.run.failed")] - PipelineRunFailed(PipelineRunFailedParams), + /// A detection failed. + #[serde(rename = "pipeline.detection.failed")] + DetectionFailed(DetectionFailedParams), } impl NotificationPayload { @@ -155,9 +157,9 @@ impl NotificationPayload { NotificationEvent::ConnectionSyncCompleted } NotificationPayload::ConnectionSyncFailed(_) => NotificationEvent::ConnectionSyncFailed, - NotificationPayload::PipelineRunAnalyzed(_) => NotificationEvent::PipelineRunAnalyzed, - NotificationPayload::PipelineRunCompleted(_) => NotificationEvent::PipelineRunCompleted, - NotificationPayload::PipelineRunFailed(_) => NotificationEvent::PipelineRunFailed, + NotificationPayload::DetectionCompleted(_) => NotificationEvent::DetectionCompleted, + NotificationPayload::RedactionCreated(_) => NotificationEvent::RedactionCreated, + NotificationPayload::DetectionFailed(_) => NotificationEvent::DetectionFailed, } } @@ -178,13 +180,13 @@ mod tests { use uuid::Uuid; use super::*; - use crate::types::RunId; + use crate::types::DetectionId; #[test] fn serializes_as_a_type_data_envelope_and_round_trips() { - let run_id = RunId::from_uuid(Uuid::now_v7()); - let payload = NotificationPayload::PipelineRunAnalyzed(PipelineRunAnalyzedParams { - run_id, + let detection_id = DetectionId::from_uuid(Uuid::now_v7()); + let payload = NotificationPayload::DetectionCompleted(DetectionCompletedParams { + detection_id, pipeline_slug: Handle::from_str("redact-invoices").unwrap(), input_file_name: Some("invoice.pdf".to_owned()), }); @@ -192,15 +194,15 @@ mod tests { let value = serde_json::to_value(&payload).unwrap(); // The durable wire shape: a `type` tag and a nested `data` object, matching // the activity payload and outbox event. Stored rows depend on it. - assert_eq!(value["type"], "pipeline.run.analyzed"); - assert_eq!(value["data"]["runId"], run_id.to_string()); + assert_eq!(value["type"], "pipeline.detection.completed"); + assert_eq!(value["data"]["detectionId"], detection_id.to_string()); assert_eq!(value["data"]["pipelineSlug"], "redact-invoices"); assert!( - value.get("runId").is_none(), + value.get("detectionId").is_none(), "params must nest under `data`" ); let decoded: NotificationPayload = serde_json::from_value(value).unwrap(); - assert_eq!(decoded.event(), NotificationEvent::PipelineRunAnalyzed); + assert_eq!(decoded.event(), NotificationEvent::DetectionCompleted); } } diff --git a/crates/nvisy-postgres/src/types/mod.rs b/crates/nvisy-postgres/src/types/mod.rs index 6177cadd..29a80ad8 100644 --- a/crates/nvisy-postgres/src/types/mod.rs +++ b/crates/nvisy-postgres/src/types/mod.rs @@ -16,29 +16,31 @@ pub use constraint::{ AccountApiTokenConstraints, AccountConstraints, AccountNotificationConstraints, ChatMessageConstraints, ChatSessionConstraints, ConstraintViolation, WorkspaceActivitiesConstraints, WorkspaceConnectionConstraints, - WorkspaceConnectionSyncConstraints, WorkspaceConstraints, WorkspaceFileConstraints, - WorkspaceInviteConstraints, WorkspaceMemberConstraints, WorkspacePipelineConstraints, - WorkspacePipelineReferenceConstraints, WorkspacePipelineRunConstraints, + WorkspaceConnectionSyncConstraints, WorkspaceConstraints, WorkspaceDetectionConstraints, + WorkspaceFileConstraints, WorkspaceInviteConstraints, WorkspaceMemberConstraints, + WorkspacePipelineConstraints, WorkspacePipelineReferenceConstraints, WorkspacePolicyConstraints, WorkspaceWebhookConstraints, }; pub use enums::{ - ActivityType, ApiTokenType, ChatRole, FileKind, InviteStatus, NotificationEvent, OutboxStatus, - PipelineRunStatus, PipelineStatus, PipelineTriggerType, ProviderType, SyncDeletionPolicy, - SyncMode, SyncStatus, SyncTriggerType, WebhookEvent, WebhookStatus, WorkspaceRole, + ActivityType, ApiTokenType, ChatRole, DetectionStatus, FileKind, InviteStatus, + NotificationEvent, OutboxStatus, PipelineStatus, PipelineTriggerType, ProviderType, + SyncDeletionPolicy, SyncMode, SyncStatus, SyncTriggerType, WebhookEvent, WebhookStatus, + WorkspaceRole, }; -pub use filtering::{FileFilter, InviteFilter, MemberFilter, RunFilter}; +pub use filtering::{DetectionFilter, FileFilter, InviteFilter, MemberFilter}; pub use handle::{HANDLE_MAX_LENGTH, HANDLE_MIN_LENGTH, Handle, HandleError}; pub use json::{ ActivityPayload, ConnectionActivityParams, ConnectionSyncCompletedParams, - ConnectionSyncFailedParams, FileActivityParams, InvalidHeader, InviteActivityParams, Json, - MemberActivityParams, MemberInvitedParams, MemberJoinedParams, NotificationPayload, OcrPolicy, - PipelineActivityParams, PipelineMetadata, PipelineRunActivityParams, PipelineRunAnalyzedParams, - PipelineRunCompletedParams, PipelineRunFailedParams, PolicyActivityParams, Retention, - RetentionOverride, RetentionScope, RetentionSettings, RunMetadata, WebhookActivityParams, - WebhookHeaders, WorkspaceActivityParams, WorkspaceMetadata, WorkspaceSettings, + ConnectionSyncFailedParams, DetectionActivityParams, DetectionCompletedParams, + DetectionFailedParams, DetectionMetadata, FileActivityParams, InvalidHeader, + InviteActivityParams, Json, MemberActivityParams, MemberInvitedParams, MemberJoinedParams, + NotificationPayload, OcrPolicy, PipelineActivityParams, PipelineMetadata, PolicyActivityParams, + RedactionActivityParams, RedactionCreatedParams, Retention, RetentionOverride, RetentionScope, + RetentionSettings, WebhookActivityParams, WebhookHeaders, WorkspaceActivityParams, + WorkspaceMetadata, WorkspaceSettings, }; pub use pagination::{Cursor, CursorPage, CursorPagination, OffsetPage, OffsetPagination}; -pub use prefixed_id::{ConnectionId, PrefixedIdError, RunId, WebhookId}; +pub use prefixed_id::{ConnectionId, DetectionId, PrefixedIdError, RedactionId, WebhookId}; pub use sorting::{ FileSortBy, FileSortField, InviteSortBy, InviteSortField, MemberSortBy, MemberSortField, SortBy, SortOrder, diff --git a/crates/nvisy-postgres/src/types/prefixed_id.rs b/crates/nvisy-postgres/src/types/prefixed_id.rs index effb46c1..d31df71a 100644 --- a/crates/nvisy-postgres/src/types/prefixed_id.rs +++ b/crates/nvisy-postgres/src/types/prefixed_id.rs @@ -137,8 +137,13 @@ prefixed_id! { } prefixed_id! { - /// Opaque identifier for a pipeline run (`run_`). - RunId, "run" + /// Opaque identifier for a detection (`detection_`). + DetectionId, "detection" +} + +prefixed_id! { + /// Opaque identifier for a redaction (`redaction_`). + RedactionId, "redaction" } #[cfg(test)] diff --git a/crates/nvisy-server/src/handler/analytics.rs b/crates/nvisy-server/src/handler/analytics.rs index 79b2f4f5..1236e86a 100644 --- a/crates/nvisy-server/src/handler/analytics.rs +++ b/crates/nvisy-server/src/handler/analytics.rs @@ -10,7 +10,7 @@ use nvisy_postgres::query::WorkspaceAnalyticsRepository; use crate::extract::{AuthProvider, AuthState, Json, Permission, Query, WorkspaceContext}; use crate::handler::request::DateWindow; -use crate::handler::response::{ErrorResponse, RunTimeSeries, WorkspaceAnalytics}; +use crate::handler::response::{DetectionTimeSeries, ErrorResponse, WorkspaceAnalytics}; use crate::handler::{Result, ServiceState}; /// Tracing target for workspace analytics operations. @@ -68,7 +68,7 @@ async fn get_run_timeseries( AuthState(auth_state): AuthState, WorkspaceContext(workspace): WorkspaceContext, Query(window): Query, -) -> Result<(StatusCode, Json)> { +) -> Result<(StatusCode, Json)> { tracing::debug!(target: TRACING_TARGET, "Computing run time series"); let window = window.resolve()?; @@ -86,7 +86,7 @@ async fn get_run_timeseries( window.to_timestamp()?, ) .await?; - let series = RunTimeSeries::from_window(window.from, window.to, points); + let series = DetectionTimeSeries::from_window(window.from, window.to, points); Ok((StatusCode::OK, Json(series))) } @@ -96,7 +96,7 @@ fn get_run_timeseries_docs(op: TransformOperation) -> TransformOperation { .description( "Returns a workspace's daily pipeline-run activity over a date window: runs per day, plus each day's error rate and durations. Every day in the window is present (quiet days report runs: 0), so the series plots as a continuous line or a contribution-style calendar. The window is `from`/`to` (inclusive, YYYY-MM-DD); it defaults to the last 30 days and is capped at 366 days.", ) - .response::<200, Json>() + .response::<200, Json>() .response::<400, Json>() .response::<401, Json>() .response::<403, Json>() diff --git a/crates/nvisy-server/src/handler/pipeline_audits.rs b/crates/nvisy-server/src/handler/detection_audits.rs similarity index 63% rename from crates/nvisy-server/src/handler/pipeline_audits.rs rename to crates/nvisy-server/src/handler/detection_audits.rs index 9ce0854c..a9e23790 100644 --- a/crates/nvisy-server/src/handler/pipeline_audits.rs +++ b/crates/nvisy-server/src/handler/detection_audits.rs @@ -1,9 +1,9 @@ -//! Pipeline-run audit handlers: read and export a run's analysis. +//! Detection audit handlers: read and export a detection's analysis. //! -//! Once a run is analyzed, its `Audit` (the decrypted map of detected findings) -//! can be reviewed inline or downloaded as JSON or a zip of CSV tables. The run -//! lifecycle itself (create, list, redact) lives in -//! [`pipeline_runs`](super::pipeline_runs). +//! Once a detection is complete, its `Audit` (the decrypted map of detected +//! findings) can be reviewed inline or downloaded as JSON or a zip of CSV tables. +//! The detection lifecycle itself (create, list, redact) lives in +//! [`detections`](super::detections). use aide::axum::ApiRouter; use aide::axum::routing::get_with; @@ -15,38 +15,38 @@ use elide_pipeline::Audit; use elide_pipeline::export::{ExportCsv, ExportJson}; use nvisy_postgres::PgClient; -use super::pipeline_runs::find_pipeline_run; +use super::detections::find_detection; use crate::extract::{AuthProvider, AuthState, Json, Path, Permission, Query, WorkspaceContext}; -use crate::handler::request::{ExportFormat, ExportQuery, PipelineRunPathParams}; +use crate::handler::request::{DetectionPathParams, ExportFormat, ExportQuery}; use crate::handler::response::ErrorResponse; use crate::handler::utility::{DownloadResponseExt, attachment_headers}; use crate::handler::{Error, ErrorKind, Result}; use crate::service::{EngineService, RunBlobStore, ServiceState}; -/// Tracing target for pipeline audit operations. -const TRACING_TARGET: &str = "nvisy_server::handler::audits"; +/// Tracing target for detection audit operations. +const TRACING_TARGET: &str = "nvisy_server::handler::detection_audits"; -/// Returns the run's analysis (the detected findings) for review. +/// Returns the detection's analysis (the detected findings) for review. /// /// Fetches and decrypts the engine's `Audit` from the audit bucket. Available -/// once the run is analyzed. Requires `ViewPipelines`. +/// once the detection is complete. Requires `ViewPipelines`. #[tracing::instrument( skip_all, fields( account_id = %auth_state.account_id, workspace_id = %workspace.id, - run_id = %path_params.run_id, + detection_id = %path_params.detection_id, ) )] -async fn get_pipeline_run_analysis( +async fn get_detection_analysis( State(pg_client): State, State(blob): State, State(engine): State, AuthState(auth_state): AuthState, WorkspaceContext(workspace): WorkspaceContext, - Path(path_params): Path, + Path(path_params): Path, ) -> Result<(StatusCode, Json)> { - tracing::debug!(target: TRACING_TARGET, "Getting pipeline run analysis"); + tracing::debug!(target: TRACING_TARGET, "Getting detection analysis"); let mut conn = pg_client.get_connection().await?; @@ -54,21 +54,23 @@ async fn get_pipeline_run_analysis( .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) .await?; - let (run, _pipeline) = - find_pipeline_run(&mut conn, workspace.id, path_params.run_id.as_uuid()).await?; + let (detection, _pipeline) = + find_detection(&mut conn, workspace.id, path_params.detection_id.as_uuid()).await?; let analyzed = blob - .load_analyzed_document(&mut conn, &engine, workspace.id, &run) + .load_analyzed_document(&mut conn, &engine, workspace.id, &detection) .await?; - tracing::debug!(target: TRACING_TARGET, "Pipeline run analysis retrieved"); + tracing::debug!(target: TRACING_TARGET, "Detection analysis retrieved"); Ok((StatusCode::OK, Json(analyzed))) } -fn get_pipeline_run_analysis_docs(op: TransformOperation) -> TransformOperation { - op.summary("Get run detections") - .description("Returns the run's detected findings (the analyzed document) for review.") +fn get_detection_analysis_docs(op: TransformOperation) -> TransformOperation { + op.summary("Get detection findings") + .description( + "Returns the detection's detected findings (the analyzed document) for review.", + ) .response::<200, Json>() .response::<401, Json>() .response::<403, Json>() @@ -76,7 +78,7 @@ fn get_pipeline_run_analysis_docs(op: TransformOperation) -> TransformOperation .response::<409, Json>() } -/// Downloads a run's audit as a file, in the requested `format`. +/// Downloads a detection's audit as a file, in the requested `format`. /// /// `json` yields a pretty-printed JSON file with the full structure — body, /// parts, context, and each entity's provenance chain. `csv` (the default) @@ -87,30 +89,30 @@ fn get_pipeline_run_analysis_docs(op: TransformOperation) -> TransformOperation fields( account_id = %auth_state.account_id, workspace_id = %workspace.id, - run_id = %path_params.run_id, + detection_id = %path_params.detection_id, format = ?query.format, ) )] -async fn download_pipeline_run_audit( +async fn download_detection_audit( State(pg_client): State, State(blob): State, State(engine): State, AuthState(auth_state): AuthState, WorkspaceContext(workspace): WorkspaceContext, - Path(path_params): Path, + Path(path_params): Path, Query(query): Query, ) -> Result<(StatusCode, HeaderMap, Body)> { - tracing::debug!(target: TRACING_TARGET, "Downloading pipeline run audit"); + tracing::debug!(target: TRACING_TARGET, "Downloading detection audit"); let mut conn = pg_client.get_connection().await?; auth_state .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) .await?; - let (run, _pipeline) = - find_pipeline_run(&mut conn, workspace.id, path_params.run_id.as_uuid()).await?; + let (detection, _pipeline) = + find_detection(&mut conn, workspace.id, path_params.detection_id.as_uuid()).await?; let audit = blob - .load_analyzed_document(&mut conn, &engine, workspace.id, &run) + .load_analyzed_document(&mut conn, &engine, workspace.id, &detection) .await?; let (content_type, filename, body) = match query.format { @@ -121,13 +123,17 @@ async fn download_pipeline_run_audit( .with_message("Failed to export audit as JSON") .with_context(err.to_string()) })?; - ("application/json", format!("audit-{}.json", run.id), buffer) + ( + "application/json", + format!("audit-{}.json", detection.id), + buffer, + ) } ExportFormat::Csv => { let archive = build_audit_csv_zip(&audit)?; ( "application/zip", - format!("audit-{}.csv.zip", run.id), + format!("audit-{}.csv.zip", detection.id), archive, ) } @@ -139,18 +145,18 @@ async fn download_pipeline_run_audit( body.len() as u64, ); - tracing::debug!(target: TRACING_TARGET, "Pipeline run audit exported"); + tracing::debug!(target: TRACING_TARGET, "Detection audit exported"); Ok((StatusCode::OK, headers, Body::from(body))) } -fn download_pipeline_run_audit_docs(op: TransformOperation) -> TransformOperation { - op.summary("Download run audit") +fn download_detection_audit_docs(op: TransformOperation) -> TransformOperation { + op.summary("Download detection audit") .description( - "Downloads the run's audit as a file. `format` is `csv` (default) — a zip of \ + "Downloads the detection's audit as a file. `format` is `csv` (default) — a zip of \ entities.csv, provenance.csv, and reviews.csv — or `json`, a pretty-printed JSON file.", ) .download_response( - "The run's exported audit.", + "The detection's exported audit.", &["application/zip", "application/json"], ) .response::<401, Json>() @@ -177,19 +183,16 @@ fn archive_error(error: impl std::fmt::Display) -> Error<'static> { .with_context(error.to_string()) } -/// Builds the pipeline-run audit routes. +/// Builds the detection audit routes. pub fn routes() -> ApiRouter { ApiRouter::new() .api_route( - "/workspaces/{workspaceSlug}/runs/{runId}/detections/", - get_with(get_pipeline_run_analysis, get_pipeline_run_analysis_docs), + "/workspaces/{workspaceSlug}/detections/{detectionId}/analysis/", + get_with(get_detection_analysis, get_detection_analysis_docs), ) .api_route( - "/workspaces/{workspaceSlug}/runs/{runId}/audit", - get_with( - download_pipeline_run_audit, - download_pipeline_run_audit_docs, - ), + "/workspaces/{workspaceSlug}/detections/{detectionId}/audit/", + get_with(download_detection_audit, download_detection_audit_docs), ) - .with_path_items(|item| item.tag("Pipeline Runs")) + .with_path_items(|item| item.tag("Detections")) } diff --git a/crates/nvisy-server/src/handler/detections.rs b/crates/nvisy-server/src/handler/detections.rs new file mode 100644 index 00000000..53adb111 --- /dev/null +++ b/crates/nvisy-server/src/handler/detections.rs @@ -0,0 +1,793 @@ +//! Detection handlers: detect, review, and redact. +//! +//! A detection is one analysis of a file through a pipeline. Detect creates the +//! detection and stores the findings; once it is complete a redaction consumes +//! the findings (with optional reviewer edits) and produces a redacted file. A +//! detection can be redacted many times. + +use aide::axum::ApiRouter; +use aide::transform::TransformOperation; +use async_stream::stream; +use axum::extract::State; +use axum::http::StatusCode; +use axum::response::sse::Event; +use futures::StreamExt; +use nvisy_postgres::model::{ + NewWorkspaceDetection, NewWorkspaceRedaction, WorkspaceDetection, WorkspacePipeline, +}; +use nvisy_postgres::query::{ + DetectionFiles, PipelineReferenceRepository, WorkspaceDetectionRepository, + WorkspaceFileRepository, WorkspacePipelineRepository, WorkspaceRedactionRepository, +}; +use nvisy_postgres::types::DetectionStatus; +use nvisy_postgres::{AsyncConnection, PgClient, PgConn}; +use uuid::Uuid; + +use crate::extract::{ + AuthProvider, AuthState, IdempotencyKey, Json, Path, Permission, Query, SecurityContext, + ValidateJson, WorkspaceContext, +}; +use crate::handler::request::{ + CreateDetection, CursorPagination, DetectionPathParams, PipelineDefinition, + PipelineDetectionsQuery, PipelinePathParams, RedactDetection, WorkspaceDetectionsQuery, +}; +use crate::handler::response::{Detection, DetectionsPage, ErrorResponse, Redaction}; +use crate::handler::utility::{SseResponse, resolve_account_ref}; +use crate::handler::{Error, ErrorKind, Result}; +use crate::service::{ + CryptoService, DetectionJob, DetectionQueue, DetectionRef, DetectionStatusEvent, EngineService, + EventEmitter, EventOrigin, FailDetection, RunBlobStore, ServiceState, WorkspaceEvent, + fail_detection, resolve_policies, +}; + +/// Tracing target for detection operations. +const TRACING_TARGET: &str = "nvisy_server::handler::detections"; + +/// Starts a detection: analyzes a file with the pipeline's configuration. +/// +/// Returns the detection holding the findings for review. A repeated request with +/// the same `Idempotency-Key` returns the existing detection instead of analyzing +/// again. Requires `RunPipelines` permission. +#[tracing::instrument( + skip_all, + fields( + account_id = %auth_state.account_id, + workspace_id = %workspace.id, + pipeline_slug = %path_params.pipeline_slug, + ) +)] +async fn create_detection( + State(pg_client): State, + State(detection): State, + AuthState(auth_state): AuthState, + WorkspaceContext(workspace): WorkspaceContext, + Path(path_params): Path, + IdempotencyKey(idempotency_key): IdempotencyKey, + security: SecurityContext, + ValidateJson(request): ValidateJson, +) -> Result<(StatusCode, Json)> { + tracing::debug!(target: TRACING_TARGET, "Starting detection"); + + let mut conn = pg_client.get_connection().await?; + + auth_state + .authorize_workspace(&mut conn, workspace.id, Permission::RunPipelines) + .await?; + + let pipeline = find_pipeline(&mut conn, workspace.id, &path_params.pipeline_slug).await?; + + // Only an enabled pipeline runs: a draft (still being configured) or a + // disabled (paused) pipeline is rejected. + if !pipeline.status.is_enabled() { + return Err(ErrorKind::Conflict + .with_message("Pipeline is not enabled") + .with_resource("pipeline")); + } + + // Idempotent replay: a repeated key returns the detection created the first + // time, attributed to whoever originally triggered it (not the current + // caller). + if let Some(key) = &idempotency_key + && let Some(existing) = conn + .find_detection_by_idempotency_key(pipeline.id, key) + .await? + { + tracing::debug!(target: TRACING_TARGET, "Replaying detection for idempotency key"); + let trigger = resolve_account_ref(&mut conn, existing.account_id).await?; + let files = conn.detection_file_names(workspace.id, &existing).await?; + return Ok(( + StatusCode::OK, + Json(Detection::from_model( + existing, + pipeline.slug.clone(), + workspace.slug.clone(), + trigger, + files, + )), + )); + } + + // Validate synchronously so a bad request fails fast (4xx) rather than as a + // detection that immediately fails in the worker. + let file = conn + .find_file_in_workspace(pipeline.workspace_id, request.file_id) + .await? + .ok_or_else(|| Error::not_found("file"))?; + + if conn.list_pipeline_policy_ids(pipeline.id).await?.is_empty() { + return Err(ErrorKind::BadRequest + .with_message("Pipeline has no policies; attach at least one before running") + .with_resource("pipeline")); + } + + // Decode the pipeline definition now so an undecodable definition fails the + // request synchronously (400) instead of returning 202 and failing later in + // the worker. The decoded value is rebuilt in the worker from the same + // stored bytes; this is validation only. + let _validated = PipelineDefinition::from_parts(pipeline.definition.clone(), Vec::new()) + .map_err(|err| { + ErrorKind::BadRequest + .with_message("Pipeline definition is invalid") + .with_resource("pipeline") + .with_context(err.to_string()) + })?; + + // Create the detection (its id is the engine correlation id) and enqueue + // analysis for the worker. The response returns immediately; the client + // learns the findings are ready via the detection's status (SSE at + // `.../events` or a re-read). + let new_detection = NewWorkspaceDetection { + pipeline_id: pipeline.id, + input_file_id: file.id, + account_id: auth_state.account_id, + status: Some(DetectionStatus::Pending), + idempotency_key: idempotency_key.clone(), + ..Default::default() + }; + + // Create the detection and record its start event in one transaction, so the + // event is never lost, nor recorded for a detection that rolled back. + let detection_row = conn + .transaction(async |conn| { + let detection_row = conn.create_workspace_detection(new_detection).await?; + conn.emit_event( + EventOrigin { + workspace_id: workspace.id, + account_id: auth_state.account_id, + security: &security, + }, + WorkspaceEvent::DetectionStarted(DetectionRef { + detection_id: detection_row.id, + pipeline_slug: pipeline.slug.clone(), + }), + ) + .await?; + Ok::<_, Error>(detection_row) + }) + .await?; + + let job = DetectionJob { + workspace_id: workspace.id, + detection_id: detection_row.id, + scope: request.scope, + }; + if let Err(err) = detection.enqueue(job).await { + // Enqueue failed, so the worker will never pick this detection up: fail it + // now rather than leaving it stuck in `Pending`. + fail_detection( + &mut conn, + &detection, + FailDetection { + workspace_id: workspace.id, + detection_id: detection_row.id, + pipeline_slug: pipeline.slug.clone(), + triggered_by: auth_state.account_id, + reason: "Failed to enqueue detection", + claim: None, + }, + ) + .await; + return Err(err); + } + + detection + .broadcast_status(detection_row.id, DetectionStatus::Pending) + .await; + + tracing::info!(target: TRACING_TARGET, detection_id = %detection_row.id, "Detection enqueued"); + + let trigger = resolve_account_ref(&mut conn, detection_row.account_id).await?; + + // The detection was just created from this file. + let files = DetectionFiles { + input: Some(file.display_name), + }; + + Ok(( + StatusCode::ACCEPTED, + Json(Detection::from_model( + detection_row, + pipeline.slug, + workspace.slug, + trigger, + files, + )), + )) +} + +fn create_detection_docs(op: TransformOperation) -> TransformOperation { + op.summary("Start a detection") + .description( + "Starts analysis for a file and returns 202 with the detection in the \ + `executing` state; the analysis runs in the background. Watch the \ + detection's status via the SSE stream at \ + `.../detections/{detectionId}/events` (or re-read the detection) and \ + fetch the findings from `.../detections/{detectionId}/analysis/` once \ + it reaches `complete`. A repeated Idempotency-Key returns the existing \ + detection.", + ) + .response::<202, Json>() + .response::<200, Json>() + .response::<400, Json>() + .response::<401, Json>() + .response::<403, Json>() + .response::<404, Json>() + .response::<409, Json>() +} + +/// Lists detections for a specific pipeline. +#[tracing::instrument( + skip_all, + fields( + account_id = %auth_state.account_id, + workspace_id = %workspace.id, + pipeline_slug = %path_params.pipeline_slug, + ) +)] +async fn list_pipeline_detections( + State(pg_client): State, + AuthState(auth_state): AuthState, + WorkspaceContext(workspace): WorkspaceContext, + Path(path_params): Path, + Query(pagination): Query, + Query(query): Query, +) -> Result<(StatusCode, Json)> { + tracing::debug!(target: TRACING_TARGET, "Listing pipeline detections"); + + let mut conn = pg_client.get_connection().await?; + + auth_state + .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) + .await?; + + let pipeline = find_pipeline(&mut conn, workspace.id, &path_params.pipeline_slug).await?; + + let page = conn + .cursor_list_pipeline_detections(pipeline.id, pagination.into(), &query.into()) + .await?; + + tracing::debug!( + target: TRACING_TARGET, + detection_count = page.items.len(), + "Pipeline detections listed" + ); + + let response = DetectionsPage::from_cursor_page(page, |row| { + Detection::from_model( + row.detection, + row.pipeline_slug, + workspace.slug.clone(), + row.account.into(), + DetectionFiles { + input: row.input_file_name, + }, + ) + }); + + Ok((StatusCode::OK, Json(response))) +} + +fn list_pipeline_detections_docs(op: TransformOperation) -> TransformOperation { + op.summary("List pipeline detections") + .description( + "Returns detections for a specific pipeline, most recent first, with \ + optional status, file, trigger-account, and trigger-type filters.", + ) + .response::<200, Json>() + .response::<401, Json>() + .response::<403, Json>() + .response::<404, Json>() +} + +/// Lists all detections across the workspace's pipelines. +/// +/// Aggregates detections from every pipeline in the workspace, most recent first, +/// with optional status and pipeline filters. Requires `ViewPipelines`. +#[tracing::instrument( + skip_all, + fields( + account_id = %auth_state.account_id, + workspace_id = %workspace.id, + ) +)] +async fn list_workspace_detections( + State(pg_client): State, + AuthState(auth_state): AuthState, + WorkspaceContext(workspace): WorkspaceContext, + Query(pagination): Query, + Query(query): Query, +) -> Result<(StatusCode, Json)> { + tracing::debug!(target: TRACING_TARGET, "Listing workspace detections"); + + let mut conn = pg_client.get_connection().await?; + + auth_state + .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) + .await?; + + let page = conn + .cursor_list_workspace_detections(workspace.id, pagination.into(), &query.into()) + .await?; + + tracing::debug!( + target: TRACING_TARGET, + detection_count = page.items.len(), + "Workspace detections listed" + ); + + Ok(( + StatusCode::OK, + Json(DetectionsPage::from_cursor_page(page, |row| { + Detection::from_model( + row.detection, + row.pipeline_slug, + workspace.slug.clone(), + row.account.into(), + DetectionFiles { + input: row.input_file_name, + }, + ) + })), + )) +} + +fn list_workspace_detections_docs(op: TransformOperation) -> TransformOperation { + op.summary("List workspace detections") + .description( + "Returns all detections across the workspace, most recent first, \ + with optional status, file, pipeline, trigger-account, and \ + trigger-type filters.", + ) + .response::<200, Json>() + .response::<401, Json>() + .response::<403, Json>() + .response::<404, Json>() +} + +/// Gets a specific detection. +#[tracing::instrument( + skip_all, + fields( + account_id = %auth_state.account_id, + workspace_id = %workspace.id, + detection_id = %path_params.detection_id, + ) +)] +async fn get_detection( + State(pg_client): State, + AuthState(auth_state): AuthState, + WorkspaceContext(workspace): WorkspaceContext, + Path(path_params): Path, +) -> Result<(StatusCode, Json)> { + tracing::debug!(target: TRACING_TARGET, "Getting detection"); + + let mut conn = pg_client.get_connection().await?; + + auth_state + .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) + .await?; + + let (detection, pipeline) = + find_detection(&mut conn, workspace.id, path_params.detection_id.as_uuid()).await?; + + let trigger = resolve_account_ref(&mut conn, detection.account_id).await?; + let files = conn.detection_file_names(workspace.id, &detection).await?; + + tracing::debug!(target: TRACING_TARGET, "Detection retrieved"); + + Ok(( + StatusCode::OK, + Json(Detection::from_model( + detection, + pipeline.slug, + workspace.slug, + trigger, + files, + )), + )) +} + +fn get_detection_docs(op: TransformOperation) -> TransformOperation { + op.summary("Get detection") + .description("Returns the detection and its status for review.") + .response::<200, Json>() + .response::<401, Json>() + .response::<403, Json>() + .response::<404, Json>() +} + +/// Streams a detection's status changes as Server-Sent Events until it settles. +/// +/// Emits one `status` event with the detection's current status immediately (so a +/// client that connects after analysis already finished still learns the state), +/// then forwards each status change. The stream ends once the detection leaves +/// the detecting phase (`pending`/`executing`) — i.e. analysis has produced +/// `complete`, or the detection `failed`. +/// +/// Live status changes arrive over a best-effort core-NATS broadcast; if none +/// arrives within a short interval the authoritative detection row is re-read from +/// the database, so a dropped broadcast never leaves the stream hanging. +/// +/// Authenticated like every other route (Bearer); browsers should consume it via +/// a `fetch` stream rather than the native `EventSource`, which cannot send an +/// `Authorization` header. +#[tracing::instrument( + skip_all, + fields( + account_id = %auth_state.account_id, + workspace_id = %workspace.id, + detection_id = %path_params.detection_id, + ) +)] +async fn stream_detection_events( + State(pg_client): State, + State(detection): State, + AuthState(auth_state): AuthState, + WorkspaceContext(workspace): WorkspaceContext, + Path(path_params): Path, +) -> Result> { + tracing::debug!(target: TRACING_TARGET, "Opening detection status stream"); + + let detection_id = path_params.detection_id.as_uuid(); + let mut conn = pg_client.get_connection().await?; + auth_state + .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) + .await?; + + // Subscribe BEFORE reading the current status: core-NATS broadcasts are not + // replayed, so a terminal status published between the read and the + // subscription going live would otherwise be lost and the stream would hang. + let mut updates = detection.subscribe_status(detection_id).await?; + + // Confirm the detection exists (and is workspace-scoped) so a bad id 404s here + // rather than opening an empty stream. + let (detection_row, _pipeline) = find_detection(&mut conn, workspace.id, detection_id).await?; + let current = detection_row.status; + drop(conn); + + let stream = stream! { + // Emit the current status first: covers the race where analysis settled + // before the subscription was live (no live event would ever arrive). + yield status_event(&DetectionStatusEvent { detection_id, status: current }); + if !current.is_detecting() { + return; + } + + loop { + match tokio::time::timeout(STATUS_POLL_INTERVAL, updates.next()).await { + // A live broadcast arrived; forward it and stop once it settles. + Ok(Some(event)) => { + let settled = !event.status.is_detecting(); + yield status_event(&event); + if settled { + break; + } + } + // The subscription ended; fall back to the DB so the client still + // learns the final status. + Ok(None) => { + if let Some(status) = reread_detection_status(&pg_client, workspace.id, detection_id).await { + yield status_event(&DetectionStatusEvent { detection_id, status }); + } + break; + } + // No broadcast within the interval: re-read the authoritative + // detection row. This recovers a dropped best-effort broadcast + // (core NATS is at-most-once) instead of hanging on keep-alive. + Err(_) => { + if let Some(status) = reread_detection_status(&pg_client, workspace.id, detection_id).await { + yield status_event(&DetectionStatusEvent { detection_id, status }); + if !status.is_detecting() { + break; + } + } + } + } + } + }; + + Ok(SseResponse::new(stream)) +} + +/// How long the status stream waits for a live broadcast before re-reading the +/// authoritative detection row from the database (the fallback for a dropped +/// best-effort broadcast). +const STATUS_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_secs(15); + +/// Re-reads a detection's current status from the database, returning `None` if +/// the detection can no longer be read (missing, or a transient error — the next +/// poll retries). +async fn reread_detection_status( + pg_client: &PgClient, + workspace_id: Uuid, + detection_id: Uuid, +) -> Option { + let mut conn = pg_client.get_connection().await.ok()?; + match find_detection(&mut conn, workspace_id, detection_id).await { + Ok((detection, _pipeline)) => Some(detection.status), + Err(err) => { + tracing::debug!(target: TRACING_TARGET, error = %err, %detection_id, "Failed to re-read detection status"); + None + } + } +} + +/// OpenAPI documentation for the detection status SSE stream. +fn stream_detection_events_docs(op: TransformOperation) -> TransformOperation { + op.summary("Stream detection status") + .description( + "Opens a Server-Sent Events stream of the detection's status changes. \ + Emits the current status immediately, then each transition, and \ + ends once the detection settles (complete or failed). Each event's \ + `data` is a `DetectionStatusEvent` (see the response schema). \ + Authenticate with a Bearer token via a `fetch`-based client; the \ + native `EventSource` cannot send an `Authorization` header.", + ) + .response::<401, Json>() + .response::<403, Json>() + .response::<404, Json>() +} + +/// Builds a `status` SSE event carrying the detection's status change. +fn status_event(event: &DetectionStatusEvent) -> Event { + Event::default() + .event("status") + .json_data(event) + .unwrap_or_else(|_| Event::default().event("status")) +} + +/// Redacts a detection using its findings, storing the result. +/// +/// Applies the pipeline's policies to the detection's stored analysis, stores the +/// redacted bytes as a new file, and emits a redaction event. Requires +/// `RunPipelines` permission. A detection can be redacted more than once. +#[tracing::instrument( + skip_all, + fields( + account_id = %auth_state.account_id, + workspace_id = %workspace.id, + detection_id = %path_params.detection_id, + ) +)] +async fn redact_detection( + State(pg_client): State, + State(blob): State, + State(crypto): State, + State(engine): State, + AuthState(auth_state): AuthState, + WorkspaceContext(workspace): WorkspaceContext, + Path(path_params): Path, + security: SecurityContext, + Json(request): Json, +) -> Result<(StatusCode, Json)> { + tracing::debug!(target: TRACING_TARGET, "Redacting detection"); + + let mut conn = pg_client.get_connection().await?; + + auth_state + .authorize_workspace(&mut conn, workspace.id, Permission::RunPipelines) + .await?; + + let (detection, pipeline) = + find_detection(&mut conn, workspace.id, path_params.detection_id.as_uuid()).await?; + + // A detection can only be redacted once its analysis is complete. + if !detection.is_complete() { + return Err(ErrorKind::Conflict + .with_message("Detection is not ready to redact") + .with_resource("detection")); + } + + // The source document is normally held back from retention while the + // detection is unfinished (see files_due_for_expiry), so this is reachable + // only if the input was explicitly deleted; surface a message that names the + // cause. + let file = conn + .find_file_in_workspace(workspace.id, detection.input_file_id) + .await? + .ok_or_else(|| { + ErrorKind::Conflict + .with_message("The detection's source document is no longer available") + .with_resource("detection") + })?; + + // The stored detection analysis is loaded into a working audit and never + // mutated on disk: reviewer edits and the redaction outcome land on this + // clone, which is persisted as the redaction's own review audit, leaving the + // detection analysis immutable and re-redactable. + let mut reviewed = blob + .load_analyzed_document(&mut conn, &engine, workspace.id, &detection) + .await?; + let policies = resolve_policies(&mut conn, &crypto, workspace.id, pipeline.id).await?; + + // Layer the reviewer's edits onto the working audit's report before + // redaction. Validation is report-relative (an unknown target or a + // self-contradiction → 400, via `EditError`'s `From` impl) so a reviewer is + // never told a decision took effect when the document says otherwise. + if let Some(edits) = &request.edits { + edits.validate(&reviewed.report)?; + edits.apply(&mut reviewed.report); + } + + let document = blob.build_document(&file, detection.id).await?; + + // No per-request key: the server does not yet drive keyed operators + // (HMAC/encrypt), whose `KeyConfig` would be supplied here. The codec params + // and document context are read back from the audit, recorded at detect time. + let redacted = engine + .anonymize(document, &policies, &mut reviewed, None) + .await?; + + // Stage both produced objects (redacted document + review audit) outside the + // transaction — object writes are not transactional — then commit their file + // rows and the redaction row together. On rollback the staged objects are + // reclaimed so no orphaned bytes accrue. + let retention = workspace.settings.or_default().retention; + let staged_output = blob + .stage_redacted_file( + &file, + &pipeline, + &retention, + auth_state.account_id, + redacted.bytes, + ) + .await?; + let staged_review = blob + .stage_review_audit(&pipeline, &retention, auth_state.account_id, &reviewed) + .await?; + + let redaction = conn + .transaction(async |conn| { + let output_file = conn.create_workspace_file(staged_output.clone()).await?; + let review_file = conn.create_workspace_file(staged_review.clone()).await?; + let redaction = conn + .create_redaction(NewWorkspaceRedaction { + detection_id: detection.id, + account_id: auth_state.account_id, + review_file_id: Some(review_file.id), + output_file_id: Some(output_file.id), + }) + .await?; + conn.emit_event( + EventOrigin { + workspace_id: workspace.id, + account_id: auth_state.account_id, + security: &security, + }, + WorkspaceEvent::RedactionCreated { + detection: DetectionRef { + detection_id: detection.id, + pipeline_slug: pipeline.slug.clone(), + }, + input_file_name: Some(file.display_name.clone()), + notify: detection.account_id, + }, + ) + .await?; + Ok::<_, Error>(redaction) + }) + .await; + + let redaction = match redaction { + Ok(redaction) => redaction, + Err(err) => { + // The rows rolled back, so their staged objects are orphans: reclaim + // both (best effort — a failure only leaves them for a later sweep). + blob.discard_staged_object(&staged_output).await.ok(); + blob.discard_staged_object(&staged_review).await.ok(); + return Err(err); + } + }; + + tracing::info!( + target: TRACING_TARGET, + detection_id = %detection.id, + redaction_id = %redaction.id, + "Detection redacted" + ); + + let requested_by = resolve_account_ref(&mut conn, redaction.account_id).await?; + + Ok(( + StatusCode::CREATED, + Json(Redaction::from_model( + redaction, + workspace.slug, + requested_by, + )), + )) +} + +fn redact_detection_docs(op: TransformOperation) -> TransformOperation { + op.summary("Redact a detection") + .description( + "Applies the pipeline's policies to the detection's stored analysis — with any \ + reviewer `edits` layered on first (suppress a false positive, retag a detection, or \ + add one the analysis missed) — and produces a new redaction: a redacted document \ + plus a review audit recording what was redacted. A detection can be redacted more \ + than once. An edit targeting a detection not in the analysis, or a set that \ + contradicts itself, is rejected (400).", + ) + .response::<201, Json>() + .response::<400, Json>() + .response::<401, Json>() + .response::<403, Json>() + .response::<404, Json>() + .response::<409, Json>() +} + +/// Finds a pipeline within a workspace by slug or returns NotFound. +async fn find_pipeline( + conn: &mut PgConn, + workspace_id: Uuid, + pipeline_slug: &str, +) -> Result { + conn.find_pipeline_in_workspace_by_slug(workspace_id, pipeline_slug) + .await? + .map(|wc| wc.item) + .ok_or_else(|| Error::not_found("pipeline")) +} + +/// Resolves a detection by its opaque id within a workspace, returning the +/// detection and its owning pipeline (for the response's pipeline slug). The +/// lookup is workspace-scoped through the owning pipeline. +pub(super) async fn find_detection( + conn: &mut PgConn, + workspace_id: Uuid, + detection_id: Uuid, +) -> Result<(WorkspaceDetection, WorkspacePipeline)> { + conn.find_workspace_detection_by_id(workspace_id, detection_id) + .await? + .ok_or_else(|| Error::not_found("detection")) +} + +/// Returns a [`Router`] with all detection routes. +/// +/// [`Router`]: axum::routing::Router +pub fn routes() -> ApiRouter { + use aide::axum::routing::*; + + ApiRouter::new() + .api_route( + "/workspaces/{workspaceSlug}/pipelines/detections/", + get_with(list_workspace_detections, list_workspace_detections_docs), + ) + .api_route( + "/workspaces/{workspaceSlug}/pipelines/{pipelineSlug}/detections/", + post_with(create_detection, create_detection_docs) + .get_with(list_pipeline_detections, list_pipeline_detections_docs), + ) + .api_route( + "/workspaces/{workspaceSlug}/detections/{detectionId}/", + get_with(get_detection, get_detection_docs), + ) + .api_route( + "/workspaces/{workspaceSlug}/detections/{detectionId}/events/", + get_with(stream_detection_events, stream_detection_events_docs), + ) + .api_route( + "/workspaces/{workspaceSlug}/detections/{detectionId}/redactions/", + post_with(redact_detection, redact_detection_docs), + ) + .with_path_items(|item| item.tag("Detections")) +} diff --git a/crates/nvisy-server/src/handler/error/engine_error.rs b/crates/nvisy-server/src/handler/error/engine_error.rs index 0f53dbd8..4baacdad 100644 --- a/crates/nvisy-server/src/handler/error/engine_error.rs +++ b/crates/nvisy-server/src/handler/error/engine_error.rs @@ -7,6 +7,7 @@ //! own message travels along as context. use elide_pipeline::ErrorKind as EngineErrorKind; +use elide_pipeline::entity::EditError; use super::http_error::{Error as HttpError, ErrorKind}; @@ -22,3 +23,15 @@ impl<'a> From for HttpError<'a> { } } } + +impl<'a> From for HttpError<'a> { + /// A reviewer edit set that does not apply to the analysis is always a client + /// error: both an unknown target (a stale or wrong-modality entity id) and a + /// self-contradiction (two edits deciding one entity differently) are the + /// caller's edits being wrong, so they map to a 400. + fn from(error: EditError) -> Self { + ErrorKind::BadRequest + .with_message("A reviewer edit does not apply to this analysis") + .with_context(error.to_string()) + } +} diff --git a/crates/nvisy-server/src/handler/error/pg_error.rs b/crates/nvisy-server/src/handler/error/pg_error.rs index 62958722..36f7665b 100644 --- a/crates/nvisy-server/src/handler/error/pg_error.rs +++ b/crates/nvisy-server/src/handler/error/pg_error.rs @@ -29,7 +29,7 @@ impl From for Error<'static> { ConstraintViolation::WorkspaceWebhook(c) => c.into(), ConstraintViolation::WorkspaceFile(c) => c.into(), ConstraintViolation::WorkspacePipeline(c) => c.into(), - ConstraintViolation::WorkspacePipelineRun(c) => c.into(), + ConstraintViolation::WorkspaceDetection(c) => c.into(), ConstraintViolation::WorkspacePipelineReference(c) => c.into(), ConstraintViolation::WorkspaceConnection(c) => c.into(), ConstraintViolation::WorkspaceConnectionSync(c) => c.into(), diff --git a/crates/nvisy-server/src/handler/error/pg_pipeline.rs b/crates/nvisy-server/src/handler/error/pg_pipeline.rs index d2f3c079..4d6fe8e9 100644 --- a/crates/nvisy-server/src/handler/error/pg_pipeline.rs +++ b/crates/nvisy-server/src/handler/error/pg_pipeline.rs @@ -2,8 +2,8 @@ use nvisy_postgres::types::{ WorkspaceConnectionConstraints, WorkspaceConnectionSyncConstraints, - WorkspacePipelineConstraints, WorkspacePipelineReferenceConstraints, - WorkspacePipelineRunConstraints, WorkspacePolicyConstraints, + WorkspaceDetectionConstraints, WorkspacePipelineConstraints, + WorkspacePipelineReferenceConstraints, WorkspacePolicyConstraints, }; use crate::handler::{Error, ErrorKind}; @@ -36,19 +36,19 @@ impl From for Error<'static> { } } -impl From for Error<'static> { - fn from(c: WorkspacePipelineRunConstraints) -> Self { +impl From for Error<'static> { + fn from(c: WorkspaceDetectionConstraints) -> Self { let error = match c { - WorkspacePipelineRunConstraints::MetadataSize => ErrorKind::BadRequest - .with_message("Pipeline run metadata size exceeds maximum limit"), - WorkspacePipelineRunConstraints::IdempotencyKeyLength => ErrorKind::BadRequest + WorkspaceDetectionConstraints::MetadataSize => ErrorKind::BadRequest + .with_message("Detection metadata size exceeds maximum limit"), + WorkspaceDetectionConstraints::IdempotencyKeyLength => ErrorKind::BadRequest .with_message("Idempotency key must be 1 to 255 characters"), - WorkspacePipelineRunConstraints::IdempotencyUnique => ErrorKind::Conflict - .with_message("A run with this idempotency key already exists"), + WorkspaceDetectionConstraints::IdempotencyUnique => ErrorKind::Conflict + .with_message("A detection with this idempotency key already exists"), }; - error.with_resource("pipeline_run") + error.with_resource("detection") } } diff --git a/crates/nvisy-server/src/handler/mod.rs b/crates/nvisy-server/src/handler/mod.rs index a16ef83f..f367ed90 100644 --- a/crates/nvisy-server/src/handler/mod.rs +++ b/crates/nvisy-server/src/handler/mod.rs @@ -12,16 +12,17 @@ mod catalog; mod chat; mod connection_syncs; mod connections; +mod detection_audits; +mod detections; mod error; mod files; mod invites; mod members; mod monitors; mod notifications; -mod pipeline_audits; -mod pipeline_runs; mod pipelines; mod policies; +mod redactions; pub mod request; pub mod response; mod tokens; @@ -89,8 +90,9 @@ fn private_routes( .merge(connection_syncs::routes()) .merge(files::routes()) .merge(pipelines::routes()) - .merge(pipeline_runs::routes()) - .merge(pipeline_audits::routes()) + .merge(detections::routes()) + .merge(detection_audits::routes()) + .merge(redactions::routes()) .merge(policies::routes()) .merge(catalog::routes()); diff --git a/crates/nvisy-server/src/handler/pipeline_runs.rs b/crates/nvisy-server/src/handler/pipeline_runs.rs deleted file mode 100644 index 68123708..00000000 --- a/crates/nvisy-server/src/handler/pipeline_runs.rs +++ /dev/null @@ -1,763 +0,0 @@ -//! Pipeline run handlers: detect, review, and redact. -//! -//! A run is one analysis of a file through a pipeline. Detect creates the run -//! and stores the findings; the run then awaits reviewer verification before -//! redact consumes the verified findings and produces a redacted file. - -use aide::axum::ApiRouter; -use aide::transform::TransformOperation; -use async_stream::stream; -use axum::extract::State; -use axum::http::StatusCode; -use axum::response::sse::Event; -use futures::StreamExt; -use nvisy_postgres::model::{ - NewWorkspacePipelineRun, UpdateWorkspacePipelineRun, WorkspacePipeline, WorkspacePipelineRun, -}; -use nvisy_postgres::query::{ - PipelineReferenceRepository, RunFiles, WorkspaceFileRepository, WorkspacePipelineRepository, - WorkspacePipelineRunRepository, -}; -use nvisy_postgres::types::PipelineRunStatus; -use nvisy_postgres::{AsyncConnection, PgClient, PgConn}; -use uuid::Uuid; - -use crate::extract::{ - AuthProvider, AuthState, IdempotencyKey, Json, Path, Permission, Query, SecurityContext, - ValidateJson, WorkspaceContext, -}; -use crate::handler::request::{ - CreatePipelineRun, CursorPagination, PipelineDefinition, PipelinePathParams, - PipelineRunPathParams, PipelineRunsQuery, WorkspaceRunsQuery, -}; -use crate::handler::response::{ErrorResponse, PipelineRun, PipelineRunsPage}; -use crate::handler::utility::{SseResponse, resolve_account_ref}; -use crate::handler::{Error, ErrorKind, Result}; -use crate::service::{ - CryptoService, DetectionJob, DetectionQueue, EngineService, EventEmitter, EventOrigin, FailRun, - PipelineRunRef, RunBlobStore, RunStatusEvent, ServiceState, WorkspaceEvent, fail_run, - resolve_policies, -}; - -/// Tracing target for pipeline run operations. -const TRACING_TARGET: &str = "nvisy_server::handler::runs"; - -/// Starts a run: analyzes a file with the pipeline's configuration (detect). -/// -/// Returns the run holding the findings for review. A repeated request with the -/// same `Idempotency-Key` returns the existing run instead of analyzing again. -/// Requires `RunPipelines` permission. -#[tracing::instrument( - skip_all, - fields( - account_id = %auth_state.account_id, - workspace_id = %workspace.id, - pipeline_slug = %path_params.pipeline_slug, - ) -)] -async fn create_pipeline_run( - State(pg_client): State, - State(detection): State, - AuthState(auth_state): AuthState, - WorkspaceContext(workspace): WorkspaceContext, - Path(path_params): Path, - IdempotencyKey(idempotency_key): IdempotencyKey, - security: SecurityContext, - ValidateJson(request): ValidateJson, -) -> Result<(StatusCode, Json)> { - tracing::debug!(target: TRACING_TARGET, "Starting pipeline run (detect)"); - - let mut conn = pg_client.get_connection().await?; - - auth_state - .authorize_workspace(&mut conn, workspace.id, Permission::RunPipelines) - .await?; - - let pipeline = find_pipeline(&mut conn, workspace.id, &path_params.pipeline_slug).await?; - - // Only an enabled pipeline runs: a draft (still being configured) or a - // disabled (paused) pipeline is rejected. - if !pipeline.status.is_enabled() { - return Err(ErrorKind::Conflict - .with_message("Pipeline is not enabled") - .with_resource("pipeline")); - } - - // Idempotent replay: a repeated key returns the run created the first time, - // attributed to whoever originally triggered it (not the current caller). - if let Some(key) = &idempotency_key - && let Some(existing) = conn - .find_pipeline_run_by_idempotency_key(pipeline.id, key) - .await? - { - tracing::debug!(target: TRACING_TARGET, "Replaying run for idempotency key"); - let trigger = resolve_account_ref(&mut conn, existing.account_id).await?; - let files = conn.run_file_names(workspace.id, &existing).await?; - return Ok(( - StatusCode::OK, - Json(PipelineRun::from_model( - existing, - pipeline.slug.clone(), - workspace.slug.clone(), - trigger, - files, - )), - )); - } - - // Validate synchronously so a bad request fails fast (4xx) rather than as a - // run that immediately fails in the worker. - let file = conn - .find_file_in_workspace(pipeline.workspace_id, request.file_id) - .await? - .ok_or_else(|| Error::not_found("file"))?; - - if conn.list_pipeline_policy_ids(pipeline.id).await?.is_empty() { - return Err(ErrorKind::BadRequest - .with_message("Pipeline has no policies; attach at least one before running") - .with_resource("pipeline")); - } - - // Decode the pipeline definition now so an undecodable definition fails the - // request synchronously (400) instead of returning 202 and failing later in - // the worker. The decoded value is rebuilt in the worker from the same - // stored bytes; this is validation only. - let _validated = PipelineDefinition::from_parts(pipeline.definition.clone(), Vec::new()) - .map_err(|err| { - ErrorKind::BadRequest - .with_message("Pipeline definition is invalid") - .with_resource("pipeline") - .with_context(err.to_string()) - })?; - - // Create the run (its id is the engine correlation id) and enqueue detection - // for the worker. The response returns immediately; the client learns the - // findings are ready via the run's status (SSE at `.../events` or a re-read). - let new_run = NewWorkspacePipelineRun { - pipeline_id: pipeline.id, - input_file_id: file.id, - account_id: auth_state.account_id, - status: Some(PipelineRunStatus::Queued), - idempotency_key: idempotency_key.clone(), - ..Default::default() - }; - - // Create the run and record its start event in one transaction, so the event - // is never lost, nor recorded for a run that rolled back. - let run = conn - .transaction(async |conn| { - let run = conn.create_workspace_pipeline_run(new_run).await?; - conn.emit_event( - EventOrigin { - workspace_id: workspace.id, - account_id: auth_state.account_id, - security: &security, - }, - WorkspaceEvent::PipelineRunStarted(PipelineRunRef { - run_id: run.id, - pipeline_slug: pipeline.slug.clone(), - }), - ) - .await?; - Ok::<_, Error>(run) - }) - .await?; - - let job = DetectionJob { - workspace_id: workspace.id, - run_id: run.id, - scope: request.scope, - }; - if let Err(err) = detection.enqueue(job).await { - // Enqueue failed, so the worker will never pick this run up: fail it now - // rather than leaving it stuck in `Queued`. - fail_run( - &mut conn, - &detection, - FailRun { - workspace_id: workspace.id, - run_id: run.id, - pipeline_slug: pipeline.slug.clone(), - triggered_by: auth_state.account_id, - reason: "Failed to enqueue detection", - claim: None, - }, - ) - .await; - return Err(err); - } - - detection - .broadcast_status(run.id, PipelineRunStatus::Queued) - .await; - - tracing::info!(target: TRACING_TARGET, run_id = %run.id, "Pipeline run enqueued for detection"); - - let trigger = resolve_account_ref(&mut conn, run.account_id).await?; - - // The run was just created from this file and has no output yet. - let files = RunFiles { - input: Some(file.display_name), - output: None, - }; - - Ok(( - StatusCode::ACCEPTED, - Json(PipelineRun::from_model( - run, - pipeline.slug, - workspace.slug, - trigger, - files, - )), - )) -} - -fn create_pipeline_run_docs(op: TransformOperation) -> TransformOperation { - op.summary("Start a run (detect)") - .description( - "Starts detection for a file and returns 202 with the run in the \ - `running` state; the analysis runs in the background. Watch the run's \ - status via the SSE stream at `.../runs/{runId}/events` (or re-read the \ - run) and fetch the findings from `.../runs/{runId}/detections/` once it \ - reaches `analyzed`. A repeated Idempotency-Key returns the existing run.", - ) - .response::<202, Json>() - .response::<200, Json>() - .response::<400, Json>() - .response::<401, Json>() - .response::<403, Json>() - .response::<404, Json>() - .response::<409, Json>() -} - -/// Lists runs for a specific pipeline. -#[tracing::instrument( - skip_all, - fields( - account_id = %auth_state.account_id, - workspace_id = %workspace.id, - pipeline_slug = %path_params.pipeline_slug, - ) -)] -async fn list_pipeline_runs( - State(pg_client): State, - AuthState(auth_state): AuthState, - WorkspaceContext(workspace): WorkspaceContext, - Path(path_params): Path, - Query(pagination): Query, - Query(query): Query, -) -> Result<(StatusCode, Json)> { - tracing::debug!(target: TRACING_TARGET, "Listing pipeline runs"); - - let mut conn = pg_client.get_connection().await?; - - auth_state - .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) - .await?; - - let pipeline = find_pipeline(&mut conn, workspace.id, &path_params.pipeline_slug).await?; - - let page = conn - .cursor_list_workspace_pipeline_runs(pipeline.id, pagination.into(), &query.into()) - .await?; - - tracing::debug!( - target: TRACING_TARGET, - run_count = page.items.len(), - "Pipeline runs listed" - ); - - let response = PipelineRunsPage::from_cursor_page(page, |row| { - PipelineRun::from_model( - row.run, - row.pipeline_slug, - workspace.slug.clone(), - row.account.into(), - RunFiles { - input: row.input_file_name, - output: None, - }, - ) - }); - - Ok((StatusCode::OK, Json(response))) -} - -fn list_pipeline_runs_docs(op: TransformOperation) -> TransformOperation { - op.summary("List pipeline runs") - .description( - "Returns runs for a specific pipeline, most recent first, with \ - optional status, file, trigger-account, and trigger-type filters.", - ) - .response::<200, Json>() - .response::<401, Json>() - .response::<403, Json>() - .response::<404, Json>() -} - -/// Lists all runs across the workspace's pipelines. -/// -/// Aggregates runs from every pipeline in the workspace, most recent first, -/// with optional status and pipeline filters. Requires `ViewPipelines`. -#[tracing::instrument( - skip_all, - fields( - account_id = %auth_state.account_id, - workspace_id = %workspace.id, - ) -)] -async fn list_workspace_runs( - State(pg_client): State, - AuthState(auth_state): AuthState, - WorkspaceContext(workspace): WorkspaceContext, - Query(pagination): Query, - Query(query): Query, -) -> Result<(StatusCode, Json)> { - tracing::debug!(target: TRACING_TARGET, "Listing workspace runs"); - - let mut conn = pg_client.get_connection().await?; - - auth_state - .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) - .await?; - - let page = conn - .cursor_list_workspace_runs(workspace.id, pagination.into(), &query.into()) - .await?; - - tracing::debug!( - target: TRACING_TARGET, - run_count = page.items.len(), - "Workspace runs listed" - ); - - Ok(( - StatusCode::OK, - Json(PipelineRunsPage::from_cursor_page(page, |row| { - PipelineRun::from_model( - row.run, - row.pipeline_slug, - workspace.slug.clone(), - row.account.into(), - RunFiles { - input: row.input_file_name, - output: None, - }, - ) - })), - )) -} - -fn list_workspace_runs_docs(op: TransformOperation) -> TransformOperation { - op.summary("List workspace runs") - .description( - "Returns all pipeline runs across the workspace, most recent first, \ - with optional status, file, pipeline, trigger-account, and \ - trigger-type filters.", - ) - .response::<200, Json>() - .response::<401, Json>() - .response::<403, Json>() - .response::<404, Json>() -} - -/// Gets a specific pipeline run. -#[tracing::instrument( - skip_all, - fields( - account_id = %auth_state.account_id, - workspace_id = %workspace.id, - run_id = %path_params.run_id, - ) -)] -async fn get_pipeline_run( - State(pg_client): State, - AuthState(auth_state): AuthState, - WorkspaceContext(workspace): WorkspaceContext, - Path(path_params): Path, -) -> Result<(StatusCode, Json)> { - tracing::debug!(target: TRACING_TARGET, "Getting pipeline run"); - - let mut conn = pg_client.get_connection().await?; - - auth_state - .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) - .await?; - - let (run, pipeline) = - find_pipeline_run(&mut conn, workspace.id, path_params.run_id.as_uuid()).await?; - - let trigger = resolve_account_ref(&mut conn, run.account_id).await?; - let files = conn.run_file_names(workspace.id, &run).await?; - - tracing::debug!(target: TRACING_TARGET, "Pipeline run retrieved"); - - Ok(( - StatusCode::OK, - Json(PipelineRun::from_model( - run, - pipeline.slug, - workspace.slug, - trigger, - files, - )), - )) -} - -fn get_pipeline_run_docs(op: TransformOperation) -> TransformOperation { - op.summary("Get pipeline run") - .description("Returns the run and its status for review.") - .response::<200, Json>() - .response::<401, Json>() - .response::<403, Json>() - .response::<404, Json>() -} - -/// Streams a run's status changes as Server-Sent Events until detection settles. -/// -/// Emits one `status` event with the run's current status immediately (so a -/// client that connects after detection already finished still learns the -/// state), then forwards each status change. The stream ends once the run leaves -/// the detecting phase (`queued`/`analyzing`) — i.e. detection has produced -/// `analyzed`, or the run `failed`/`cancelled`. -/// -/// Live status changes arrive over a best-effort core-NATS broadcast; if none -/// arrives within a short interval the authoritative run row is re-read from the -/// database, so a dropped broadcast never leaves the stream hanging. -/// -/// Authenticated like every other route (Bearer); browsers should consume it via -/// a `fetch` stream rather than the native `EventSource`, which cannot send an -/// `Authorization` header. -#[tracing::instrument( - skip_all, - fields( - account_id = %auth_state.account_id, - workspace_id = %workspace.id, - run_id = %path_params.run_id, - ) -)] -async fn stream_pipeline_run_events( - State(pg_client): State, - State(detection): State, - AuthState(auth_state): AuthState, - WorkspaceContext(workspace): WorkspaceContext, - Path(path_params): Path, -) -> Result> { - tracing::debug!(target: TRACING_TARGET, "Opening run status stream"); - - let run_id = path_params.run_id.as_uuid(); - let mut conn = pg_client.get_connection().await?; - auth_state - .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) - .await?; - - // Subscribe BEFORE reading the current status: core-NATS broadcasts are not - // replayed, so a terminal status published between the read and the - // subscription going live would otherwise be lost and the stream would hang. - let mut updates = detection.subscribe_status(run_id).await?; - - // Confirm the run exists (and is workspace-scoped) so a bad id 404s here - // rather than opening an empty stream. - let (run, _pipeline) = find_pipeline_run(&mut conn, workspace.id, run_id).await?; - let current = run.status; - drop(conn); - - let stream = stream! { - // Emit the current status first: covers the race where detection settled - // before the subscription was live (no live event would ever arrive). - yield status_event(&RunStatusEvent { run_id, status: current }); - if !current.is_detecting() { - return; - } - - loop { - match tokio::time::timeout(STATUS_POLL_INTERVAL, updates.next()).await { - // A live broadcast arrived; forward it and stop once detection settles. - Ok(Some(event)) => { - let settled = !event.status.is_detecting(); - yield status_event(&event); - if settled { - break; - } - } - // The subscription ended; fall back to the DB so the client still - // learns the final status. - Ok(None) => { - if let Some(status) = reread_run_status(&pg_client, workspace.id, run_id).await { - yield status_event(&RunStatusEvent { run_id, status }); - } - break; - } - // No broadcast within the interval: re-read the authoritative run - // row. This recovers a dropped best-effort broadcast (core NATS is - // at-most-once) instead of hanging on keep-alive forever. - Err(_) => { - if let Some(status) = reread_run_status(&pg_client, workspace.id, run_id).await { - yield status_event(&RunStatusEvent { run_id, status }); - if !status.is_detecting() { - break; - } - } - } - } - } - }; - - Ok(SseResponse::new(stream)) -} - -/// How long the status stream waits for a live broadcast before re-reading the -/// authoritative run row from the database (the fallback for a dropped -/// best-effort broadcast). -const STATUS_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_secs(15); - -/// Re-reads a run's current status from the database, returning `None` if the -/// run can no longer be read (missing, or a transient error — the next poll -/// retries). -async fn reread_run_status( - pg_client: &PgClient, - workspace_id: Uuid, - run_id: Uuid, -) -> Option { - let mut conn = pg_client.get_connection().await.ok()?; - match find_pipeline_run(&mut conn, workspace_id, run_id).await { - Ok((run, _pipeline)) => Some(run.status), - Err(err) => { - tracing::debug!(target: TRACING_TARGET, error = %err, %run_id, "Failed to re-read run status"); - None - } - } -} - -/// OpenAPI documentation for the run status SSE stream. -fn stream_pipeline_run_events_docs(op: TransformOperation) -> TransformOperation { - op.summary("Stream pipeline run status") - .description( - "Opens a Server-Sent Events stream of the run's status changes. \ - Emits the current status immediately, then each transition, and \ - ends once the run settles (analyzed, failed, or cancelled). Each \ - event's `data` is a `RunStatusEvent` (see the response schema). \ - Authenticate with a Bearer token via a `fetch`-based client; the \ - native `EventSource` cannot send an `Authorization` header.", - ) - .response::<401, Json>() - .response::<403, Json>() - .response::<404, Json>() -} - -/// Builds a `status` SSE event carrying the run's status change. -fn status_event(event: &RunStatusEvent) -> Event { - Event::default() - .event("status") - .json_data(event) - .unwrap_or_else(|_| Event::default().event("status")) -} - -/// Redacts a run using the reviewer-verified findings, storing the result. -/// -/// Consumes the analyzed run (which must be awaiting review), applies the -/// pipeline's policies to the verified findings, stores the redacted bytes as a -/// new file, and completes the run. Requires `RunPipelines` permission. -#[tracing::instrument( - skip_all, - fields( - account_id = %auth_state.account_id, - workspace_id = %workspace.id, - run_id = %path_params.run_id, - ) -)] -async fn redact_pipeline_run( - State(pg_client): State, - State(blob): State, - State(crypto): State, - State(engine): State, - AuthState(auth_state): AuthState, - WorkspaceContext(workspace): WorkspaceContext, - Path(path_params): Path, - security: SecurityContext, -) -> Result<(StatusCode, Json)> { - tracing::debug!(target: TRACING_TARGET, "Redacting pipeline run"); - - let mut conn = pg_client.get_connection().await?; - - auth_state - .authorize_workspace(&mut conn, workspace.id, Permission::RunPipelines) - .await?; - - let (run, pipeline) = - find_pipeline_run(&mut conn, workspace.id, path_params.run_id.as_uuid()).await?; - - // A run can only be redacted once, after detection. - if !run.is_analyzed() { - return Err(ErrorKind::Conflict - .with_message("Run is not awaiting redaction") - .with_resource("pipeline_run")); - } - - // The source document is normally held back from retention while the run is - // unfinished (see files_due_for_expiry), so this is reachable only if the - // input was explicitly deleted; surface a message that names the cause. - let file = conn - .find_file_in_workspace(workspace.id, run.input_file_id) - .await? - .ok_or_else(|| { - ErrorKind::Conflict - .with_message("The run's source document is no longer available") - .with_resource("pipeline_run") - })?; - - // The stored analysis is the source of truth for what gets redacted. It - // carries the scope (label catalog) analyze resolved, so redaction compiles - // against the same vocabulary without re-deriving it. - let mut analyzed = blob - .load_analyzed_document(&mut conn, &engine, workspace.id, &run) - .await?; - let policies = resolve_policies(&mut conn, &crypto, workspace.id, pipeline.id).await?; - let document = blob.build_document(&file, run.id).await?; - - // No per-request key: the server does not yet drive keyed operators - // (HMAC/encrypt), whose `KeyConfig` would be supplied here. The codec params - // and document context are read back from the audit, recorded at detect time. - let redacted = engine - .anonymize(document, &policies, &mut analyzed, None) - .await?; - - // Store the redacted bytes as a new workspace file and link it to the run. - let workspace_settings = workspace.settings.or_default().retention; - let output_file = blob - .store_redacted_file( - &mut conn, - &file, - &pipeline, - &workspace_settings, - auth_state.account_id, - redacted.bytes, - ) - .await?; - - // Complete the run and record its completion event in one transaction, so the - // event is never lost, nor recorded for an update that rolled back. - let run = conn - .transaction(async |conn| { - let run = conn - .update_workspace_pipeline_run( - run.id, - UpdateWorkspacePipelineRun { - status: Some(PipelineRunStatus::Completed), - output_file_id: Some(Some(output_file.id)), - completed_at: Some(Some(jiff::Timestamp::now().into())), - ..Default::default() - }, - ) - .await?; - conn.emit_event( - EventOrigin { - workspace_id: workspace.id, - account_id: auth_state.account_id, - security: &security, - }, - WorkspaceEvent::PipelineRunCompleted { - run: PipelineRunRef { - run_id: run.id, - pipeline_slug: pipeline.slug.clone(), - }, - input_file_name: Some(file.display_name.clone()), - notify: run.account_id, - }, - ) - .await?; - Ok::<_, Error>(run) - }) - .await?; - - tracing::info!( - target: TRACING_TARGET, - run_id = %run.id, - output_file_id = %output_file.id, - "Pipeline run redacted" - ); - - let trigger = resolve_account_ref(&mut conn, run.account_id).await?; - let files = conn.run_file_names(workspace.id, &run).await?; - - Ok(( - StatusCode::OK, - Json(PipelineRun::from_model( - run, - pipeline.slug, - workspace.slug, - trigger, - files, - )), - )) -} - -fn redact_pipeline_run_docs(op: TransformOperation) -> TransformOperation { - op.summary("Redact a run") - .description( - "Applies the pipeline's policies to the run's stored analysis, stores \ - the redacted file, and completes the run.", - ) - .response::<200, Json>() - .response::<401, Json>() - .response::<403, Json>() - .response::<404, Json>() - .response::<409, Json>() -} - -/// Finds a pipeline within a workspace by slug or returns NotFound. -async fn find_pipeline( - conn: &mut PgConn, - workspace_id: Uuid, - pipeline_slug: &str, -) -> Result { - conn.find_pipeline_in_workspace_by_slug(workspace_id, pipeline_slug) - .await? - .map(|wc| wc.item) - .ok_or_else(|| Error::not_found("pipeline")) -} - -/// Resolves a run by its opaque id within a workspace, returning the run and its -/// owning pipeline (for the response's pipeline slug). The lookup is -/// workspace-scoped through the owning pipeline. -pub(super) async fn find_pipeline_run( - conn: &mut PgConn, - workspace_id: Uuid, - run_id: Uuid, -) -> Result<(WorkspacePipelineRun, WorkspacePipeline)> { - conn.find_workspace_run_by_id(workspace_id, run_id) - .await? - .ok_or_else(|| Error::not_found("pipeline_run")) -} - -/// Returns a [`Router`] with all pipeline run routes. -/// -/// [`Router`]: axum::routing::Router -pub fn routes() -> ApiRouter { - use aide::axum::routing::*; - - ApiRouter::new() - .api_route( - "/workspaces/{workspaceSlug}/pipelines/runs/", - get_with(list_workspace_runs, list_workspace_runs_docs), - ) - .api_route( - "/workspaces/{workspaceSlug}/pipelines/{pipelineSlug}/runs/", - post_with(create_pipeline_run, create_pipeline_run_docs) - .get_with(list_pipeline_runs, list_pipeline_runs_docs), - ) - .api_route( - "/workspaces/{workspaceSlug}/runs/{runId}/", - get_with(get_pipeline_run, get_pipeline_run_docs), - ) - .api_route( - "/workspaces/{workspaceSlug}/runs/{runId}/events", - get_with(stream_pipeline_run_events, stream_pipeline_run_events_docs), - ) - .api_route( - "/workspaces/{workspaceSlug}/runs/{runId}/redactions/", - post_with(redact_pipeline_run, redact_pipeline_run_docs), - ) - .with_path_items(|item| item.tag("Pipeline Runs")) -} diff --git a/crates/nvisy-server/src/handler/redactions.rs b/crates/nvisy-server/src/handler/redactions.rs new file mode 100644 index 00000000..3b6a2608 --- /dev/null +++ b/crates/nvisy-server/src/handler/redactions.rs @@ -0,0 +1,179 @@ +//! Redaction handlers: list a detection's redactions and read a redaction's +//! review audit. +//! +//! A redaction is produced by `POST /detections/{detectionId}/redactions/` (in +//! [`detections`](super::detections)); these endpoints read them back. + +use aide::axum::ApiRouter; +use aide::axum::routing::get_with; +use aide::transform::TransformOperation; +use axum::extract::State; +use axum::http::StatusCode; +use elide_pipeline::Audit; +use nvisy_postgres::query::WorkspaceRedactionRepository; +use nvisy_postgres::{PgClient, PgConn}; +use uuid::Uuid; + +use super::detections::find_detection; +use crate::extract::{AuthProvider, AuthState, Json, Path, Permission, Query, WorkspaceContext}; +use crate::handler::request::{ + CursorPagination, DetectionPathParams, DetectionRedactionPathParams, +}; +use crate::handler::response::{ErrorResponse, Redaction, RedactionsPage}; +use crate::handler::utility::resolve_account_ref; +use crate::handler::{ErrorKind, Result, ServiceState}; +use crate::service::{EngineService, RunBlobStore}; + +/// Tracing target for redaction operations. +const TRACING_TARGET: &str = "nvisy_server::handler::redactions"; + +/// Lists a detection's redactions, most recent first, cursor-paginated. +#[tracing::instrument( + skip_all, + fields( + account_id = %auth_state.account_id, + workspace_id = %workspace.id, + detection_id = %path_params.detection_id, + ) +)] +async fn list_detection_redactions( + State(pg_client): State, + AuthState(auth_state): AuthState, + WorkspaceContext(workspace): WorkspaceContext, + Path(path_params): Path, + Query(pagination): Query, +) -> Result<(StatusCode, Json)> { + tracing::debug!(target: TRACING_TARGET, "Listing detection redactions"); + + let mut conn = pg_client.get_connection().await?; + + auth_state + .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) + .await?; + + // Confirm the detection exists in this workspace (404 otherwise) before + // listing its redactions. + let (detection, _pipeline) = + find_detection(&mut conn, workspace.id, path_params.detection_id.as_uuid()).await?; + + let page = conn + .cursor_list_detection_redactions(detection.id, pagination.into()) + .await?; + + // Resolve the requesting account per row. A detection's redactions are few + // (one per manual redact request), so a per-row lookup is acceptable here. + let mut items = Vec::with_capacity(page.items.len()); + for redaction in page.items { + let requested_by = resolve_account_ref(&mut conn, redaction.account_id).await?; + items.push(Redaction::from_model( + redaction, + workspace.slug.clone(), + requested_by, + )); + } + let response = RedactionsPage::new(items, page.total, page.next_cursor); + + Ok((StatusCode::OK, Json(response))) +} + +fn list_detection_redactions_docs(op: TransformOperation) -> TransformOperation { + op.summary("List detection redactions") + .description( + "Returns a detection's redactions, most recent first, cursor-paginated. Each \ + redaction is one redact pass with its own reviewer edits, output document, and \ + review audit.", + ) + .response::<200, Json>() + .response::<401, Json>() + .response::<403, Json>() + .response::<404, Json>() +} + +/// Returns a redaction's review audit: the analysis with the reviewer's edits +/// applied and the per-entity redaction outcome recorded. +#[tracing::instrument( + skip_all, + fields( + account_id = %auth_state.account_id, + workspace_id = %workspace.id, + detection_id = %path_params.detection_id, + redaction_id = %path_params.redaction_id, + ) +)] +async fn get_redaction_review( + State(pg_client): State, + State(blob): State, + State(engine): State, + AuthState(auth_state): AuthState, + WorkspaceContext(workspace): WorkspaceContext, + Path(path_params): Path, +) -> Result<(StatusCode, Json)> { + tracing::debug!(target: TRACING_TARGET, "Getting redaction review audit"); + + let mut conn = pg_client.get_connection().await?; + + auth_state + .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) + .await?; + + let redaction = find_redaction( + &mut conn, + workspace.id, + path_params.detection_id.as_uuid(), + path_params.redaction_id.as_uuid(), + ) + .await?; + + let review = blob + .load_review_audit(&mut conn, &engine, workspace.id, redaction.review_file_id) + .await?; + + Ok((StatusCode::OK, Json(review))) +} + +fn get_redaction_review_docs(op: TransformOperation) -> TransformOperation { + op.summary("Get redaction review") + .description( + "Returns the redaction's review audit: the detection analysis with the reviewer's \ + edits applied and the per-entity redaction outcome recorded.", + ) + .response::<200, Json>() + .response::<401, Json>() + .response::<403, Json>() + .response::<404, Json>() + .response::<409, Json>() +} + +/// Loads a redaction scoped to its detection and workspace, mapping a missing +/// detection or redaction to a 404. +async fn find_redaction( + conn: &mut PgConn, + workspace_id: Uuid, + detection_id: Uuid, + redaction_id: Uuid, +) -> Result { + // Confirm the detection is in this workspace first, so a redaction id cannot + // be probed against detections in another workspace. + find_detection(conn, workspace_id, detection_id).await?; + conn.find_redaction_by_id(detection_id, redaction_id) + .await? + .ok_or_else(|| { + ErrorKind::NotFound + .with_message("Redaction not found") + .with_resource("redaction") + }) +} + +/// Builds the redaction routes. +pub fn routes() -> ApiRouter { + ApiRouter::new() + .api_route( + "/workspaces/{workspaceSlug}/detections/{detectionId}/redactions/", + get_with(list_detection_redactions, list_detection_redactions_docs), + ) + .api_route( + "/workspaces/{workspaceSlug}/detections/{detectionId}/redactions/{redactionId}/review", + get_with(get_redaction_review, get_redaction_review_docs), + ) + .with_path_items(|item| item.tag("Redactions")) +} diff --git a/crates/nvisy-server/src/handler/request/detections.rs b/crates/nvisy-server/src/handler/request/detections.rs new file mode 100644 index 00000000..bd18a52e --- /dev/null +++ b/crates/nvisy-server/src/handler/request/detections.rs @@ -0,0 +1,102 @@ +//! Detection request types (detect and redact). + +use elide_pipeline::DocumentContext; +use elide_pipeline::entity::EditSet; +use nvisy_postgres::types::{DetectionFilter, DetectionStatus, PipelineTriggerType}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; +use validator::Validate; + +/// Query parameters for listing detections across a workspace. +/// +/// Every field is an optional filter; unset fields impose no constraint. +#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct WorkspaceDetectionsQuery { + /// Filter by detection status. + pub status: Option, + /// Filter by the source file the detection analyzes. + pub file_id: Option, + /// Filter by the owning pipeline. + pub pipeline_id: Option, + /// Filter by the account that triggered the detection. + pub triggered_by: Option, + /// Filter by how the detection was initiated (user vs system). + pub trigger_type: Option, +} + +impl From for DetectionFilter { + fn from(query: WorkspaceDetectionsQuery) -> Self { + DetectionFilter { + status: query.status, + input_file_id: query.file_id, + pipeline_id: query.pipeline_id, + account_id: query.triggered_by, + trigger_type: query.trigger_type, + } + } +} + +/// Query parameters for listing a single pipeline's detections. +/// +/// The pipeline is fixed by the route, so it narrows only by status, file, +/// trigger account, and trigger type. +#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct PipelineDetectionsQuery { + /// Filter by detection status. + pub status: Option, + /// Filter by the source file the detection analyzes. + pub file_id: Option, + /// Filter by the account that triggered the detection. + pub triggered_by: Option, + /// Filter by how the detection was initiated (user vs system). + pub trigger_type: Option, +} + +impl From for DetectionFilter { + fn from(query: PipelineDetectionsQuery) -> Self { + DetectionFilter { + status: query.status, + input_file_id: query.file_id, + pipeline_id: None, + account_id: query.triggered_by, + trigger_type: query.trigger_type, + } + } +} + +/// Request payload to start a detection over a file. +/// +/// Analyzes the file with the pipeline's configuration and returns the +/// detection, which holds the findings for review before redaction. +#[must_use] +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Validate)] +#[serde(rename_all = "camelCase")] +pub struct CreateDetection { + /// The file to analyze. + pub file_id: Uuid, + /// Per-document scope (languages, jurisdictions, document labels). + /// + /// Overrides the pipeline's `defaultScope` when present; absent falls back to + /// the pipeline default. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scope: Option, +} + +/// Request payload to redact a detection. +/// +/// The reviewer's edits layer over the detection's analysis before redaction: +/// suppress a false positive, retag a detection, or add one the analysis missed. +/// Omit `edits` to redact with the policy decisions exactly as detected. Each +/// redact request produces a new redaction. +#[must_use] +#[derive(Debug, Clone, Default, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct RedactDetection { + /// Reviewer edits to apply before redaction, grouped by modality. Omit to + /// redact with the policy decisions exactly as detected. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub edits: Option, +} diff --git a/crates/nvisy-server/src/handler/request/mod.rs b/crates/nvisy-server/src/handler/request/mod.rs index 8a4e16cd..b9e63363 100644 --- a/crates/nvisy-server/src/handler/request/mod.rs +++ b/crates/nvisy-server/src/handler/request/mod.rs @@ -6,13 +6,13 @@ mod authentications; mod chat; mod connection_syncs; mod connections; +mod detections; mod exports; mod files; mod invites; mod members; mod paginations; mod paths; -mod pipeline_runs; mod pipelines; mod policies; mod tokens; @@ -27,13 +27,13 @@ pub use authentications::*; pub use chat::*; pub use connection_syncs::*; pub use connections::*; +pub use detections::*; pub use exports::*; pub use files::*; pub use invites::*; pub use members::*; pub use paginations::*; pub use paths::*; -pub use pipeline_runs::*; pub use pipelines::*; pub use policies::*; pub use tokens::*; diff --git a/crates/nvisy-server/src/handler/request/paths.rs b/crates/nvisy-server/src/handler/request/paths.rs index ed49cc43..b1167b2a 100644 --- a/crates/nvisy-server/src/handler/request/paths.rs +++ b/crates/nvisy-server/src/handler/request/paths.rs @@ -1,6 +1,6 @@ //! Path parameter types for HTTP handlers. -use nvisy_postgres::types::{Handle, RunId, WebhookId}; +use nvisy_postgres::types::{DetectionId, Handle, RedactionId, WebhookId}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -83,13 +83,24 @@ pub struct PipelinePathParams { pub pipeline_slug: String, } -/// Path parameters for pipeline run operations. +/// Path parameters for detection operations. #[must_use] #[derive(Debug, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] -pub struct PipelineRunPathParams { - /// Opaque identifier of the run. - pub run_id: RunId, +pub struct DetectionPathParams { + /// Opaque identifier of the detection. + pub detection_id: DetectionId, +} + +/// Path parameters for a redaction nested under its detection. +#[must_use] +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct DetectionRedactionPathParams { + /// Opaque identifier of the detection. + pub detection_id: DetectionId, + /// Opaque identifier of the redaction. + pub redaction_id: RedactionId, } /// Path parameters for notification operations. diff --git a/crates/nvisy-server/src/handler/request/pipeline_runs.rs b/crates/nvisy-server/src/handler/request/pipeline_runs.rs deleted file mode 100644 index ec823bc2..00000000 --- a/crates/nvisy-server/src/handler/request/pipeline_runs.rs +++ /dev/null @@ -1,85 +0,0 @@ -//! Pipeline run request types (detect). - -use elide_pipeline::DocumentContext; -use nvisy_postgres::types::{PipelineRunStatus, PipelineTriggerType, RunFilter}; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use uuid::Uuid; -use validator::Validate; - -/// Query parameters for listing runs across a workspace. -/// -/// Every field is an optional filter; unset fields impose no constraint. -#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub struct WorkspaceRunsQuery { - /// Filter by run status. - pub status: Option, - /// Filter by the source file the run analyzes. - pub file_id: Option, - /// Filter by the owning pipeline. - pub pipeline_id: Option, - /// Filter by the account that triggered the run. - pub triggered_by: Option, - /// Filter by how the run was initiated (user vs system). - pub trigger_type: Option, -} - -impl From for RunFilter { - fn from(query: WorkspaceRunsQuery) -> Self { - RunFilter { - status: query.status, - input_file_id: query.file_id, - pipeline_id: query.pipeline_id, - account_id: query.triggered_by, - trigger_type: query.trigger_type, - } - } -} - -/// Query parameters for listing a single pipeline's runs. -/// -/// The pipeline is fixed by the route, so it narrows only by status, file, -/// trigger account, and trigger type. -#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub struct PipelineRunsQuery { - /// Filter by run status. - pub status: Option, - /// Filter by the source file the run analyzes. - pub file_id: Option, - /// Filter by the account that triggered the run. - pub triggered_by: Option, - /// Filter by how the run was initiated (user vs system). - pub trigger_type: Option, -} - -impl From for RunFilter { - fn from(query: PipelineRunsQuery) -> Self { - RunFilter { - status: query.status, - input_file_id: query.file_id, - pipeline_id: None, - account_id: query.triggered_by, - trigger_type: query.trigger_type, - } - } -} - -/// Request payload to start a run (detect) over a file. -/// -/// Analyzes the file with the pipeline's configuration and returns the run, -/// which holds the findings for review before redaction. -#[must_use] -#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Validate)] -#[serde(rename_all = "camelCase")] -pub struct CreatePipelineRun { - /// The file to analyze. - pub file_id: Uuid, - /// Per-document scope (languages, jurisdictions, document labels). - /// - /// Overrides the pipeline's `defaultScope` when present; absent falls back to - /// the pipeline default. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub scope: Option, -} diff --git a/crates/nvisy-server/src/handler/response/analytics.rs b/crates/nvisy-server/src/handler/response/analytics.rs index 4b481901..b11169cc 100644 --- a/crates/nvisy-server/src/handler/response/analytics.rs +++ b/crates/nvisy-server/src/handler/response/analytics.rs @@ -9,7 +9,7 @@ use std::collections::BTreeMap; use jiff::ToSpan; use jiff::civil::Date; use nvisy_postgres::query::{AnalyticsSnapshot, RunDayPoint}; -use nvisy_postgres::types::{FileKind, PipelineRunStatus}; +use nvisy_postgres::types::{DetectionStatus, FileKind}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use strum::IntoEnumIterator; @@ -21,8 +21,8 @@ use strum::IntoEnumIterator; pub struct WorkspaceAnalytics { /// Stored-file totals and their per-kind breakdown. pub storage: StorageAnalytics, - /// Pipeline-run health: volume, status mix, and durations. - pub runs: RunAnalytics, + /// Detection health: volume, status mix, and durations. + pub detections: DetectionAnalytics, /// Inference token usage: workspace totals and a per-model breakdown. pub usage: UsageAnalytics, } @@ -84,12 +84,12 @@ pub struct StorageKindEntry { /// Pipeline-run health for a workspace. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] -pub struct RunAnalytics { +pub struct DetectionAnalytics { /// Total number of runs. pub total: i64, /// Per-status breakdown, one entry per run status (zero-filled), in a stable /// order. - pub by_status: Vec, + pub by_status: Vec, /// Failed / (completed + failed). Omitted when no run has reached a terminal /// state (genuinely no signal, not zero). #[serde(skip_serializing_if = "Option::is_none")] @@ -106,9 +106,9 @@ pub struct RunAnalytics { /// One status's share of a workspace's runs. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] -pub struct RunStatusEntry { +pub struct DetectionStatusEntry { /// The run status. - pub status: PipelineRunStatus, + pub status: DetectionStatus, /// Number of runs in this status. pub count: i64, } @@ -136,8 +136,8 @@ impl WorkspaceAnalytics { let total_bytes = by_kind.iter().map(|e| e.total_bytes).sum(); let file_count = by_kind.iter().map(|e| e.file_count).sum(); - let by_status: Vec = PipelineRunStatus::iter() - .map(|status| RunStatusEntry { + let by_status: Vec = DetectionStatus::iter() + .map(|status| DetectionStatusEntry { status, count: runs .iter() @@ -147,8 +147,8 @@ impl WorkspaceAnalytics { .collect(); let total = by_status.iter().map(|e| e.count).sum(); - let completed = count_of(&by_status, PipelineRunStatus::Completed); - let failed = count_of(&by_status, PipelineRunStatus::Failed); + let completed = count_of(&by_status, DetectionStatus::Complete); + let failed = count_of(&by_status, DetectionStatus::Failed); let terminal = completed + failed; let error_rate = (terminal > 0).then(|| failed as f64 / terminal as f64); @@ -174,7 +174,7 @@ impl WorkspaceAnalytics { file_count, by_kind, }, - runs: RunAnalytics { + detections: DetectionAnalytics { total, by_status, error_rate, @@ -190,15 +190,15 @@ impl WorkspaceAnalytics { /// (quiet days included with `runs: 0`), ready to plot as a continuous series. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] -pub struct RunTimeSeries { +pub struct DetectionTimeSeries { /// One entry per day in the requested window, oldest first. - pub points: Vec, + pub points: Vec, } /// A single day of run activity. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] -pub struct RunDayEntry { +pub struct DetectionDayEntry { /// The day (`YYYY-MM-DD`, UTC). pub date: Date, /// Runs started this day (`0` on a quiet day). @@ -224,7 +224,7 @@ pub struct RunDayEntry { pub total_tokens: Option, } -impl RunTimeSeries { +impl DetectionTimeSeries { /// Builds a dense daily series over `[from, to]` from the sparse per-day rows /// the query returns. Every day in the window is emitted in order; a day with /// no runs reports `runs: 0` and omits the rate/duration/token fields. The @@ -242,7 +242,7 @@ impl RunTimeSeries { let mut date = from; while date <= to { days.push(match by_day.remove(&date) { - Some(p) => RunDayEntry { + Some(p) => DetectionDayEntry { date, runs: p.runs, error_rate: (p.terminal > 0).then(|| p.failed as f64 / p.terminal as f64), @@ -252,7 +252,7 @@ impl RunTimeSeries { output_tokens: p.output_tokens, total_tokens: p.total_tokens, }, - None => RunDayEntry { + None => DetectionDayEntry { date, runs: 0, error_rate: None, @@ -274,7 +274,7 @@ impl RunTimeSeries { } /// Looks up a status's count in the assembled breakdown. -fn count_of(by_status: &[RunStatusEntry], status: PipelineRunStatus) -> i64 { +fn count_of(by_status: &[DetectionStatusEntry], status: DetectionStatus) -> i64 { by_status .iter() .find(|e| e.status == status) @@ -303,11 +303,11 @@ mod tests { ]; let runs = vec![ RunStatusCount { - status: PipelineRunStatus::Completed, + status: DetectionStatus::Complete, count: 3, }, RunStatusCount { - status: PipelineRunStatus::Failed, + status: DetectionStatus::Failed, count: 1, }, ]; @@ -337,10 +337,13 @@ mod tests { usage, }); - // Every FileKind / PipelineRunStatus is present (zero-filled), so the + // Every FileKind / DetectionStatus is present (zero-filled), so the // breakdown lengths equal the enum sizes. assert_eq!(a.storage.by_kind.len(), FileKind::iter().count()); - assert_eq!(a.runs.by_status.len(), PipelineRunStatus::iter().count()); + assert_eq!( + a.detections.by_status.len(), + DetectionStatus::iter().count() + ); // Absent audit kind is zero, not missing. let audit = a .storage @@ -353,11 +356,11 @@ mod tests { // Totals sum the breakdown. assert_eq!(a.storage.total_bytes, 350); assert_eq!(a.storage.file_count, 3); - assert_eq!(a.runs.total, 4); + assert_eq!(a.detections.total, 4); // error_rate = failed / (completed + failed) = 1 / 4. - assert_eq!(a.runs.error_rate, Some(0.25)); - assert_eq!(a.runs.avg_duration_ms, Some(30_000)); + assert_eq!(a.detections.error_rate, Some(0.25)); + assert_eq!(a.detections.avg_duration_ms, Some(30_000)); // Usage: per-model entries preserved, workspace totals summed with // never-reported fields treated as 0 (not conflated across fields). @@ -390,7 +393,8 @@ mod tests { total_tokens: None, }]; - let series = RunTimeSeries::from_window(date("2026-01-05"), date("2026-01-07"), sparse); + let series = + DetectionTimeSeries::from_window(date("2026-01-05"), date("2026-01-07"), sparse); // Dense: every day in the window present, in order. assert_eq!(series.points.len(), 3); @@ -417,7 +421,7 @@ mod tests { fn error_rate_is_none_with_no_terminal_runs() { // Only active runs, no files, no completed durations. let runs = vec![RunStatusCount { - status: PipelineRunStatus::Queued, + status: DetectionStatus::Pending, count: 5, }]; let a = WorkspaceAnalytics::from_snapshot(AnalyticsSnapshot { @@ -430,11 +434,11 @@ mod tests { usage: Vec::new(), }); - assert_eq!(a.runs.error_rate, None); - assert_eq!(a.runs.avg_duration_ms, None); + assert_eq!(a.detections.error_rate, None); + assert_eq!(a.detections.avg_duration_ms, None); assert_eq!(a.storage.total_bytes, 0); // Still every kind/status present, all zero except queued. assert_eq!(a.storage.by_kind.len(), FileKind::iter().count()); - assert_eq!(a.runs.total, 5); + assert_eq!(a.detections.total, 5); } } diff --git a/crates/nvisy-server/src/handler/response/detections.rs b/crates/nvisy-server/src/handler/response/detections.rs new file mode 100644 index 00000000..8b45da4d --- /dev/null +++ b/crates/nvisy-server/src/handler/response/detections.rs @@ -0,0 +1,91 @@ +//! Detection response types. + +use jiff::Timestamp; +use nvisy_postgres::model::WorkspaceDetection as DetectionModel; +use nvisy_postgres::query::DetectionFiles; +use nvisy_postgres::types::{ + DetectionId, DetectionMetadata, DetectionStatus, Handle, PipelineTriggerType, +}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use super::{AccountRef, Page}; + +/// Response type for a detection. +/// +/// A detection is addressed by its own opaque id; the owning pipeline and +/// workspace slugs are carried for context. Redacted outputs are not here — a +/// detection produces many redactions, each fetched from its `redactions` +/// endpoint. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct Detection { + /// Opaque identifier of the detection. + pub id: DetectionId, + /// Handle of the pipeline this detection belongs to. + pub pipeline_slug: Handle, + /// Handle of the workspace this detection belongs to. + pub workspace_slug: Handle, + /// Source document this detection analyzes. + pub input_file_id: Uuid, + /// Display name of the source document, for showing the detection without a + /// separate file lookup. `None` if the file was removed (e.g. by retention). + #[serde(skip_serializing_if = "Option::is_none")] + pub input_file_name: Option, + /// Account that triggered the detection. + pub triggered_by: AccountRef, + /// How the detection was triggered. + pub trigger_type: PipelineTriggerType, + /// Current detection status. + /// + /// The detections are available to fetch from the detection's `analysis` + /// endpoint once this reaches `complete`. + pub status: DetectionStatus, + /// Human-readable failure reason, present only when the detection `failed`. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// Non-encrypted metadata for filtering/display. + pub metadata: DetectionMetadata, + /// When the detection started. + pub started_at: Timestamp, + /// When the detection completed analysis. + #[serde(skip_serializing_if = "Option::is_none")] + pub completed_at: Option, +} + +/// Paginated response for detections. +pub type DetectionsPage = Page; + +impl Detection { + /// Creates a detection response from the database model, the slugs of its + /// owning pipeline and workspace, the triggering account, and the resolved + /// input file display name. + pub fn from_model( + detection: DetectionModel, + pipeline_slug: Handle, + workspace_slug: Handle, + triggered_by: AccountRef, + files: DetectionFiles, + ) -> Self { + // Surface the failure reason (written to metadata.error by the worker / + // enqueue-failure path) as a dedicated field for a failed detection. + let metadata = detection.metadata.or_default(); + let error = metadata.error.clone(); + + Self { + id: DetectionId::from_uuid(detection.id), + pipeline_slug, + workspace_slug, + input_file_id: detection.input_file_id, + input_file_name: files.input, + triggered_by, + trigger_type: detection.trigger_type, + status: detection.status, + error, + metadata, + started_at: detection.started_at.into(), + completed_at: detection.completed_at.map(Into::into), + } + } +} diff --git a/crates/nvisy-server/src/handler/response/mod.rs b/crates/nvisy-server/src/handler/response/mod.rs index 30f9c83b..053e8729 100644 --- a/crates/nvisy-server/src/handler/response/mod.rs +++ b/crates/nvisy-server/src/handler/response/mod.rs @@ -13,15 +13,16 @@ mod catalog; mod chat; mod connection_syncs; mod connections; +mod detections; mod errors; mod files; mod invites; mod members; mod monitors; mod notifications; -mod pipeline_runs; mod pipelines; mod policies; +mod redactions; mod tokens; mod webhooks; mod workspaces; @@ -35,15 +36,16 @@ pub use catalog::*; pub use chat::*; pub use connection_syncs::*; pub use connections::*; +pub use detections::*; pub use errors::*; pub use files::*; pub use invites::*; pub use members::*; pub use monitors::*; pub use notifications::*; -pub use pipeline_runs::*; pub use pipelines::*; pub use policies::*; +pub use redactions::*; pub use tokens::*; pub use webhooks::*; pub use workspaces::*; diff --git a/crates/nvisy-server/src/handler/response/pipeline_runs.rs b/crates/nvisy-server/src/handler/response/pipeline_runs.rs deleted file mode 100644 index 8e591867..00000000 --- a/crates/nvisy-server/src/handler/response/pipeline_runs.rs +++ /dev/null @@ -1,95 +0,0 @@ -//! Pipeline run response types. - -use jiff::Timestamp; -use nvisy_postgres::model::WorkspacePipelineRun as PipelineRunModel; -use nvisy_postgres::query::RunFiles; -use nvisy_postgres::types::{Handle, PipelineRunStatus, PipelineTriggerType, RunId, RunMetadata}; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use uuid::Uuid; - -use super::{AccountRef, Page}; - -/// Response type for a pipeline run. -/// -/// A run is addressed by its own opaque id; the owning pipeline and workspace -/// slugs are carried for context. -#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub struct PipelineRun { - /// Opaque identifier of the run. - pub id: RunId, - /// Handle of the pipeline this run belongs to. - pub pipeline_slug: Handle, - /// Handle of the workspace this run belongs to. - pub workspace_slug: Handle, - /// Source document this run analyzes / redacts. - pub input_file_id: Uuid, - /// Display name of the source document, for showing the run without a - /// separate file lookup. `None` if the file was removed (e.g. by retention). - #[serde(skip_serializing_if = "Option::is_none")] - pub input_file_name: Option, - /// Redacted document produced by the run, once it completes. - #[serde(skip_serializing_if = "Option::is_none")] - pub output_file_id: Option, - /// Display name of the redacted output, once the run completes. - #[serde(skip_serializing_if = "Option::is_none")] - pub output_file_name: Option, - /// Account that triggered the run. - pub triggered_by: AccountRef, - /// How the run was triggered. - pub trigger_type: PipelineTriggerType, - /// Current run status. - /// - /// The detections are available to fetch from the run's `detections` - /// endpoint once this reaches `analyzed`. - pub status: PipelineRunStatus, - /// Human-readable failure reason, present only when the run `failed`. - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, - /// Non-encrypted metadata for filtering/display. - pub metadata: RunMetadata, - /// When the run started. - pub started_at: Timestamp, - /// When the run completed. - #[serde(skip_serializing_if = "Option::is_none")] - pub completed_at: Option, -} - -/// Paginated response for pipeline runs. -pub type PipelineRunsPage = Page; - -impl PipelineRun { - /// Creates a pipeline run response from the database model, the slugs of its - /// owning pipeline and workspace, the triggering account, and the resolved - /// input/output file display names. - pub fn from_model( - run: PipelineRunModel, - pipeline_slug: Handle, - workspace_slug: Handle, - triggered_by: AccountRef, - files: RunFiles, - ) -> Self { - // Surface the failure reason (written to metadata.error by the worker / - // enqueue-failure path) as a dedicated field for a failed run. - let metadata = run.metadata.or_default(); - let error = metadata.error.clone(); - - Self { - id: RunId::from_uuid(run.id), - pipeline_slug, - workspace_slug, - input_file_id: run.input_file_id, - input_file_name: files.input, - output_file_id: run.output_file_id, - output_file_name: files.output, - triggered_by, - trigger_type: run.trigger_type, - status: run.status, - error, - metadata, - started_at: run.started_at.into(), - completed_at: run.completed_at.map(Into::into), - } - } -} diff --git a/crates/nvisy-server/src/handler/response/redactions.rs b/crates/nvisy-server/src/handler/response/redactions.rs new file mode 100644 index 00000000..9dad3a27 --- /dev/null +++ b/crates/nvisy-server/src/handler/response/redactions.rs @@ -0,0 +1,57 @@ +//! Redaction response types. + +use jiff::Timestamp; +use nvisy_postgres::model::WorkspaceRedaction as RedactionModel; +use nvisy_postgres::types::{DetectionId, Handle, RedactionId}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use super::{AccountRef, Page}; + +/// Response type for a redaction. +/// +/// A redaction is one redact pass over a detection, produced with a specific set +/// of reviewer edits. It owns the redacted output document (downloadable through +/// the normal file endpoints) and a review audit recording what was redacted and +/// why (fetched from the redaction's `review` endpoint). +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct Redaction { + /// Opaque identifier of the redaction. + pub id: RedactionId, + /// The detection this redaction was produced from. + pub detection_id: DetectionId, + /// Handle of the workspace this redaction belongs to. + pub workspace_slug: Handle, + /// Redacted output document this redaction produced. `None` only if the file + /// was removed (e.g. by retention). + #[serde(skip_serializing_if = "Option::is_none")] + pub output_file_id: Option, + /// Account that requested the redaction. + pub requested_by: AccountRef, + /// When the redaction was created. + pub created_at: Timestamp, +} + +/// Paginated response for redactions. +pub type RedactionsPage = Page; + +impl Redaction { + /// Creates a redaction response from the database model, the owning + /// workspace slug, and the requesting account. + pub fn from_model( + redaction: RedactionModel, + workspace_slug: Handle, + requested_by: AccountRef, + ) -> Self { + Self { + id: RedactionId::from_uuid(redaction.id), + detection_id: DetectionId::from_uuid(redaction.detection_id), + workspace_slug, + output_file_id: redaction.output_file_id, + requested_by, + created_at: redaction.created_at.into(), + } + } +} diff --git a/crates/nvisy-server/src/middleware/specification.rs b/crates/nvisy-server/src/middleware/specification.rs index e20b9fb8..2e58bf12 100644 --- a/crates/nvisy-server/src/middleware/specification.rs +++ b/crates/nvisy-server/src/middleware/specification.rs @@ -281,8 +281,13 @@ fn api_docs(api: TransformOpenApi) -> TransformOpenApi { ..Default::default() }) .tag(Tag { - name: "Pipeline Runs".into(), - description: Some("Pipeline run execution and review".into()), + name: "Detections".into(), + description: Some("Detection analysis and redaction".into()), + ..Default::default() + }) + .tag(Tag { + name: "Redactions".into(), + description: Some("Redactions produced from a detection, with reviewer edits".into()), ..Default::default() }) .tag(Tag { diff --git a/crates/nvisy-server/src/service/detection/job.rs b/crates/nvisy-server/src/service/detection/job.rs index 5c03f5ef..a4ef570a 100644 --- a/crates/nvisy-server/src/service/detection/job.rs +++ b/crates/nvisy-server/src/service/detection/job.rs @@ -1,44 +1,45 @@ -//! Detection job and run-status event types. +//! Detection job and detection-status event types. use elide_pipeline::DocumentContext; -use nvisy_postgres::types::PipelineRunStatus; +use nvisy_postgres::types::DetectionStatus; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use uuid::Uuid; -/// A queued request to run detection for a pipeline run. +/// A queued request to run a detection. /// -/// Published to the `DetectionStream` work-queue by the create-run handler and -/// consumed by the [`DetectionWorker`](super::DetectionWorker). The worker -/// re-loads the run, pipeline, file, and policies from the ids; only the +/// Published to the `DetectionStream` work-queue by the create-detection handler +/// and consumed by the [`DetectionWorker`](super::DetectionWorker). The worker +/// re-loads the detection, pipeline, file, and policies from the ids; only the /// caller-supplied per-request scope, which is not otherwise persisted, travels /// on the job. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DetectionJob { - /// Workspace owning the run. + /// Workspace owning the detection. pub workspace_id: Uuid, - /// The run to analyze. - pub run_id: Uuid, + /// The detection to analyze. + pub detection_id: Uuid, /// Caller-supplied per-request scope override, if any. #[serde(default, skip_serializing_if = "Option::is_none")] pub scope: Option, } -/// A run's status change, broadcast on the core-NATS subject [`run_subject`]. +/// A detection's status change, broadcast on the core-NATS subject +/// [`detection_subject`]. /// -/// Fan-out to any watching SSE connections; the run row in Postgres remains the -/// source of truth, so a missed broadcast is recoverable by re-reading the run. +/// Fan-out to any watching SSE connections; the detection row in Postgres remains +/// the source of truth, so a missed broadcast is recoverable by re-reading it. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] -pub struct RunStatusEvent { - /// The run whose status changed. - pub run_id: Uuid, - /// The run's new status. - pub status: PipelineRunStatus, +pub struct DetectionStatusEvent { + /// The detection whose status changed. + pub detection_id: Uuid, + /// The detection's new status. + pub status: DetectionStatus, } -/// The core-NATS subject a run's status changes are broadcast on. +/// The core-NATS subject a detection's status changes are broadcast on. #[must_use] -pub fn run_subject(run_id: Uuid) -> String { - format!("pipeline.runs.{run_id}.status") +pub fn detection_subject(detection_id: Uuid) -> String { + format!("pipeline.detections.{detection_id}.status") } diff --git a/crates/nvisy-server/src/service/detection/mod.rs b/crates/nvisy-server/src/service/detection/mod.rs index 4c154608..22b41c6d 100644 --- a/crates/nvisy-server/src/service/detection/mod.rs +++ b/crates/nvisy-server/src/service/detection/mod.rs @@ -1,17 +1,16 @@ //! Async pipeline detection. //! -//! A pipeline run is created synchronously by the API, then its detection -//! (analyze) is enqueued to the `DetectionStream` work-queue and handled by the -//! [`DetectionWorker`] off the request thread. Status changes are broadcast on a -//! core-NATS subject (see [`run_subject`]) for SSE watchers and emitted as -//! webhook events. +//! A detection is created synchronously by the API, then its analysis is enqueued +//! to the `DetectionStream` work-queue and handled by the [`DetectionWorker`] off +//! the request thread. Status changes are broadcast on a core-NATS subject (see +//! [`detection_subject`]) for SSE watchers and emitted as webhook events. mod job; mod service; mod support; mod worker; -pub use job::{DetectionJob, RunStatusEvent, run_subject}; +pub use job::{DetectionJob, DetectionStatusEvent, detection_subject}; pub use service::DetectionQueue; -pub(crate) use support::{FailRun, fail_run, resolve_policies}; +pub(crate) use support::{FailDetection, fail_detection, resolve_policies}; pub use worker::DetectionWorker; diff --git a/crates/nvisy-server/src/service/detection/service.rs b/crates/nvisy-server/src/service/detection/service.rs index 361f675f..cacc34ad 100644 --- a/crates/nvisy-server/src/service/detection/service.rs +++ b/crates/nvisy-server/src/service/detection/service.rs @@ -1,19 +1,20 @@ //! Detection enqueue service. //! //! The request-side counterpart to the [`DetectionWorker`](super::DetectionWorker): -//! publishes a run's detection to the `DetectionStream` work-queue and broadcasts -//! its status on the run's core-NATS subject. Injected into the create-run -//! handler so the handler stays thin and the NATS wiring lives in one place. +//! publishes a detection's analysis to the `DetectionStream` work-queue and +//! broadcasts its status on the detection's core-NATS subject. Injected into the +//! create-detection handler so the handler stays thin and the NATS wiring lives in +//! one place. use nvisy_nats::stream::{BroadcastStream, DetectionStream}; -use nvisy_postgres::types::PipelineRunStatus; +use nvisy_postgres::types::DetectionStatus; use uuid::Uuid; -use super::job::{DetectionJob, RunStatusEvent, run_subject}; +use super::job::{DetectionJob, DetectionStatusEvent, detection_subject}; use crate::handler::Result; use crate::service::Infra; -/// Enqueues pipeline detection jobs and broadcasts run-status changes. +/// Enqueues detection jobs and broadcasts detection-status changes. /// /// Cheaply cloneable (holds the shared [`Infra`] clients, all `Arc`-backed). #[derive(Clone)] @@ -28,7 +29,8 @@ impl DetectionQueue { Self { infra } } - /// Enqueues a run's detection onto the work-queue for the worker to pick up. + /// Enqueues a detection's analysis onto the work-queue for the worker to pick + /// up. pub async fn enqueue(&self, job: DetectionJob) -> Result<()> { let publisher = self .infra @@ -39,32 +41,40 @@ impl DetectionQueue { Ok(()) } - /// Broadcasts a run's status change on its core-NATS subject (best-effort; - /// the run row is authoritative, so a dropped broadcast is recoverable). - pub async fn broadcast_status(&self, run_id: Uuid, status: PipelineRunStatus) { - let event = RunStatusEvent { run_id, status }; + /// Broadcasts a detection's status change on its core-NATS subject + /// (best-effort; the detection row is authoritative, so a dropped broadcast is + /// recoverable). + pub async fn broadcast_status(&self, detection_id: Uuid, status: DetectionStatus) { + let event = DetectionStatusEvent { + detection_id, + status, + }; if let Err(err) = self .infra .nats - .publish_broadcast(run_subject(run_id), &event) + .publish_broadcast(detection_subject(detection_id), &event) .await { tracing::debug!( target: "nvisy_server::service::detection", error = %err, - "Failed to broadcast run status", + "Failed to broadcast detection status", ); } } - /// Subscribes to a run's status broadcasts, yielding each [`RunStatusEvent`]. + /// Subscribes to a detection's status broadcasts, yielding each + /// [`DetectionStatusEvent`]. /// /// Used by the SSE endpoint to forward status changes to a watching client. - pub async fn subscribe_status(&self, run_id: Uuid) -> Result> { + pub async fn subscribe_status( + &self, + detection_id: Uuid, + ) -> Result> { let stream = self .infra .nats - .subscribe_broadcast::(run_subject(run_id)) + .subscribe_broadcast::(detection_subject(detection_id)) .await?; Ok(stream) } diff --git a/crates/nvisy-server/src/service/detection/support.rs b/crates/nvisy-server/src/service/detection/support.rs index c65abc2b..cddc6579 100644 --- a/crates/nvisy-server/src/service/detection/support.rs +++ b/crates/nvisy-server/src/service/detection/support.rs @@ -1,45 +1,47 @@ -//! Shared detection helpers used by both the create-run handler and the worker. +//! Shared detection helpers used by both the create-detection handler and the +//! worker. use elide_pipeline::policy::PolicyDefinition; -use nvisy_postgres::model::{NewWorkspacePipelineRunUsage, UpdateWorkspacePipelineRun}; +use nvisy_postgres::model::{NewWorkspaceDetectionUsage, UpdateWorkspaceDetection}; use nvisy_postgres::query::{ - PipelineReferenceRepository, WorkspacePipelineRunRepository, WorkspacePolicyRepository, + PipelineReferenceRepository, WorkspaceDetectionRepository, WorkspacePolicyRepository, }; -use nvisy_postgres::types::{Handle, Json, PipelineRunStatus, RunMetadata}; +use nvisy_postgres::types::{DetectionMetadata, DetectionStatus, Handle, Json}; use uuid::Uuid; use super::service::DetectionQueue; use crate::extract::SecurityContext; use crate::handler::Result; -use crate::service::{CryptoService, EventEmitter, EventOrigin, PipelineRunRef, WorkspaceEvent}; +use crate::service::{CryptoService, DetectionRef, EventEmitter, EventOrigin, WorkspaceEvent}; /// Tracing target for shared detection operations. const TRACING_TARGET: &str = "nvisy_server::service::detection"; -/// A run's inference usage, extracted from the engine's [`Audit`]: the per-model -/// token rows for the usage table and the full report as JSON for the metadata -/// blob. Absent when the run used no model-based recognizers. +/// A detection's inference usage, extracted from the engine's [`Audit`]: the +/// per-model token rows for the usage table and the full report as JSON for the +/// metadata blob. Absent when the detection used no model-based recognizers. /// /// [`Audit`]: elide_pipeline::Audit -pub(crate) struct RunUsage { +pub(crate) struct DetectionUsage { /// One row per distinct model, tokens summed across the recognizers that /// shared it. Empty when no recognizer used a model. - pub per_model: Vec, - /// The full per-recognizer usage report, serialized for `RunMetadata.usage`. + pub per_model: Vec, + /// The full per-recognizer usage report, serialized for + /// `DetectionMetadata.usage`. pub report: serde_json::Value, } -/// Extracts a run's inference usage from the analysis, or `None` when the report -/// is empty (a purely deterministic run spends no tokens). +/// Extracts a detection's inference usage from the analysis, or `None` when the +/// report is empty (a purely deterministic detection spends no tokens). /// /// Recognizers are grouped by `(model, version)` and their token counts summed — /// `input`, `output`, and `total` independently, since a provider may report a /// `total` that is not `input + output` (cached/reasoning tokens). Durations are /// summed into milliseconds. The whole report is also kept as JSON for drill-down. -pub(crate) fn extract_run_usage( - run_id: Uuid, +pub(crate) fn extract_detection_usage( + detection_id: Uuid, analyzed: &elide_pipeline::Audit, -) -> Option { +) -> Option { use std::collections::BTreeMap; let usage = &analyzed.usage; @@ -80,8 +82,8 @@ pub(crate) fn extract_run_usage( let per_model = by_model .into_iter() - .map(|((model, version), acc)| NewWorkspacePipelineRunUsage { - run_id, + .map(|((model, version), acc)| NewWorkspaceDetectionUsage { + detection_id, model, version, input_tokens: acc.input.map(to_i64), @@ -93,7 +95,7 @@ pub(crate) fn extract_run_usage( let report = serde_json::to_value(usage).unwrap_or(serde_json::Value::Null); - Some(RunUsage { per_model, report }) + Some(DetectionUsage { per_model, report }) } /// Clamps an upstream `u64` token count into the `i64` column; token counts are @@ -102,57 +104,58 @@ fn to_i64(value: u64) -> i64 { value.try_into().unwrap_or(i64::MAX) } -/// Identifies the run to fail and how, so [`fail_run`] takes one bundle rather -/// than a long positional list. -pub(crate) struct FailRun<'a> { - /// The workspace the run belongs to. +/// Identifies the detection to fail and how, so [`fail_detection`] takes one +/// bundle rather than a long positional list. +pub(crate) struct FailDetection<'a> { + /// The workspace the detection belongs to. pub workspace_id: Uuid, - /// The run to fail. - pub run_id: Uuid, - /// Slug of the run's pipeline, for the emitted event. + /// The detection to fail. + pub detection_id: Uuid, + /// Slug of the detection's pipeline, for the emitted event. pub pipeline_slug: Handle, - /// The account that triggered the run (the failure's actor and notify target). + /// The account that triggered the detection (the failure's actor and notify + /// target). pub triggered_by: Uuid, - /// Human-readable failure reason, stored in the run's metadata. + /// Human-readable failure reason, stored in the detection's metadata. pub reason: &'a str, /// The worker's claim timestamp, guarding the transition; `None` on the /// handler path, which has no claim to fence. pub claim: Option, } -/// Marks a run `Failed` (best effort), recording the reason in its metadata, -/// broadcasting the terminal status for SSE watchers, and emitting the -/// `PipelineRunFailed` event (activity log, webhook, and owner notification). +/// Marks a detection `Failed` (best effort), recording the reason in its +/// metadata, broadcasting the terminal status for SSE watchers, and emitting the +/// `DetectionFailed` event (activity log, webhook, and owner notification). /// -/// Shared by the create-run handler (enqueue failed) and the worker (analysis -/// failed) so a failure takes the same steps on every path. +/// Shared by the create-detection handler (enqueue failed) and the worker +/// (analysis failed) so a failure takes the same steps on every path. /// -/// `params.claim` fences the worker path: when `Some(claimed_at)`, the run is -/// failed only while that claim still holds (still `Analyzing`, `claimed_at` +/// `params.claim` fences the worker path: when `Some(claimed_at)`, the detection +/// is failed only while that claim still holds (still `Executing`, `claimed_at` /// unchanged), and the broadcast/event fire only if it did — so a worker whose -/// lease expired mid-analysis cannot fail, or announce the failure of, a run -/// another worker now owns. The handler passes `None`: it fails the run it just -/// created, with no claim to guard. -pub(crate) async fn fail_run( +/// lease expired mid-analysis cannot fail, or announce the failure of, a +/// detection another worker now owns. The handler passes `None`: it fails the +/// detection it just created, with no claim to guard. +pub(crate) async fn fail_detection( conn: &mut nvisy_postgres::PgConn, detection: &DetectionQueue, - params: FailRun<'_>, + params: FailDetection<'_>, ) { - let FailRun { + let FailDetection { workspace_id, - run_id, + detection_id, pipeline_slug, triggered_by, reason, claim, } = params; - let metadata = RunMetadata { + let metadata = DetectionMetadata { error: Some(reason.to_owned()), ..Default::default() }; - let update = UpdateWorkspacePipelineRun { - status: Some(PipelineRunStatus::Failed), + let update = UpdateWorkspaceDetection { + status: Some(DetectionStatus::Failed), metadata: Some(Json::encode(&metadata)), completed_at: Some(Some(jiff::Timestamp::now().into())), ..Default::default() @@ -160,28 +163,28 @@ pub(crate) async fn fail_run( match claim { // Worker path: guard on the claim. A stale claim fails nothing and stays - // silent — the new owner drives the run to its own outcome. - Some(claimed_at) => match conn.finalize_failed_run(run_id, claimed_at, update).await { + // silent — the new owner drives the detection to its own outcome. + Some(claimed_at) => match conn.fail_detection(detection_id, claimed_at, update).await { Ok(true) => {} Ok(false) => { - tracing::warn!(target: TRACING_TARGET, %run_id, "Claim went stale before failure; another worker owns the run"); + tracing::warn!(target: TRACING_TARGET, %detection_id, "Claim went stale before failure; another worker owns the detection"); return; } Err(err) => { - tracing::warn!(target: TRACING_TARGET, error = %err, %run_id, "Failed to mark run failed"); + tracing::warn!(target: TRACING_TARGET, error = %err, %detection_id, "Failed to mark detection failed"); return; } }, // Handler path: no claim to guard. None => { - if let Err(err) = conn.update_workspace_pipeline_run(run_id, update).await { - tracing::warn!(target: TRACING_TARGET, error = %err, %run_id, "Failed to mark run failed"); + if let Err(err) = conn.update_workspace_detection(detection_id, update).await { + tracing::warn!(target: TRACING_TARGET, error = %err, %detection_id, "Failed to mark detection failed"); } } } detection - .broadcast_status(run_id, PipelineRunStatus::Failed) + .broadcast_status(detection_id, DetectionStatus::Failed) .await; if let Err(err) = conn @@ -191,9 +194,9 @@ pub(crate) async fn fail_run( account_id: triggered_by, security: &SecurityContext::default(), }, - WorkspaceEvent::PipelineRunFailed { - run: PipelineRunRef { - run_id, + WorkspaceEvent::DetectionFailed { + detection: DetectionRef { + detection_id, pipeline_slug, }, input_file_name: None, @@ -203,7 +206,7 @@ pub(crate) async fn fail_run( ) .await { - tracing::warn!(target: TRACING_TARGET, error = %err, %run_id, "Failed to record run-failed event"); + tracing::warn!(target: TRACING_TARGET, error = %err, %detection_id, "Failed to record detection-failed event"); } } diff --git a/crates/nvisy-server/src/service/detection/worker.rs b/crates/nvisy-server/src/service/detection/worker.rs index 58a68646..e258b4d8 100644 --- a/crates/nvisy-server/src/service/detection/worker.rs +++ b/crates/nvisy-server/src/service/detection/worker.rs @@ -1,32 +1,33 @@ //! Pipeline detection worker. //! //! Consumes [`DetectionJob`]s from the `DetectionStream` work-queue and runs a -//! run's detection (analyze) in the background: builds the document, analyzes it -//! with the pipeline's policies, stores the encrypted audit, and marks the run -//! `Analyzed` (or `Failed`). Each terminal transition is broadcast on the run's -//! core-NATS status subject (for SSE watchers) and emitted as a webhook event. +//! detection's analysis in the background: builds the document, analyzes it with +//! the pipeline's policies, stores the encrypted audit, and marks the detection +//! `Complete` (or `Failed`). Each terminal transition is broadcast on the +//! detection's core-NATS status subject (for SSE watchers) and emitted as a +//! webhook event. use std::time::Duration; use elide_pipeline::RasterMode; use nvisy_nats::stream::DetectionStream; -use nvisy_postgres::model::{UpdateWorkspacePipelineRun, WorkspacePipeline, WorkspacePipelineRun}; +use nvisy_postgres::model::{UpdateWorkspaceDetection, WorkspaceDetection, WorkspacePipeline}; use nvisy_postgres::query::{ - EventOutboxRepository, WorkspaceFileRepository, WorkspacePipelineRunRepository, + EventOutboxRepository, WorkspaceDetectionRepository, WorkspaceFileRepository, WorkspaceRepository, }; -use nvisy_postgres::types::{Json, OcrPolicy, PipelineRunStatus, WorkspaceSettings}; +use nvisy_postgres::types::{DetectionStatus, Json, OcrPolicy, WorkspaceSettings}; use nvisy_postgres::{AsyncConnection, DieselError, PgConn, PgError}; use tokio_util::sync::CancellationToken; use super::job::DetectionJob; use super::service::DetectionQueue; -use super::support::{FailRun, extract_run_usage, fail_run, resolve_policies}; +use super::support::{FailDetection, extract_detection_usage, fail_detection, resolve_policies}; use crate::extract::SecurityContext; use crate::handler::request::PipelineDefinition; use crate::handler::{ErrorKind, Result}; use crate::service::{ - EngineService, EventOrigin, Infra, PipelineRunRef, RunBlobStore, Worker, WorkspaceEvent, + DetectionRef, EngineService, EventOrigin, Infra, RunBlobStore, Worker, WorkspaceEvent, event_outbox_row, }; @@ -34,9 +35,10 @@ use crate::service::{ const TRACING_TARGET: &str = "nvisy_server::worker::detection"; /// How long a detection claim stays valid before another delivery may re-claim -/// the run. Set above `DetectionStream::ACK_WAIT` (15 min) so a slow-but-healthy -/// worker whose job is redelivered keeps its claim; only a run whose worker died -/// (no progress past the lease) is re-claimed and re-analyzed. +/// the detection. Set above `DetectionStream::ACK_WAIT` (15 min) so a +/// slow-but-healthy worker whose job is redelivered keeps its claim; only a +/// detection whose worker died (no progress past the lease) is re-claimed and +/// re-analyzed. const DETECTION_LEASE: Duration = Duration::from_secs(30 * 60); /// Background worker that runs pipeline detection off the request thread. @@ -90,11 +92,11 @@ impl DetectionWorker { /// Consumes detection jobs until cancelled. /// /// At-least-once with an explicit claim: a job is acked once it reaches a - /// terminal outcome (analyzed, or marked failed), and nacked for redelivery - /// on a transient error (a DB/pool blip before the run could even be - /// claimed), so a run is never silently stranded in a non-terminal state. - /// The claim (`claim_run_for_detection`) makes redelivery idempotent: a run - /// already being analyzed under a fresh lease is skipped. + /// terminal outcome (complete, or marked failed), and nacked for redelivery + /// on a transient error (a DB/pool blip before the detection could even be + /// claimed), so a detection is never silently stranded in a non-terminal + /// state. The claim (`claim_detection`) makes redelivery idempotent: a + /// detection already being analyzed under a fresh lease is skipped. async fn run_inner(&self, cancel: CancellationToken) -> Result<()> { let subscriber = self .infra @@ -136,72 +138,73 @@ impl DetectionWorker { Ok(()) } - /// Runs one detection job: claims the run, analyzes, and records the result. + /// Runs one detection job: claims the detection, analyzes, and records the + /// result. /// /// Returns [`JobOutcome::Retry`] when the job should be redelivered (a - /// transient error before the run was claimed), and [`JobOutcome::Done`] - /// when it reached a terminal outcome or is safe to drop (missing run, + /// transient error before the detection was claimed), and [`JobOutcome::Done`] + /// when it reached a terminal outcome or is safe to drop (missing detection, /// already claimed, already settled). - #[tracing::instrument(skip_all, fields(run_id = %job.run_id, workspace_id = %job.workspace_id))] + #[tracing::instrument(skip_all, fields(detection_id = %job.detection_id, workspace_id = %job.workspace_id))] async fn run_job(&self, job: DetectionJob) -> JobOutcome { let mut conn = match self.infra.postgres.get_connection().await { Ok(conn) => conn, Err(err) => { - // No connection: the run is still queued. Redeliver so it is not - // stranded in a non-terminal state. + // No connection: the detection is still pending. Redeliver so it + // is not stranded in a non-terminal state. tracing::error!(target: TRACING_TARGET, error = %err, "Failed to get connection for detection job"); return JobOutcome::Retry; } }; - let (run, pipeline) = match conn - .find_workspace_run_by_id(job.workspace_id, job.run_id) + let (detection, pipeline) = match conn + .find_workspace_detection_by_id(job.workspace_id, job.detection_id) .await { Ok(Some(pair)) => pair, Ok(None) => { - tracing::warn!(target: TRACING_TARGET, "Detection job for a missing run; dropping"); + tracing::warn!(target: TRACING_TARGET, "Detection job for a missing detection; dropping"); return JobOutcome::Done; } Err(err) => { - // Transient load error: redeliver rather than strand the run. - tracing::error!(target: TRACING_TARGET, error = %err, "Failed to load run for detection job"); + // Transient load error: redeliver rather than strand the detection. + tracing::error!(target: TRACING_TARGET, error = %err, "Failed to load detection for detection job"); return JobOutcome::Retry; } }; - // Nothing to do for a run already past the queued/analyzing phase. - if !run.status.is_detecting() { - tracing::debug!(target: TRACING_TARGET, status = %run.status, "Run is not detecting; dropping"); + // Nothing to do for a detection already past the pending/executing phase. + if !detection.status.is_detecting() { + tracing::debug!(target: TRACING_TARGET, status = %detection.status, "Detection is not detecting; dropping"); return JobOutcome::Done; } - // Atomically claim the run (queued -> analyzing). A redelivery whose - // claim is still fresh matches no row and is skipped, so a slow job is - // never analyzed twice; only a run whose worker died (stale lease) is - // re-claimed. + // Atomically claim the detection (pending -> executing). A redelivery + // whose claim is still fresh matches no row and is skipped, so a slow job + // is never analyzed twice; only a detection whose worker died (stale + // lease) is re-claimed. let stale_before = jiff::Timestamp::now() - DETECTION_LEASE; - let claimed = match conn.claim_run_for_detection(run.id, stale_before).await { + let claimed = match conn.claim_detection(detection.id, stale_before).await { Ok(Some(claimed)) => claimed, Ok(None) => { - tracing::debug!(target: TRACING_TARGET, "Run already claimed by another worker; skipping"); + tracing::debug!(target: TRACING_TARGET, "Detection already claimed by another worker; skipping"); return JobOutcome::Done; } Err(err) => { - tracing::error!(target: TRACING_TARGET, error = %err, "Failed to claim run for detection"); + tracing::error!(target: TRACING_TARGET, error = %err, "Failed to claim detection"); return JobOutcome::Retry; } }; // The claim stamped `claimed_at`; it fences the finalize (and a failure) // against a concurrent re-claim if this analysis outlives the lease. let Some(claim_token) = claimed.claimed_at else { - tracing::error!(target: TRACING_TARGET, "Claimed run has no claim timestamp; skipping"); + tracing::error!(target: TRACING_TARGET, "Claimed detection has no claim timestamp; skipping"); return JobOutcome::Done; }; let claim_token: jiff::Timestamp = claim_token.into(); self.detection - .broadcast_status(run.id, PipelineRunStatus::Analyzing) + .broadcast_status(detection.id, DetectionStatus::Executing) .await; if let Err(err) = self @@ -209,14 +212,14 @@ impl DetectionWorker { .await { tracing::warn!(target: TRACING_TARGET, error = %err, "Detection failed"); - fail_run( + fail_detection( &mut conn, &self.detection, - FailRun { + FailDetection { workspace_id: job.workspace_id, - run_id: run.id, + detection_id: detection.id, pipeline_slug: pipeline.slug.clone(), - triggered_by: run.account_id, + triggered_by: detection.account_id, reason: &err.to_string(), claim: Some(claim_token), }, @@ -229,7 +232,7 @@ impl DetectionWorker { /// Best-effort reclaim of a staged audit object whose file row did not /// commit. A failure only defers cleanup, so it is logged, never propagated. async fn discard_staged_audit(&self, staged: &nvisy_postgres::model::NewWorkspaceFile) { - if let Err(err) = self.blob.discard_staged_audit(staged).await { + if let Err(err) = self.blob.discard_staged_object(staged).await { tracing::warn!( target: TRACING_TARGET, error = %err, @@ -239,12 +242,12 @@ impl DetectionWorker { } } - /// Performs the analysis and records the run as `Analyzed`. + /// Performs the analysis and records the detection as `Complete`. async fn detect( &self, conn: &mut PgConn, job: &DetectionJob, - run: &WorkspacePipelineRun, + detection: &WorkspaceDetection, pipeline: &WorkspacePipeline, claim_token: jiff::Timestamp, ) -> Result<()> { @@ -253,7 +256,7 @@ impl DetectionWorker { .await? .ok_or_else(|| ErrorKind::NotFound.with_message("Workspace not found"))?; let file = conn - .find_file_in_workspace(job.workspace_id, run.input_file_id) + .find_file_in_workspace(job.workspace_id, detection.input_file_id) .await? .ok_or_else(|| ErrorKind::NotFound.with_message("Input file not found"))?; @@ -270,7 +273,7 @@ impl DetectionWorker { self.engine .request_context(&definition, job.scope.clone(), raster_mode_of(&settings)); - let document = self.blob.build_document(&file, run.id).await?; + let document = self.blob.build_document(&file, detection.id).await?; let policies = resolve_policies(conn, &self.infra.crypto, job.workspace_id, pipeline.id).await?; @@ -283,51 +286,58 @@ impl DetectionWorker { let analyzed = self.engine.analyze(document, &policies, &request).await?; // Write the (non-transactional) audit object first, then commit its file - // row together with the run's usage and status in one transaction below. + // row together with the detection's usage and status in one transaction + // below. let audit_file = self .blob - .stage_analyzed_document(pipeline, &settings.retention, run.account_id, &analyzed) + .stage_analyzed_document( + pipeline, + &settings.retention, + detection.account_id, + &analyzed, + ) .await?; // Record inference usage: per-model token rows into the usage table (the // usage aggregation surface) and the full per-recognizer report into - // metadata for drill-down. Absent for a purely deterministic run. The - // report is layered onto the run's existing metadata so tags and any - // recorded error survive the write. - let usage = extract_run_usage(run.id, &analyzed); + // metadata for drill-down. Absent for a purely deterministic detection. + // The report is layered onto the detection's existing metadata so tags and + // any recorded error survive the write. + let usage = extract_detection_usage(detection.id, &analyzed); let metadata = usage.as_ref().map(|u| { - let mut current = run.metadata.or_default(); + let mut current = detection.metadata.or_default(); current.usage = Some(u.report.clone()); Json::encode(¤t) }); - // Persist the audit file row, per-model usage, and the run's transition to - // `Analyzed` atomically: a partial failure would otherwise strand usage - // rows or an audit pointer on a run still marked `Analyzing`. The finalize - // is fenced on our claim; if it went stale (another worker re-claimed the - // run past the lease), the whole transaction rolls back so we do not stamp - // over the new owner's work or leak usage/audit rows for a run we lost. - // Kept to reclaim the just-staged object if the transaction does not - // commit: on rollback its `workspace_files` row never lands, so the - // row-driven reaper could never find the object otherwise. - // Build the outbox row here so the finalize transaction is `PgError`-typed - // for its rollback sentinel, and insert it alongside the finalize so the - // `Analyzed` event commits atomically with the run. - let analyzed_event = WorkspaceEvent::PipelineRunAnalyzed { - run: PipelineRunRef { - run_id: run.id, + // Persist the audit file row, per-model usage, and the detection's + // transition to `Complete` atomically: a partial failure would otherwise + // strand usage rows or an audit pointer on a detection still marked + // `Executing`. The finalize is fenced on our claim; if it went stale + // (another worker re-claimed the detection past the lease), the whole + // transaction rolls back so we do not stamp over the new owner's work or + // leak usage/audit rows for a detection we lost. Kept to reclaim the + // just-staged object if the transaction does not commit: on rollback its + // `workspace_files` row never lands, so the row-driven reaper could never + // find the object otherwise. Build the outbox row here so the finalize + // transaction is `PgError`-typed for its rollback sentinel, and insert it + // alongside the finalize so the `Complete` event commits atomically with + // the detection. + let completed_event = WorkspaceEvent::DetectionCompleted { + detection: DetectionRef { + detection_id: detection.id, pipeline_slug: pipeline.slug.clone(), }, input_file_name: Some(file.display_name.clone()), - notify: run.account_id, + notify: detection.account_id, }; let outbox_row = event_outbox_row( EventOrigin { workspace_id: job.workspace_id, - account_id: run.account_id, + account_id: detection.account_id, security: &SecurityContext::default(), }, - &analyzed_event, + &completed_event, )?; let staged_audit = audit_file.clone(); @@ -335,13 +345,13 @@ impl DetectionWorker { .transaction(async |conn| { let audit_file_id = conn.create_workspace_file(audit_file).await?.id; if let Some(usage) = &usage { - conn.record_run_usage(&usage.per_model).await?; + conn.record_detection_usage(&usage.per_model).await?; } let finalized = conn - .finalize_analyzed_run( - run.id, + .finalize_detection( + detection.id, claim_token, - UpdateWorkspacePipelineRun { + UpdateWorkspaceDetection { audit_file_id: Some(Some(audit_file_id)), metadata, ..Default::default() @@ -349,9 +359,9 @@ impl DetectionWorker { ) .await?; if !finalized { - // Abort the audit-file and usage inserts: the run is no longer - // ours to finalize. `RollbackTransaction` unwinds the writes - // without being a real error; it is matched below. + // Abort the audit-file and usage inserts: the detection is no + // longer ours to finalize. `RollbackTransaction` unwinds the + // writes without being a real error; it is matched below. return Err(PgError::Query(DieselError::RollbackTransaction)); } conn.insert_event_outbox(outbox_row).await?; @@ -363,7 +373,7 @@ impl DetectionWorker { Ok(()) => {} Err(PgError::Query(DieselError::RollbackTransaction)) => { self.discard_staged_audit(&staged_audit).await; - tracing::warn!(target: TRACING_TARGET, run_id = %run.id, "Claim went stale before finalize; another worker owns the run"); + tracing::warn!(target: TRACING_TARGET, detection_id = %detection.id, "Claim went stale before finalize; another worker owns the detection"); return Ok(()); } Err(err) => { @@ -374,9 +384,9 @@ impl DetectionWorker { } } - tracing::info!(target: TRACING_TARGET, run_id = %run.id, "Run analyzed"); + tracing::info!(target: TRACING_TARGET, detection_id = %detection.id, "Detection complete"); self.detection - .broadcast_status(run.id, PipelineRunStatus::Analyzed) + .broadcast_status(detection.id, DetectionStatus::Complete) .await; Ok(()) @@ -388,12 +398,12 @@ impl DetectionWorker { enum JobOutcome { /// Reached a terminal outcome or is safe to drop; ack the message. Done, - /// Transient error before the run was claimed; nack for redelivery. + /// Transient error before the detection was claimed; nack for redelivery. Retry, } -/// Maps a workspace's OCR policy to the engine's per-run page-rasterisation -/// mode. +/// Maps a workspace's OCR policy to the engine's per-detection +/// page-rasterisation mode. fn raster_mode_of(settings: &WorkspaceSettings) -> RasterMode { match settings.ocr { OcrPolicy::Auto => RasterMode::Auto, diff --git a/crates/nvisy-server/src/service/event/drainer.rs b/crates/nvisy-server/src/service/event/drainer.rs index e361cade..2b5c9125 100644 --- a/crates/nvisy-server/src/service/event/drainer.rs +++ b/crates/nvisy-server/src/service/event/drainer.rs @@ -21,11 +21,11 @@ use nvisy_postgres::model::{EventOutbox, NewWorkspaceActivity}; use nvisy_postgres::query::{EventOutboxRepository, WorkspaceActivityRepository}; use nvisy_postgres::types::{ ActivityPayload, ConnectionActivityParams, ConnectionId, ConnectionSyncCompletedParams, - ConnectionSyncFailedParams, FileActivityParams, Handle, InviteActivityParams, Json, - MemberActivityParams, NotificationPayload, PipelineActivityParams, PipelineRunActivityParams, - PipelineRunAnalyzedParams, PipelineRunCompletedParams, PipelineRunFailedParams, - PolicyActivityParams, RunId, WebhookActivityParams, WebhookEvent, WebhookId, - WorkspaceActivityParams, + ConnectionSyncFailedParams, DetectionActivityParams, DetectionCompletedParams, + DetectionFailedParams, DetectionId, FileActivityParams, Handle, InviteActivityParams, Json, + MemberActivityParams, NotificationPayload, PipelineActivityParams, PolicyActivityParams, + RedactionActivityParams, RedactionCreatedParams, RedactionId, WebhookActivityParams, + WebhookEvent, WebhookId, WorkspaceActivityParams, }; use nvisy_postgres::{AsyncConnection, PgConn}; use serde_json::Value; @@ -34,7 +34,7 @@ use uuid::Uuid; use crate::handler::{Error, Result}; use crate::service::event::{ - ConnectionRef, FileRef, InviteRef, MemberRef, PipelineRunRef, PolicyRef, WebhookRef, + ConnectionRef, DetectionRef, FileRef, InviteRef, MemberRef, PolicyRef, WebhookRef, WorkspaceEvent, WorkspaceRef, }; use crate::service::{Infra, NotificationEmitter, WebhookEmitter, Worker}; @@ -326,9 +326,17 @@ fn activity_of(event: &WorkspaceEvent) -> ActivityPayload { let pipeline = |pipeline_slug: &Handle| PipelineActivityParams { pipeline_slug: pipeline_slug.clone(), }; - let run = |run: &PipelineRunRef| PipelineRunActivityParams { - pipeline_slug: run.pipeline_slug.clone(), - run_id: RunId::from_uuid(run.run_id), + let detection = |detection: &DetectionRef| DetectionActivityParams { + pipeline_slug: detection.pipeline_slug.clone(), + detection_id: DetectionId::from_uuid(detection.detection_id), + }; + // TODO(redaction-feature): once the redact handler persists a redaction row, + // `RedactionCreated` should carry a real `RedactionId`; until then the + // activity params reuse the detection id as a placeholder so the projection + // compiles. The parent reworks the redact emission path. + let redaction = |detection: &DetectionRef| RedactionActivityParams { + pipeline_slug: detection.pipeline_slug.clone(), + redaction_id: RedactionId::from_uuid(detection.detection_id), }; let policy = |p: &PolicyRef| PolicyActivityParams { policy_id: p.policy_id, @@ -374,10 +382,12 @@ fn activity_of(event: &WorkspaceEvent) -> ActivityPayload { E::PipelineCreated(p) => ActivityPayload::PipelineCreated(pipeline(&p.pipeline_slug)), E::PipelineUpdated(p) => ActivityPayload::PipelineUpdated(pipeline(&p.pipeline_slug)), E::PipelineDeleted(p) => ActivityPayload::PipelineDeleted(pipeline(&p.pipeline_slug)), - E::PipelineRunStarted(r) => ActivityPayload::PipelineRunStarted(run(r)), - E::PipelineRunAnalyzed { run: r, .. } => ActivityPayload::PipelineRunAnalyzed(run(r)), - E::PipelineRunCompleted { run: r, .. } => ActivityPayload::PipelineRunCompleted(run(r)), - E::PipelineRunFailed { run: r, .. } => ActivityPayload::PipelineRunFailed(run(r)), + E::DetectionStarted(d) => ActivityPayload::DetectionStarted(detection(d)), + E::DetectionCompleted { detection: d, .. } => { + ActivityPayload::DetectionCompleted(detection(d)) + } + E::DetectionFailed { detection: d, .. } => ActivityPayload::DetectionFailed(detection(d)), + E::RedactionCreated { detection: d, .. } => ActivityPayload::RedactionCreated(redaction(d)), E::PolicyCreated(p) => ActivityPayload::PolicyCreated(policy(p)), E::PolicyUpdated(p) => ActivityPayload::PolicyUpdated(policy(p)), E::PolicyDeleted(p) => ActivityPayload::PolicyDeleted(policy(p)), @@ -418,10 +428,10 @@ fn webhook_of(event: &WorkspaceEvent) -> Option<(WebhookEvent, Option)> { E::PipelineCreated(..) => (WebhookEvent::PipelineCreated, None), E::PipelineUpdated(..) => (WebhookEvent::PipelineUpdated, None), E::PipelineDeleted(..) => (WebhookEvent::PipelineDeleted, None), - E::PipelineRunStarted(..) => (WebhookEvent::PipelineRunStarted, None), - E::PipelineRunAnalyzed { .. } => (WebhookEvent::PipelineRunAnalyzed, None), - E::PipelineRunCompleted { .. } => (WebhookEvent::PipelineRunCompleted, None), - E::PipelineRunFailed { .. } => (WebhookEvent::PipelineRunFailed, None), + E::DetectionStarted(..) => (WebhookEvent::DetectionStarted, None), + E::DetectionCompleted { .. } => (WebhookEvent::DetectionCompleted, None), + E::DetectionFailed { .. } => (WebhookEvent::DetectionFailed, None), + E::RedactionCreated { .. } => (WebhookEvent::RedactionCreated, None), E::PolicyCreated(..) => (WebhookEvent::PolicyCreated, None), E::PolicyUpdated(..) => (WebhookEvent::PolicyUpdated, None), E::PolicyDeleted(..) => (WebhookEvent::PolicyDeleted, None), @@ -474,40 +484,44 @@ fn notification_of(event: WorkspaceEvent) -> Option<(Uuid, NotificationPayload)> }), ) }), - E::PipelineRunAnalyzed { - run, + E::DetectionCompleted { + detection, input_file_name, notify, } => Some(( notify, - NotificationPayload::PipelineRunAnalyzed(PipelineRunAnalyzedParams { - run_id: RunId::from_uuid(run.run_id), - pipeline_slug: run.pipeline_slug, + NotificationPayload::DetectionCompleted(DetectionCompletedParams { + detection_id: DetectionId::from_uuid(detection.detection_id), + pipeline_slug: detection.pipeline_slug, input_file_name, }), )), - E::PipelineRunCompleted { - run, + // TODO(redaction-feature): `RedactionCreated` should carry a real + // `RedactionId`; until the redact handler persists a redaction row the + // detection id stands in as a placeholder so the projection compiles. + E::RedactionCreated { + detection, input_file_name, notify, } => Some(( notify, - NotificationPayload::PipelineRunCompleted(PipelineRunCompletedParams { - run_id: RunId::from_uuid(run.run_id), - pipeline_slug: run.pipeline_slug, + NotificationPayload::RedactionCreated(RedactionCreatedParams { + redaction_id: RedactionId::from_uuid(detection.detection_id), + detection_id: DetectionId::from_uuid(detection.detection_id), + pipeline_slug: detection.pipeline_slug, input_file_name, }), )), - E::PipelineRunFailed { - run, + E::DetectionFailed { + detection, input_file_name, error, notify, } => Some(( notify, - NotificationPayload::PipelineRunFailed(PipelineRunFailedParams { - run_id: RunId::from_uuid(run.run_id), - pipeline_slug: run.pipeline_slug, + NotificationPayload::DetectionFailed(DetectionFailedParams { + detection_id: DetectionId::from_uuid(detection.detection_id), + pipeline_slug: detection.pipeline_slug, input_file_name, error, }), @@ -537,7 +551,7 @@ fn notification_of(event: WorkspaceEvent) -> Option<(Uuid, NotificationPayload)> | E::PipelineCreated(_) | E::PipelineUpdated(_) | E::PipelineDeleted(_) - | E::PipelineRunStarted(_) + | E::DetectionStarted(_) | E::PolicyCreated(_) | E::PolicyUpdated(_) | E::PolicyDeleted(_) => None, @@ -565,10 +579,10 @@ fn resource_id_of(event: &WorkspaceEvent) -> Uuid { E::FileCreated { file, .. } => file.file_id, E::FileUpdated(f) | E::FileDeleted(f) => f.file_id, E::PipelineCreated(p) | E::PipelineUpdated(p) | E::PipelineDeleted(p) => p.pipeline_id, - E::PipelineRunStarted(r) => r.run_id, - E::PipelineRunAnalyzed { run, .. } - | E::PipelineRunCompleted { run, .. } - | E::PipelineRunFailed { run, .. } => run.run_id, + E::DetectionStarted(d) => d.detection_id, + E::DetectionCompleted { detection, .. } + | E::DetectionFailed { detection, .. } + | E::RedactionCreated { detection, .. } => detection.detection_id, E::PolicyCreated(p) | E::PolicyUpdated(p) | E::PolicyDeleted(p) => p.policy_id, } } diff --git a/crates/nvisy-server/src/service/event/mod.rs b/crates/nvisy-server/src/service/event/mod.rs index e12ce816..8921208a 100644 --- a/crates/nvisy-server/src/service/event/mod.rs +++ b/crates/nvisy-server/src/service/event/mod.rs @@ -19,8 +19,8 @@ use crate::extract::SecurityContext; pub use crate::service::event::drainer::EventOutboxDrainer; pub use crate::service::event::emitter::{EventEmitter, event_outbox_row}; pub use crate::service::event::workspace_event::{ - ConnectionRef, FileRef, InviteRef, MemberRef, PipelineRef, PipelineRunRef, PolicyRef, - WebhookRef, WorkspaceEvent, WorkspaceRef, + ConnectionRef, DetectionRef, FileRef, InviteRef, MemberRef, PipelineRef, PolicyRef, WebhookRef, + WorkspaceEvent, WorkspaceRef, }; /// Who raised an event and where. diff --git a/crates/nvisy-server/src/service/event/workspace_event.rs b/crates/nvisy-server/src/service/event/workspace_event.rs index e9ee6ef2..815f193b 100644 --- a/crates/nvisy-server/src/service/event/workspace_event.rs +++ b/crates/nvisy-server/src/service/event/workspace_event.rs @@ -99,29 +99,31 @@ pub enum WorkspaceEvent { #[serde(rename = "pipeline.deleted")] PipelineDeleted(PipelineRef), - // Pipeline runs - #[serde(rename = "pipeline.run.started")] - PipelineRunStarted(PipelineRunRef), - #[serde(rename = "pipeline.run.analyzed")] - PipelineRunAnalyzed { + // Detections + #[serde(rename = "pipeline.detection.started")] + DetectionStarted(DetectionRef), + #[serde(rename = "pipeline.detection.completed")] + DetectionCompleted { #[serde(flatten)] - run: PipelineRunRef, + detection: DetectionRef, input_file_name: Option, notify: Uuid, }, - #[serde(rename = "pipeline.run.completed")] - PipelineRunCompleted { + #[serde(rename = "pipeline.detection.failed")] + DetectionFailed { #[serde(flatten)] - run: PipelineRunRef, + detection: DetectionRef, input_file_name: Option, + error: Option, notify: Uuid, }, - #[serde(rename = "pipeline.run.failed")] - PipelineRunFailed { + + // Redactions + #[serde(rename = "pipeline.redaction.created")] + RedactionCreated { #[serde(flatten)] - run: PipelineRunRef, + detection: DetectionRef, input_file_name: Option, - error: Option, notify: Uuid, }, @@ -186,10 +188,10 @@ pub struct PipelineRef { pub pipeline_slug: Handle, } -/// A pipeline run and its pipeline's slug. +/// A detection and its pipeline's slug. #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PipelineRunRef { - pub run_id: Uuid, +pub struct DetectionRef { + pub detection_id: Uuid, pub pipeline_slug: Handle, } diff --git a/crates/nvisy-server/src/service/mod.rs b/crates/nvisy-server/src/service/mod.rs index a32c6614..2ec1d725 100644 --- a/crates/nvisy-server/src/service/mod.rs +++ b/crates/nvisy-server/src/service/mod.rs @@ -34,14 +34,13 @@ pub use crate::service::connection_config::ConnectionConfig; pub use crate::service::crypto::{CryptoConfig, CryptoService}; pub(crate) use crate::service::crypto::{CryptoError, HashingReader, Measurements}; pub use crate::service::detection::{ - DetectionJob, DetectionQueue, DetectionWorker, RunStatusEvent, run_subject, + DetectionJob, DetectionQueue, DetectionStatusEvent, DetectionWorker, detection_subject, }; -pub(crate) use crate::service::detection::{FailRun, fail_run, resolve_policies}; +pub(crate) use crate::service::detection::{FailDetection, fail_detection, resolve_policies}; pub use crate::service::engine::{EngineConfig, EngineService, UnknownFormatToken}; pub use crate::service::event::{ - ConnectionRef, EventEmitter, EventOrigin, EventOutboxDrainer, FileRef, InviteRef, MemberRef, - PipelineRef, PipelineRunRef, PolicyRef, WebhookRef, WorkspaceEvent, WorkspaceRef, - event_outbox_row, + ConnectionRef, DetectionRef, EventEmitter, EventOrigin, EventOutboxDrainer, FileRef, InviteRef, + MemberRef, PipelineRef, PolicyRef, WebhookRef, WorkspaceEvent, WorkspaceRef, event_outbox_row, }; pub use crate::service::external_object_store::ExternalObjectStore; pub use crate::service::file_reaper::FileReaper; diff --git a/crates/nvisy-server/src/service/run_blob_store.rs b/crates/nvisy-server/src/service/run_blob_store.rs index 11ab215d..39fa31c5 100644 --- a/crates/nvisy-server/src/service/run_blob_store.rs +++ b/crates/nvisy-server/src/service/run_blob_store.rs @@ -16,7 +16,7 @@ use elide_pipeline::{Audit, Engine}; use nvisy_nats::object::{AuditBucket, AuditKey, FileKey, FilesBucket, ObjectBucket}; use nvisy_postgres::PgConn; use nvisy_postgres::model::{ - NewWorkspaceFile, WorkspaceFile, WorkspacePipeline, WorkspacePipelineRun, + NewWorkspaceFile, WorkspaceDetection, WorkspaceFile, WorkspacePipeline, }; use nvisy_postgres::query::WorkspaceFileRepository; use nvisy_postgres::types::{FileKind, RetentionScope, RetentionSettings}; @@ -216,7 +216,7 @@ impl RunBlobStore { .resolve(RetentionScope::RedactedDocuments, over.as_ref()) .expires_at(jiff::Timestamp::now()); - let redacted_name = format!("{}.redacted", source.display_name); + let redacted_name = redacted_display_name(&source.display_name, &source.file_extension); let new_file = NewWorkspaceFile { workspace_id: source.workspace_id, account_id, @@ -250,7 +250,7 @@ impl RunBlobStore { /// inserted (with the run's other writes) atomically. A rollback therefore /// leaves at worst an orphan object in the bucket, never a file row that points /// at bytes that were never written; the caller reclaims that orphan via - /// [`discard_staged_audit`](Self::discard_staged_audit). + /// [`discard_staged_object`](Self::discard_staged_object). pub async fn stage_analyzed_document( &self, pipeline: &WorkspacePipeline, @@ -299,15 +299,17 @@ impl RunBlobStore { }) } - /// Deletes a staged audit object whose file row was never committed. + /// Deletes a staged object whose file row was never committed. /// - /// [`stage_analyzed_document`](Self::stage_analyzed_document) writes the object - /// before its `workspace_files` row; if the committing transaction rolls back, - /// the object has no row and the row-driven reaper can never find it. The - /// caller invokes this on that path so the orphan is removed immediately - /// instead of accumulating. Best effort: a failure here only leaves the object - /// for a later manual sweep, so callers log rather than propagate. - pub async fn discard_staged_audit(&self, staged: &NewWorkspaceFile) -> Result<()> { + /// The `stage_*` methods write an object before its `workspace_files` row; if + /// the committing transaction rolls back, the object has no row and the + /// row-driven reaper can never find it. The caller invokes this on that path + /// so the orphan is removed immediately instead of accumulating. It deletes + /// from whichever bucket the staged row names, so it reclaims a staged audit, + /// review audit, or redacted document alike. Best effort: a failure here only + /// leaves the object for a later manual sweep, so callers log rather than + /// propagate. + pub async fn discard_staged_object(&self, staged: &NewWorkspaceFile) -> Result<()> { self.delete_object(&staged.storage_bucket, &staged.storage_path) .await } @@ -319,30 +321,30 @@ impl RunBlobStore { /// entity group by modality name and only the engine's registry can map those /// back to concrete types. /// - /// Errors if the run was never analyzed (409) or its analysis has since been - /// deleted (404). + /// Errors if the detection never analyzed (409) or its analysis has since + /// been deleted (404). pub async fn load_analyzed_document( &self, conn: &mut PgConn, engine: &Engine, workspace_id: Uuid, - run: &WorkspacePipelineRun, + detection: &WorkspaceDetection, ) -> Result { - // A NULL reference means the run never produced an analysis; a reference - // to a now-deleted file means it did, but the analysis has been removed. - // These are distinct states, so they map to distinct responses. - let audit_file_id = run.audit_file_id.ok_or_else(|| { + // A NULL reference means the detection never produced an analysis; a + // reference to a now-deleted file means it did, but the analysis has been + // removed. These are distinct states, so they map to distinct responses. + let audit_file_id = detection.audit_file_id.ok_or_else(|| { ErrorKind::Conflict - .with_message("Run has no analysis yet") - .with_resource("pipeline_run") + .with_message("Detection has no analysis yet") + .with_resource("detection") })?; let audit_file = conn .find_file_in_workspace(workspace_id, audit_file_id) .await? .ok_or_else(|| { ErrorKind::NotFound - .with_message("The analysis for this run has been deleted") - .with_resource("pipeline_run") + .with_message("The analysis for this detection has been deleted") + .with_resource("detection") })?; let key = AuditKey::from_str(&audit_file.storage_path).map_err(|err| { ErrorKind::InternalServerError @@ -379,6 +381,184 @@ impl RunBlobStore { .with_context(err.to_string()) }) } + + /// Encrypts a redaction's review audit, writes it to the audit bucket, and + /// builds the `review`-kind [`WorkspaceFile`] row that will point at it — + /// without inserting the row. + /// + /// The review audit is the post-redaction [`Audit`]: the detection's analysis + /// with the reviewer's edits applied and the redaction outcome recorded per + /// entity. It is the redaction's counterpart to + /// [`stage_analyzed_document`](Self::stage_analyzed_document) — same staged + /// object-then-row protocol, reclaimed on rollback via + /// [`discard_staged_object`](Self::discard_staged_object) — but a distinct + /// file kind so it is never confused with the immutable detection audit. + pub async fn stage_review_audit( + &self, + pipeline: &WorkspacePipeline, + workspace_settings: &RetentionSettings, + account_id: Uuid, + reviewed: &Audit, + ) -> Result { + let workspace_id = pipeline.workspace_id; + let plaintext = serde_json::to_vec(reviewed).map_err(analysis_serde_error)?; + let hash = Sha256::digest(&plaintext).to_vec(); + let size = plaintext.len() as i64; + let ciphertext = self + .infra + .crypto + .encrypt(workspace_id, &plaintext) + .map_err(|err| { + ErrorKind::InternalServerError + .with_message("Failed to encrypt review audit") + .with_context(err.to_string()) + })?; + + let store = self.infra.nats.object_store::().await?; + let key = AuditKey::generate(workspace_id); + store.put(&key, Cursor::new(ciphertext)).await?; + + // A review audit shares the audit-logs retention scope with the detection + // audit (workspace baseline, pipeline override if set). + let over = pipeline.metadata.or_default().retention; + let expires_at = workspace_settings + .resolve(RetentionScope::AuditLogs, over.as_ref()) + .expires_at(jiff::Timestamp::now()); + + Ok(NewWorkspaceFile { + workspace_id, + account_id, + display_name: Some("review.audit".to_owned()), + original_filename: Some("review.audit".to_owned()), + file_extension: Some("json".to_owned()), + file_kind: Some(FileKind::Review), + file_size_bytes: size, + file_hash_sha256: hash, + storage_path: key.to_string(), + storage_bucket: store.bucket().to_owned(), + expires_at: expires_at.map(Into::into), + ..Default::default() + }) + } + + /// Encrypts redacted bytes, writes them to the files bucket, and builds the + /// `redacted`-kind [`WorkspaceFile`] row that will point at them — without + /// inserting the row. + /// + /// The staged counterpart to + /// [`store_redacted_file`](Self::store_redacted_file): a redaction commits its + /// output row, review-audit row, and redaction row together in one + /// transaction, so the output file is staged rather than inserted here. The + /// redacted file is a first-class file (a sibling of the source), downloadable + /// through the normal file endpoints. + pub async fn stage_redacted_file( + &self, + source: &WorkspaceFile, + pipeline: &WorkspacePipeline, + workspace_settings: &RetentionSettings, + account_id: Uuid, + bytes: Bytes, + ) -> Result { + let plaintext_size = bytes.len() as i64; + let plaintext_hash = Sha256::digest(&bytes).to_vec(); + let ciphertext = self + .infra + .crypto + .encrypt(source.workspace_id, &bytes) + .map_err(|err| { + ErrorKind::InternalServerError + .with_message("Failed to encrypt redacted file") + .with_context(err.to_string()) + })?; + + let store = self.infra.nats.object_store::().await?; + let key = FileKey::generate(source.workspace_id); + store.put(&key, Cursor::new(ciphertext)).await?; + + let over = pipeline.metadata.or_default().retention; + let expires_at = workspace_settings + .resolve(RetentionScope::RedactedDocuments, over.as_ref()) + .expires_at(jiff::Timestamp::now()); + + let redacted_name = redacted_display_name(&source.display_name, &source.file_extension); + Ok(NewWorkspaceFile { + workspace_id: source.workspace_id, + account_id, + parent_id: Some(source.id), + display_name: Some(redacted_name), + original_filename: Some(source.original_filename.clone()), + file_extension: Some(source.file_extension.clone()), + file_kind: Some(FileKind::Redacted), + file_size_bytes: plaintext_size, + file_hash_sha256: plaintext_hash, + storage_path: key.to_string(), + storage_bucket: store.bucket().to_owned(), + expires_at: expires_at.map(Into::into), + ..Default::default() + }) + } + + /// Fetches and decrypts a redaction's stored review [`Audit`] by its file id. + /// + /// The review counterpart to + /// [`load_analyzed_document`](Self::load_analyzed_document); the `engine` + /// rebuilds the report from its serialized form. Errors if the redaction has + /// no review audit (409) or it has since been deleted (404). + pub async fn load_review_audit( + &self, + conn: &mut PgConn, + engine: &Engine, + workspace_id: Uuid, + review_file_id: Option, + ) -> Result { + let review_file_id = review_file_id.ok_or_else(|| { + ErrorKind::Conflict + .with_message("Redaction has no review audit") + .with_resource("redaction") + })?; + let review_file = conn + .find_file_in_workspace(workspace_id, review_file_id) + .await? + .ok_or_else(|| { + ErrorKind::NotFound + .with_message("The review audit for this redaction has been deleted") + .with_resource("redaction") + })?; + let key = AuditKey::from_str(&review_file.storage_path).map_err(|err| { + ErrorKind::InternalServerError + .with_message("Invalid review audit storage key") + .with_context(err.to_string()) + })?; + + let store = self.infra.nats.object_store::().await?; + let data = store.get(&key).await?.ok_or_else(|| { + ErrorKind::InternalServerError.with_message("Review audit is missing from storage") + })?; + let mut reader = data.into_reader(); + let mut ciphertext = Vec::new(); + reader.read_to_end(&mut ciphertext).await.map_err(|err| { + ErrorKind::InternalServerError + .with_message("Failed to read review audit") + .with_context(err.to_string()) + })?; + + let plaintext = self + .infra + .crypto + .decrypt(workspace_id, &ciphertext) + .map_err(|err| { + ErrorKind::InternalServerError + .with_message("Failed to decrypt review audit") + .with_context(err.to_string()) + })?; + engine + .deserialize_audit(&mut serde_json::Deserializer::from_slice(&plaintext)) + .map_err(|err| { + ErrorKind::InternalServerError + .with_message("Failed to decode review audit") + .with_context(err.to_string()) + }) + } } /// Maps an analysis (de)serialization failure to an internal error. @@ -387,3 +567,61 @@ fn analysis_serde_error(error: serde_json::Error) -> Error<'static> { .with_message("Failed to process analysis") .with_context(error.to_string()) } + +/// Builds the redacted file's display name by inserting a `redacted` marker +/// before the extension: `report.pdf` becomes `report.redacted.pdf`. +/// +/// The stem is taken by stripping a trailing `.{extension}` (case-insensitive) +/// from the display name; a name that does not end in its own extension (or has +/// none) simply gains a `.redacted` suffix. +fn redacted_display_name(display_name: &str, extension: &str) -> String { + let suffix = format!(".{extension}"); + match display_name + .len() + .checked_sub(suffix.len()) + .filter(|_| !extension.is_empty() && display_name.to_lowercase().ends_with(&suffix)) + { + Some(stem_len) => format!("{}.redacted.{extension}", &display_name[..stem_len]), + None => format!("{display_name}.redacted"), + } +} + +#[cfg(test)] +mod tests { + use super::redacted_display_name; + + #[test] + fn inserts_marker_before_the_extension() { + assert_eq!( + redacted_display_name("report.pdf", "pdf"), + "report.redacted.pdf" + ); + } + + #[test] + fn matches_the_extension_case_insensitively() { + assert_eq!( + redacted_display_name("Report.PDF", "pdf"), + "Report.redacted.pdf" + ); + } + + #[test] + fn preserves_a_multi_dot_stem() { + assert_eq!( + redacted_display_name("2026.q1.report.pdf", "pdf"), + "2026.q1.report.redacted.pdf" + ); + } + + #[test] + fn appends_when_the_name_lacks_its_extension() { + // A display name that does not end in `.{extension}` just gains the + // marker, so no extension is fabricated. + assert_eq!(redacted_display_name("report", "pdf"), "report.redacted"); + assert_eq!( + redacted_display_name("report.txt", "pdf"), + "report.txt.redacted" + ); + } +} diff --git a/migrations/2025-05-27-011852_files/up.sql b/migrations/2025-05-27-011852_files/up.sql index 8feb3620..1c612eef 100644 --- a/migrations/2025-05-27-011852_files/up.sql +++ b/migrations/2025-05-27-011852_files/up.sql @@ -5,11 +5,12 @@ -- Role of a file: drives data-retention scope and whether it is user-facing. CREATE TYPE FILE_KIND AS ENUM ( 'original', -- Source document (uploaded or imported) - 'redacted', -- Redacted output produced by a pipeline - 'audit' -- Engine analysis blob (not shown in file lists) + 'redacted', -- Redacted output produced by a redaction + 'audit', -- Engine detection analysis blob (not shown in file lists) + 'review' -- Engine analysis after reviewer edits + redaction (not shown in file lists) ); -COMMENT ON TYPE FILE_KIND IS 'The role of a file: original document, redacted output, or audit blob.'; +COMMENT ON TYPE FILE_KIND IS 'The role of a file: original document, redacted output, detection audit, or review (redaction) audit.'; -- Workspace files table: one stored document, with version tracking and dedup. CREATE TABLE workspace_files ( diff --git a/migrations/2026-01-19-045014_pipelines/down.sql b/migrations/2026-01-19-045014_pipelines/down.sql index 8edca28a..ede23721 100644 --- a/migrations/2026-01-19-045014_pipelines/down.sql +++ b/migrations/2026-01-19-045014_pipelines/down.sql @@ -1,10 +1,11 @@ -- Revert the pipelines tables. -- Objects are dropped in reverse order of creation. -DROP TABLE IF EXISTS workspace_pipeline_run_usage; -DROP TABLE IF EXISTS workspace_pipeline_runs; +DROP TABLE IF EXISTS workspace_redactions; +DROP TABLE IF EXISTS workspace_detection_usage; +DROP TABLE IF EXISTS workspace_detections; DROP TABLE IF EXISTS workspace_pipelines; DROP TYPE IF EXISTS PIPELINE_TRIGGER_TYPE; -DROP TYPE IF EXISTS PIPELINE_RUN_STATUS; +DROP TYPE IF EXISTS DETECTION_STATUS; DROP TYPE IF EXISTS PIPELINE_STATUS; diff --git a/migrations/2026-01-19-045014_pipelines/up.sql b/migrations/2026-01-19-045014_pipelines/up.sql index c36e3479..4951f93c 100644 --- a/migrations/2026-01-19-045014_pipelines/up.sql +++ b/migrations/2026-01-19-045014_pipelines/up.sql @@ -1,6 +1,8 @@ --- Pipelines: redaction pipeline definitions, their runs, and produced files. --- A pipeline is a workspace-scoped detection/redaction config; a run is one --- pass of a file through that config. Policy references live in a join table +-- Pipelines: redaction pipeline definitions, their detections, and the +-- redactions produced from each. A pipeline is a workspace-scoped +-- detection/redaction config; a detection is one analysis pass of a file +-- through that config, and each detection can produce many redactions (one per +-- reviewer-edited redact request). Policy references live in a join table -- declared alongside policies, not embedded here. -- Lifecycle status of a pipeline definition. @@ -12,25 +14,23 @@ CREATE TYPE PIPELINE_STATUS AS ENUM ( COMMENT ON TYPE PIPELINE_STATUS IS 'Lifecycle status of a pipeline definition: draft, enabled, or disabled.'; --- Execution status of a pipeline run. -CREATE TYPE PIPELINE_RUN_STATUS AS ENUM ( - 'queued', -- Enqueued for detection; no worker has picked it up yet - 'analyzing', -- A worker is actively analyzing the document - 'analyzed', -- Detection done; awaiting reviewer verification - 'completed', -- Redaction applied; run finished - 'failed', -- Run failed with error - 'cancelled' -- Run was cancelled by user +-- Execution status of a detection (analysis pass). +CREATE TYPE DETECTION_STATUS AS ENUM ( + 'pending', -- Enqueued for detection; no worker has picked it up yet + 'executing', -- A worker is actively analyzing the document + 'complete', -- Detection done; ready to redact + 'failed' -- Detection failed with error ); -COMMENT ON TYPE PIPELINE_RUN_STATUS IS 'Execution status of a pipeline run.'; +COMMENT ON TYPE DETECTION_STATUS IS 'Execution status of a detection: pending, executing, complete, or failed.'; --- How a pipeline run was initiated. +-- How a detection was initiated. CREATE TYPE PIPELINE_TRIGGER_TYPE AS ENUM ( 'user', -- Started directly by a user 'system' -- Started automatically (e.g. a file upload auto-redacted) ); -COMMENT ON TYPE PIPELINE_TRIGGER_TYPE IS 'How a pipeline run was initiated: by a user or by the system.'; +COMMENT ON TYPE PIPELINE_TRIGGER_TYPE IS 'How a detection was initiated: by a user or by the system.'; -- Pipeline definitions table: a workspace's detection/redaction configs. CREATE TABLE workspace_pipelines ( @@ -119,8 +119,8 @@ COMMENT ON COLUMN workspace_pipelines.created_at IS 'Pipeline creation timestamp COMMENT ON COLUMN workspace_pipelines.updated_at IS 'Last modification timestamp'; COMMENT ON COLUMN workspace_pipelines.deleted_at IS 'Soft-deletion timestamp; NULL means live'; --- Pipeline runs table: one pass of a file through a pipeline. -CREATE TABLE workspace_pipeline_runs ( +-- Detections table: one analysis pass of a file through a pipeline. +CREATE TABLE workspace_detections ( -- Primary identifier id UUID PRIMARY KEY DEFAULT gen_random_uuid(), @@ -128,117 +128,162 @@ CREATE TABLE workspace_pipeline_runs ( pipeline_id UUID NOT NULL REFERENCES workspace_pipelines (id) ON DELETE CASCADE, account_id UUID NOT NULL REFERENCES accounts (id) ON DELETE CASCADE, - -- The three files a run relates to, each a distinct role. The input is the - -- source document (required); the audit blob and redacted output are produced - -- by the run, so they are null until their phase completes. - -- input: the original document being analyzed/redacted. + -- The two files a detection relates to, each a distinct role. The input is + -- the source document (required); the audit blob is produced by the analysis, + -- so it is null until the pass completes. Redacted outputs are not here — a + -- detection produces many redactions, each owning its own output (see the + -- workspace_redactions table below). + -- input: the original document being analyzed. -- audit: the engine's analysis (Audit), a `file_kind = audit` file held -- between detect and redact; redact reads it as the source of truth. - -- output: the redacted document produced by redact. - -- A run is append-only audit history: its file references survive the files - -- themselves. Files are only ever soft-deleted (their objects purged), so - -- these ON DELETE actions fire only on a hard delete. The app never hard- + -- A detection is append-only audit history: its file references survive the + -- files themselves. Files are only ever soft-deleted (their objects purged), + -- so these ON DELETE actions fire only on a hard delete. The app never hard- -- deletes an individual file; the one hard delete is a whole-workspace - -- teardown, which cascades files and runs away together. The input therefore - -- cascades — a workspace deletion that removes the source document should - -- remove its runs too. Produced files use ON DELETE SET NULL so that, in that - -- same teardown, they clear rather than cascade (redundant here, but the - -- correct action for a produced artifact). + -- teardown, which cascades files and detections away together. The input + -- therefore cascades — a workspace deletion that removes the source document + -- should remove its detections too. The produced audit uses ON DELETE SET + -- NULL so that, in that same teardown, it clears rather than cascades + -- (redundant here, but the correct action for a produced artifact). input_file_id UUID NOT NULL REFERENCES workspace_files (id) ON DELETE CASCADE, audit_file_id UUID DEFAULT NULL REFERENCES workspace_files (id) ON DELETE SET NULL, - output_file_id UUID DEFAULT NULL REFERENCES workspace_files (id) ON DELETE SET NULL, - -- Run attributes + -- Detection attributes trigger_type PIPELINE_TRIGGER_TYPE NOT NULL DEFAULT 'user', - status PIPELINE_RUN_STATUS NOT NULL DEFAULT 'queued', + status DETECTION_STATUS NOT NULL DEFAULT 'pending', -- Idempotency key from the initiating detect request; a repeat replays the - -- existing run instead of analyzing twice. + -- existing detection instead of analyzing twice. idempotency_key TEXT DEFAULT NULL, - CONSTRAINT workspace_pipeline_runs_idempotency_key_length CHECK (idempotency_key IS NULL OR length(idempotency_key) BETWEEN 1 AND 255), + CONSTRAINT workspace_detections_idempotency_key_length CHECK (idempotency_key IS NULL OR length(idempotency_key) BETWEEN 1 AND 255), -- Non-encrypted metadata for filtering and display. The engine's full -- per-recognizer usage report (durations, per-model token counts) is kept -- here under `usage` for drill-down; per-model token totals for usage - -- aggregation live in the workspace_pipeline_run_usage table below. + -- aggregation live in the workspace_detection_usage table below. metadata JSONB NOT NULL DEFAULT '{}', - CONSTRAINT workspace_pipeline_runs_metadata_size CHECK (length(metadata::TEXT) BETWEEN 2 AND 65536), + CONSTRAINT workspace_detections_metadata_size CHECK (length(metadata::TEXT) BETWEEN 2 AND 65536), - -- Detection lease: when a worker last claimed this run. A redelivered job - -- whose claim is still fresh is skipped (no double-analyze); a stale claim + -- Detection lease: when a worker last claimed this detection. A redelivered + -- job whose claim is still fresh is skipped (no double-analyze); a stale claim -- (a worker that died mid-analysis) can be re-claimed. Null until claimed. claimed_at TIMESTAMPTZ DEFAULT NULL, -- Timing started_at TIMESTAMPTZ NOT NULL DEFAULT current_timestamp, completed_at TIMESTAMPTZ DEFAULT NULL, - CONSTRAINT workspace_pipeline_runs_completed_after_started CHECK (completed_at IS NULL OR completed_at >= started_at) + CONSTRAINT workspace_detections_completed_after_started CHECK (completed_at IS NULL OR completed_at >= started_at) ); --- A pipeline's runs, newest first (the run list). -CREATE INDEX workspace_pipeline_runs_pipeline_idx - ON workspace_pipeline_runs (pipeline_id, started_at DESC); +-- A pipeline's detections, newest first (the detection list). +CREATE INDEX workspace_detections_pipeline_idx + ON workspace_detections (pipeline_id, started_at DESC); --- Runs triggered by an account, newest first. -CREATE INDEX workspace_pipeline_runs_account_idx - ON workspace_pipeline_runs (account_id, started_at DESC); +-- Detections triggered by an account, newest first. +CREATE INDEX workspace_detections_account_idx + ON workspace_detections (account_id, started_at DESC); --- In-flight runs by status (queue and review backlog). -CREATE INDEX workspace_pipeline_runs_status_idx - ON workspace_pipeline_runs (status, started_at DESC) - WHERE status IN ('queued', 'analyzing', 'analyzed'); +-- In-flight detections by status (queue and ready backlog). +CREATE INDEX workspace_detections_status_idx + ON workspace_detections (status, started_at DESC) + WHERE status IN ('pending', 'executing', 'complete'); --- Runs analyzing a given input file, newest first. -CREATE INDEX workspace_pipeline_runs_input_file_idx - ON workspace_pipeline_runs (input_file_id, started_at DESC); +-- Detections analyzing a given input file, newest first. +CREATE INDEX workspace_detections_input_file_idx + ON workspace_detections (input_file_id, started_at DESC); --- Idempotent detect: at most one run per (pipeline, idempotency key). -CREATE UNIQUE INDEX workspace_pipeline_runs_idempotency_idx - ON workspace_pipeline_runs (pipeline_id, idempotency_key) +-- Idempotent detect: at most one detection per (pipeline, idempotency key). +CREATE UNIQUE INDEX workspace_detections_idempotency_idx + ON workspace_detections (pipeline_id, idempotency_key) WHERE idempotency_key IS NOT NULL; --- A run is append-only history: its file references (input/audit/output) are --- kept even after those files are deleted, so the record of what a run analyzed --- and produced survives. The `ON DELETE SET NULL` FKs would fire only on a hard --- file delete, which never happens (files are soft-deleted); a reference to a --- soft-deleted file resolves to "gone" at read time, distinct from a NULL that --- means the run never had one. - -COMMENT ON TABLE workspace_pipeline_runs IS 'Detect/redact runs: one pass of a file through a pipeline.'; -COMMENT ON COLUMN workspace_pipeline_runs.id IS 'Unique run identifier'; -COMMENT ON COLUMN workspace_pipeline_runs.pipeline_id IS 'Pipeline whose config drove the run'; -COMMENT ON COLUMN workspace_pipeline_runs.account_id IS 'Account that triggered the run'; -COMMENT ON COLUMN workspace_pipeline_runs.input_file_id IS 'Source document the run analyzes / redacts'; -COMMENT ON COLUMN workspace_pipeline_runs.audit_file_id IS 'Audit file (file_kind=audit) holding the analysis between detect and redact'; -COMMENT ON COLUMN workspace_pipeline_runs.output_file_id IS 'Redacted document produced by redact; NULL until completed'; -COMMENT ON COLUMN workspace_pipeline_runs.trigger_type IS 'How the run was initiated'; -COMMENT ON COLUMN workspace_pipeline_runs.status IS 'Current run status'; -COMMENT ON COLUMN workspace_pipeline_runs.idempotency_key IS 'Detect idempotency key (dedupes retries)'; -COMMENT ON COLUMN workspace_pipeline_runs.metadata IS 'Non-encrypted metadata for filtering/display; holds the full per-recognizer usage report under `usage`'; -COMMENT ON COLUMN workspace_pipeline_runs.claimed_at IS 'Detection lease: when a worker last claimed this run'; -COMMENT ON COLUMN workspace_pipeline_runs.started_at IS 'When the run started'; -COMMENT ON COLUMN workspace_pipeline_runs.completed_at IS 'When the run completed; NULL while in flight'; - --- Per-model inference usage for a run: one row per distinct model a run's --- recognizers used. Token counts are aggregated across the recognizers that --- shared a model, letting usage analytics report tokens broken down by model --- (a run's summed tokens cannot, since a run may mix models). The full --- per-recognizer report is kept on the run (metadata.usage) for drill-down; --- this table is the aggregation surface. -CREATE TABLE workspace_pipeline_run_usage ( +-- A detection is append-only history: its file references (input/audit) are kept +-- even after those files are deleted, so the record of what it analyzed +-- survives. The `ON DELETE SET NULL` FK would fire only on a hard file delete, +-- which never happens (files are soft-deleted); a reference to a soft-deleted +-- file resolves to "gone" at read time, distinct from a NULL that means the +-- detection never had one. + +COMMENT ON TABLE workspace_detections IS 'Detections: one analysis pass of a file through a pipeline.'; +COMMENT ON COLUMN workspace_detections.id IS 'Unique detection identifier'; +COMMENT ON COLUMN workspace_detections.pipeline_id IS 'Pipeline whose config drove the detection'; +COMMENT ON COLUMN workspace_detections.account_id IS 'Account that triggered the detection'; +COMMENT ON COLUMN workspace_detections.input_file_id IS 'Source document the detection analyzes'; +COMMENT ON COLUMN workspace_detections.audit_file_id IS 'Audit file (file_kind=audit) holding the analysis between detect and redact'; +COMMENT ON COLUMN workspace_detections.trigger_type IS 'How the detection was initiated'; +COMMENT ON COLUMN workspace_detections.status IS 'Current detection status'; +COMMENT ON COLUMN workspace_detections.idempotency_key IS 'Detect idempotency key (dedupes retries)'; +COMMENT ON COLUMN workspace_detections.metadata IS 'Non-encrypted metadata for filtering/display; holds the full per-recognizer usage report under `usage`'; +COMMENT ON COLUMN workspace_detections.claimed_at IS 'Detection lease: when a worker last claimed this detection'; +COMMENT ON COLUMN workspace_detections.started_at IS 'When the detection started'; +COMMENT ON COLUMN workspace_detections.completed_at IS 'When the detection completed; NULL while in flight'; + +-- Redactions table: one redact pass over a detection's analysis. A detection can +-- be redacted many times — each redact request may carry a different set of +-- reviewer edits — so each is its own row owning the edited audit it applied and +-- the redacted document it produced. +CREATE TABLE workspace_redactions ( + -- Primary identifier + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + -- The detection this redaction was produced from; redactions are deleted + -- with their detection. + detection_id UUID NOT NULL REFERENCES workspace_detections (id) ON DELETE CASCADE, + + -- Account that requested the redaction. + account_id UUID NOT NULL REFERENCES accounts (id) ON DELETE CASCADE, + + -- The two files a redaction produces. Both are set at creation (a redaction + -- always yields a review audit and a redacted document), and both use + -- ON DELETE SET NULL for the same append-only-history reasons as a detection. + -- review: the engine's Audit after the reviewer edits were applied and + -- redaction ran (`file_kind = review`); the record of exactly what + -- was redacted and why. + -- output: the redacted document this redaction produced. + review_file_id UUID DEFAULT NULL REFERENCES workspace_files (id) ON DELETE SET NULL, + output_file_id UUID DEFAULT NULL REFERENCES workspace_files (id) ON DELETE SET NULL, + + -- Timing + created_at TIMESTAMPTZ NOT NULL DEFAULT current_timestamp +); + +-- A detection's redactions, newest first (the redaction list). +CREATE INDEX workspace_redactions_detection_idx + ON workspace_redactions (detection_id, created_at DESC); + +-- Redactions requested by an account, newest first. +CREATE INDEX workspace_redactions_account_idx + ON workspace_redactions (account_id, created_at DESC); + +COMMENT ON TABLE workspace_redactions IS 'Redactions: one redact pass over a detection, with its own reviewer edits, edited audit, and output.'; +COMMENT ON COLUMN workspace_redactions.id IS 'Unique redaction identifier'; +COMMENT ON COLUMN workspace_redactions.detection_id IS 'Detection this redaction was produced from'; +COMMENT ON COLUMN workspace_redactions.account_id IS 'Account that requested the redaction'; +COMMENT ON COLUMN workspace_redactions.review_file_id IS 'Review audit (file_kind=review) recording the applied edits and redaction outcome'; +COMMENT ON COLUMN workspace_redactions.output_file_id IS 'Redacted document this redaction produced'; +COMMENT ON COLUMN workspace_redactions.created_at IS 'When the redaction was created'; + +-- Per-model inference usage for a detection: one row per distinct model a +-- detection's recognizers used. Token counts are aggregated across the +-- recognizers that shared a model, letting usage analytics report tokens broken +-- down by model (a detection's summed tokens cannot, since it may mix models). +-- The full per-recognizer report is kept on the detection (metadata.usage) for +-- drill-down; this table is the aggregation surface. +CREATE TABLE workspace_detection_usage ( -- Primary identifier id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - -- The run this usage belongs to; usage is deleted with its run. - run_id UUID NOT NULL REFERENCES workspace_pipeline_runs (id) ON DELETE CASCADE, + -- The detection this usage belongs to; usage is deleted with its detection. + detection_id UUID NOT NULL REFERENCES workspace_detections (id) ON DELETE CASCADE, - -- The model and its optional version. Identity is (run, model, version): - -- a run may use the same model at more than one version, and each is its own - -- row (uniqueness enforced by an index below that normalizes a NULL version). + -- The model and its optional version. Identity is (detection, model, + -- version): a detection may use the same model at more than one version, and + -- each is its own row (uniqueness enforced by an index below that normalizes + -- a NULL version). model TEXT NOT NULL, version TEXT DEFAULT NULL, - CONSTRAINT workspace_pipeline_run_usage_model_length CHECK (length(model) BETWEEN 1 AND 255), - CONSTRAINT workspace_pipeline_run_usage_version_length CHECK (version IS NULL OR length(version) BETWEEN 1 AND 255), + CONSTRAINT workspace_detection_usage_model_length CHECK (length(model) BETWEEN 1 AND 255), + CONSTRAINT workspace_detection_usage_version_length CHECK (version IS NULL OR length(version) BETWEEN 1 AND 255), -- Token counts as the provider reported them. Each is independently nullable: -- `total` is NOT necessarily input + output (a provider may report only a @@ -247,31 +292,31 @@ CREATE TABLE workspace_pipeline_run_usage ( input_tokens BIGINT DEFAULT NULL, output_tokens BIGINT DEFAULT NULL, total_tokens BIGINT DEFAULT NULL, - CONSTRAINT workspace_pipeline_run_usage_input_non_negative CHECK (input_tokens IS NULL OR input_tokens >= 0), - CONSTRAINT workspace_pipeline_run_usage_output_non_negative CHECK (output_tokens IS NULL OR output_tokens >= 0), - CONSTRAINT workspace_pipeline_run_usage_total_non_negative CHECK (total_tokens IS NULL OR total_tokens >= 0), + CONSTRAINT workspace_detection_usage_input_non_negative CHECK (input_tokens IS NULL OR input_tokens >= 0), + CONSTRAINT workspace_detection_usage_output_non_negative CHECK (output_tokens IS NULL OR output_tokens >= 0), + CONSTRAINT workspace_detection_usage_total_non_negative CHECK (total_tokens IS NULL OR total_tokens >= 0), -- Wall-clock time this model's recognizers spent, in milliseconds. duration_ms BIGINT NOT NULL DEFAULT 0, - CONSTRAINT workspace_pipeline_run_usage_duration_non_negative CHECK (duration_ms >= 0) + CONSTRAINT workspace_detection_usage_duration_non_negative CHECK (duration_ms >= 0) ); --- One row per (run, model, version); a NULL version is normalized so two +-- One row per (detection, model, version); a NULL version is normalized so two -- unversioned rows for the same model collide instead of both being inserted. --- Leads with run_id, so it also serves per-run drill-down lookups. -CREATE UNIQUE INDEX workspace_pipeline_run_usage_run_model_version_key - ON workspace_pipeline_run_usage (run_id, model, COALESCE(version, '')); - --- Usage rollups by model across runs. -CREATE INDEX workspace_pipeline_run_usage_model_idx - ON workspace_pipeline_run_usage (model); - -COMMENT ON TABLE workspace_pipeline_run_usage IS 'Per-model inference token usage for a pipeline run.'; -COMMENT ON COLUMN workspace_pipeline_run_usage.id IS 'Unique usage row identifier'; -COMMENT ON COLUMN workspace_pipeline_run_usage.run_id IS 'Run this usage belongs to'; -COMMENT ON COLUMN workspace_pipeline_run_usage.model IS 'Model identifier the recognizers used'; -COMMENT ON COLUMN workspace_pipeline_run_usage.version IS 'Model version, if the provider reported one'; -COMMENT ON COLUMN workspace_pipeline_run_usage.input_tokens IS 'Input/prompt tokens for this model; NULL if not reported'; -COMMENT ON COLUMN workspace_pipeline_run_usage.output_tokens IS 'Output/completion tokens for this model; NULL if not reported'; -COMMENT ON COLUMN workspace_pipeline_run_usage.total_tokens IS 'Total tokens as reported (not necessarily input + output); NULL if not reported'; -COMMENT ON COLUMN workspace_pipeline_run_usage.duration_ms IS 'Wall-clock time this model spent, in milliseconds'; +-- Leads with detection_id, so it also serves per-detection drill-down lookups. +CREATE UNIQUE INDEX workspace_detection_usage_detection_model_version_key + ON workspace_detection_usage (detection_id, model, COALESCE(version, '')); + +-- Usage rollups by model across detections. +CREATE INDEX workspace_detection_usage_model_idx + ON workspace_detection_usage (model); + +COMMENT ON TABLE workspace_detection_usage IS 'Per-model inference token usage for a detection.'; +COMMENT ON COLUMN workspace_detection_usage.id IS 'Unique usage row identifier'; +COMMENT ON COLUMN workspace_detection_usage.detection_id IS 'Detection this usage belongs to'; +COMMENT ON COLUMN workspace_detection_usage.model IS 'Model identifier the recognizers used'; +COMMENT ON COLUMN workspace_detection_usage.version IS 'Model version, if the provider reported one'; +COMMENT ON COLUMN workspace_detection_usage.input_tokens IS 'Input/prompt tokens for this model; NULL if not reported'; +COMMENT ON COLUMN workspace_detection_usage.output_tokens IS 'Output/completion tokens for this model; NULL if not reported'; +COMMENT ON COLUMN workspace_detection_usage.total_tokens IS 'Total tokens as reported (not necessarily input + output); NULL if not reported'; +COMMENT ON COLUMN workspace_detection_usage.duration_ms IS 'Wall-clock time this model spent, in milliseconds'; From 55755c062bdf0274cd490eb049a42220d51c20bf Mon Sep 17 00:00:00 2001 From: Oleh Martsokha Date: Wed, 26 Aug 2026 15:20:59 +0200 Subject: [PATCH 2/7] Fix redaction addressing, event enum values, and detection lifecycle gaps Follow-up fixes on the detections/redactions branch: - Rename the redaction response type Redaction -> RedactionResult; it collided with the engine audit's `Redaction` event in the generated OpenAPI (Redaction2). - Address a redaction flat: GET /workspaces/{slug}/redactions/{redactionId}/review (a RedactionId is globally unique, like a DetectionId), resolved in-workspace via a new find_redaction_in_workspace query; the list stays under its detection. - Fix the event wire strings in the SQL enums (activities/webhooks/notifications) and the workspaces notification defaults: they still listed pipeline.run.* while the code emits pipeline.detection.*/pipeline.redaction.*, so the outbox drainer rejected every event. Now match the Rust db_rename values. - Stamp completed_at at every terminal detection transition, inside finalize_detection and fail_detection (was set only on the failure path via the caller). Guard the enqueue-failure handler path on status=Pending via a new fail_pending_detection so it no-ops once a worker has claimed the detection. - Backfill expires_at for the Review file kind on retention changes at both the workspace and pipeline level (the query arm existed but no caller passed Review, so review audits kept stale expiry). - Remove the now-dead store_redacted_file and update_workspace_detection; fix the redacted display name (report.pdf.redacted -> report.redacted.pdf) and the create-detection doc (pending, not executing). - Update elide-runtime to the latest revision (API-compatible). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018bKk1YEG4tZ69jzYVQvQL8 --- Cargo.lock | 52 ++++++------- .../src/query/workspace_detection.rs | 74 ++++++++++++------- .../src/query/workspace_redaction.rs | 26 +++++-- crates/nvisy-server/src/handler/detections.rs | 10 +-- crates/nvisy-server/src/handler/pipelines.rs | 12 +-- crates/nvisy-server/src/handler/redactions.rs | 35 ++++----- .../nvisy-server/src/handler/request/paths.rs | 9 ++- .../src/handler/response/redactions.rs | 10 ++- crates/nvisy-server/src/handler/workspaces.rs | 3 + .../src/service/detection/support.rs | 21 ++++-- .../src/service/run_blob_store.rs | 69 ++--------------- .../2025-05-21-121132_notifications/up.sql | 18 ++--- .../2025-05-21-222840_workspaces/up.sql | 2 +- .../2025-05-21-222841_activities/up.sql | 10 +-- migrations/2025-05-21-222842_webhooks/up.sql | 10 +-- 15 files changed, 171 insertions(+), 190 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c82dfecd..8ef09f31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2880,7 +2880,7 @@ checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" [[package]] name = "elide" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "async-trait", "elide-codec", @@ -2918,7 +2918,7 @@ dependencies = [ [[package]] name = "elide-codec" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "async-trait", "bytes", @@ -2940,7 +2940,7 @@ dependencies = [ [[package]] name = "elide-context" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "async-trait", "elide-core", @@ -2951,7 +2951,7 @@ dependencies = [ [[package]] name = "elide-core" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "async-trait", "blake3", @@ -2971,7 +2971,7 @@ dependencies = [ [[package]] name = "elide-detection" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "elide-core", "futures", @@ -2983,7 +2983,7 @@ dependencies = [ [[package]] name = "elide-engine" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "bytes", "elide-codec", @@ -3001,7 +3001,7 @@ dependencies = [ [[package]] name = "elide-export" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide-runtime?branch=main#ff20eb27d6246185d7f491d24a5b9e601cda8a55" +source = "git+https://github.com/nvisycom/elide-runtime?branch=main#1b5bf516ac1b053bdc2e5e6cca27fe37456b025f" dependencies = [ "csv", "elide", @@ -3013,7 +3013,7 @@ dependencies = [ [[package]] name = "elide-fake" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "async-trait", "elide-core", @@ -3024,7 +3024,7 @@ dependencies = [ [[package]] name = "elide-governance" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide-runtime?branch=main#ff20eb27d6246185d7f491d24a5b9e601cda8a55" +source = "git+https://github.com/nvisycom/elide-runtime?branch=main#1b5bf516ac1b053bdc2e5e6cca27fe37456b025f" dependencies = [ "elide-core", "elide-operator", @@ -3038,7 +3038,7 @@ dependencies = [ [[package]] name = "elide-lingua" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "async-trait", "elide-core", @@ -3049,7 +3049,7 @@ dependencies = [ [[package]] name = "elide-llm" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "async-trait", "derive_builder", @@ -3070,7 +3070,7 @@ dependencies = [ [[package]] name = "elide-ner" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "async-trait", "derive_builder", @@ -3084,7 +3084,7 @@ dependencies = [ [[package]] name = "elide-ocr" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "async-trait", "derive_builder", @@ -3096,7 +3096,7 @@ dependencies = [ [[package]] name = "elide-office" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "bytes", "hipstr", @@ -3108,7 +3108,7 @@ dependencies = [ [[package]] name = "elide-operator" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "aes-gcm", "async-trait", @@ -3130,7 +3130,7 @@ dependencies = [ [[package]] name = "elide-pattern" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "aho-corasick", "async-trait", @@ -3150,7 +3150,7 @@ dependencies = [ [[package]] name = "elide-pdf" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "bytes", "hipstr", @@ -3162,7 +3162,7 @@ dependencies = [ [[package]] name = "elide-pipeline" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide-runtime?branch=main#ff20eb27d6246185d7f491d24a5b9e601cda8a55" +source = "git+https://github.com/nvisycom/elide-runtime?branch=main#1b5bf516ac1b053bdc2e5e6cca27fe37456b025f" dependencies = [ "bytes", "elide", @@ -3183,7 +3183,7 @@ dependencies = [ [[package]] name = "elide-provider" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide-runtime?branch=main#ff20eb27d6246185d7f491d24a5b9e601cda8a55" +source = "git+https://github.com/nvisycom/elide-runtime?branch=main#1b5bf516ac1b053bdc2e5e6cca27fe37456b025f" dependencies = [ "elide", "elide-bento", @@ -3197,7 +3197,7 @@ dependencies = [ [[package]] name = "elide-redaction" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "elide-core", "hipstr", @@ -3207,7 +3207,7 @@ dependencies = [ [[package]] name = "elide-review" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide-runtime?branch=main#ff20eb27d6246185d7f491d24a5b9e601cda8a55" +source = "git+https://github.com/nvisycom/elide-runtime?branch=main#1b5bf516ac1b053bdc2e5e6cca27fe37456b025f" dependencies = [ "elide", "elide-governance", @@ -3219,7 +3219,7 @@ dependencies = [ [[package]] name = "elide-stt" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "async-trait", "derive_builder", @@ -3231,7 +3231,7 @@ dependencies = [ [[package]] name = "elide-template" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide-runtime?branch=main#ff20eb27d6246185d7f491d24a5b9e601cda8a55" +source = "git+https://github.com/nvisycom/elide-runtime?branch=main#1b5bf516ac1b053bdc2e5e6cca27fe37456b025f" dependencies = [ "elide-core", "elide-governance", @@ -6987,7 +6987,7 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "itertools 0.14.0", "log", "multimap", @@ -8547,7 +8547,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.119", @@ -8559,7 +8559,7 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "287f59010008f0d7cf5e3b03196d666c1acc46c8d3e9cf34c28a1a7157601e72" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.119", diff --git a/crates/nvisy-postgres/src/query/workspace_detection.rs b/crates/nvisy-postgres/src/query/workspace_detection.rs index aa29453f..0d2c8b22 100644 --- a/crates/nvisy-postgres/src/query/workspace_detection.rs +++ b/crates/nvisy-postgres/src/query/workspace_detection.rs @@ -126,13 +126,6 @@ pub trait WorkspaceDetectionRepository { detection: &WorkspaceDetection, ) -> impl Future> + Send; - /// Updates a workspace detection with new data. - fn update_workspace_detection( - &mut self, - detection_id: Uuid, - updates: UpdateWorkspaceDetection, - ) -> impl Future> + Send; - /// Transitions a detection to `Complete` only while the caller still holds /// its claim — the detection is still `Executing` and its `claimed_at` matches /// the value stamped when the caller claimed it. Returns `true` on success, @@ -162,6 +155,19 @@ pub trait WorkspaceDetectionRepository { updates: UpdateWorkspaceDetection, ) -> impl Future> + Send; + /// Transitions a detection to `Failed` only while it is still `Pending` — no + /// worker has claimed it. For the enqueue-failure path in the create handler: + /// if enqueue reported an error but the job was in fact delivered, a worker + /// may already have claimed the detection (moving it to `Executing`), and this + /// guard makes the handler's failure a no-op so it never clobbers the outcome + /// the worker will drive. Returns `true` if it failed the detection, `false` + /// if it was no longer `Pending`. Forces `status` and `completed_at`. + fn fail_pending_detection( + &mut self, + detection_id: Uuid, + updates: UpdateWorkspaceDetection, + ) -> impl Future> + Send; + /// Records a detection's per-model inference usage. A no-op for an empty slice /// (a deterministic detection spends no tokens). Inserted once, at analyze /// time. @@ -532,24 +538,6 @@ impl WorkspaceDetectionRepository for PgConnection { Ok(DetectionFiles { input }) } - async fn update_workspace_detection( - &mut self, - detection_id: Uuid, - updates: UpdateWorkspaceDetection, - ) -> PgResult { - use schema::workspace_detections::{self, dsl}; - - let detection = - diesel::update(workspace_detections::table.filter(dsl::id.eq(detection_id))) - .set(&updates) - .returning(WorkspaceDetection::as_returning()) - .get_result(self) - .await - .map_err(PgError::from)?; - - Ok(detection) - } - async fn finalize_detection( &mut self, detection_id: Uuid, @@ -558,9 +546,11 @@ impl WorkspaceDetectionRepository for PgConnection { ) -> PgResult { use schema::workspace_detections::{self, dsl}; - // Force the terminal transition here; the guard makes it a no-op unless we - // still own the claim. + // Force the terminal transition and stamp the terminal time here, so every + // terminal outcome records `completed_at` regardless of the caller; the + // guard makes it a no-op unless we still own the claim. updates.status = Some(DetectionStatus::Complete); + updates.completed_at = Some(Some(jiff::Timestamp::now().into())); let claimed_at = jiff_diesel::Timestamp::from(claimed_at); // Guard on the claim we hold: same detection, still `Executing`, and the @@ -589,7 +579,10 @@ impl WorkspaceDetectionRepository for PgConnection { ) -> PgResult { use schema::workspace_detections::{self, dsl}; + // Force the terminal transition and stamp the terminal time here, mirroring + // `finalize_detection`, so both terminal outcomes record `completed_at`. updates.status = Some(DetectionStatus::Failed); + updates.completed_at = Some(Some(jiff::Timestamp::now().into())); let claimed_at = jiff_diesel::Timestamp::from(claimed_at); // Same claim guard as the complete finalize: only our still-live claim @@ -608,6 +601,33 @@ impl WorkspaceDetectionRepository for PgConnection { Ok(updated == 1) } + async fn fail_pending_detection( + &mut self, + detection_id: Uuid, + mut updates: UpdateWorkspaceDetection, + ) -> PgResult { + use schema::workspace_detections::{self, dsl}; + + // Force the terminal transition and stamp the terminal time, as the other + // finalize methods do. + updates.status = Some(DetectionStatus::Failed); + updates.completed_at = Some(Some(jiff::Timestamp::now().into())); + + // Guard on `Pending`: once a worker claims the detection (moving it to + // `Executing`), this matches no row and the worker owns the outcome. + let updated = diesel::update( + workspace_detections::table + .filter(dsl::id.eq(detection_id)) + .filter(dsl::status.eq(DetectionStatus::Pending)), + ) + .set(&updates) + .execute(self) + .await + .map_err(PgError::from)?; + + Ok(updated == 1) + } + async fn record_detection_usage( &mut self, usage: &[NewWorkspaceDetectionUsage], diff --git a/crates/nvisy-postgres/src/query/workspace_redaction.rs b/crates/nvisy-postgres/src/query/workspace_redaction.rs index 68def03f..2ea9ea79 100644 --- a/crates/nvisy-postgres/src/query/workspace_redaction.rs +++ b/crates/nvisy-postgres/src/query/workspace_redaction.rs @@ -22,10 +22,14 @@ pub trait WorkspaceRedactionRepository { new_redaction: NewWorkspaceRedaction, ) -> impl Future> + Send; - /// Finds a redaction by its id, scoped to its owning detection. - fn find_redaction_by_id( + /// Finds a redaction by its id, scoped to a workspace. + /// + /// A [`RedactionId`](crate::types::RedactionId) is globally unique, so a + /// redaction is addressable by id alone; this resolves it only within the + /// given workspace by joining through its detection's pipeline. + fn find_redaction_in_workspace( &mut self, - detection_id: Uuid, + workspace_id: Uuid, redaction_id: Uuid, ) -> impl Future>> + Send; @@ -54,16 +58,22 @@ impl WorkspaceRedactionRepository for PgConnection { Ok(redaction) } - async fn find_redaction_by_id( + async fn find_redaction_in_workspace( &mut self, - detection_id: Uuid, + workspace_id: Uuid, redaction_id: Uuid, ) -> PgResult> { - use schema::workspace_redactions::{self, dsl}; + use schema::workspace_redactions::dsl as redactions; + use schema::{workspace_detections, workspace_pipelines, workspace_redactions}; + // Redactions carry no workspace column; scope through the detection's + // pipeline so the id resolves only within its workspace, and only while + // that pipeline is live (a soft-deleted pipeline hides its redactions). let redaction = workspace_redactions::table - .filter(dsl::id.eq(redaction_id)) - .filter(dsl::detection_id.eq(detection_id)) + .inner_join(workspace_detections::table.inner_join(workspace_pipelines::table)) + .filter(redactions::id.eq(redaction_id)) + .filter(workspace_pipelines::workspace_id.eq(workspace_id)) + .filter(workspace_pipelines::deleted_at.is_null()) .select(WorkspaceRedaction::as_select()) .first(self) .await diff --git a/crates/nvisy-server/src/handler/detections.rs b/crates/nvisy-server/src/handler/detections.rs index 53adb111..a07f5cdd 100644 --- a/crates/nvisy-server/src/handler/detections.rs +++ b/crates/nvisy-server/src/handler/detections.rs @@ -31,7 +31,7 @@ use crate::handler::request::{ CreateDetection, CursorPagination, DetectionPathParams, PipelineDefinition, PipelineDetectionsQuery, PipelinePathParams, RedactDetection, WorkspaceDetectionsQuery, }; -use crate::handler::response::{Detection, DetectionsPage, ErrorResponse, Redaction}; +use crate::handler::response::{Detection, DetectionsPage, ErrorResponse, RedactionResult}; use crate::handler::utility::{SseResponse, resolve_account_ref}; use crate::handler::{Error, ErrorKind, Result}; use crate::service::{ @@ -219,7 +219,7 @@ fn create_detection_docs(op: TransformOperation) -> TransformOperation { op.summary("Start a detection") .description( "Starts analysis for a file and returns 202 with the detection in the \ - `executing` state; the analysis runs in the background. Watch the \ + `pending` state; the analysis runs in the background. Watch the \ detection's status via the SSE stream at \ `.../detections/{detectionId}/events` (or re-read the detection) and \ fetch the findings from `.../detections/{detectionId}/analysis/` once \ @@ -579,7 +579,7 @@ async fn redact_detection( Path(path_params): Path, security: SecurityContext, Json(request): Json, -) -> Result<(StatusCode, Json)> { +) -> Result<(StatusCode, Json)> { tracing::debug!(target: TRACING_TARGET, "Redacting detection"); let mut conn = pg_client.get_connection().await?; @@ -710,7 +710,7 @@ async fn redact_detection( Ok(( StatusCode::CREATED, - Json(Redaction::from_model( + Json(RedactionResult::from_model( redaction, workspace.slug, requested_by, @@ -728,7 +728,7 @@ fn redact_detection_docs(op: TransformOperation) -> TransformOperation { than once. An edit targeting a detection not in the analysis, or a set that \ contradicts itself, is rejected (400).", ) - .response::<201, Json>() + .response::<201, Json>() .response::<400, Json>() .response::<401, Json>() .response::<403, Json>() diff --git a/crates/nvisy-server/src/handler/pipelines.rs b/crates/nvisy-server/src/handler/pipelines.rs index bd5d1794..5c7d26a7 100644 --- a/crates/nvisy-server/src/handler/pipelines.rs +++ b/crates/nvisy-server/src/handler/pipelines.rs @@ -261,6 +261,9 @@ async fn update_pipeline( let backfill = retention_override.map(|over| { let workspace_retention = workspace.settings.or_default().retention; let now = jiff::Timestamp::now(); + let audit_logs_expiry = workspace_retention + .resolve(RetentionScope::AuditLogs, Some(&over)) + .expires_at(now); [ ( FileKind::Redacted, @@ -268,12 +271,9 @@ async fn update_pipeline( .resolve(RetentionScope::RedactedDocuments, Some(&over)) .expires_at(now), ), - ( - FileKind::Audit, - workspace_retention - .resolve(RetentionScope::AuditLogs, Some(&over)) - .expires_at(now), - ), + (FileKind::Audit, audit_logs_expiry), + // Review audits share the audit-logs scope with detection audits. + (FileKind::Review, audit_logs_expiry), ] }); diff --git a/crates/nvisy-server/src/handler/redactions.rs b/crates/nvisy-server/src/handler/redactions.rs index 3b6a2608..41880e4a 100644 --- a/crates/nvisy-server/src/handler/redactions.rs +++ b/crates/nvisy-server/src/handler/redactions.rs @@ -16,10 +16,8 @@ use uuid::Uuid; use super::detections::find_detection; use crate::extract::{AuthProvider, AuthState, Json, Path, Permission, Query, WorkspaceContext}; -use crate::handler::request::{ - CursorPagination, DetectionPathParams, DetectionRedactionPathParams, -}; -use crate::handler::response::{ErrorResponse, Redaction, RedactionsPage}; +use crate::handler::request::{CursorPagination, DetectionPathParams, RedactionPathParams}; +use crate::handler::response::{ErrorResponse, RedactionResult, RedactionsPage}; use crate::handler::utility::resolve_account_ref; use crate::handler::{ErrorKind, Result, ServiceState}; use crate::service::{EngineService, RunBlobStore}; @@ -65,7 +63,7 @@ async fn list_detection_redactions( let mut items = Vec::with_capacity(page.items.len()); for redaction in page.items { let requested_by = resolve_account_ref(&mut conn, redaction.account_id).await?; - items.push(Redaction::from_model( + items.push(RedactionResult::from_model( redaction, workspace.slug.clone(), requested_by, @@ -96,7 +94,6 @@ fn list_detection_redactions_docs(op: TransformOperation) -> TransformOperation fields( account_id = %auth_state.account_id, workspace_id = %workspace.id, - detection_id = %path_params.detection_id, redaction_id = %path_params.redaction_id, ) )] @@ -106,7 +103,7 @@ async fn get_redaction_review( State(engine): State, AuthState(auth_state): AuthState, WorkspaceContext(workspace): WorkspaceContext, - Path(path_params): Path, + Path(path_params): Path, ) -> Result<(StatusCode, Json)> { tracing::debug!(target: TRACING_TARGET, "Getting redaction review audit"); @@ -116,13 +113,8 @@ async fn get_redaction_review( .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) .await?; - let redaction = find_redaction( - &mut conn, - workspace.id, - path_params.detection_id.as_uuid(), - path_params.redaction_id.as_uuid(), - ) - .await?; + let redaction = + find_redaction(&mut conn, workspace.id, path_params.redaction_id.as_uuid()).await?; let review = blob .load_review_audit(&mut conn, &engine, workspace.id, redaction.review_file_id) @@ -144,18 +136,17 @@ fn get_redaction_review_docs(op: TransformOperation) -> TransformOperation { .response::<409, Json>() } -/// Loads a redaction scoped to its detection and workspace, mapping a missing -/// detection or redaction to a 404. +/// Loads a redaction scoped to the workspace, mapping a missing one to a 404. +/// +/// A [`RedactionId`](nvisy_postgres::types::RedactionId) is globally unique, so +/// the redaction is addressed by id alone and resolved within the workspace via +/// its detection's pipeline. async fn find_redaction( conn: &mut PgConn, workspace_id: Uuid, - detection_id: Uuid, redaction_id: Uuid, ) -> Result { - // Confirm the detection is in this workspace first, so a redaction id cannot - // be probed against detections in another workspace. - find_detection(conn, workspace_id, detection_id).await?; - conn.find_redaction_by_id(detection_id, redaction_id) + conn.find_redaction_in_workspace(workspace_id, redaction_id) .await? .ok_or_else(|| { ErrorKind::NotFound @@ -172,7 +163,7 @@ pub fn routes() -> ApiRouter { get_with(list_detection_redactions, list_detection_redactions_docs), ) .api_route( - "/workspaces/{workspaceSlug}/detections/{detectionId}/redactions/{redactionId}/review", + "/workspaces/{workspaceSlug}/redactions/{redactionId}/review", get_with(get_redaction_review, get_redaction_review_docs), ) .with_path_items(|item| item.tag("Redactions")) diff --git a/crates/nvisy-server/src/handler/request/paths.rs b/crates/nvisy-server/src/handler/request/paths.rs index b1167b2a..5c0f70a3 100644 --- a/crates/nvisy-server/src/handler/request/paths.rs +++ b/crates/nvisy-server/src/handler/request/paths.rs @@ -92,13 +92,14 @@ pub struct DetectionPathParams { pub detection_id: DetectionId, } -/// Path parameters for a redaction nested under its detection. +/// Path parameters for a redaction. +/// +/// The redaction id is globally unique, so a redaction is addressed by id alone +/// and resolved within the workspace by the query. #[must_use] #[derive(Debug, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] -pub struct DetectionRedactionPathParams { - /// Opaque identifier of the detection. - pub detection_id: DetectionId, +pub struct RedactionPathParams { /// Opaque identifier of the redaction. pub redaction_id: RedactionId, } diff --git a/crates/nvisy-server/src/handler/response/redactions.rs b/crates/nvisy-server/src/handler/response/redactions.rs index 9dad3a27..be26d66a 100644 --- a/crates/nvisy-server/src/handler/response/redactions.rs +++ b/crates/nvisy-server/src/handler/response/redactions.rs @@ -15,9 +15,13 @@ use super::{AccountRef, Page}; /// of reviewer edits. It owns the redacted output document (downloadable through /// the normal file endpoints) and a review audit recording what was redacted and /// why (fetched from the redaction's `review` endpoint). +/// +/// Named `RedactionResult` rather than `Redaction` because the engine's audit +/// schema already carries a `Redaction` (an audit event), and the two must not +/// collide in the generated OpenAPI. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] -pub struct Redaction { +pub struct RedactionResult { /// Opaque identifier of the redaction. pub id: RedactionId, /// The detection this redaction was produced from. @@ -35,9 +39,9 @@ pub struct Redaction { } /// Paginated response for redactions. -pub type RedactionsPage = Page; +pub type RedactionsPage = Page; -impl Redaction { +impl RedactionResult { /// Creates a redaction response from the database model, the owning /// workspace slug, and the requesting account. pub fn from_model( diff --git a/crates/nvisy-server/src/handler/workspaces.rs b/crates/nvisy-server/src/handler/workspaces.rs index 2191f21d..9d207bc5 100644 --- a/crates/nvisy-server/src/handler/workspaces.rs +++ b/crates/nvisy-server/src/handler/workspaces.rs @@ -51,6 +51,9 @@ async fn backfill_retention( (RetentionScope::OriginalDocuments, FileKind::Original), (RetentionScope::RedactedDocuments, FileKind::Redacted), (RetentionScope::AuditLogs, FileKind::Audit), + // Review audits share the audit-logs scope with detection audits; a + // redaction stages them under `AuditLogs`, so they backfill under it too. + (RetentionScope::AuditLogs, FileKind::Review), ] { let expires_at = retention.get(scope).expires_at(now); conn.backfill_files_expiry(workspace_id, kind, expires_at) diff --git a/crates/nvisy-server/src/service/detection/support.rs b/crates/nvisy-server/src/service/detection/support.rs index cddc6579..809d3185 100644 --- a/crates/nvisy-server/src/service/detection/support.rs +++ b/crates/nvisy-server/src/service/detection/support.rs @@ -154,10 +154,11 @@ pub(crate) async fn fail_detection( error: Some(reason.to_owned()), ..Default::default() }; + // `status` and `completed_at` are forced by `fail_detection` itself (the + // terminal transition owns the terminal timestamp), so only the failure + // reason is supplied here. let update = UpdateWorkspaceDetection { - status: Some(DetectionStatus::Failed), metadata: Some(Json::encode(&metadata)), - completed_at: Some(Some(jiff::Timestamp::now().into())), ..Default::default() }; @@ -175,12 +176,20 @@ pub(crate) async fn fail_detection( return; } }, - // Handler path: no claim to guard. - None => { - if let Err(err) = conn.update_workspace_detection(detection_id, update).await { + // Handler path (enqueue failure): guard on `Pending` so this no-ops if a + // worker already claimed the detection — enqueue can report an error even + // when the job was delivered, and the worker then owns the outcome. + None => match conn.fail_pending_detection(detection_id, update).await { + Ok(true) => {} + Ok(false) => { + tracing::warn!(target: TRACING_TARGET, %detection_id, "Detection was already claimed before enqueue-failure handling; the worker owns it"); + return; + } + Err(err) => { tracing::warn!(target: TRACING_TARGET, error = %err, %detection_id, "Failed to mark detection failed"); + return; } - } + }, } detection diff --git a/crates/nvisy-server/src/service/run_blob_store.rs b/crates/nvisy-server/src/service/run_blob_store.rs index 39fa31c5..0d82460d 100644 --- a/crates/nvisy-server/src/service/run_blob_store.rs +++ b/crates/nvisy-server/src/service/run_blob_store.rs @@ -178,64 +178,6 @@ impl RunBlobStore { Ok(Document::new(bytes, file.file_extension.clone()).with_correlation_id(correlation_id)) } - /// Stores redacted bytes as a new workspace file (the run's output). - /// - /// The redacted file is a first-class file — a sibling of the source — so it - /// is downloadable through the normal file endpoints. - pub async fn store_redacted_file( - &self, - conn: &mut PgConn, - source: &WorkspaceFile, - pipeline: &WorkspacePipeline, - workspace_settings: &RetentionSettings, - account_id: Uuid, - bytes: Bytes, - ) -> Result { - // Record the plaintext size and hash before encrypting; storage holds - // only the ciphertext. - let plaintext_size = bytes.len() as i64; - let plaintext_hash = Sha256::digest(&bytes).to_vec(); - let ciphertext = self - .infra - .crypto - .encrypt(source.workspace_id, &bytes) - .map_err(|err| { - ErrorKind::InternalServerError - .with_message("Failed to encrypt redacted file") - .with_context(err.to_string()) - })?; - - let store = self.infra.nats.object_store::().await?; - let key = FileKey::generate(source.workspace_id); - store.put(&key, Cursor::new(ciphertext)).await?; - - // Retention expiry for the redacted-documents scope (workspace baseline, - // pipeline override if set). - let over = pipeline.metadata.or_default().retention; - let expires_at = workspace_settings - .resolve(RetentionScope::RedactedDocuments, over.as_ref()) - .expires_at(jiff::Timestamp::now()); - - let redacted_name = redacted_display_name(&source.display_name, &source.file_extension); - let new_file = NewWorkspaceFile { - workspace_id: source.workspace_id, - account_id, - parent_id: Some(source.id), - display_name: Some(redacted_name), - original_filename: Some(source.original_filename.clone()), - file_extension: Some(source.file_extension.clone()), - file_kind: Some(FileKind::Redacted), - file_size_bytes: plaintext_size, - file_hash_sha256: plaintext_hash, - storage_path: key.to_string(), - storage_bucket: store.bucket().to_owned(), - expires_at: expires_at.map(Into::into), - ..Default::default() - }; - - Ok(conn.create_workspace_file(new_file).await?) - } - /// Encrypts the analysis, writes it to the audit bucket, and builds the /// `audit`-kind [`WorkspaceFile`] row that will point at it — but does not /// insert the row. @@ -445,11 +387,12 @@ impl RunBlobStore { /// `redacted`-kind [`WorkspaceFile`] row that will point at them — without /// inserting the row. /// - /// The staged counterpart to - /// [`store_redacted_file`](Self::store_redacted_file): a redaction commits its - /// output row, review-audit row, and redaction row together in one - /// transaction, so the output file is staged rather than inserted here. The - /// redacted file is a first-class file (a sibling of the source), downloadable + /// A redaction commits its output row, review-audit row, and redaction row + /// together in one transaction, so the output file is staged (object written, + /// row returned for the caller to insert) rather than inserted here, and is + /// reclaimed on rollback via + /// [`discard_staged_object`](Self::discard_staged_object). The redacted file + /// is a first-class file (a sibling of the source), downloadable /// through the normal file endpoints. pub async fn stage_redacted_file( &self, diff --git a/migrations/2025-05-21-121132_notifications/up.sql b/migrations/2025-05-21-121132_notifications/up.sql index 3832be8b..a2c0c56e 100644 --- a/migrations/2025-05-21-121132_notifications/up.sql +++ b/migrations/2025-05-21-121132_notifications/up.sql @@ -1,18 +1,18 @@ -- Notifications: per-account notification inbox for member, connection-sync, --- and pipeline-run events. Account-scoped but a standalone feature; the client --- renders copy from the event type and its typed params. +-- detection, and redaction events. Account-scoped but a standalone feature; the +-- client renders copy from the event type and its typed params. -- Type of a notification event: what happened that the account is told about. CREATE TYPE NOTIFICATION_EVENT AS ENUM ( - 'member.invited', -- User was invited to a workspace - 'member.joined', -- A new member joined a workspace + 'member.invited', -- User was invited to a workspace + 'member.joined', -- A new member joined a workspace - 'connection.sync.completed', -- A connection sync completed - 'connection.sync.failed', -- A connection sync failed + 'connection.sync.completed', -- A connection sync completed + 'connection.sync.failed', -- A connection sync failed - 'pipeline.run.analyzed', -- A run finished detection, awaiting review - 'pipeline.run.completed', -- A run completed (redaction produced) - 'pipeline.run.failed' -- A run failed + 'pipeline.detection.completed', -- A detection finished analysis, ready to redact + 'pipeline.redaction.created', -- A redaction was created (redacted output produced) + 'pipeline.detection.failed' -- A detection failed ); COMMENT ON TYPE NOTIFICATION_EVENT IS 'Type of a notification event delivered to an account.'; diff --git a/migrations/2025-05-21-222840_workspaces/up.sql b/migrations/2025-05-21-222840_workspaces/up.sql index 72a5adf0..def8bae9 100644 --- a/migrations/2025-05-21-222840_workspaces/up.sql +++ b/migrations/2025-05-21-222840_workspaces/up.sql @@ -117,7 +117,7 @@ CREATE TABLE workspace_members ( notification_events_app NOTIFICATION_EVENT[] NOT NULL DEFAULT ARRAY[ 'member.invited', 'member.joined', 'connection.sync.completed', 'connection.sync.failed', - 'pipeline.run.analyzed', 'pipeline.run.completed', 'pipeline.run.failed' + 'pipeline.detection.completed', 'pipeline.redaction.created', 'pipeline.detection.failed' ]::NOTIFICATION_EVENT[], notification_events_email NOTIFICATION_EVENT[] NOT NULL DEFAULT '{}', diff --git a/migrations/2025-05-21-222841_activities/up.sql b/migrations/2025-05-21-222841_activities/up.sql index 67fd3c48..8df66a14 100644 --- a/migrations/2025-05-21-222841_activities/up.sql +++ b/migrations/2025-05-21-222841_activities/up.sql @@ -38,14 +38,14 @@ CREATE TYPE ACTIVITY_TYPE AS ENUM ( 'file.updated', 'file.deleted', - -- Pipeline activities + -- Pipeline, detection, and redaction activities 'pipeline.created', 'pipeline.updated', 'pipeline.deleted', - 'pipeline.run.started', - 'pipeline.run.analyzed', - 'pipeline.run.completed', - 'pipeline.run.failed', + 'pipeline.detection.started', + 'pipeline.detection.completed', + 'pipeline.detection.failed', + 'pipeline.redaction.created', -- Policy activities 'policy.created', diff --git a/migrations/2025-05-21-222842_webhooks/up.sql b/migrations/2025-05-21-222842_webhooks/up.sql index 1d1bc8c7..387e6189 100644 --- a/migrations/2025-05-21-222842_webhooks/up.sql +++ b/migrations/2025-05-21-222842_webhooks/up.sql @@ -32,14 +32,14 @@ CREATE TYPE WEBHOOK_EVENT AS ENUM ( 'connection.sync.completed', 'connection.sync.failed', - -- Pipeline events + -- Pipeline, detection, and redaction events 'pipeline.created', 'pipeline.updated', 'pipeline.deleted', - 'pipeline.run.started', - 'pipeline.run.analyzed', - 'pipeline.run.completed', - 'pipeline.run.failed', + 'pipeline.detection.started', + 'pipeline.detection.completed', + 'pipeline.detection.failed', + 'pipeline.redaction.created', -- Policy events 'policy.created', From b32b46a7f11ed0de51c97bcc90ecdba19966ffaf Mon Sep 17 00:00:00 2001 From: Oleh Martsokha Date: Wed, 26 Aug 2026 15:56:58 +0200 Subject: [PATCH 3/7] Address PR review: redaction id, retention holds, analytics naming Fixes from the CodeRabbit review on the detections/redactions PR. - Carry the real redaction id in the RedactionCreated event and thread it through the activity and notification projections, so repeatable redactions are identified correctly (was substituting the detection id). - Stop holding a Complete detection's input and audit files from expiry forever: Complete is terminal, so those files now expire per their own retention; re-redaction is bounded by that. Add DetectionStatus::TERMINAL / IN_PROGRESS consts (renamed from OUTCOMES). - Preserve existing detection metadata (e.g. tags) on the failure path by layering the error onto the passed metadata instead of replacing the blob. - Reclaim the staged redacted output when review-audit staging fails, before the early return, so no object is stranded without a row. - Match the redacted-file extension case-insensitively on both sides (report.PDF -> report.redacted.PDF), with tests. - Rename the analytics Run* family to Detection* (types, fields, internal helpers, the daily-series trait method) and the last route /analytics/runs/timeseries/ -> /analytics/detections/timeseries/, so no run terminology remains. - Extract the duplicated detection-listing filter block into a scoped closure; add a partial index on workspace_detections.audit_file_id for the expiry sweep; align the redaction file-column comment with their nullability; fix the create-detection doc (pending, not executing). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018bKk1YEG4tZ69jzYVQvQL8 --- crates/nvisy-postgres/src/query/analytics.rs | 223 +++++++++--------- crates/nvisy-postgres/src/query/mod.rs | 4 +- .../src/query/workspace_detection.rs | 72 +++--- .../src/query/workspace_file.rs | 37 ++- .../src/types/enums/detection_status.rs | 13 +- crates/nvisy-server/src/handler/analytics.rs | 22 +- crates/nvisy-server/src/handler/detections.rs | 15 +- .../src/handler/response/analytics.rs | 107 +++++---- .../src/service/detection/support.rs | 17 +- .../src/service/detection/worker.rs | 1 + .../nvisy-server/src/service/event/drainer.rs | 20 +- .../src/service/event/workspace_event.rs | 3 + .../src/service/run_blob_store.rs | 26 +- migrations/2026-01-19-045014_pipelines/up.sql | 16 +- 14 files changed, 309 insertions(+), 267 deletions(-) diff --git a/crates/nvisy-postgres/src/query/analytics.rs b/crates/nvisy-postgres/src/query/analytics.rs index f07765f9..727c10b2 100644 --- a/crates/nvisy-postgres/src/query/analytics.rs +++ b/crates/nvisy-postgres/src/query/analytics.rs @@ -1,13 +1,13 @@ -//! Workspace analytics: aggregate queries over a workspace's files and pipeline -//! runs, for the analytics endpoint. +//! Workspace analytics: aggregate queries over a workspace's files and +//! detections, for the analytics endpoint. //! //! Two entry points, each reading a consistent snapshot in one read-only -//! transaction: `snapshot` (storage, run health, token usage) and `runs_by_day` -//! (the daily time series). Both compose several grouped aggregates from private -//! `load_*` helpers. Rows come back sparse (one per group that has data, never -//! zero-filled here) — the handler maps them onto the full set of enum values, -//! filling absent groups with zero, so the response is stable regardless of what -//! data exists. +//! transaction: `snapshot` (storage, detection health, token usage) and +//! `detections_by_day` (the daily time series). Both compose several grouped +//! aggregates from private `load_*` helpers. Rows come back sparse (one per group +//! that has data, never zero-filled here) — the handler maps them onto the full +//! set of enum values, filling absent groups with zero, so the response is stable +//! regardless of what data exists. use std::future::Future; @@ -20,11 +20,12 @@ use uuid::Uuid; use crate::types::{DetectionStatus, FileKind}; use crate::{PgConnection, PgError, PgResult, schema}; -/// Per-day run counts and durations, as loaded from the grouped run query. +/// Per-day detection counts and durations, as loaded from the grouped detection +/// query. #[derive(Debug, Clone, Queryable)] -struct RunDayCounts { +struct DetectionDayCounts { day: jiff_diesel::Timestamp, - runs: i64, + detections: i64, // Conditional counts are `sum(CASE WHEN cond THEN 1 ELSE 0 END)`; Postgres // `sum` of bigint returns numeric, hence BigDecimal (nullable over no rows). // Converted to i64 when assembling the point. @@ -37,15 +38,15 @@ struct RunDayCounts { /// Per-day token totals, as loaded from the grouped usage query. #[derive(Debug, Clone, Queryable)] -struct RunDayTokens { +struct DetectionDayTokens { day: jiff_diesel::Timestamp, input: Option, output: Option, total: Option, } -/// A day's token totals converted to `i64`, keyed by day while merging the run -/// and usage series. +/// A day's token totals converted to `i64`, keyed by day while merging the +/// detection and usage series. #[derive(Debug, Clone, Copy, Default)] struct DayTokens { input: Option, @@ -53,28 +54,28 @@ struct DayTokens { total: Option, } -/// One day of pipeline-run activity for a workspace. Only days with at least one -/// run are produced; the caller fills the gaps to a dense series over its window. -/// `day` is the UTC day (midnight) the run counts fall in. +/// One day of detection activity for a workspace. Only days with at least one +/// detection are produced; the caller fills the gaps to a dense series over its +/// window. `day` is the UTC day (midnight) the detection counts fall in. #[derive(Debug, Clone)] -pub struct RunDayPoint { +pub struct DetectionDayPoint { /// The day (UTC midnight) this point covers. pub day: jiff::Timestamp, - /// Runs started on this day. - pub runs: i64, - /// Runs that reached a terminal state (`completed` or `failed`) this day. + /// Detections started on this day. + pub detections: i64, + /// Detections that reached a terminal state (`complete` or `failed`) this day. pub terminal: i64, - /// Runs that failed this day. + /// Detections that failed this day. pub failed: i64, - /// Mean duration (milliseconds) of runs that completed this day; null if none did. + /// Mean duration (milliseconds) of detections that completed this day; null if none did. pub avg_ms: Option, - /// 95th-percentile duration (milliseconds) of runs completed this day; null if none. + /// 95th-percentile duration (milliseconds) of detections completed this day; null if none. pub p95_ms: Option, - /// Input/prompt tokens across models used by this day's runs; null if none. + /// Input/prompt tokens across models used by this day's detections; null if none. pub input_tokens: Option, - /// Output/completion tokens across models used by this day's runs; null if none. + /// Output/completion tokens across models used by this day's detections; null if none. pub output_tokens: Option, - /// Reported total tokens across models used by this day's runs; null if none. + /// Reported total tokens across models used by this day's detections; null if none. pub total_tokens: Option, } @@ -88,14 +89,14 @@ struct UsageByModelRow { total_tokens: Option, } -/// Inference token totals for one model across a workspace's runs. Each field is -/// independently nullable because a provider may report only some of them (a -/// `total` that is not `input + output`, or no breakdown at all). +/// Inference token totals for one model across a workspace's detections. Each +/// field is independently nullable because a provider may report only some of +/// them (a `total` that is not `input + output`, or no breakdown at all). #[derive(Debug, Clone)] pub struct UsageByModel { /// The model these totals are for. pub model: String, - /// Summed input/prompt tokens, or `None` if no run reported them for this model. + /// Summed input/prompt tokens, or `None` if no detection reported them for this model. pub input_tokens: Option, /// Summed output/completion tokens, or `None` if none reported. pub output_tokens: Option, @@ -126,43 +127,43 @@ pub struct StorageByKind { /// Detection count for one `status` in a workspace. #[derive(Debug, Clone, Queryable)] -pub struct RunStatusCount { +pub struct DetectionStatusCount { /// The detection status this row aggregates. pub status: DetectionStatus, /// Number of detections in this status. pub count: i64, } -/// Completed-run duration summary for a workspace, in milliseconds. Both are -/// `None` until at least one run has completed. +/// Completed-detection duration summary for a workspace, in milliseconds. Both +/// are `None` until at least one detection has completed. #[derive(Debug, Clone)] -pub struct RunDurations { - /// Mean wall-clock duration of completed runs. +pub struct DetectionDurations { + /// Mean wall-clock duration of completed detections. pub avg_ms: Option, - /// 95th-percentile duration of completed runs. + /// 95th-percentile duration of completed detections. pub p95_ms: Option, } -/// A workspace's point-in-time analytics: storage, run health, and token usage -/// read together so the parts agree. Produced by +/// A workspace's point-in-time analytics: storage, detection health, and token +/// usage read together so the parts agree. Produced by /// [`snapshot`](WorkspaceAnalyticsRepository::snapshot). #[derive(Debug, Clone)] pub struct AnalyticsSnapshot { /// Live-file counts and byte totals, one per `file_kind` present. pub storage: Vec, - /// Run counts, one per `status` present. - pub runs: Vec, - /// Completed-run duration summary. - pub durations: RunDurations, + /// Detection counts, one per `status` present. + pub detections: Vec, + /// Completed-detection duration summary. + pub durations: DetectionDurations, /// Inference token totals, one per model used. pub usage: Vec, } /// Read-only aggregate queries backing the workspace analytics endpoint. pub trait WorkspaceAnalyticsRepository { - /// A workspace's point-in-time analytics — storage by kind, run counts by - /// status, completed-run durations, and per-model token usage — read in one - /// read-only, repeatable-read transaction so the parts reflect a single + /// A workspace's point-in-time analytics — storage by kind, detection counts + /// by status, completed-detection durations, and per-model token usage — read + /// in one read-only, repeatable-read transaction so the parts reflect a single /// snapshot. Each breakdown lists only the groups that have data; the caller /// zero-fills the rest. fn snapshot( @@ -170,66 +171,67 @@ pub trait WorkspaceAnalyticsRepository { workspace_id: Uuid, ) -> impl Future> + Send; - /// Daily pipeline-run activity for a workspace over `[from, to)` (UTC day + /// Daily detection activity for a workspace over `[from, to)` (UTC day /// boundaries; `to` exclusive). Returns one row per day that has at least one - /// run — the series is sparse — so the caller gap-fills the window to a dense - /// series (see `RunTimeSeries::from_window`). The window is bucketed by - /// `date_trunc('day', started_at)`; runs are scoped through their live - /// pipeline. The caller is responsible for bounding the window. + /// detection — the series is sparse — so the caller gap-fills the window to a + /// dense series (see `DetectionTimeSeries::from_window`). The window is + /// bucketed by `date_trunc('day', started_at)`; detections are scoped through + /// their live pipeline. The caller is responsible for bounding the window. /// - /// Run counts, durations, and token totals are read in one transaction so the - /// two grouped statements observe the same snapshot. - fn runs_by_day( + /// Detection counts, durations, and token totals are read in one transaction + /// so the two grouped statements observe the same snapshot. + fn detections_by_day( &mut self, workspace_id: Uuid, from: jiff::Timestamp, to: jiff::Timestamp, - ) -> impl Future>> + Send; + ) -> impl Future>> + Send; } impl WorkspaceAnalyticsRepository for PgConnection { async fn snapshot(&mut self, workspace_id: Uuid) -> PgResult { // The four reads run in one read-only, repeatable-read transaction so the - // snapshot is internally consistent: a run cannot appear in the status - // counts while its tokens are missing from the usage totals because a - // write landed between two of the reads. + // snapshot is internally consistent: a detection cannot appear in the + // status counts while its tokens are missing from the usage totals because + // a write landed between two of the reads. self.build_transaction() .read_only() .repeatable_read() .run(async |conn| { Ok(AnalyticsSnapshot { storage: load_storage_by_kind(conn, workspace_id).await?, - runs: load_runs_by_status(conn, workspace_id).await?, - durations: load_run_durations(conn, workspace_id).await?, + detections: load_detections_by_status(conn, workspace_id).await?, + durations: load_detection_durations(conn, workspace_id).await?, usage: load_usage_by_model(conn, workspace_id).await?, }) }) .await } - async fn runs_by_day( + async fn detections_by_day( &mut self, workspace_id: Uuid, from: jiff::Timestamp, to: jiff::Timestamp, - ) -> PgResult> { + ) -> PgResult> { use std::collections::BTreeMap; let from = jiff_diesel::Timestamp::from(from); let to = jiff_diesel::Timestamp::from(to); - // Read the run counts/durations and the token totals in one read-only, - // repeatable-read transaction so both grouped statements observe the same - // snapshot; otherwise a run written between them could show in one series - // but not the other. - let (run_rows, token_rows): (Vec, Vec) = self + // Read the detection counts/durations and the token totals in one + // read-only, repeatable-read transaction so both grouped statements + // observe the same snapshot; otherwise a detection written between them + // could show in one series but not the other. + let (detection_rows, token_rows): (Vec, Vec) = self .build_transaction() .read_only() .repeatable_read() .run(async |conn| { - let run_rows = load_run_day_counts(conn, workspace_id, from, to).await?; - let token_rows = load_run_day_tokens(conn, workspace_id, from, to).await?; - Ok::<_, PgError>((run_rows, token_rows)) + let detection_rows = + load_detection_day_counts(conn, workspace_id, from, to).await?; + let token_rows = load_detection_day_tokens(conn, workspace_id, from, to).await?; + Ok::<_, PgError>((detection_rows, token_rows)) }) .await?; @@ -248,14 +250,14 @@ impl WorkspaceAnalyticsRepository for PgConnection { ); } - let points = run_rows + let points = detection_rows .into_iter() .map(|row| { let day: jiff::Timestamp = row.day.into(); let t = tokens.remove(&day).unwrap_or_default(); - RunDayPoint { + DetectionDayPoint { day, - runs: row.runs, + detections: row.detections, terminal: to_i64(row.terminal).unwrap_or(0), failed: to_i64(row.failed).unwrap_or(0), avg_ms: row.avg_ms, @@ -304,12 +306,12 @@ async fn load_storage_by_kind( .collect()) } -/// Run count per `status`, scoped through the live pipeline. Only statuses with -/// a run appear. -async fn load_runs_by_status( +/// Detection count per `status`, scoped through the live pipeline. Only statuses +/// with a detection appear. +async fn load_detections_by_status( conn: &mut PgConnection, workspace_id: Uuid, -) -> PgResult> { +) -> PgResult> { use diesel::dsl::count_star; use schema::workspace_detections::dsl as detections; use schema::workspace_pipelines::dsl as pipelines; @@ -327,8 +329,11 @@ async fn load_runs_by_status( } /// Mean and 95th-percentile duration (milliseconds) of the workspace's completed -/// runs. Both `None` when no run has completed. -async fn load_run_durations(conn: &mut PgConnection, workspace_id: Uuid) -> PgResult { +/// detections. Both `None` when no detection has completed. +async fn load_detection_durations( + conn: &mut PgConnection, + workspace_id: Uuid, +) -> PgResult { use diesel::dsl::sql; use diesel::sql_types::{BigInt, Nullable}; use schema::workspace_detections::dsl as detections; @@ -360,12 +365,12 @@ async fn load_run_durations(conn: &mut PgConnection, workspace_id: Uuid) -> PgRe .await .map_err(PgError::from)?; - Ok(RunDurations { avg_ms, p95_ms }) + Ok(DetectionDurations { avg_ms, p95_ms }) } -/// Inference token totals per model across the workspace's runs, scoped through -/// the live pipeline. Only models actually used appear. Kept per-model rather -/// than summed to one total, since a run may mix models. +/// Inference token totals per model across the workspace's detections, scoped +/// through the live pipeline. Only models actually used appear. Kept per-model +/// rather than summed to one total, since a detection may mix models. async fn load_usage_by_model( conn: &mut PgConnection, workspace_id: Uuid, @@ -405,24 +410,24 @@ async fn load_usage_by_model( .collect()) } -/// The day bucket a run's start falls in. Shared by both daily queries so the -/// truncation text cannot drift — they must group on the same expression for the -/// per-day merge to line up. The column is table-qualified so the join to +/// The day bucket a detection's start falls in. Shared by both daily queries so +/// the truncation text cannot drift — they must group on the same expression for +/// the per-day merge to line up. The column is table-qualified so the join to /// pipelines can never make it ambiguous. Returns a fresh fragment per call, as /// the builder consumes it in both `group_by` and `select`. -fn run_day() -> diesel::expression::SqlLiteral { +fn detection_day() -> diesel::expression::SqlLiteral { diesel::dsl::sql::("date_trunc('day', workspace_detections.started_at)") } -/// Per-day run counts and durations over `[from, to)`, scoped through the live -/// pipeline. Sparse: only days that have a run. Shared with the token query so -/// both observe the same snapshot inside one transaction. -async fn load_run_day_counts( +/// Per-day detection counts and durations over `[from, to)`, scoped through the +/// live pipeline. Sparse: only days that have a detection. Shared with the token +/// query so both observe the same snapshot inside one transaction. +async fn load_detection_day_counts( conn: &mut PgConnection, workspace_id: Uuid, from: jiff_diesel::Timestamp, to: jiff_diesel::Timestamp, -) -> PgResult> { +) -> PgResult> { use diesel::dsl::{case_when, count_star, sql, sum}; use diesel::sql_types::{BigInt, Nullable as SqlNullable}; use schema::workspace_detections::dsl as detections; @@ -454,12 +459,12 @@ async fn load_run_day_counts( .filter(pipelines::deleted_at.is_null()) .filter(detections::started_at.ge(from)) .filter(detections::started_at.lt(to)) - .group_by(run_day()) + .group_by(detection_day()) .select(( - run_day(), + detection_day(), count_star(), sum(case_when::<_, _, BigInt>( - detections::status.eq_any(DetectionStatus::OUTCOMES), + detections::status.eq_any(DetectionStatus::TERMINAL), 1i64, ) .otherwise(0i64)), @@ -476,31 +481,31 @@ async fn load_run_day_counts( } /// Per-day inference token totals over `[from, to)`, scoped through the live -/// pipeline. Usage is per-model-per-run, so each token field is summed per run -/// first (a correlated subquery) before day-grouping, otherwise a run's multiple -/// model rows would multiply the day totals. Sparse: only days with a run (token -/// sums are null on days with no usage). -async fn load_run_day_tokens( +/// pipeline. Usage is per-model-per-detection, so each token field is summed per +/// detection first (a correlated subquery) before day-grouping, otherwise a +/// detection's multiple model rows would multiply the day totals. Sparse: only +/// days with a detection (token sums are null on days with no usage). +async fn load_detection_day_tokens( conn: &mut PgConnection, workspace_id: Uuid, from: jiff_diesel::Timestamp, to: jiff_diesel::Timestamp, -) -> PgResult> { +) -> PgResult> { use diesel::dsl::sum; use schema::workspace_detection_usage::dsl as usage; use schema::workspace_detections::dsl as detections; use schema::workspace_pipelines::dsl as pipelines; use schema::{workspace_detection_usage, workspace_detections, workspace_pipelines}; - let per_run_input = workspace_detection_usage::table + let per_detection_input = workspace_detection_usage::table .filter(usage::detection_id.eq(detections::id)) .select(sum(usage::input_tokens)) .single_value(); - let per_run_output = workspace_detection_usage::table + let per_detection_output = workspace_detection_usage::table .filter(usage::detection_id.eq(detections::id)) .select(sum(usage::output_tokens)) .single_value(); - let per_run_total = workspace_detection_usage::table + let per_detection_total = workspace_detection_usage::table .filter(usage::detection_id.eq(detections::id)) .select(sum(usage::total_tokens)) .single_value(); @@ -511,12 +516,12 @@ async fn load_run_day_tokens( .filter(pipelines::deleted_at.is_null()) .filter(detections::started_at.ge(from)) .filter(detections::started_at.lt(to)) - .group_by(run_day()) + .group_by(detection_day()) .select(( - run_day(), - sum(per_run_input), - sum(per_run_output), - sum(per_run_total), + detection_day(), + sum(per_detection_input), + sum(per_detection_output), + sum(per_detection_total), )) .load(conn) .await diff --git a/crates/nvisy-postgres/src/query/mod.rs b/crates/nvisy-postgres/src/query/mod.rs index 98a3e94c..8e436400 100644 --- a/crates/nvisy-postgres/src/query/mod.rs +++ b/crates/nvisy-postgres/src/query/mod.rs @@ -40,8 +40,8 @@ pub use account::AccountRepository; pub use account_api_token::AccountApiTokenRepository; pub use account_notification::AccountNotificationRepository; pub use analytics::{ - AnalyticsSnapshot, RunDayPoint, RunDurations, RunStatusCount, StorageByKind, UsageByModel, - WorkspaceAnalyticsRepository, + AnalyticsSnapshot, DetectionDayPoint, DetectionDurations, DetectionStatusCount, StorageByKind, + UsageByModel, WorkspaceAnalyticsRepository, }; pub use chat_message::{AppendSessionUpdate, ChatMessageRepository}; pub use chat_session::ChatSessionRepository; diff --git a/crates/nvisy-postgres/src/query/workspace_detection.rs b/crates/nvisy-postgres/src/query/workspace_detection.rs index 0d2c8b22..270d1130 100644 --- a/crates/nvisy-postgres/src/query/workspace_detection.rs +++ b/crates/nvisy-postgres/src/query/workspace_detection.rs @@ -251,28 +251,37 @@ impl WorkspaceDetectionRepository for PgConnection { use schema::workspace_detections::dsl; use schema::{accounts, workspace_detections, workspace_files, workspace_pipelines}; - // Build base query with filters. The listing is already scoped to one - // pipeline, so `filter.pipeline_id` is not applied here. - let mut base_query = workspace_detections::table - .filter(dsl::pipeline_id.eq(pipeline_id)) - .into_boxed(); - - if let Some(status) = filter.status { - base_query = base_query.filter(dsl::status.eq(status)); - } - if let Some(file_id) = filter.input_file_id { - base_query = base_query.filter(dsl::input_file_id.eq(file_id)); - } - if let Some(account_id) = filter.account_id { - base_query = base_query.filter(dsl::account_id.eq(account_id)); - } - if let Some(trigger_type) = filter.trigger_type { - base_query = base_query.filter(dsl::trigger_type.eq(trigger_type)); - } + // One scoped builder for both the count and the page, so a future filter + // cannot be added to one and forgotten on the other. The listing is + // already scoped to one pipeline, so `filter.pipeline_id` is not applied. + // Join the owning pipeline (for its slug) and the input file (to name the + // detection's analyzed document) so a row is self-contained; the file is + // LEFT-joined so one removed by retention yields a null name. + let scoped = || { + let mut query = workspace_detections::table + .inner_join(accounts::table) + .inner_join(workspace_pipelines::table) + .left_join(workspace_files::table.on(dsl::input_file_id.eq(workspace_files::id))) + .filter(dsl::pipeline_id.eq(pipeline_id)) + .into_boxed(); + if let Some(status) = filter.status { + query = query.filter(dsl::status.eq(status)); + } + if let Some(file_id) = filter.input_file_id { + query = query.filter(dsl::input_file_id.eq(file_id)); + } + if let Some(account_id) = filter.account_id { + query = query.filter(dsl::account_id.eq(account_id)); + } + if let Some(trigger_type) = filter.trigger_type { + query = query.filter(dsl::trigger_type.eq(trigger_type)); + } + query + }; let total = if pagination.include_count { Some( - base_query + scoped() .count() .get_result::(self) .await @@ -282,30 +291,7 @@ impl WorkspaceDetectionRepository for PgConnection { None }; - // Rebuild query for fetching items. Join the owning pipeline (for its - // slug) and the input file (to name the detection's analyzed document) so - // a row is self-contained; a LEFT JOIN on the file tolerates one removed - // by retention, yielding a null name. - let mut query = workspace_detections::table - .inner_join(accounts::table) - .inner_join(workspace_pipelines::table) - .left_join(workspace_files::table.on(dsl::input_file_id.eq(workspace_files::id))) - .filter(dsl::pipeline_id.eq(pipeline_id)) - .into_boxed(); - - if let Some(status) = filter.status { - query = query.filter(dsl::status.eq(status)); - } - if let Some(file_id) = filter.input_file_id { - query = query.filter(dsl::input_file_id.eq(file_id)); - } - if let Some(account_id) = filter.account_id { - query = query.filter(dsl::account_id.eq(account_id)); - } - if let Some(trigger_type) = filter.trigger_type { - query = query.filter(dsl::trigger_type.eq(trigger_type)); - } - + let query = scoped(); let limit = pagination.fetch_limit(); let selection = ( WorkspaceDetection::as_select(), diff --git a/crates/nvisy-postgres/src/query/workspace_file.rs b/crates/nvisy-postgres/src/query/workspace_file.rs index 68c935e8..c4c57ede 100644 --- a/crates/nvisy-postgres/src/query/workspace_file.rs +++ b/crates/nvisy-postgres/src/query/workspace_file.rs @@ -135,8 +135,8 @@ pub trait WorkspaceFileRepository { ) -> impl Future> + Send; /// Recomputes `expires_at` for live files of `kind` produced by a specific - /// pipeline's runs (redacted outputs via `output_file_id`, audit blobs via - /// `audit_file_id`), returning the number updated. Used to backfill when a + /// pipeline's detections and redactions (detection audits, redaction outputs, + /// and review audits), returning the number updated. Used to backfill when a /// pipeline's own retention override changes, without touching other /// pipelines' files. `None` clears the expiry. fn backfill_pipeline_files_expiry( @@ -365,26 +365,23 @@ impl WorkspaceFileRepository for PgConnection { use schema::workspace_detections::dsl as detections; use schema::{workspace_detections, workspace_files}; - // A detection that is still pending, executing, or complete needs its - // input document and audit blob: pending/executing may still analyze - // them, and a complete detection can still be redacted (redaction reads - // the audit and input). So those files are held back from expiry until the - // detection fails. Otherwise an in-flight detect/redact could lose its - // source or analysis mid-flight and get stuck. Redaction outputs are not - // protected here — they belong to redactions, not the detection. + // A detection that is still analyzing (pending or executing) needs its + // input document and audit blob, so those files are held back from expiry + // until analysis reaches a terminal state — otherwise an in-flight detect + // could lose its source or analysis mid-flight and get stuck. A `Complete` + // detection is NOT held: it is terminal (redaction never changes its + // status), so holding it would pin the input and audit forever and defeat + // retention. Re-redaction of a complete detection is bounded by those + // files' own `expires_at` — retention itself decides how long they remain + // redactable. Redaction outputs are not protected here — they belong to + // redactions, not the detection. let active_detection_holds_file = exists( workspace_detections::table.filter( - detections::status - .eq_any([ - DetectionStatus::Pending, - DetectionStatus::Executing, - DetectionStatus::Complete, - ]) - .and( - detections::input_file_id - .eq(workspace_files::id) - .or(detections::audit_file_id.eq(workspace_files::id.nullable())), - ), + detections::status.eq_any(DetectionStatus::IN_PROGRESS).and( + detections::input_file_id + .eq(workspace_files::id) + .or(detections::audit_file_id.eq(workspace_files::id.nullable())), + ), ), ); diff --git a/crates/nvisy-postgres/src/types/enums/detection_status.rs b/crates/nvisy-postgres/src/types/enums/detection_status.rs index ee100640..db90a05b 100644 --- a/crates/nvisy-postgres/src/types/enums/detection_status.rs +++ b/crates/nvisy-postgres/src/types/enums/detection_status.rs @@ -39,10 +39,15 @@ pub enum DetectionStatus { } impl DetectionStatus { - /// Statuses that carry a success/failure outcome: a detection reached one of - /// these iff it either finished analysis or failed. This is the correct basis - /// for an error rate (`failed / (complete + failed)`). - pub const OUTCOMES: [DetectionStatus; 2] = [DetectionStatus::Complete, DetectionStatus::Failed]; + /// In-progress statuses: a detection is still analyzing — enqueued or + /// running — so its input and audit files must not expire yet. `Complete` is + /// excluded (it is terminal), so holding it would pin those files forever. + pub const IN_PROGRESS: [DetectionStatus; 2] = + [DetectionStatus::Pending, DetectionStatus::Executing]; + /// Terminal statuses: a detection reached one of these iff it either finished + /// analysis or failed, and its status will not change again. This is the + /// correct basis for an error rate (`failed / (complete + failed)`). + pub const TERMINAL: [DetectionStatus; 2] = [DetectionStatus::Complete, DetectionStatus::Failed]; /// Returns whether analysis is done and the detection is ready to redact. #[inline] diff --git a/crates/nvisy-server/src/handler/analytics.rs b/crates/nvisy-server/src/handler/analytics.rs index 1236e86a..c57990f0 100644 --- a/crates/nvisy-server/src/handler/analytics.rs +++ b/crates/nvisy-server/src/handler/analytics.rs @@ -1,5 +1,5 @@ //! Workspace analytics handler: aggregate metrics over a workspace's files and -//! pipeline runs, for a dashboard. +//! detections, for a dashboard. use aide::axum::ApiRouter; use aide::transform::TransformOperation; @@ -47,7 +47,7 @@ async fn get_analytics( fn get_analytics_docs(op: TransformOperation) -> TransformOperation { op.summary("Workspace analytics") .description( - "Returns aggregate analytics for a workspace: stored-file totals with a per-kind breakdown, pipeline-run health (status mix, error rate, and durations), and inference token usage (workspace totals plus a per-model breakdown). Breakdowns list every kind/status, zero-filled, in a stable order.", + "Returns aggregate analytics for a workspace: stored-file totals with a per-kind breakdown, detection health (status mix, error rate, and durations), and inference token usage (workspace totals plus a per-model breakdown). Breakdowns list every kind/status, zero-filled, in a stable order.", ) .response::<200, Json>() .response::<401, Json>() @@ -55,7 +55,7 @@ fn get_analytics_docs(op: TransformOperation) -> TransformOperation { .response::<404, Json>() } -/// Returns a workspace's daily run activity over a date window. +/// Returns a workspace's daily detection activity over a date window. #[tracing::instrument( skip_all, fields( @@ -63,13 +63,13 @@ fn get_analytics_docs(op: TransformOperation) -> TransformOperation { workspace_id = %workspace.id, ) )] -async fn get_run_timeseries( +async fn get_detection_timeseries( State(pg_client): State, AuthState(auth_state): AuthState, WorkspaceContext(workspace): WorkspaceContext, Query(window): Query, ) -> Result<(StatusCode, Json)> { - tracing::debug!(target: TRACING_TARGET, "Computing run time series"); + tracing::debug!(target: TRACING_TARGET, "Computing detection time series"); let window = window.resolve()?; @@ -80,7 +80,7 @@ async fn get_run_timeseries( .await?; let points = conn - .runs_by_day( + .detections_by_day( workspace.id, window.from_timestamp()?, window.to_timestamp()?, @@ -91,10 +91,10 @@ async fn get_run_timeseries( Ok((StatusCode::OK, Json(series))) } -fn get_run_timeseries_docs(op: TransformOperation) -> TransformOperation { - op.summary("Workspace run time series") +fn get_detection_timeseries_docs(op: TransformOperation) -> TransformOperation { + op.summary("Workspace detection time series") .description( - "Returns a workspace's daily pipeline-run activity over a date window: runs per day, plus each day's error rate and durations. Every day in the window is present (quiet days report runs: 0), so the series plots as a continuous line or a contribution-style calendar. The window is `from`/`to` (inclusive, YYYY-MM-DD); it defaults to the last 30 days and is capped at 366 days.", + "Returns a workspace's daily detection activity over a date window: detections per day, plus each day's error rate and durations. Every day in the window is present (quiet days report detections: 0), so the series plots as a continuous line or a contribution-style calendar. The window is `from`/`to` (inclusive, YYYY-MM-DD); it defaults to the last 30 days and is capped at 366 days.", ) .response::<200, Json>() .response::<400, Json>() @@ -113,8 +113,8 @@ pub fn routes() -> ApiRouter { get_with(get_analytics, get_analytics_docs), ) .api_route( - "/workspaces/{workspaceSlug}/analytics/runs/timeseries/", - get_with(get_run_timeseries, get_run_timeseries_docs), + "/workspaces/{workspaceSlug}/analytics/detections/timeseries/", + get_with(get_detection_timeseries, get_detection_timeseries_docs), ) .with_path_items(|item| item.tag("Analytics")) } diff --git a/crates/nvisy-server/src/handler/detections.rs b/crates/nvisy-server/src/handler/detections.rs index a07f5cdd..53f6f64e 100644 --- a/crates/nvisy-server/src/handler/detections.rs +++ b/crates/nvisy-server/src/handler/detections.rs @@ -183,6 +183,7 @@ async fn create_detection( pipeline_slug: pipeline.slug.clone(), triggered_by: auth_state.account_id, reason: "Failed to enqueue detection", + metadata: detection_row.metadata.or_default(), claim: None, }, ) @@ -652,9 +653,18 @@ async fn redact_detection( redacted.bytes, ) .await?; - let staged_review = blob + // Staging the review audit after the output means a failure here would strand + // the already-written output object (no row to reclaim it); discard it first. + let staged_review = match blob .stage_review_audit(&pipeline, &retention, auth_state.account_id, &reviewed) - .await?; + .await + { + Ok(staged) => staged, + Err(err) => { + blob.discard_staged_object(&staged_output).await.ok(); + return Err(err); + } + }; let redaction = conn .transaction(async |conn| { @@ -679,6 +689,7 @@ async fn redact_detection( detection_id: detection.id, pipeline_slug: pipeline.slug.clone(), }, + redaction_id: redaction.id, input_file_name: Some(file.display_name.clone()), notify: detection.account_id, }, diff --git a/crates/nvisy-server/src/handler/response/analytics.rs b/crates/nvisy-server/src/handler/response/analytics.rs index b11169cc..5280e90f 100644 --- a/crates/nvisy-server/src/handler/response/analytics.rs +++ b/crates/nvisy-server/src/handler/response/analytics.rs @@ -8,13 +8,13 @@ use std::collections::BTreeMap; use jiff::ToSpan; use jiff::civil::Date; -use nvisy_postgres::query::{AnalyticsSnapshot, RunDayPoint}; +use nvisy_postgres::query::{AnalyticsSnapshot, DetectionDayPoint}; use nvisy_postgres::types::{DetectionStatus, FileKind}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use strum::IntoEnumIterator; -/// Aggregate analytics for a workspace: what it stores, how its runs fare, and +/// Aggregate analytics for a workspace: what it stores, how its detections fare, and /// the inference tokens they spent. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] @@ -27,7 +27,7 @@ pub struct WorkspaceAnalytics { pub usage: UsageAnalytics, } -/// Inference token usage across a workspace's runs. +/// Inference token usage across a workspace's detections. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct UsageAnalytics { @@ -42,7 +42,7 @@ pub struct UsageAnalytics { pub by_model: Vec, } -/// One model's token usage across a workspace's runs. +/// One model's token usage across a workspace's detections. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct ModelUsageEntry { @@ -81,35 +81,36 @@ pub struct StorageKindEntry { pub total_bytes: i64, } -/// Pipeline-run health for a workspace. +/// Detection health for a workspace. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct DetectionAnalytics { - /// Total number of runs. + /// Total number of detections. pub total: i64, - /// Per-status breakdown, one entry per run status (zero-filled), in a stable - /// order. + /// Per-status breakdown, one entry per detection status (zero-filled), in a + /// stable order. pub by_status: Vec, - /// Failed / (completed + failed). Omitted when no run has reached a terminal - /// state (genuinely no signal, not zero). + /// Failed / (completed + failed). Omitted when no detection has reached a + /// terminal state (genuinely no signal, not zero). #[serde(skip_serializing_if = "Option::is_none")] pub error_rate: Option, - /// Mean completed-run duration in milliseconds; omitted until a run completes. + /// Mean completed-detection duration in milliseconds; omitted until a + /// detection completes. #[serde(skip_serializing_if = "Option::is_none")] pub avg_duration_ms: Option, - /// 95th-percentile completed-run duration in milliseconds; omitted until a run - /// completes. + /// 95th-percentile completed-detection duration in milliseconds; omitted until + /// a detection completes. #[serde(skip_serializing_if = "Option::is_none")] pub p95_duration_ms: Option, } -/// One status's share of a workspace's runs. +/// One status's share of a workspace's detections. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct DetectionStatusEntry { - /// The run status. + /// The detection status. pub status: DetectionStatus, - /// Number of runs in this status. + /// Number of detections in this status. pub count: i64, } @@ -119,7 +120,7 @@ impl WorkspaceAnalytics { pub fn from_snapshot(snapshot: AnalyticsSnapshot) -> Self { let AnalyticsSnapshot { storage, - runs, + detections, durations, usage, } = snapshot; @@ -139,7 +140,7 @@ impl WorkspaceAnalytics { let by_status: Vec = DetectionStatus::iter() .map(|status| DetectionStatusEntry { status, - count: runs + count: detections .iter() .find(|r| r.status == status) .map_or(0, |r| r.count), @@ -186,8 +187,9 @@ impl WorkspaceAnalytics { } } -/// A workspace's daily run activity over a window: one point per day, dense -/// (quiet days included with `runs: 0`), ready to plot as a continuous series. +/// A workspace's daily detection activity over a window: one point per day, dense +/// (quiet days included with `detections: 0`), ready to plot as a continuous +/// series. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct DetectionTimeSeries { @@ -195,25 +197,25 @@ pub struct DetectionTimeSeries { pub points: Vec, } -/// A single day of run activity. +/// A single day of detection activity. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct DetectionDayEntry { /// The day (`YYYY-MM-DD`, UTC). pub date: Date, - /// Runs started this day (`0` on a quiet day). - pub runs: i64, - /// Failed / (completed + failed) for this day; omitted when no run reached a - /// terminal state that day. + /// Detections started this day (`0` on a quiet day). + pub detections: i64, + /// Failed / (completed + failed) for this day; omitted when no detection + /// reached a terminal state that day. #[serde(skip_serializing_if = "Option::is_none")] pub error_rate: Option, - /// Mean completed-run duration (milliseconds) this day; omitted if none completed. + /// Mean completed-detection duration (milliseconds) this day; omitted if none completed. #[serde(skip_serializing_if = "Option::is_none")] pub avg_duration_ms: Option, - /// 95th-percentile completed-run duration (milliseconds) this day; omitted if none. + /// 95th-percentile completed-detection duration (milliseconds) this day; omitted if none. #[serde(skip_serializing_if = "Option::is_none")] pub p95_duration_ms: Option, - /// Input/prompt tokens spent by this day's runs; omitted when none used a model. + /// Input/prompt tokens spent by this day's detections; omitted when none used a model. #[serde(skip_serializing_if = "Option::is_none")] pub input_tokens: Option, /// Output/completion tokens spent this day; omitted when none used a model. @@ -227,12 +229,13 @@ pub struct DetectionDayEntry { impl DetectionTimeSeries { /// Builds a dense daily series over `[from, to]` from the sparse per-day rows /// the query returns. Every day in the window is emitted in order; a day with - /// no runs reports `runs: 0` and omits the rate/duration/token fields. The - /// query only produces days that had a run, so gap-filling happens here. - pub fn from_window(from: Date, to: Date, points: Vec) -> Self { + /// no detections reports `detections: 0` and omits the rate/duration/token + /// fields. The query only produces days that had a detection, so gap-filling + /// happens here. + pub fn from_window(from: Date, to: Date, points: Vec) -> Self { // Index the sparse rows by their UTC day for O(1) lookup while walking the // window. - let mut by_day: BTreeMap = BTreeMap::new(); + let mut by_day: BTreeMap = BTreeMap::new(); for p in points { let date = p.day.to_zoned(jiff::tz::TimeZone::UTC).date(); by_day.insert(date, p); @@ -244,7 +247,7 @@ impl DetectionTimeSeries { days.push(match by_day.remove(&date) { Some(p) => DetectionDayEntry { date, - runs: p.runs, + detections: p.detections, error_rate: (p.terminal > 0).then(|| p.failed as f64 / p.terminal as f64), avg_duration_ms: p.avg_ms, p95_duration_ms: p.p95_ms, @@ -254,7 +257,7 @@ impl DetectionTimeSeries { }, None => DetectionDayEntry { date, - runs: 0, + detections: 0, error_rate: None, avg_duration_ms: None, p95_duration_ms: None, @@ -283,7 +286,9 @@ fn count_of(by_status: &[DetectionStatusEntry], status: DetectionStatus) -> i64 #[cfg(test)] mod tests { - use nvisy_postgres::query::{RunDurations, RunStatusCount, StorageByKind, UsageByModel}; + use nvisy_postgres::query::{ + DetectionDurations, DetectionStatusCount, StorageByKind, UsageByModel, + }; use super::*; @@ -301,17 +306,17 @@ mod tests { total_bytes: 50, }, ]; - let runs = vec![ - RunStatusCount { + let detections = vec![ + DetectionStatusCount { status: DetectionStatus::Complete, count: 3, }, - RunStatusCount { + DetectionStatusCount { status: DetectionStatus::Failed, count: 1, }, ]; - let durations = RunDurations { + let durations = DetectionDurations { avg_ms: Some(30_000), p95_ms: Some(56_000), }; @@ -332,7 +337,7 @@ mod tests { ]; let a = WorkspaceAnalytics::from_snapshot(AnalyticsSnapshot { storage, - runs, + detections, durations, usage, }); @@ -380,10 +385,10 @@ mod tests { .timestamp() }; // Only the active day is returned by the query (sparse); Jan 6 and 7 have - // no runs and must be gap-filled by from_window over the Jan 5..7 window. - let sparse = vec![RunDayPoint { + // no detections and must be gap-filled by from_window over the Jan 5..7 window. + let sparse = vec![DetectionDayPoint { day: day_ts("2026-01-05"), - runs: 3, + detections: 3, terminal: 3, failed: 1, avg_ms: Some(20_000), @@ -403,31 +408,31 @@ mod tests { // Active day: derived error rate + tokens carried through. let d5 = &series.points[0]; - assert_eq!(d5.runs, 3); + assert_eq!(d5.detections, 3); assert_eq!(d5.error_rate, Some(1.0 / 3.0)); assert_eq!(d5.avg_duration_ms, Some(20_000)); assert_eq!(d5.input_tokens, Some(1000)); assert_eq!(d5.total_tokens, None); - // Gap-filled days: runs 0, everything else omitted. + // Gap-filled days: detections 0, everything else omitted. let d6 = &series.points[1]; - assert_eq!(d6.runs, 0); + assert_eq!(d6.detections, 0); assert_eq!(d6.error_rate, None); assert_eq!(d6.avg_duration_ms, None); assert_eq!(d6.input_tokens, None); } #[test] - fn error_rate_is_none_with_no_terminal_runs() { - // Only active runs, no files, no completed durations. - let runs = vec![RunStatusCount { + fn error_rate_is_none_with_no_terminal_detections() { + // Only in-progress detections, no files, no completed durations. + let detections = vec![DetectionStatusCount { status: DetectionStatus::Pending, count: 5, }]; let a = WorkspaceAnalytics::from_snapshot(AnalyticsSnapshot { storage: vec![], - runs, - durations: RunDurations { + detections, + durations: DetectionDurations { avg_ms: None, p95_ms: None, }, diff --git a/crates/nvisy-server/src/service/detection/support.rs b/crates/nvisy-server/src/service/detection/support.rs index 809d3185..dec06530 100644 --- a/crates/nvisy-server/src/service/detection/support.rs +++ b/crates/nvisy-server/src/service/detection/support.rs @@ -118,6 +118,9 @@ pub(crate) struct FailDetection<'a> { pub triggered_by: Uuid, /// Human-readable failure reason, stored in the detection's metadata. pub reason: &'a str, + /// The detection's current metadata, so the failure reason is layered onto it + /// rather than replacing recorded fields such as tags. + pub metadata: DetectionMetadata, /// The worker's claim timestamp, guarding the transition; `None` on the /// handler path, which has no claim to fence. pub claim: Option, @@ -147,16 +150,16 @@ pub(crate) async fn fail_detection( pipeline_slug, triggered_by, reason, + mut metadata, claim, } = params; - let metadata = DetectionMetadata { - error: Some(reason.to_owned()), - ..Default::default() - }; - // `status` and `completed_at` are forced by `fail_detection` itself (the - // terminal transition owns the terminal timestamp), so only the failure - // reason is supplied here. + // Layer the failure reason onto the detection's existing metadata so recorded + // fields (e.g. reviewer tags) survive the failure write. + metadata.error = Some(reason.to_owned()); + // `status` and `completed_at` are forced by the finalize methods themselves + // (the terminal transition owns the terminal timestamp), so only the metadata + // is supplied here. let update = UpdateWorkspaceDetection { metadata: Some(Json::encode(&metadata)), ..Default::default() diff --git a/crates/nvisy-server/src/service/detection/worker.rs b/crates/nvisy-server/src/service/detection/worker.rs index e258b4d8..72cd9956 100644 --- a/crates/nvisy-server/src/service/detection/worker.rs +++ b/crates/nvisy-server/src/service/detection/worker.rs @@ -221,6 +221,7 @@ impl DetectionWorker { pipeline_slug: pipeline.slug.clone(), triggered_by: detection.account_id, reason: &err.to_string(), + metadata: detection.metadata.or_default(), claim: Some(claim_token), }, ) diff --git a/crates/nvisy-server/src/service/event/drainer.rs b/crates/nvisy-server/src/service/event/drainer.rs index 2b5c9125..9cbb7bd2 100644 --- a/crates/nvisy-server/src/service/event/drainer.rs +++ b/crates/nvisy-server/src/service/event/drainer.rs @@ -330,13 +330,9 @@ fn activity_of(event: &WorkspaceEvent) -> ActivityPayload { pipeline_slug: detection.pipeline_slug.clone(), detection_id: DetectionId::from_uuid(detection.detection_id), }; - // TODO(redaction-feature): once the redact handler persists a redaction row, - // `RedactionCreated` should carry a real `RedactionId`; until then the - // activity params reuse the detection id as a placeholder so the projection - // compiles. The parent reworks the redact emission path. - let redaction = |detection: &DetectionRef| RedactionActivityParams { + let redaction = |detection: &DetectionRef, redaction_id: Uuid| RedactionActivityParams { pipeline_slug: detection.pipeline_slug.clone(), - redaction_id: RedactionId::from_uuid(detection.detection_id), + redaction_id: RedactionId::from_uuid(redaction_id), }; let policy = |p: &PolicyRef| PolicyActivityParams { policy_id: p.policy_id, @@ -387,7 +383,11 @@ fn activity_of(event: &WorkspaceEvent) -> ActivityPayload { ActivityPayload::DetectionCompleted(detection(d)) } E::DetectionFailed { detection: d, .. } => ActivityPayload::DetectionFailed(detection(d)), - E::RedactionCreated { detection: d, .. } => ActivityPayload::RedactionCreated(redaction(d)), + E::RedactionCreated { + detection: d, + redaction_id, + .. + } => ActivityPayload::RedactionCreated(redaction(d, *redaction_id)), E::PolicyCreated(p) => ActivityPayload::PolicyCreated(policy(p)), E::PolicyUpdated(p) => ActivityPayload::PolicyUpdated(policy(p)), E::PolicyDeleted(p) => ActivityPayload::PolicyDeleted(policy(p)), @@ -496,17 +496,15 @@ fn notification_of(event: WorkspaceEvent) -> Option<(Uuid, NotificationPayload)> input_file_name, }), )), - // TODO(redaction-feature): `RedactionCreated` should carry a real - // `RedactionId`; until the redact handler persists a redaction row the - // detection id stands in as a placeholder so the projection compiles. E::RedactionCreated { detection, + redaction_id, input_file_name, notify, } => Some(( notify, NotificationPayload::RedactionCreated(RedactionCreatedParams { - redaction_id: RedactionId::from_uuid(detection.detection_id), + redaction_id: RedactionId::from_uuid(redaction_id), detection_id: DetectionId::from_uuid(detection.detection_id), pipeline_slug: detection.pipeline_slug, input_file_name, diff --git a/crates/nvisy-server/src/service/event/workspace_event.rs b/crates/nvisy-server/src/service/event/workspace_event.rs index 815f193b..4932275a 100644 --- a/crates/nvisy-server/src/service/event/workspace_event.rs +++ b/crates/nvisy-server/src/service/event/workspace_event.rs @@ -123,6 +123,9 @@ pub enum WorkspaceEvent { RedactionCreated { #[serde(flatten)] detection: DetectionRef, + /// The redaction that was produced (its own id, distinct from the + /// detection's — a detection can produce many redactions). + redaction_id: Uuid, input_file_name: Option, notify: Uuid, }, diff --git a/crates/nvisy-server/src/service/run_blob_store.rs b/crates/nvisy-server/src/service/run_blob_store.rs index 0d82460d..75011d89 100644 --- a/crates/nvisy-server/src/service/run_blob_store.rs +++ b/crates/nvisy-server/src/service/run_blob_store.rs @@ -518,13 +518,20 @@ fn analysis_serde_error(error: serde_json::Error) -> Error<'static> { /// from the display name; a name that does not end in its own extension (or has /// none) simply gains a `.redacted` suffix. fn redacted_display_name(display_name: &str, extension: &str) -> String { + // Split off a trailing `.{extension}`, matched case-insensitively on both + // sides so an upper- or mixed-case extension (`Report.PDF`) still has the + // marker inserted before it, not appended after. let suffix = format!(".{extension}"); - match display_name + let stem = display_name .len() .checked_sub(suffix.len()) - .filter(|_| !extension.is_empty() && display_name.to_lowercase().ends_with(&suffix)) - { - Some(stem_len) => format!("{}.redacted.{extension}", &display_name[..stem_len]), + .filter(|_| !extension.is_empty()) + .filter(|&at| display_name.is_char_boundary(at)) + .map(|at| display_name.split_at(at)) + .filter(|(_, tail)| tail.eq_ignore_ascii_case(&suffix)) + .map(|(stem, _)| stem); + match stem { + Some(stem) => format!("{stem}.redacted.{extension}"), None => format!("{display_name}.redacted"), } } @@ -543,10 +550,21 @@ mod tests { #[test] fn matches_the_extension_case_insensitively() { + // The name's extension case differs from the passed extension... assert_eq!( redacted_display_name("Report.PDF", "pdf"), "Report.redacted.pdf" ); + // ...and the passed extension itself may be upper- or mixed-case; the + // marker still lands before it, and the original extension text is kept. + assert_eq!( + redacted_display_name("Report.PDF", "PDF"), + "Report.redacted.PDF" + ); + assert_eq!( + redacted_display_name("report.pdf", "PDF"), + "report.redacted.PDF" + ); } #[test] diff --git a/migrations/2026-01-19-045014_pipelines/up.sql b/migrations/2026-01-19-045014_pipelines/up.sql index 4951f93c..414ee122 100644 --- a/migrations/2026-01-19-045014_pipelines/up.sql +++ b/migrations/2026-01-19-045014_pipelines/up.sql @@ -192,6 +192,14 @@ CREATE INDEX workspace_detections_status_idx CREATE INDEX workspace_detections_input_file_idx ON workspace_detections (input_file_id, started_at DESC); +-- The file-expiry sweep's hold check matches a candidate file against a +-- detection's input OR audit file; the input side is covered above, this covers +-- the audit side. Partial, since most detections eventually carry an audit but +-- the column is NULL until analysis writes it. +CREATE INDEX workspace_detections_audit_file_idx + ON workspace_detections (audit_file_id) + WHERE audit_file_id IS NOT NULL; + -- Idempotent detect: at most one detection per (pipeline, idempotency key). CREATE UNIQUE INDEX workspace_detections_idempotency_idx ON workspace_detections (pipeline_id, idempotency_key) @@ -233,9 +241,11 @@ CREATE TABLE workspace_redactions ( -- Account that requested the redaction. account_id UUID NOT NULL REFERENCES accounts (id) ON DELETE CASCADE, - -- The two files a redaction produces. Both are set at creation (a redaction - -- always yields a review audit and a redacted document), and both use - -- ON DELETE SET NULL for the same append-only-history reasons as a detection. + -- The two files a redaction produces. A redaction always yields both, so the + -- app sets them at creation; they are nullable only because `ON DELETE SET + -- NULL` clears a reference if its file is ever hard-deleted (for the same + -- append-only-history reasons as a detection — a soft-deleted file resolves to + -- "gone" at read time, distinct from a NULL that means the file was purged). -- review: the engine's Audit after the reviewer edits were applied and -- redaction ran (`file_kind = review`); the record of exactly what -- was redacted and why. From 9f930d37235d001122f420fb6d6c8f9544183493 Mon Sep 17 00:00:00 2001 From: Oleh Martsokha Date: Wed, 26 Aug 2026 16:33:01 +0200 Subject: [PATCH 4/7] Address PR review: retry on persist failure, durations exclude failures - Retry the detection job when the failure-finalization write does not persist: fail_detection now returns a FailOutcome, and the worker NACKs on PersistFailed instead of ACKing, so a detection is never left Executing with no queued job to reclaim its lease. Event-emission failures are kept separate and do not trigger a retry. - Exclude failed detections from the duration aggregates: completed_at is stamped on failure too, so the avg/p95 loaders (snapshot and daily series) now filter on status = Complete rather than completed_at IS NOT NULL, matching the "completed detections only" contract. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018bKk1YEG4tZ69jzYVQvQL8 --- crates/nvisy-postgres/src/query/analytics.rs | 12 +++- .../src/service/detection/support.rs | 62 ++++++++++++------- .../src/service/detection/worker.rs | 13 +++- 3 files changed, 58 insertions(+), 29 deletions(-) diff --git a/crates/nvisy-postgres/src/query/analytics.rs b/crates/nvisy-postgres/src/query/analytics.rs index 727c10b2..80f51585 100644 --- a/crates/nvisy-postgres/src/query/analytics.rs +++ b/crates/nvisy-postgres/src/query/analytics.rs @@ -359,7 +359,10 @@ async fn load_detection_durations( .inner_join(workspace_pipelines::table) .filter(pipelines::workspace_id.eq(workspace_id)) .filter(pipelines::deleted_at.is_null()) - .filter(detections::completed_at.is_not_null()) + // Durations describe successful analysis only. `completed_at` is stamped + // on failure too, so filter on the terminal `Complete` status, not merely + // on the timestamp being present. + .filter(detections::status.eq(DetectionStatus::Complete)) .select((avg_ms, p95_ms)) .first(conn) .await @@ -441,16 +444,19 @@ async fn load_detection_day_counts( // Conditional counts are `sum(CASE WHEN cond THEN 1 ELSE 0 END)` since Diesel's // aggregate FILTER is not available on `count(*)`. Columns are table-qualified // so the join to pipelines can never make them ambiguous. + // Durations describe successful analysis only, so filter the aggregates on the + // terminal `complete` status: `completed_at` is stamped on failure too, so a + // presence check alone would fold failed detections into avg/p95. let avg_ms = sql::>( "round(avg(EXTRACT(EPOCH FROM (workspace_detections.completed_at \ - workspace_detections.started_at))) \ - FILTER (WHERE workspace_detections.completed_at IS NOT NULL) * 1000)::bigint", + FILTER (WHERE workspace_detections.status = 'complete') * 1000)::bigint", ); let p95_ms = sql::>( "round(percentile_cont(0.95) WITHIN GROUP \ (ORDER BY EXTRACT(EPOCH FROM (workspace_detections.completed_at \ - workspace_detections.started_at))) \ - FILTER (WHERE workspace_detections.completed_at IS NOT NULL) * 1000)::bigint", + FILTER (WHERE workspace_detections.status = 'complete') * 1000)::bigint", ); workspace_detections::table diff --git a/crates/nvisy-server/src/service/detection/support.rs b/crates/nvisy-server/src/service/detection/support.rs index dec06530..0fbc8b3b 100644 --- a/crates/nvisy-server/src/service/detection/support.rs +++ b/crates/nvisy-server/src/service/detection/support.rs @@ -143,7 +143,7 @@ pub(crate) async fn fail_detection( conn: &mut nvisy_postgres::PgConn, detection: &DetectionQueue, params: FailDetection<'_>, -) { +) -> FailOutcome { let FailDetection { workspace_id, detection_id, @@ -165,34 +165,30 @@ pub(crate) async fn fail_detection( ..Default::default() }; - match claim { + let persisted = match claim { // Worker path: guard on the claim. A stale claim fails nothing and stays // silent — the new owner drives the detection to its own outcome. - Some(claimed_at) => match conn.fail_detection(detection_id, claimed_at, update).await { - Ok(true) => {} - Ok(false) => { - tracing::warn!(target: TRACING_TARGET, %detection_id, "Claim went stale before failure; another worker owns the detection"); - return; - } - Err(err) => { - tracing::warn!(target: TRACING_TARGET, error = %err, %detection_id, "Failed to mark detection failed"); - return; - } - }, + Some(claimed_at) => conn.fail_detection(detection_id, claimed_at, update).await, // Handler path (enqueue failure): guard on `Pending` so this no-ops if a // worker already claimed the detection — enqueue can report an error even // when the job was delivered, and the worker then owns the outcome. - None => match conn.fail_pending_detection(detection_id, update).await { - Ok(true) => {} - Ok(false) => { - tracing::warn!(target: TRACING_TARGET, %detection_id, "Detection was already claimed before enqueue-failure handling; the worker owns it"); - return; - } - Err(err) => { - tracing::warn!(target: TRACING_TARGET, error = %err, %detection_id, "Failed to mark detection failed"); - return; - } - }, + None => conn.fail_pending_detection(detection_id, update).await, + }; + match persisted { + Ok(true) => {} + // The guard didn't match: another owner (or an already-terminal detection) + // drives the outcome. Nothing to persist, announce, or retry. + Ok(false) => { + tracing::warn!(target: TRACING_TARGET, %detection_id, "Detection no longer owned at failure; another owner drives it"); + return FailOutcome::NotOwned; + } + // The terminal write itself failed: the detection is still `Executing` + // with no queued job to reclaim it, so the caller must retry (redeliver) + // rather than treat the failure as handled. + Err(err) => { + tracing::warn!(target: TRACING_TARGET, error = %err, %detection_id, "Failed to persist detection failure; will retry"); + return FailOutcome::PersistFailed; + } } detection @@ -220,6 +216,24 @@ pub(crate) async fn fail_detection( { tracing::warn!(target: TRACING_TARGET, error = %err, %detection_id, "Failed to record detection-failed event"); } + + // The state transition is persisted; a lost event is recoverable from the + // outbox and does not change the outcome. + FailOutcome::Failed +} + +/// The result of attempting to fail a detection, so a caller driving a work +/// queue can decide whether to redeliver. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum FailOutcome { + /// The detection was transitioned to `Failed` and its failure announced. + Failed, + /// The failure was not applied because the detection is no longer owned by + /// the caller (another owner, or an already-terminal detection). No retry. + NotOwned, + /// The terminal write failed to persist, leaving the detection mid-flight; + /// the caller should redeliver so the stale lease is reclaimed. + PersistFailed, } /// Resolves a pipeline's live policy references into decrypted engine policies. diff --git a/crates/nvisy-server/src/service/detection/worker.rs b/crates/nvisy-server/src/service/detection/worker.rs index 72cd9956..67a305f3 100644 --- a/crates/nvisy-server/src/service/detection/worker.rs +++ b/crates/nvisy-server/src/service/detection/worker.rs @@ -22,7 +22,9 @@ use tokio_util::sync::CancellationToken; use super::job::DetectionJob; use super::service::DetectionQueue; -use super::support::{FailDetection, extract_detection_usage, fail_detection, resolve_policies}; +use super::support::{ + FailDetection, FailOutcome, extract_detection_usage, fail_detection, resolve_policies, +}; use crate::extract::SecurityContext; use crate::handler::request::PipelineDefinition; use crate::handler::{ErrorKind, Result}; @@ -212,7 +214,7 @@ impl DetectionWorker { .await { tracing::warn!(target: TRACING_TARGET, error = %err, "Detection failed"); - fail_detection( + let outcome = fail_detection( &mut conn, &self.detection, FailDetection { @@ -226,6 +228,13 @@ impl DetectionWorker { }, ) .await; + // If the failure state could not be persisted, the detection is left + // `Executing` with no queued job to reclaim its lease: redeliver so a + // later attempt drives it to a terminal state rather than ack'ing a + // detection that will hang. + if outcome == FailOutcome::PersistFailed { + return JobOutcome::Retry; + } } JobOutcome::Done } From d248a805242e102c45e82ac8ea7ba4d81520eb76 Mon Sep 17 00:00:00 2001 From: Oleh Martsokha Date: Wed, 26 Aug 2026 17:04:59 +0200 Subject: [PATCH 5/7] Enqueue detections through a transactional outbox Remove the dual-write between creating a detection and publishing its analysis job: the create-detection handler no longer publishes to NATS inline (and no longer marks a detection failed when that publish errored). Instead it inserts a job-outbox row in the same transaction as the detection, and a background DetectionOutboxDrainer relays each pending row to the detection work-queue. Publishing at-least-once inside the claim transaction, absorbed by the worker's existing claim-based dedup, means a detection is never lost to a publish that failed after the row committed, nor marked failed for a publish that in fact went through. - New workspace_detection_jobs outbox table (mirrors event_outbox; reuses the shared OUTBOX_STATUS type), its model, and a DetectionJobOutboxRepository with the same claim/mark/defer/dead-letter operations. - New DetectionOutboxDrainer worker (mirrors EventOutboxDrainer), spawned alongside the event drainer; it decodes each row's DetectionJob and publishes it to the existing DetectionStream, so the worker is unchanged. - create_detection inserts the outbox row transactionally and drops the inline enqueue + fail-on-enqueue path. Migration restructure (pre-launch, edit-in-place): move the whole event_outbox migration to right after webhooks so it owns the shared OUTBOX_STATUS type before either outbox table needs it, and split the monolithic pipelines migration into policies -> pipelines (+ the pipeline-policies join) -> detections (+ the job outbox) -> redactions, each feature self-contained. The full chain applies and reverts cleanly. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018bKk1YEG4tZ69jzYVQvQL8 --- crates/nvisy-postgres/src/model/mod.rs | 2 + .../src/model/workspace_detection_job.rs | 50 +++++ crates/nvisy-postgres/src/query/mod.rs | 2 + .../src/query/workspace_detection_job.rs | 148 ++++++++++++ crates/nvisy-postgres/src/schema.rs | 18 ++ crates/nvisy-server/src/handler/detections.rs | 69 +++--- .../src/service/detection/drainer.rs | 200 +++++++++++++++++ .../nvisy-server/src/service/detection/mod.rs | 4 +- crates/nvisy-server/src/service/mod.rs | 6 +- .../down.sql | 0 .../up.sql | 0 .../2026-01-19-045014_policies/down.sql | 2 + .../up.sql | 29 --- .../2026-01-19-045015_pipelines/down.sql | 7 + migrations/2026-01-19-045015_pipelines/up.sql | 129 +++++++++++ .../2026-01-19-045015_policies/down.sql | 7 - .../down.sql | 6 +- .../up.sql | 210 +++++------------- .../2026-01-19-045017_redactions/down.sql | 4 + .../2026-01-19-045017_redactions/up.sql | 46 ++++ 20 files changed, 709 insertions(+), 230 deletions(-) create mode 100644 crates/nvisy-postgres/src/model/workspace_detection_job.rs create mode 100644 crates/nvisy-postgres/src/query/workspace_detection_job.rs create mode 100644 crates/nvisy-server/src/service/detection/drainer.rs rename migrations/{2026-08-21-045211_event_outbox => 2025-05-21-222843_event_outbox}/down.sql (100%) rename migrations/{2026-08-21-045211_event_outbox => 2025-05-21-222843_event_outbox}/up.sql (100%) create mode 100644 migrations/2026-01-19-045014_policies/down.sql rename migrations/{2026-01-19-045015_policies => 2026-01-19-045014_policies}/up.sql (72%) create mode 100644 migrations/2026-01-19-045015_pipelines/down.sql create mode 100644 migrations/2026-01-19-045015_pipelines/up.sql delete mode 100644 migrations/2026-01-19-045015_policies/down.sql rename migrations/{2026-01-19-045014_pipelines => 2026-01-19-045016_detections}/down.sql (59%) rename migrations/{2026-01-19-045014_pipelines => 2026-01-19-045016_detections}/up.sql (57%) create mode 100644 migrations/2026-01-19-045017_redactions/down.sql create mode 100644 migrations/2026-01-19-045017_redactions/up.sql diff --git a/crates/nvisy-postgres/src/model/mod.rs b/crates/nvisy-postgres/src/model/mod.rs index 8c22b32a..4be94262 100644 --- a/crates/nvisy-postgres/src/model/mod.rs +++ b/crates/nvisy-postgres/src/model/mod.rs @@ -16,6 +16,7 @@ mod workspace_connection; mod workspace_connection_schedule; mod workspace_connection_sync; mod workspace_detection; +mod workspace_detection_job; mod workspace_detection_usage; mod workspace_file; mod workspace_file_imports; @@ -53,6 +54,7 @@ pub use workspace_connection_sync::{ pub use workspace_detection::{ NewWorkspaceDetection, UpdateWorkspaceDetection, WorkspaceDetection, }; +pub use workspace_detection_job::{NewWorkspaceDetectionJob, WorkspaceDetectionJob}; pub use workspace_detection_usage::{NewWorkspaceDetectionUsage, WorkspaceDetectionUsage}; pub use workspace_file::{NewWorkspaceFile, UpdateWorkspaceFile, WorkspaceFile}; pub use workspace_file_imports::{NewWorkspaceFileImport, WorkspaceFileImport}; diff --git a/crates/nvisy-postgres/src/model/workspace_detection_job.rs b/crates/nvisy-postgres/src/model/workspace_detection_job.rs new file mode 100644 index 00000000..b8192f84 --- /dev/null +++ b/crates/nvisy-postgres/src/model/workspace_detection_job.rs @@ -0,0 +1,50 @@ +//! Transactional-outbox model for detection jobs. + +use diesel::prelude::*; +use jiff_diesel::Timestamp; +use uuid::Uuid; + +use crate::schema::workspace_detection_jobs; +use crate::types::OutboxStatus; + +/// A pending or processed detection-job outbox row: a serialized `DetectionJob` +/// awaiting (or past) publication to the detection work-queue. +/// +/// The `job` column is an opaque JSON blob to this layer — a serialized +/// server-side `DetectionJob` — so the ORM stays free of the job vocabulary; the +/// drainer decodes it and publishes it. +#[derive(Debug, Clone, Queryable, Selectable)] +#[diesel(table_name = workspace_detection_jobs)] +#[diesel(check_for_backend(diesel::pg::Pg))] +pub struct WorkspaceDetectionJob { + /// Unique outbox row identifier. + pub id: Uuid, + /// The detection this job analyzes. + pub detection_id: Uuid, + /// The serialized detection job. + pub job: serde_json::Value, + /// Processing state: pending, processed, or failed (dead-lettered). + pub status: OutboxStatus, + /// Number of publish attempts the drainer has made. + pub attempts: i32, + /// Earliest time the row may next be claimed; advanced by a backoff after + /// each failed attempt. + pub next_attempt_at: Timestamp, + /// When the job was queued. + pub created_at: Timestamp, + /// When a terminal (processed or failed) row was resolved by an operator; + /// `None` until then. A manual affordance for inspecting the outbox. + pub resolved_at: Option, +} + +/// A new detection-job outbox row, inserted in the same transaction as the +/// detection it queues. +#[derive(Debug, Clone, Insertable)] +#[diesel(table_name = workspace_detection_jobs)] +#[diesel(check_for_backend(diesel::pg::Pg))] +pub struct NewWorkspaceDetectionJob { + /// The detection this job analyzes. + pub detection_id: Uuid, + /// The serialized detection job. + pub job: serde_json::Value, +} diff --git a/crates/nvisy-postgres/src/query/mod.rs b/crates/nvisy-postgres/src/query/mod.rs index 8e436400..809b7de9 100644 --- a/crates/nvisy-postgres/src/query/mod.rs +++ b/crates/nvisy-postgres/src/query/mod.rs @@ -28,6 +28,7 @@ mod workspace_connection; mod workspace_connection_schedule; mod workspace_connection_sync; mod workspace_detection; +mod workspace_detection_job; mod workspace_file; mod workspace_invite; mod workspace_member; @@ -53,6 +54,7 @@ pub use workspace_connection::{ScheduledConnection, WorkspaceConnectionRepositor pub use workspace_connection_schedule::WorkspaceConnectionScheduleRepository; pub use workspace_connection_sync::WorkspaceConnectionSyncRepository; pub use workspace_detection::{DetectionFiles, DetectionListRow, WorkspaceDetectionRepository}; +pub use workspace_detection_job::DetectionJobOutboxRepository; pub use workspace_file::{ExpiredFileRef, ImportedFileRef, WorkspaceFileRepository}; pub use workspace_invite::WorkspaceInviteRepository; pub use workspace_member::WorkspaceMemberRepository; diff --git a/crates/nvisy-postgres/src/query/workspace_detection_job.rs b/crates/nvisy-postgres/src/query/workspace_detection_job.rs new file mode 100644 index 00000000..174e4d90 --- /dev/null +++ b/crates/nvisy-postgres/src/query/workspace_detection_job.rs @@ -0,0 +1,148 @@ +//! Detection-job outbox repository: the write side (insert in the create-detection +//! transaction) and the drainer side (claim a due batch, mark processed, defer or +//! dead-letter a failure). +//! +//! The drainer runs the claim and the subsequent `mark_*`/`defer_*` inside one +//! transaction per batch (see the detection-job drainer), so the `FOR UPDATE SKIP +//! LOCKED` locks are held from claim through completion: no other drainer takes +//! the same rows, and a row's state transition commits atomically with its +//! publication. + +use std::future::Future; + +use diesel::prelude::*; +use diesel::sql_types::{BigInt, Timestamptz}; +use diesel_async::RunQueryDsl; +use uuid::Uuid; + +use crate::model::{NewWorkspaceDetectionJob, WorkspaceDetectionJob}; +use crate::types::OutboxStatus; +use crate::{PgConnection, PgError, PgResult, schema}; + +/// Read and write operations on the detection-job outbox. +pub trait DetectionJobOutboxRepository { + /// Inserts one job outbox row. Called in the same transaction as the + /// detection it queues, so the two commit atomically. + fn insert_detection_job( + &mut self, + row: NewWorkspaceDetectionJob, + ) -> impl Future> + Send; + + /// Claims up to `limit` due pending rows for publication, oldest first. + /// + /// Due means unprocessed, not dead-lettered, and past its `next_attempt_at`, + /// so a row deferred by a backoff is skipped until its time arrives. Locks the + /// claimed rows with `FOR UPDATE SKIP LOCKED` so concurrent drainers take + /// disjoint batches without blocking each other; the lock is held for the + /// caller's transaction. Must run inside that transaction. + fn claim_detection_job_batch( + &mut self, + limit: i64, + ) -> impl Future>> + Send; + + /// Marks a row processed (its job durably published), taking it out of the + /// pending set. Runs in the drainer's batch transaction. + fn mark_detection_job_processed( + &mut self, + id: Uuid, + ) -> impl Future> + Send; + + /// Records a failed attempt: increments `attempts` and defers the next attempt + /// to `now() + backoff_secs` (computed by the database clock, so a drainer's + /// wall-clock skew cannot mis-schedule it), leaving the row pending for a + /// later retry. Runs in the drainer's batch transaction. + fn defer_detection_job_attempt( + &mut self, + id: Uuid, + backoff_secs: i64, + ) -> impl Future> + Send; + + /// Dead-letters a row: increments `attempts` and marks it `Failed`, taking it + /// out of the pending set so a job that can never publish stops consuming + /// drain cycles. The row is retained for inspection. Runs in the drainer's + /// batch transaction. + fn mark_detection_job_failed(&mut self, id: Uuid) -> impl Future> + Send; +} + +impl DetectionJobOutboxRepository for PgConnection { + async fn insert_detection_job( + &mut self, + row: NewWorkspaceDetectionJob, + ) -> PgResult { + use schema::workspace_detection_jobs; + + diesel::insert_into(workspace_detection_jobs::table) + .values(&row) + .returning(WorkspaceDetectionJob::as_returning()) + .get_result(self) + .await + .map_err(PgError::from) + } + + async fn claim_detection_job_batch( + &mut self, + limit: i64, + ) -> PgResult> { + use schema::workspace_detection_jobs::{self, dsl}; + + workspace_detection_jobs::table + .filter(dsl::status.eq(OutboxStatus::Pending)) + .filter(dsl::next_attempt_at.le(diesel::dsl::now)) + .order((dsl::next_attempt_at.asc(), dsl::created_at.asc())) + .limit(limit) + .select(WorkspaceDetectionJob::as_select()) + .for_update() + .skip_locked() + .load(self) + .await + .map_err(PgError::from) + } + + async fn mark_detection_job_processed(&mut self, id: Uuid) -> PgResult<()> { + use schema::workspace_detection_jobs::{self, dsl}; + + diesel::update(workspace_detection_jobs::table.filter(dsl::id.eq(id))) + .set(( + dsl::status.eq(OutboxStatus::Processed), + dsl::attempts.eq(dsl::attempts + 1), + )) + .execute(self) + .await + .map_err(PgError::from)?; + Ok(()) + } + + async fn defer_detection_job_attempt(&mut self, id: Uuid, backoff_secs: i64) -> PgResult<()> { + use schema::workspace_detection_jobs::{self, dsl}; + + // The row stays `Pending`; only its attempt count and next-due time move. + // `now() + (backoff_secs * interval '1 second')` schedules the next attempt + // by the database clock, not the drainer's. + let next_attempt_at = diesel::dsl::sql::("now() + (") + .bind::(backoff_secs) + .sql(" * interval '1 second')"); + diesel::update(workspace_detection_jobs::table.filter(dsl::id.eq(id))) + .set(( + dsl::attempts.eq(dsl::attempts + 1), + dsl::next_attempt_at.eq(next_attempt_at), + )) + .execute(self) + .await + .map_err(PgError::from)?; + Ok(()) + } + + async fn mark_detection_job_failed(&mut self, id: Uuid) -> PgResult<()> { + use schema::workspace_detection_jobs::{self, dsl}; + + diesel::update(workspace_detection_jobs::table.filter(dsl::id.eq(id))) + .set(( + dsl::status.eq(OutboxStatus::Failed), + dsl::attempts.eq(dsl::attempts + 1), + )) + .execute(self) + .await + .map_err(PgError::from)?; + Ok(()) + } +} diff --git a/crates/nvisy-postgres/src/schema.rs b/crates/nvisy-postgres/src/schema.rs index 43d26ef2..ae81a1c8 100644 --- a/crates/nvisy-postgres/src/schema.rs +++ b/crates/nvisy-postgres/src/schema.rs @@ -247,6 +247,22 @@ diesel::table! { } } +diesel::table! { + use diesel::sql_types::*; + use super::sql_types::OutboxStatus; + + workspace_detection_jobs (id) { + id -> Uuid, + detection_id -> Uuid, + job -> Jsonb, + status -> OutboxStatus, + attempts -> Int4, + next_attempt_at -> Timestamptz, + created_at -> Timestamptz, + resolved_at -> Nullable, + } +} + diesel::table! { use diesel::sql_types::*; @@ -477,6 +493,7 @@ diesel::joinable!(workspace_connection_syncs -> accounts (account_id)); diesel::joinable!(workspace_connection_syncs -> workspace_connection_schedule (connection_id)); diesel::joinable!(workspace_connections -> accounts (account_id)); diesel::joinable!(workspace_connections -> workspaces (workspace_id)); +diesel::joinable!(workspace_detection_jobs -> workspace_detections (detection_id)); diesel::joinable!(workspace_detection_usage -> workspace_detections (detection_id)); diesel::joinable!(workspace_detections -> accounts (account_id)); diesel::joinable!(workspace_detections -> workspace_pipelines (pipeline_id)); @@ -508,6 +525,7 @@ diesel::allow_tables_to_appear_in_same_query!( workspace_connection_schedule, workspace_connection_syncs, workspace_connections, + workspace_detection_jobs, workspace_detection_usage, workspace_detections, workspace_file_imports, diff --git a/crates/nvisy-server/src/handler/detections.rs b/crates/nvisy-server/src/handler/detections.rs index 53f6f64e..850cb51a 100644 --- a/crates/nvisy-server/src/handler/detections.rs +++ b/crates/nvisy-server/src/handler/detections.rs @@ -13,11 +13,13 @@ use axum::http::StatusCode; use axum::response::sse::Event; use futures::StreamExt; use nvisy_postgres::model::{ - NewWorkspaceDetection, NewWorkspaceRedaction, WorkspaceDetection, WorkspacePipeline, + NewWorkspaceDetection, NewWorkspaceDetectionJob, NewWorkspaceRedaction, WorkspaceDetection, + WorkspacePipeline, }; use nvisy_postgres::query::{ - DetectionFiles, PipelineReferenceRepository, WorkspaceDetectionRepository, - WorkspaceFileRepository, WorkspacePipelineRepository, WorkspaceRedactionRepository, + DetectionFiles, DetectionJobOutboxRepository, PipelineReferenceRepository, + WorkspaceDetectionRepository, WorkspaceFileRepository, WorkspacePipelineRepository, + WorkspaceRedactionRepository, }; use nvisy_postgres::types::DetectionStatus; use nvisy_postgres::{AsyncConnection, PgClient, PgConn}; @@ -36,8 +38,7 @@ use crate::handler::utility::{SseResponse, resolve_account_ref}; use crate::handler::{Error, ErrorKind, Result}; use crate::service::{ CryptoService, DetectionJob, DetectionQueue, DetectionRef, DetectionStatusEvent, EngineService, - EventEmitter, EventOrigin, FailDetection, RunBlobStore, ServiceState, WorkspaceEvent, - fail_detection, resolve_policies, + EventEmitter, EventOrigin, RunBlobStore, ServiceState, WorkspaceEvent, resolve_policies, }; /// Tracing target for detection operations. @@ -132,10 +133,9 @@ async fn create_detection( .with_context(err.to_string()) })?; - // Create the detection (its id is the engine correlation id) and enqueue - // analysis for the worker. The response returns immediately; the client - // learns the findings are ready via the detection's status (SSE at - // `.../events` or a re-read). + // Create the detection (its id is the engine correlation id). The response + // returns immediately; the client learns the findings are ready via the + // detection's status (SSE at `.../events` or a re-read). let new_detection = NewWorkspaceDetection { pipeline_id: pipeline.id, input_file_id: file.id, @@ -145,8 +145,13 @@ async fn create_detection( ..Default::default() }; - // Create the detection and record its start event in one transaction, so the - // event is never lost, nor recorded for a detection that rolled back. + // Create the detection, record its start event, and queue its analysis in one + // transaction, so all three commit or roll back together. The job goes onto + // the outbox (not published inline) so the detection is never lost to a + // publish that failed after the row committed, nor marked failed for a publish + // that in fact went through: the drainer relays the outbox row to the + // work-queue, and the worker's claim dedups an at-least-once redelivery. + let scope = request.scope; let detection_row = conn .transaction(async |conn| { let detection_row = conn.create_workspace_detection(new_detection).await?; @@ -162,40 +167,30 @@ async fn create_detection( }), ) .await?; + let job = DetectionJob { + workspace_id: workspace.id, + detection_id: detection_row.id, + scope, + }; + conn.insert_detection_job(NewWorkspaceDetectionJob { + detection_id: detection_row.id, + job: serde_json::to_value(&job).map_err(|err| { + ErrorKind::InternalServerError + .with_message("Failed to encode detection job") + .with_context(err.to_string()) + })?, + }) + .await?; Ok::<_, Error>(detection_row) }) .await?; - let job = DetectionJob { - workspace_id: workspace.id, - detection_id: detection_row.id, - scope: request.scope, - }; - if let Err(err) = detection.enqueue(job).await { - // Enqueue failed, so the worker will never pick this detection up: fail it - // now rather than leaving it stuck in `Pending`. - fail_detection( - &mut conn, - &detection, - FailDetection { - workspace_id: workspace.id, - detection_id: detection_row.id, - pipeline_slug: pipeline.slug.clone(), - triggered_by: auth_state.account_id, - reason: "Failed to enqueue detection", - metadata: detection_row.metadata.or_default(), - claim: None, - }, - ) - .await; - return Err(err); - } - + // Best-effort UI hint; the detection row is authoritative. detection .broadcast_status(detection_row.id, DetectionStatus::Pending) .await; - tracing::info!(target: TRACING_TARGET, detection_id = %detection_row.id, "Detection enqueued"); + tracing::info!(target: TRACING_TARGET, detection_id = %detection_row.id, "Detection queued"); let trigger = resolve_account_ref(&mut conn, detection_row.account_id).await?; diff --git a/crates/nvisy-server/src/service/detection/drainer.rs b/crates/nvisy-server/src/service/detection/drainer.rs new file mode 100644 index 00000000..416bfc4c --- /dev/null +++ b/crates/nvisy-server/src/service/detection/drainer.rs @@ -0,0 +1,200 @@ +//! Detection-job outbox drainer. +//! +//! Publishes each pending detection-job outbox row onto the detection work-queue, +//! so an analysis queued transactionally with its detection reaches the worker +//! even if the process crashes between the commit and the publish. This is the +//! relay half of the transactional outbox: the create-detection handler writes +//! the row in the detection's transaction, and this drains it to NATS. + +use std::time::Duration; + +use nvisy_postgres::AsyncConnection; +use nvisy_postgres::model::WorkspaceDetectionJob; +use nvisy_postgres::query::DetectionJobOutboxRepository; +use tokio_util::sync::CancellationToken; + +use super::job::DetectionJob; +use super::service::DetectionQueue; +use crate::handler::{Error, Result}; +use crate::service::{Infra, Worker}; + +/// Tracing target for the detection-job drainer. +const TRACING_TARGET: &str = "nvisy_server::service::detection::drainer"; + +/// How often the drainer polls for due jobs. Short, since it is the enqueue +/// latency between creating a detection and the worker picking it up. +const TICK_INTERVAL: Duration = Duration::from_secs(5); + +/// Maximum jobs drained per tick, bounding the work (and lock hold) per pass. +const DRAIN_BATCH: i64 = 100; + +/// Base unit of the retry backoff (seconds): a failed row's next attempt is +/// deferred by `RETRY_BACKOFF_BASE_SECS * attempts` (linear), capped at +/// [`RETRY_BACKOFF_MAX_SECS`]. +const RETRY_BACKOFF_BASE_SECS: i64 = 30; + +/// Ceiling on the retry backoff (seconds), so a long-failing row still retries +/// periodically rather than backing off unboundedly. +const RETRY_BACKOFF_MAX_SECS: i64 = 60 * 60; + +/// How many failed attempts a row gets before the drainer dead-letters it, so a +/// job that can never publish (e.g. an undecodable payload) stops consuming drain +/// cycles instead of retrying forever. +const MAX_ATTEMPTS: i32 = 10; + +/// Drains the detection-job outbox, publishing each pending job to the work-queue. +pub struct DetectionOutboxDrainer { + infra: Infra, + queue: DetectionQueue, +} + +/// The tally of one [`drain_batch`](DetectionOutboxDrainer::drain_batch) pass: of +/// the rows claimed, how many published, how many were deferred for a later retry, +/// and how many were dead-lettered. +struct DrainPass { + claimed: usize, + processed: usize, + deferred: usize, + dead_lettered: usize, +} + +impl Worker for DetectionOutboxDrainer { + type Output = Result<()>; + + fn name(&self) -> &'static str { + "detection_outbox_drainer" + } + + async fn run(&self, cancel: CancellationToken) -> Result<()> { + tracing::info!(target: TRACING_TARGET, "Starting detection-job drainer"); + + let mut ticker = tokio::time::interval(TICK_INTERVAL); + loop { + tokio::select! { + _ = cancel.cancelled() => break, + _ = ticker.tick() => self.tick(&cancel).await, + } + } + + tracing::info!(target: TRACING_TARGET, "Detection-job drainer stopped"); + Ok(()) + } +} + +impl DetectionOutboxDrainer { + /// Creates a new [`DetectionOutboxDrainer`]. + pub fn new(infra: Infra) -> Self { + Self { + queue: DetectionQueue::new(infra.clone()), + infra, + } + } + + /// One drain pass: claim and publish batches until a short page signals the + /// due set is drained, or until cancellation is requested. + /// + /// The cancellation check between batches keeps shutdown prompt even under a + /// sustained backlog. + async fn tick(&self, cancel: &CancellationToken) { + loop { + if cancel.is_cancelled() { + break; + } + match self.drain_batch().await { + Ok(pass) => { + if pass.deferred > 0 || pass.dead_lettered > 0 { + tracing::warn!( + target: TRACING_TARGET, + claimed = pass.claimed, + processed = pass.processed, + deferred = pass.deferred, + dead_lettered = pass.dead_lettered, + "Detection-job drain pass had failing jobs", + ); + } else if pass.claimed > 0 { + tracing::debug!(target: TRACING_TARGET, processed = pass.processed, "Detection-job drain pass published jobs"); + } + if pass.claimed < DRAIN_BATCH as usize { + break; + } + } + Err(err) => { + tracing::error!(target: TRACING_TARGET, error = %err, "Detection-job drain pass failed"); + break; + } + } + } + } + + /// Drains one batch: claims due rows and publishes each to the work-queue, all + /// in one transaction. Returns the [`DrainPass`] tally. + /// + /// The transaction holds the claim's `FOR UPDATE SKIP LOCKED` locks through + /// completion, so the claim and each row's state transition commit atomically + /// and no other drainer takes the same rows. The publish runs inside the + /// transaction and gates `mark_processed`: this is at-least-once (a crash after + /// publish but before commit re-publishes on the next pass), which the worker's + /// claim-based dedup absorbs — the same detection is only ever analyzed once. + async fn drain_batch(&self) -> Result { + let mut conn = self.infra.postgres.get_connection().await?; + + let pass = conn + .transaction(async |conn| { + let batch = conn.claim_detection_job_batch(DRAIN_BATCH).await?; + let mut pass = DrainPass { + claimed: batch.len(), + processed: 0, + deferred: 0, + dead_lettered: 0, + }; + + for row in batch { + match self.publish(&row).await { + Ok(()) => { + conn.mark_detection_job_processed(row.id).await?; + pass.processed += 1; + } + // `attempts` counts prior failures; this attempt makes it + // `attempts + 1`. Once that reaches the cap, dead-letter the + // row instead of deferring it forever. + Err(()) if row.attempts + 1 >= MAX_ATTEMPTS => { + tracing::error!(target: TRACING_TARGET, id = %row.id, attempts = row.attempts + 1, "Dead-lettering detection job after too many failed attempts"); + conn.mark_detection_job_failed(row.id).await?; + pass.dead_lettered += 1; + } + Err(()) => { + conn.defer_detection_job_attempt(row.id, retry_backoff(row.attempts)) + .await?; + pass.deferred += 1; + } + } + } + + Ok::<_, Error>(pass) + }) + .await?; + + Ok(pass) + } + + /// Decodes a row's job and publishes it to the work-queue. Returns `Err` if the + /// payload cannot decode or the publish fails, so the caller defers or + /// dead-letters it. + async fn publish(&self, row: &WorkspaceDetectionJob) -> std::result::Result<(), ()> { + let job = serde_json::from_value::(row.job.clone()).map_err(|err| { + tracing::error!(target: TRACING_TARGET, error = %err, id = %row.id, "Failed to decode detection job"); + })?; + self.queue.enqueue(job).await.map_err(|err| { + tracing::warn!(target: TRACING_TARGET, error = %err, id = %row.id, "Failed to publish detection job; deferring"); + }) + } +} + +/// The delay in seconds before a failed row's next attempt: linear in `attempts` +/// (the count before this failure), capped at [`RETRY_BACKOFF_MAX_SECS`]. +fn retry_backoff(attempts: i32) -> i64 { + let steps = i64::from(attempts.max(0)) + 1; + RETRY_BACKOFF_BASE_SECS + .saturating_mul(steps) + .min(RETRY_BACKOFF_MAX_SECS) +} diff --git a/crates/nvisy-server/src/service/detection/mod.rs b/crates/nvisy-server/src/service/detection/mod.rs index 22b41c6d..e098d459 100644 --- a/crates/nvisy-server/src/service/detection/mod.rs +++ b/crates/nvisy-server/src/service/detection/mod.rs @@ -5,12 +5,14 @@ //! the request thread. Status changes are broadcast on a core-NATS subject (see //! [`detection_subject`]) for SSE watchers and emitted as webhook events. +mod drainer; mod job; mod service; mod support; mod worker; +pub use drainer::DetectionOutboxDrainer; pub use job::{DetectionJob, DetectionStatusEvent, detection_subject}; pub use service::DetectionQueue; -pub(crate) use support::{FailDetection, fail_detection, resolve_policies}; +pub(crate) use support::resolve_policies; pub use worker::DetectionWorker; diff --git a/crates/nvisy-server/src/service/mod.rs b/crates/nvisy-server/src/service/mod.rs index 2ec1d725..de5ff09e 100644 --- a/crates/nvisy-server/src/service/mod.rs +++ b/crates/nvisy-server/src/service/mod.rs @@ -33,10 +33,11 @@ pub use crate::service::chat::{ChatService, TurnLocation}; pub use crate::service::connection_config::ConnectionConfig; pub use crate::service::crypto::{CryptoConfig, CryptoService}; pub(crate) use crate::service::crypto::{CryptoError, HashingReader, Measurements}; +pub(crate) use crate::service::detection::resolve_policies; pub use crate::service::detection::{ - DetectionJob, DetectionQueue, DetectionStatusEvent, DetectionWorker, detection_subject, + DetectionJob, DetectionOutboxDrainer, DetectionQueue, DetectionStatusEvent, DetectionWorker, + detection_subject, }; -pub(crate) use crate::service::detection::{FailDetection, fail_detection, resolve_policies}; pub use crate::service::engine::{EngineConfig, EngineService, UnknownFormatToken}; pub use crate::service::event::{ ConnectionRef, DetectionRef, EventEmitter, EventOrigin, EventOutboxDrainer, FileRef, InviteRef, @@ -173,6 +174,7 @@ impl ServiceState { )); workers.spawn(FileReaper::new(self.infra.clone())); workers.spawn(EventOutboxDrainer::new(self.infra.clone())); + workers.spawn(DetectionOutboxDrainer::new(self.infra.clone())); workers.spawn(DetectionWorker::new( self.infra.clone(), self.engine.clone(), diff --git a/migrations/2026-08-21-045211_event_outbox/down.sql b/migrations/2025-05-21-222843_event_outbox/down.sql similarity index 100% rename from migrations/2026-08-21-045211_event_outbox/down.sql rename to migrations/2025-05-21-222843_event_outbox/down.sql diff --git a/migrations/2026-08-21-045211_event_outbox/up.sql b/migrations/2025-05-21-222843_event_outbox/up.sql similarity index 100% rename from migrations/2026-08-21-045211_event_outbox/up.sql rename to migrations/2025-05-21-222843_event_outbox/up.sql diff --git a/migrations/2026-01-19-045014_policies/down.sql b/migrations/2026-01-19-045014_policies/down.sql new file mode 100644 index 00000000..2d22722a --- /dev/null +++ b/migrations/2026-01-19-045014_policies/down.sql @@ -0,0 +1,2 @@ +-- Revert the policies table. +DROP TABLE IF EXISTS workspace_policies; diff --git a/migrations/2026-01-19-045015_policies/up.sql b/migrations/2026-01-19-045014_policies/up.sql similarity index 72% rename from migrations/2026-01-19-045015_policies/up.sql rename to migrations/2026-01-19-045014_policies/up.sql index 7f10fa5d..1dcb42f5 100644 --- a/migrations/2026-01-19-045015_policies/up.sql +++ b/migrations/2026-01-19-045014_policies/up.sql @@ -80,32 +80,3 @@ COMMENT ON COLUMN workspace_policies.metadata IS 'Metadata for filtering/display COMMENT ON COLUMN workspace_policies.created_at IS 'Creation timestamp'; COMMENT ON COLUMN workspace_policies.updated_at IS 'Last modification timestamp'; COMMENT ON COLUMN workspace_policies.deleted_at IS 'Soft-deletion timestamp; NULL means live'; - --- Pipeline -> policy join table: redaction policies a pipeline applies. --- Lives here (after both workspace_pipelines and workspace_policies exist); the --- shared workspace_id in both composite foreign keys enforces that a pipeline --- can only reference policies from its own workspace. Drops before its parent in --- down.sql. -CREATE TABLE workspace_pipeline_policies ( - -- References - workspace_id UUID NOT NULL REFERENCES workspaces (id) ON DELETE CASCADE, - pipeline_id UUID NOT NULL, - policy_id UUID NOT NULL, - - PRIMARY KEY (pipeline_id, policy_id), - - -- Composite foreign keys, both sharing workspace_id, so a pipeline can only - -- reference policies from its own workspace. - CONSTRAINT workspace_pipeline_policies_pipeline_fkey FOREIGN KEY (workspace_id, pipeline_id) - REFERENCES workspace_pipelines (workspace_id, id) ON DELETE CASCADE, - CONSTRAINT workspace_pipeline_policies_policy_fkey FOREIGN KEY (workspace_id, policy_id) - REFERENCES workspace_policies (workspace_id, id) ON DELETE CASCADE -); - --- All pipelines that apply a given policy (back the policy composite FK). -CREATE INDEX workspace_pipeline_policies_policy_idx ON workspace_pipeline_policies (policy_id); - -COMMENT ON TABLE workspace_pipeline_policies IS 'Policies a pipeline applies at redaction. CASCADE cleans up on hard delete.'; -COMMENT ON COLUMN workspace_pipeline_policies.workspace_id IS 'Workspace shared by the pipeline and policy'; -COMMENT ON COLUMN workspace_pipeline_policies.pipeline_id IS 'Pipeline that applies the policy'; -COMMENT ON COLUMN workspace_pipeline_policies.policy_id IS 'Policy applied by the pipeline'; diff --git a/migrations/2026-01-19-045015_pipelines/down.sql b/migrations/2026-01-19-045015_pipelines/down.sql new file mode 100644 index 00000000..aee1463a --- /dev/null +++ b/migrations/2026-01-19-045015_pipelines/down.sql @@ -0,0 +1,7 @@ +-- Revert the pipelines tables. +-- Objects are dropped in reverse order of creation. + +DROP TABLE IF EXISTS workspace_pipeline_policies; +DROP TABLE IF EXISTS workspace_pipelines; + +DROP TYPE IF EXISTS PIPELINE_STATUS; diff --git a/migrations/2026-01-19-045015_pipelines/up.sql b/migrations/2026-01-19-045015_pipelines/up.sql new file mode 100644 index 00000000..35312489 --- /dev/null +++ b/migrations/2026-01-19-045015_pipelines/up.sql @@ -0,0 +1,129 @@ +-- Pipelines: redaction pipeline definitions. A pipeline is a workspace-scoped +-- detection/redaction config that references workspace policies (via the join +-- table below) and drives the detections created from it. + +-- Lifecycle status of a pipeline definition. +CREATE TYPE PIPELINE_STATUS AS ENUM ( + 'draft', -- Pipeline is being configured + 'enabled', -- Pipeline is ready to run + 'disabled' -- Pipeline is turned off +); + +COMMENT ON TYPE PIPELINE_STATUS IS 'Lifecycle status of a pipeline definition: draft, enabled, or disabled.'; + +-- Pipeline definitions table: a workspace's detection/redaction configs. +CREATE TABLE workspace_pipelines ( + -- Primary identifier + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + -- References + workspace_id UUID NOT NULL REFERENCES workspaces (id) ON DELETE CASCADE, + account_id UUID NOT NULL REFERENCES accounts (id) ON DELETE CASCADE, + + -- Composite key target for workspace-scoped foreign keys (join tables). + CONSTRAINT workspace_pipelines_workspace_id_id_key UNIQUE (workspace_id, id), + + -- URL identity, unique within the workspace (among live pipelines; enforced + -- by a partial index below so a slug frees up after soft deletion): lowercase + -- alphanumeric with single internal dashes, 3-32 characters. + slug TEXT NOT NULL, + CONSTRAINT workspace_pipelines_slug_length CHECK (length(slug) BETWEEN 3 AND 32), + CONSTRAINT workspace_pipelines_slug_format CHECK (slug ~ '^[a-z0-9]+(-[a-z0-9]+)*$'), + + -- Core attributes + display_name TEXT NOT NULL, + CONSTRAINT workspace_pipelines_display_name_length CHECK (length(trim(display_name)) BETWEEN 2 AND 128), + description TEXT DEFAULT NULL, + CONSTRAINT workspace_pipelines_description_length CHECK (description IS NULL OR length(description) <= 500), + status PIPELINE_STATUS NOT NULL DEFAULT 'draft', + + -- Engine detection + redaction config (nvisy_schema plan as JSON): + -- recognizers, enrichers, deduplication, label catalog, default scope. + -- Policy references are relational (workspace_pipeline_policies, declared + -- alongside policies), not embedded here. + definition JSONB NOT NULL, + CONSTRAINT workspace_pipelines_definition_size CHECK (length(definition::TEXT) BETWEEN 2 AND 1048576), + + -- Free-form metadata for filtering and display. + metadata JSONB NOT NULL DEFAULT '{}', + CONSTRAINT workspace_pipelines_metadata_size CHECK (length(metadata::TEXT) BETWEEN 2 AND 65536), + + -- Lifecycle timestamps + created_at TIMESTAMPTZ NOT NULL DEFAULT current_timestamp, + updated_at TIMESTAMPTZ NOT NULL DEFAULT current_timestamp, + deleted_at TIMESTAMPTZ DEFAULT NULL, + CONSTRAINT workspace_pipelines_updated_after_created CHECK (updated_at >= created_at), + CONSTRAINT workspace_pipelines_deleted_after_created CHECK (deleted_at IS NULL OR deleted_at >= created_at) +); + +-- Maintain updated_at on every row modification. +SELECT setup_updated_at('workspace_pipelines'); + +-- One live pipeline per slug within a workspace (slug frees up after deletion). +CREATE UNIQUE INDEX workspace_pipelines_slug_unique_idx + ON workspace_pipelines (workspace_id, slug) + WHERE deleted_at IS NULL; + +-- Live pipelines of a workspace, newest first (the pipeline list). +CREATE INDEX workspace_pipelines_workspace_idx + ON workspace_pipelines (workspace_id, created_at DESC) + WHERE deleted_at IS NULL; + +-- Live pipelines created by an account, newest first. +CREATE INDEX workspace_pipelines_account_idx + ON workspace_pipelines (account_id, created_at DESC) + WHERE deleted_at IS NULL; + +-- Filter live pipelines by lifecycle status within a workspace. +CREATE INDEX workspace_pipelines_status_idx + ON workspace_pipelines (status, workspace_id) + WHERE deleted_at IS NULL; + +-- Trigram search over live pipeline display names. +CREATE INDEX workspace_pipelines_display_name_trgm_idx + ON workspace_pipelines USING gin (display_name gin_trgm_ops) + WHERE deleted_at IS NULL; + +COMMENT ON TABLE workspace_pipelines IS 'Workspace-scoped redaction pipeline definitions.'; +COMMENT ON COLUMN workspace_pipelines.id IS 'Unique pipeline identifier'; +COMMENT ON COLUMN workspace_pipelines.workspace_id IS 'Workspace this pipeline belongs to'; +COMMENT ON COLUMN workspace_pipelines.account_id IS 'Account that created the pipeline'; +COMMENT ON COLUMN workspace_pipelines.slug IS 'URL identity, unique among live pipelines in the workspace'; +COMMENT ON COLUMN workspace_pipelines.display_name IS 'Pipeline display name (2-128 chars)'; +COMMENT ON COLUMN workspace_pipelines.description IS 'Pipeline description (up to 500 chars)'; +COMMENT ON COLUMN workspace_pipelines.status IS 'Pipeline lifecycle status'; +COMMENT ON COLUMN workspace_pipelines.definition IS 'Detection/redaction config (nvisy_schema plan as JSON)'; +COMMENT ON COLUMN workspace_pipelines.metadata IS 'Free-form metadata for filtering/display'; +COMMENT ON COLUMN workspace_pipelines.created_at IS 'Pipeline creation timestamp'; +COMMENT ON COLUMN workspace_pipelines.updated_at IS 'Last modification timestamp'; +COMMENT ON COLUMN workspace_pipelines.deleted_at IS 'Soft-deletion timestamp; NULL means live'; + + +-- Pipeline -> policy join table: redaction policies a pipeline applies. +-- Lives here (after both workspace_pipelines and workspace_policies exist); the +-- shared workspace_id in both composite foreign keys enforces that a pipeline +-- can only reference policies from its own workspace. Drops before its parent in +-- down.sql. +CREATE TABLE workspace_pipeline_policies ( + -- References + workspace_id UUID NOT NULL REFERENCES workspaces (id) ON DELETE CASCADE, + pipeline_id UUID NOT NULL, + policy_id UUID NOT NULL, + + PRIMARY KEY (pipeline_id, policy_id), + + -- Composite foreign keys, both sharing workspace_id, so a pipeline can only + -- reference policies from its own workspace. + CONSTRAINT workspace_pipeline_policies_pipeline_fkey FOREIGN KEY (workspace_id, pipeline_id) + REFERENCES workspace_pipelines (workspace_id, id) ON DELETE CASCADE, + CONSTRAINT workspace_pipeline_policies_policy_fkey FOREIGN KEY (workspace_id, policy_id) + REFERENCES workspace_policies (workspace_id, id) ON DELETE CASCADE +); + +-- All pipelines that apply a given policy (back the policy composite FK). +CREATE INDEX workspace_pipeline_policies_policy_idx ON workspace_pipeline_policies (policy_id); + +COMMENT ON TABLE workspace_pipeline_policies IS 'Policies a pipeline applies at redaction. CASCADE cleans up on hard delete.'; +COMMENT ON COLUMN workspace_pipeline_policies.workspace_id IS 'Workspace shared by the pipeline and policy'; +COMMENT ON COLUMN workspace_pipeline_policies.pipeline_id IS 'Pipeline that applies the policy'; +COMMENT ON COLUMN workspace_pipeline_policies.policy_id IS 'Policy applied by the pipeline'; diff --git a/migrations/2026-01-19-045015_policies/down.sql b/migrations/2026-01-19-045015_policies/down.sql deleted file mode 100644 index 13919a95..00000000 --- a/migrations/2026-01-19-045015_policies/down.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Revert the policies tables. --- Objects are dropped in reverse order of creation. - --- workspace_pipeline_policies is the pipeline-to-policy join; drop it before --- its parent policy table. -DROP TABLE IF EXISTS workspace_pipeline_policies; -DROP TABLE IF EXISTS workspace_policies; diff --git a/migrations/2026-01-19-045014_pipelines/down.sql b/migrations/2026-01-19-045016_detections/down.sql similarity index 59% rename from migrations/2026-01-19-045014_pipelines/down.sql rename to migrations/2026-01-19-045016_detections/down.sql index ede23721..1eee3a1d 100644 --- a/migrations/2026-01-19-045014_pipelines/down.sql +++ b/migrations/2026-01-19-045016_detections/down.sql @@ -1,11 +1,9 @@ --- Revert the pipelines tables. +-- Revert the detections tables. -- Objects are dropped in reverse order of creation. -DROP TABLE IF EXISTS workspace_redactions; +DROP TABLE IF EXISTS workspace_detection_jobs; DROP TABLE IF EXISTS workspace_detection_usage; DROP TABLE IF EXISTS workspace_detections; -DROP TABLE IF EXISTS workspace_pipelines; DROP TYPE IF EXISTS PIPELINE_TRIGGER_TYPE; DROP TYPE IF EXISTS DETECTION_STATUS; -DROP TYPE IF EXISTS PIPELINE_STATUS; diff --git a/migrations/2026-01-19-045014_pipelines/up.sql b/migrations/2026-01-19-045016_detections/up.sql similarity index 57% rename from migrations/2026-01-19-045014_pipelines/up.sql rename to migrations/2026-01-19-045016_detections/up.sql index 414ee122..0110aa78 100644 --- a/migrations/2026-01-19-045014_pipelines/up.sql +++ b/migrations/2026-01-19-045016_detections/up.sql @@ -1,20 +1,7 @@ --- Pipelines: redaction pipeline definitions, their detections, and the --- redactions produced from each. A pipeline is a workspace-scoped --- detection/redaction config; a detection is one analysis pass of a file --- through that config, and each detection can produce many redactions (one per --- reviewer-edited redact request). Policy references live in a join table --- declared alongside policies, not embedded here. - --- Lifecycle status of a pipeline definition. -CREATE TYPE PIPELINE_STATUS AS ENUM ( - 'draft', -- Pipeline is being configured - 'enabled', -- Pipeline is ready to run - 'disabled' -- Pipeline is turned off -); - -COMMENT ON TYPE PIPELINE_STATUS IS 'Lifecycle status of a pipeline definition: draft, enabled, or disabled.'; +-- Detections: one analysis pass of a file through a pipeline, its per-model +-- usage, and the transactional outbox that queues each analysis. A detection +-- can produce many redactions (see the redactions migration). --- Execution status of a detection (analysis pass). CREATE TYPE DETECTION_STATUS AS ENUM ( 'pending', -- Enqueued for detection; no worker has picked it up yet 'executing', -- A worker is actively analyzing the document @@ -32,93 +19,6 @@ CREATE TYPE PIPELINE_TRIGGER_TYPE AS ENUM ( COMMENT ON TYPE PIPELINE_TRIGGER_TYPE IS 'How a detection was initiated: by a user or by the system.'; --- Pipeline definitions table: a workspace's detection/redaction configs. -CREATE TABLE workspace_pipelines ( - -- Primary identifier - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - - -- References - workspace_id UUID NOT NULL REFERENCES workspaces (id) ON DELETE CASCADE, - account_id UUID NOT NULL REFERENCES accounts (id) ON DELETE CASCADE, - - -- Composite key target for workspace-scoped foreign keys (join tables). - CONSTRAINT workspace_pipelines_workspace_id_id_key UNIQUE (workspace_id, id), - - -- URL identity, unique within the workspace (among live pipelines; enforced - -- by a partial index below so a slug frees up after soft deletion): lowercase - -- alphanumeric with single internal dashes, 3-32 characters. - slug TEXT NOT NULL, - CONSTRAINT workspace_pipelines_slug_length CHECK (length(slug) BETWEEN 3 AND 32), - CONSTRAINT workspace_pipelines_slug_format CHECK (slug ~ '^[a-z0-9]+(-[a-z0-9]+)*$'), - - -- Core attributes - display_name TEXT NOT NULL, - CONSTRAINT workspace_pipelines_display_name_length CHECK (length(trim(display_name)) BETWEEN 2 AND 128), - description TEXT DEFAULT NULL, - CONSTRAINT workspace_pipelines_description_length CHECK (description IS NULL OR length(description) <= 500), - status PIPELINE_STATUS NOT NULL DEFAULT 'draft', - - -- Engine detection + redaction config (nvisy_schema plan as JSON): - -- recognizers, enrichers, deduplication, label catalog, default scope. - -- Policy references are relational (workspace_pipeline_policies, declared - -- alongside policies), not embedded here. - definition JSONB NOT NULL, - CONSTRAINT workspace_pipelines_definition_size CHECK (length(definition::TEXT) BETWEEN 2 AND 1048576), - - -- Free-form metadata for filtering and display. - metadata JSONB NOT NULL DEFAULT '{}', - CONSTRAINT workspace_pipelines_metadata_size CHECK (length(metadata::TEXT) BETWEEN 2 AND 65536), - - -- Lifecycle timestamps - created_at TIMESTAMPTZ NOT NULL DEFAULT current_timestamp, - updated_at TIMESTAMPTZ NOT NULL DEFAULT current_timestamp, - deleted_at TIMESTAMPTZ DEFAULT NULL, - CONSTRAINT workspace_pipelines_updated_after_created CHECK (updated_at >= created_at), - CONSTRAINT workspace_pipelines_deleted_after_created CHECK (deleted_at IS NULL OR deleted_at >= created_at) -); - --- Maintain updated_at on every row modification. -SELECT setup_updated_at('workspace_pipelines'); - --- One live pipeline per slug within a workspace (slug frees up after deletion). -CREATE UNIQUE INDEX workspace_pipelines_slug_unique_idx - ON workspace_pipelines (workspace_id, slug) - WHERE deleted_at IS NULL; - --- Live pipelines of a workspace, newest first (the pipeline list). -CREATE INDEX workspace_pipelines_workspace_idx - ON workspace_pipelines (workspace_id, created_at DESC) - WHERE deleted_at IS NULL; - --- Live pipelines created by an account, newest first. -CREATE INDEX workspace_pipelines_account_idx - ON workspace_pipelines (account_id, created_at DESC) - WHERE deleted_at IS NULL; - --- Filter live pipelines by lifecycle status within a workspace. -CREATE INDEX workspace_pipelines_status_idx - ON workspace_pipelines (status, workspace_id) - WHERE deleted_at IS NULL; - --- Trigram search over live pipeline display names. -CREATE INDEX workspace_pipelines_display_name_trgm_idx - ON workspace_pipelines USING gin (display_name gin_trgm_ops) - WHERE deleted_at IS NULL; - -COMMENT ON TABLE workspace_pipelines IS 'Workspace-scoped redaction pipeline definitions.'; -COMMENT ON COLUMN workspace_pipelines.id IS 'Unique pipeline identifier'; -COMMENT ON COLUMN workspace_pipelines.workspace_id IS 'Workspace this pipeline belongs to'; -COMMENT ON COLUMN workspace_pipelines.account_id IS 'Account that created the pipeline'; -COMMENT ON COLUMN workspace_pipelines.slug IS 'URL identity, unique among live pipelines in the workspace'; -COMMENT ON COLUMN workspace_pipelines.display_name IS 'Pipeline display name (2-128 chars)'; -COMMENT ON COLUMN workspace_pipelines.description IS 'Pipeline description (up to 500 chars)'; -COMMENT ON COLUMN workspace_pipelines.status IS 'Pipeline lifecycle status'; -COMMENT ON COLUMN workspace_pipelines.definition IS 'Detection/redaction config (nvisy_schema plan as JSON)'; -COMMENT ON COLUMN workspace_pipelines.metadata IS 'Free-form metadata for filtering/display'; -COMMENT ON COLUMN workspace_pipelines.created_at IS 'Pipeline creation timestamp'; -COMMENT ON COLUMN workspace_pipelines.updated_at IS 'Last modification timestamp'; -COMMENT ON COLUMN workspace_pipelines.deleted_at IS 'Soft-deletion timestamp; NULL means live'; - -- Detections table: one analysis pass of a file through a pipeline. CREATE TABLE workspace_detections ( -- Primary identifier @@ -226,53 +126,6 @@ COMMENT ON COLUMN workspace_detections.claimed_at IS 'Detection lease: when a wo COMMENT ON COLUMN workspace_detections.started_at IS 'When the detection started'; COMMENT ON COLUMN workspace_detections.completed_at IS 'When the detection completed; NULL while in flight'; --- Redactions table: one redact pass over a detection's analysis. A detection can --- be redacted many times — each redact request may carry a different set of --- reviewer edits — so each is its own row owning the edited audit it applied and --- the redacted document it produced. -CREATE TABLE workspace_redactions ( - -- Primary identifier - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - - -- The detection this redaction was produced from; redactions are deleted - -- with their detection. - detection_id UUID NOT NULL REFERENCES workspace_detections (id) ON DELETE CASCADE, - - -- Account that requested the redaction. - account_id UUID NOT NULL REFERENCES accounts (id) ON DELETE CASCADE, - - -- The two files a redaction produces. A redaction always yields both, so the - -- app sets them at creation; they are nullable only because `ON DELETE SET - -- NULL` clears a reference if its file is ever hard-deleted (for the same - -- append-only-history reasons as a detection — a soft-deleted file resolves to - -- "gone" at read time, distinct from a NULL that means the file was purged). - -- review: the engine's Audit after the reviewer edits were applied and - -- redaction ran (`file_kind = review`); the record of exactly what - -- was redacted and why. - -- output: the redacted document this redaction produced. - review_file_id UUID DEFAULT NULL REFERENCES workspace_files (id) ON DELETE SET NULL, - output_file_id UUID DEFAULT NULL REFERENCES workspace_files (id) ON DELETE SET NULL, - - -- Timing - created_at TIMESTAMPTZ NOT NULL DEFAULT current_timestamp -); - --- A detection's redactions, newest first (the redaction list). -CREATE INDEX workspace_redactions_detection_idx - ON workspace_redactions (detection_id, created_at DESC); - --- Redactions requested by an account, newest first. -CREATE INDEX workspace_redactions_account_idx - ON workspace_redactions (account_id, created_at DESC); - -COMMENT ON TABLE workspace_redactions IS 'Redactions: one redact pass over a detection, with its own reviewer edits, edited audit, and output.'; -COMMENT ON COLUMN workspace_redactions.id IS 'Unique redaction identifier'; -COMMENT ON COLUMN workspace_redactions.detection_id IS 'Detection this redaction was produced from'; -COMMENT ON COLUMN workspace_redactions.account_id IS 'Account that requested the redaction'; -COMMENT ON COLUMN workspace_redactions.review_file_id IS 'Review audit (file_kind=review) recording the applied edits and redaction outcome'; -COMMENT ON COLUMN workspace_redactions.output_file_id IS 'Redacted document this redaction produced'; -COMMENT ON COLUMN workspace_redactions.created_at IS 'When the redaction was created'; - -- Per-model inference usage for a detection: one row per distinct model a -- detection's recognizers used. Token counts are aggregated across the -- recognizers that shared a model, letting usage analytics report tokens broken @@ -330,3 +183,60 @@ COMMENT ON COLUMN workspace_detection_usage.input_tokens IS 'Input/prompt tokens COMMENT ON COLUMN workspace_detection_usage.output_tokens IS 'Output/completion tokens for this model; NULL if not reported'; COMMENT ON COLUMN workspace_detection_usage.total_tokens IS 'Total tokens as reported (not necessarily input + output); NULL if not reported'; COMMENT ON COLUMN workspace_detection_usage.duration_ms IS 'Wall-clock time this model spent, in milliseconds'; + +-- Detection-job outbox: the transactional-outbox queue for detection analysis. +-- Creating a detection inserts one row here in the same transaction as the +-- detection, so the two commit or roll back together; a background drainer then +-- publishes each pending row onto the detection NATS work-queue. This removes the +-- dual-write between the detection row and the queue: an analysis is never lost +-- to an enqueue that failed after the row committed, nor is a detection ever +-- marked failed for an enqueue that in fact went through. +CREATE TABLE workspace_detection_jobs ( + -- Primary identifier + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + -- The detection this job analyzes; the row is deleted with its detection. + detection_id UUID NOT NULL REFERENCES workspace_detections (id) ON DELETE CASCADE, + + -- The job. A serialized `DetectionJob`: the workspace, detection, and the + -- optional per-request scope the drainer publishes to the worker. + job JSONB NOT NULL, + CONSTRAINT workspace_detection_jobs_job_size CHECK (length(job::TEXT) BETWEEN 2 AND 16384), + + -- Drainer bookkeeping: the row's processing state, how many publish attempts + -- it has taken, and the earliest time it may next be claimed (advanced by a + -- backoff on each failed attempt so a failing row does not spin at the head of + -- the queue). + status OUTBOX_STATUS NOT NULL DEFAULT 'pending', + attempts INTEGER NOT NULL DEFAULT 0, + CONSTRAINT workspace_detection_jobs_attempts_non_negative CHECK (attempts >= 0), + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT current_timestamp, + + -- Lifecycle timestamps + created_at TIMESTAMPTZ NOT NULL DEFAULT current_timestamp, + + -- When a terminal row (processed or failed) was resolved by an operator; NULL + -- until then. A manual affordance for inspecting the outbox after the fact. + resolved_at TIMESTAMPTZ DEFAULT NULL, + CONSTRAINT workspace_detection_jobs_resolved_only_when_terminal + CHECK (resolved_at IS NULL OR status IN ('processed', 'failed')), + CONSTRAINT workspace_detection_jobs_resolved_after_created + CHECK (resolved_at IS NULL OR resolved_at >= created_at) +); + +-- The drainer's claim queue: pending rows ordered by due time then age, so a +-- batch claims the oldest due rows. Partial so it stays small as processed and +-- failed rows accumulate. +CREATE INDEX workspace_detection_jobs_pending_idx + ON workspace_detection_jobs (next_attempt_at, created_at) + WHERE status = 'pending'; + +COMMENT ON TABLE workspace_detection_jobs IS 'Transactional outbox of detection jobs, drained to the detection NATS work-queue.'; +COMMENT ON COLUMN workspace_detection_jobs.id IS 'Unique outbox row identifier'; +COMMENT ON COLUMN workspace_detection_jobs.detection_id IS 'Detection this job analyzes'; +COMMENT ON COLUMN workspace_detection_jobs.job IS 'Serialized DetectionJob published to the worker (JSON, 2B-16KB)'; +COMMENT ON COLUMN workspace_detection_jobs.status IS 'Processing state: pending, processed, or failed (dead-lettered)'; +COMMENT ON COLUMN workspace_detection_jobs.attempts IS 'Number of publish attempts the drainer has made'; +COMMENT ON COLUMN workspace_detection_jobs.next_attempt_at IS 'Earliest time the row may next be claimed; advanced by a backoff after each failed attempt'; +COMMENT ON COLUMN workspace_detection_jobs.created_at IS 'Timestamp when the job was queued'; +COMMENT ON COLUMN workspace_detection_jobs.resolved_at IS 'When a terminal (processed or failed) row was resolved by an operator; NULL until then. A manual affordance for inspecting the outbox after the fact'; diff --git a/migrations/2026-01-19-045017_redactions/down.sql b/migrations/2026-01-19-045017_redactions/down.sql new file mode 100644 index 00000000..07532f98 --- /dev/null +++ b/migrations/2026-01-19-045017_redactions/down.sql @@ -0,0 +1,4 @@ +-- Revert the redactions table. +-- Objects are dropped in reverse order of creation. + +DROP TABLE IF EXISTS workspace_redactions; diff --git a/migrations/2026-01-19-045017_redactions/up.sql b/migrations/2026-01-19-045017_redactions/up.sql new file mode 100644 index 00000000..113bd14a --- /dev/null +++ b/migrations/2026-01-19-045017_redactions/up.sql @@ -0,0 +1,46 @@ +-- Redactions: one redact pass over a detection's analysis. A detection can be +-- redacted many times, so each redaction is its own row owning the review audit +-- it applied and the redacted document it produced. + +CREATE TABLE workspace_redactions ( + -- Primary identifier + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + -- The detection this redaction was produced from; redactions are deleted + -- with their detection. + detection_id UUID NOT NULL REFERENCES workspace_detections (id) ON DELETE CASCADE, + + -- Account that requested the redaction. + account_id UUID NOT NULL REFERENCES accounts (id) ON DELETE CASCADE, + + -- The two files a redaction produces. A redaction always yields both, so the + -- app sets them at creation; they are nullable only because `ON DELETE SET + -- NULL` clears a reference if its file is ever hard-deleted (for the same + -- append-only-history reasons as a detection — a soft-deleted file resolves to + -- "gone" at read time, distinct from a NULL that means the file was purged). + -- review: the engine's Audit after the reviewer edits were applied and + -- redaction ran (`file_kind = review`); the record of exactly what + -- was redacted and why. + -- output: the redacted document this redaction produced. + review_file_id UUID DEFAULT NULL REFERENCES workspace_files (id) ON DELETE SET NULL, + output_file_id UUID DEFAULT NULL REFERENCES workspace_files (id) ON DELETE SET NULL, + + -- Timing + created_at TIMESTAMPTZ NOT NULL DEFAULT current_timestamp +); + +-- A detection's redactions, newest first (the redaction list). +CREATE INDEX workspace_redactions_detection_idx + ON workspace_redactions (detection_id, created_at DESC); + +-- Redactions requested by an account, newest first. +CREATE INDEX workspace_redactions_account_idx + ON workspace_redactions (account_id, created_at DESC); + +COMMENT ON TABLE workspace_redactions IS 'Redactions: one redact pass over a detection, with its own reviewer edits, review audit, and output.'; +COMMENT ON COLUMN workspace_redactions.id IS 'Unique redaction identifier'; +COMMENT ON COLUMN workspace_redactions.detection_id IS 'Detection this redaction was produced from'; +COMMENT ON COLUMN workspace_redactions.account_id IS 'Account that requested the redaction'; +COMMENT ON COLUMN workspace_redactions.review_file_id IS 'Review audit (file_kind=review) recording the applied edits and redaction outcome'; +COMMENT ON COLUMN workspace_redactions.output_file_id IS 'Redacted document this redaction produced'; +COMMENT ON COLUMN workspace_redactions.created_at IS 'When the redaction was created'; From d5c89511b6e73e59231a6b236154118ac4d4ce75 Mon Sep 17 00:00:00 2001 From: Oleh Martsokha Date: Wed, 26 Aug 2026 17:26:21 +0200 Subject: [PATCH 6/7] Address PR review: fail detection on job dead-letter; drainer hardening - Fail the detection when its outbox job is dead-lettered. The job never published, so the worker would never drive the detection terminal, leaving it Pending forever. The drainer now fails the detection (guarded on Pending) in the same transaction with a reason in metadata, and broadcasts the Failed status to SSE watchers after commit. - Add an index on workspace_detection_jobs.detection_id so a detection delete cascades without scanning the outbox (the partial claim index doesn't cover it; redaction/usage tables already lead their indexes with detection_id). - Share the `complete`-status FILTER clause between the two duration SQL fragments so they cannot drift. - Bound the drainer's NATS publish with a timeout so a hung NATS cannot hold the batch transaction's row locks open indefinitely; a timed-out publish is a failed attempt (deferred with backoff), releasing the locks. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018bKk1YEG4tZ69jzYVQvQL8 --- crates/nvisy-postgres/src/query/analytics.rs | 22 +++--- .../src/service/detection/drainer.rs | 69 ++++++++++++++++--- .../2026-01-19-045016_detections/up.sql | 6 ++ 3 files changed, 77 insertions(+), 20 deletions(-) diff --git a/crates/nvisy-postgres/src/query/analytics.rs b/crates/nvisy-postgres/src/query/analytics.rs index 80f51585..049ad8d7 100644 --- a/crates/nvisy-postgres/src/query/analytics.rs +++ b/crates/nvisy-postgres/src/query/analytics.rs @@ -444,20 +444,20 @@ async fn load_detection_day_counts( // Conditional counts are `sum(CASE WHEN cond THEN 1 ELSE 0 END)` since Diesel's // aggregate FILTER is not available on `count(*)`. Columns are table-qualified // so the join to pipelines can never make them ambiguous. - // Durations describe successful analysis only, so filter the aggregates on the - // terminal `complete` status: `completed_at` is stamped on failure too, so a - // presence check alone would fold failed detections into avg/p95. - let avg_ms = sql::>( + // Durations describe successful analysis only, so both aggregates filter on + // the terminal `complete` status: `completed_at` is stamped on failure too, so + // a presence check alone would fold failed detections into avg/p95. The FILTER + // clause is shared so the two fragments cannot drift apart. + const COMPLETE_FILTER: &str = " FILTER (WHERE workspace_detections.status = 'complete')"; + let avg_ms = sql::>(&format!( "round(avg(EXTRACT(EPOCH FROM (workspace_detections.completed_at \ - - workspace_detections.started_at))) \ - FILTER (WHERE workspace_detections.status = 'complete') * 1000)::bigint", - ); - let p95_ms = sql::>( + - workspace_detections.started_at))){COMPLETE_FILTER} * 1000)::bigint" + )); + let p95_ms = sql::>(&format!( "round(percentile_cont(0.95) WITHIN GROUP \ (ORDER BY EXTRACT(EPOCH FROM (workspace_detections.completed_at \ - - workspace_detections.started_at))) \ - FILTER (WHERE workspace_detections.status = 'complete') * 1000)::bigint", - ); + - workspace_detections.started_at))){COMPLETE_FILTER} * 1000)::bigint" + )); workspace_detections::table .inner_join(workspace_pipelines::table) diff --git a/crates/nvisy-server/src/service/detection/drainer.rs b/crates/nvisy-server/src/service/detection/drainer.rs index 416bfc4c..f95f9a36 100644 --- a/crates/nvisy-server/src/service/detection/drainer.rs +++ b/crates/nvisy-server/src/service/detection/drainer.rs @@ -9,8 +9,9 @@ use std::time::Duration; use nvisy_postgres::AsyncConnection; -use nvisy_postgres::model::WorkspaceDetectionJob; -use nvisy_postgres::query::DetectionJobOutboxRepository; +use nvisy_postgres::model::{UpdateWorkspaceDetection, WorkspaceDetectionJob}; +use nvisy_postgres::query::{DetectionJobOutboxRepository, WorkspaceDetectionRepository}; +use nvisy_postgres::types::{DetectionMetadata, DetectionStatus, Json}; use tokio_util::sync::CancellationToken; use super::job::DetectionJob; @@ -42,6 +43,15 @@ const RETRY_BACKOFF_MAX_SECS: i64 = 60 * 60; /// cycles instead of retrying forever. const MAX_ATTEMPTS: i32 = 10; +/// The failure reason recorded on a detection whose job the drainer gave up +/// publishing, so the detection's terminal state explains why it never analyzed. +const DEAD_LETTER_REASON: &str = "Detection could not be queued for analysis"; + +/// Cap on a single publish, so a slow or unavailable NATS server cannot hold the +/// batch transaction's row locks open indefinitely. A publish that exceeds this is +/// treated as a failed attempt (deferred with a backoff), releasing the locks. +const PUBLISH_TIMEOUT: Duration = Duration::from_secs(5); + /// Drains the detection-job outbox, publishing each pending job to the work-queue. pub struct DetectionOutboxDrainer { infra: Infra, @@ -138,7 +148,7 @@ impl DetectionOutboxDrainer { async fn drain_batch(&self) -> Result { let mut conn = self.infra.postgres.get_connection().await?; - let pass = conn + let outcome = conn .transaction(async |conn| { let batch = conn.claim_detection_job_batch(DRAIN_BATCH).await?; let mut pass = DrainPass { @@ -147,6 +157,9 @@ impl DetectionOutboxDrainer { deferred: 0, dead_lettered: 0, }; + // Detections failed by a dead-lettered job, so their `Failed` + // status can be broadcast after the transaction commits. + let mut dead_lettered = Vec::new(); for row in batch { match self.publish(&row).await { @@ -156,10 +169,28 @@ impl DetectionOutboxDrainer { } // `attempts` counts prior failures; this attempt makes it // `attempts + 1`. Once that reaches the cap, dead-letter the - // row instead of deferring it forever. + // row instead of deferring it forever. The job never + // published, so the worker will never drive its detection to + // a terminal state: fail the detection here too (same + // transaction) so it does not hang `Pending` forever. The + // guard on `Pending` makes it a no-op in the unlikely case a + // worker already claimed the detection. Err(()) if row.attempts + 1 >= MAX_ATTEMPTS => { - tracing::error!(target: TRACING_TARGET, id = %row.id, attempts = row.attempts + 1, "Dead-lettering detection job after too many failed attempts"); + tracing::error!(target: TRACING_TARGET, id = %row.id, detection_id = %row.detection_id, attempts = row.attempts + 1, "Dead-lettering detection job after too many failed attempts; failing the detection"); conn.mark_detection_job_failed(row.id).await?; + let metadata = DetectionMetadata { + error: Some(DEAD_LETTER_REASON.to_owned()), + ..Default::default() + }; + conn.fail_pending_detection( + row.detection_id, + UpdateWorkspaceDetection { + metadata: Some(Json::encode(&metadata)), + ..Default::default() + }, + ) + .await?; + dead_lettered.push(row.detection_id); pass.dead_lettered += 1; } Err(()) => { @@ -170,10 +201,20 @@ impl DetectionOutboxDrainer { } } - Ok::<_, Error>(pass) + Ok::<_, Error>((pass, dead_lettered)) }) .await?; + // Announce each failed detection's terminal status to any SSE watcher, + // after the transaction that committed it. Best-effort: the detection row + // is authoritative, so a dropped broadcast is recoverable by a re-read. + let (pass, dead_lettered) = outcome; + for detection_id in dead_lettered { + self.queue + .broadcast_status(detection_id, DetectionStatus::Failed) + .await; + } + Ok(pass) } @@ -184,9 +225,19 @@ impl DetectionOutboxDrainer { let job = serde_json::from_value::(row.job.clone()).map_err(|err| { tracing::error!(target: TRACING_TARGET, error = %err, id = %row.id, "Failed to decode detection job"); })?; - self.queue.enqueue(job).await.map_err(|err| { - tracing::warn!(target: TRACING_TARGET, error = %err, id = %row.id, "Failed to publish detection job; deferring"); - }) + // Bound the publish so a hung NATS cannot hold the batch transaction's + // locks open; a timeout is a failed attempt like any other. + match tokio::time::timeout(PUBLISH_TIMEOUT, self.queue.enqueue(job)).await { + Ok(Ok(())) => Ok(()), + Ok(Err(err)) => { + tracing::warn!(target: TRACING_TARGET, error = %err, id = %row.id, "Failed to publish detection job; deferring"); + Err(()) + } + Err(_elapsed) => { + tracing::warn!(target: TRACING_TARGET, id = %row.id, "Detection-job publish timed out; deferring"); + Err(()) + } + } } } diff --git a/migrations/2026-01-19-045016_detections/up.sql b/migrations/2026-01-19-045016_detections/up.sql index 0110aa78..3285d591 100644 --- a/migrations/2026-01-19-045016_detections/up.sql +++ b/migrations/2026-01-19-045016_detections/up.sql @@ -231,6 +231,12 @@ CREATE INDEX workspace_detection_jobs_pending_idx ON workspace_detection_jobs (next_attempt_at, created_at) WHERE status = 'pending'; +-- Back the detection foreign key so a detection delete cascades without scanning +-- the whole outbox (Postgres does not index a referencing column automatically, +-- and the partial claim index above does not cover it). +CREATE INDEX workspace_detection_jobs_detection_idx + ON workspace_detection_jobs (detection_id); + COMMENT ON TABLE workspace_detection_jobs IS 'Transactional outbox of detection jobs, drained to the detection NATS work-queue.'; COMMENT ON COLUMN workspace_detection_jobs.id IS 'Unique outbox row identifier'; COMMENT ON COLUMN workspace_detection_jobs.detection_id IS 'Detection this job analyzes'; From 8de49e7ae9b91fa8834ec2b05aa1363ee2846bd1 Mon Sep 17 00:00:00 2001 From: Oleh Martsokha Date: Wed, 26 Aug 2026 17:59:04 +0200 Subject: [PATCH 7/7] Broadcast Failed only when the dead-letter actually failed the detection fail_pending_detection returns false when a worker already claimed the detection (a publish timeout does not prove the job never reached a worker). The drainer was broadcasting Failed regardless, so a subscriber could see a false terminal status while the detection is really Executing or Complete. Collect the detection for broadcast only when the guarded transition returned true. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018bKk1YEG4tZ69jzYVQvQL8 --- .../src/service/detection/drainer.rs | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/crates/nvisy-server/src/service/detection/drainer.rs b/crates/nvisy-server/src/service/detection/drainer.rs index f95f9a36..f096ec49 100644 --- a/crates/nvisy-server/src/service/detection/drainer.rs +++ b/crates/nvisy-server/src/service/detection/drainer.rs @@ -182,15 +182,23 @@ impl DetectionOutboxDrainer { error: Some(DEAD_LETTER_REASON.to_owned()), ..Default::default() }; - conn.fail_pending_detection( - row.detection_id, - UpdateWorkspaceDetection { - metadata: Some(Json::encode(&metadata)), - ..Default::default() - }, - ) - .await?; - dead_lettered.push(row.detection_id); + // Only broadcast `Failed` if this actually transitioned + // the detection. A publish timeout does not prove the job + // never reached a worker, so a worker may already own the + // detection (no longer `Pending`); the guard then no-ops + // and we must not announce a false terminal status. + let failed = conn + .fail_pending_detection( + row.detection_id, + UpdateWorkspaceDetection { + metadata: Some(Json::encode(&metadata)), + ..Default::default() + }, + ) + .await?; + if failed { + dead_lettered.push(row.detection_id); + } pass.dead_lettered += 1; } Err(()) => { @@ -205,9 +213,10 @@ impl DetectionOutboxDrainer { }) .await?; - // Announce each failed detection's terminal status to any SSE watcher, - // after the transaction that committed it. Best-effort: the detection row - // is authoritative, so a dropped broadcast is recoverable by a re-read. + // Announce the terminal status of each detection this pass actually failed + // (only those the guarded transition committed as `Failed` are collected), + // after its transaction commits. Best-effort: the detection row is + // authoritative, so a dropped broadcast is recoverable by a re-read. let (pass, dead_lettered) = outcome; for detection_id in dead_lettered { self.queue