diff --git a/Cargo.lock b/Cargo.lock index 554b9ae1..8ef09f31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2880,7 +2880,7 @@ checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" [[package]] name = "elide" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "async-trait", "elide-codec", @@ -2918,7 +2918,7 @@ dependencies = [ [[package]] name = "elide-codec" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "async-trait", "bytes", @@ -2940,7 +2940,7 @@ dependencies = [ [[package]] name = "elide-context" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "async-trait", "elide-core", @@ -2951,7 +2951,7 @@ dependencies = [ [[package]] name = "elide-core" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "async-trait", "blake3", @@ -2971,7 +2971,7 @@ dependencies = [ [[package]] name = "elide-detection" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "elide-core", "futures", @@ -2983,7 +2983,7 @@ dependencies = [ [[package]] name = "elide-engine" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "bytes", "elide-codec", @@ -3001,7 +3001,7 @@ dependencies = [ [[package]] name = "elide-export" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide-runtime?branch=main#cf0758d50a9697c0cc1530f856bf0472709ed80f" +source = "git+https://github.com/nvisycom/elide-runtime?branch=main#1b5bf516ac1b053bdc2e5e6cca27fe37456b025f" dependencies = [ "csv", "elide", @@ -3013,7 +3013,7 @@ dependencies = [ [[package]] name = "elide-fake" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "async-trait", "elide-core", @@ -3024,7 +3024,7 @@ dependencies = [ [[package]] name = "elide-governance" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide-runtime?branch=main#cf0758d50a9697c0cc1530f856bf0472709ed80f" +source = "git+https://github.com/nvisycom/elide-runtime?branch=main#1b5bf516ac1b053bdc2e5e6cca27fe37456b025f" dependencies = [ "elide-core", "elide-operator", @@ -3038,7 +3038,7 @@ dependencies = [ [[package]] name = "elide-lingua" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "async-trait", "elide-core", @@ -3049,7 +3049,7 @@ dependencies = [ [[package]] name = "elide-llm" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "async-trait", "derive_builder", @@ -3070,7 +3070,7 @@ dependencies = [ [[package]] name = "elide-ner" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "async-trait", "derive_builder", @@ -3084,7 +3084,7 @@ dependencies = [ [[package]] name = "elide-ocr" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "async-trait", "derive_builder", @@ -3096,7 +3096,7 @@ dependencies = [ [[package]] name = "elide-office" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "bytes", "hipstr", @@ -3108,7 +3108,7 @@ dependencies = [ [[package]] name = "elide-operator" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "aes-gcm", "async-trait", @@ -3130,7 +3130,7 @@ dependencies = [ [[package]] name = "elide-pattern" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "aho-corasick", "async-trait", @@ -3150,7 +3150,7 @@ dependencies = [ [[package]] name = "elide-pdf" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "bytes", "hipstr", @@ -3162,7 +3162,7 @@ dependencies = [ [[package]] name = "elide-pipeline" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide-runtime?branch=main#cf0758d50a9697c0cc1530f856bf0472709ed80f" +source = "git+https://github.com/nvisycom/elide-runtime?branch=main#1b5bf516ac1b053bdc2e5e6cca27fe37456b025f" dependencies = [ "bytes", "elide", @@ -3183,7 +3183,7 @@ dependencies = [ [[package]] name = "elide-provider" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide-runtime?branch=main#cf0758d50a9697c0cc1530f856bf0472709ed80f" +source = "git+https://github.com/nvisycom/elide-runtime?branch=main#1b5bf516ac1b053bdc2e5e6cca27fe37456b025f" dependencies = [ "elide", "elide-bento", @@ -3197,7 +3197,7 @@ dependencies = [ [[package]] name = "elide-redaction" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "elide-core", "hipstr", @@ -3207,7 +3207,7 @@ dependencies = [ [[package]] name = "elide-review" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide-runtime?branch=main#cf0758d50a9697c0cc1530f856bf0472709ed80f" +source = "git+https://github.com/nvisycom/elide-runtime?branch=main#1b5bf516ac1b053bdc2e5e6cca27fe37456b025f" dependencies = [ "elide", "elide-governance", @@ -3219,7 +3219,7 @@ dependencies = [ [[package]] name = "elide-stt" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#39fedecf65d823128f13a46232efecc4321bfa05" +source = "git+https://github.com/nvisycom/elide?branch=main#41884e3a27a945649728389240969e72db41c7f0" dependencies = [ "async-trait", "derive_builder", @@ -3231,7 +3231,7 @@ dependencies = [ [[package]] name = "elide-template" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide-runtime?branch=main#cf0758d50a9697c0cc1530f856bf0472709ed80f" +source = "git+https://github.com/nvisycom/elide-runtime?branch=main#1b5bf516ac1b053bdc2e5e6cca27fe37456b025f" dependencies = [ "elide-core", "elide-governance", @@ -6987,8 +6987,8 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ - "heck 0.4.1", - "itertools 0.13.0", + "heck 0.5.0", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -7007,7 +7007,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.119", @@ -8547,7 +8547,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.119", @@ -8559,7 +8559,7 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "287f59010008f0d7cf5e3b03196d666c1acc46c8d3e9cf34c28a1a7157601e72" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.119", diff --git a/crates/nvisy-postgres/src/model/mod.rs b/crates/nvisy-postgres/src/model/mod.rs index 00a818cb..4be94262 100644 --- a/crates/nvisy-postgres/src/model/mod.rs +++ b/crates/nvisy-postgres/src/model/mod.rs @@ -15,14 +15,16 @@ mod workspace_activity; mod workspace_connection; mod workspace_connection_schedule; mod workspace_connection_sync; +mod workspace_detection; +mod workspace_detection_job; +mod workspace_detection_usage; mod workspace_file; mod workspace_file_imports; mod workspace_invite; mod workspace_member; mod workspace_pipeline; -mod workspace_pipeline_run; -mod workspace_pipeline_run_usage; mod workspace_policy; +mod workspace_redaction; mod workspace_webhook; // Account models @@ -48,15 +50,17 @@ pub use workspace_connection_schedule::{ pub use workspace_connection_sync::{ NewWorkspaceConnectionSync, UpdateWorkspaceConnectionSync, WorkspaceConnectionSync, }; +// Detection / pipeline models +pub use workspace_detection::{ + NewWorkspaceDetection, UpdateWorkspaceDetection, WorkspaceDetection, +}; +pub use workspace_detection_job::{NewWorkspaceDetectionJob, WorkspaceDetectionJob}; +pub use workspace_detection_usage::{NewWorkspaceDetectionUsage, WorkspaceDetectionUsage}; pub use workspace_file::{NewWorkspaceFile, UpdateWorkspaceFile, WorkspaceFile}; pub use workspace_file_imports::{NewWorkspaceFileImport, WorkspaceFileImport}; pub use workspace_invite::{NewWorkspaceInvite, UpdateWorkspaceInvite, WorkspaceInvite}; pub use workspace_member::{NewWorkspaceMember, UpdateWorkspaceMember, WorkspaceMember}; pub use workspace_pipeline::{NewWorkspacePipeline, UpdateWorkspacePipeline, WorkspacePipeline}; -// Pipeline models -pub use workspace_pipeline_run::{ - NewWorkspacePipelineRun, UpdateWorkspacePipelineRun, WorkspacePipelineRun, -}; -pub use workspace_pipeline_run_usage::{NewWorkspacePipelineRunUsage, WorkspacePipelineRunUsage}; pub use workspace_policy::{NewWorkspacePolicy, UpdateWorkspacePolicy, WorkspacePolicy}; +pub use workspace_redaction::{NewWorkspaceRedaction, WorkspaceRedaction}; pub use workspace_webhook::{NewWorkspaceWebhook, UpdateWorkspaceWebhook, WorkspaceWebhook}; diff --git a/crates/nvisy-postgres/src/model/workspace_detection.rs b/crates/nvisy-postgres/src/model/workspace_detection.rs new file mode 100644 index 00000000..66b51d39 --- /dev/null +++ b/crates/nvisy-postgres/src/model/workspace_detection.rs @@ -0,0 +1,94 @@ +//! Workspace detection model for PostgreSQL database operations. + +use diesel::prelude::*; +use jiff_diesel::Timestamp; +use uuid::Uuid; + +use crate::schema::workspace_detections; +use crate::types::{DetectionMetadata, DetectionStatus, Json, PipelineTriggerType}; + +/// A detection: one analysis pass of a file through a pipeline. +/// +/// Detect creates the detection and stores the engine's `Audit` in the object +/// store, keeping its file id here; the detection then stays `Complete` and can +/// be redacted any number of times (each redaction is its own row). +#[derive(Debug, Clone, PartialEq, Queryable, Selectable)] +#[diesel(table_name = workspace_detections)] +#[diesel(check_for_backend(diesel::pg::Pg))] +pub struct WorkspaceDetection { + /// Unique detection identifier. + pub id: Uuid, + /// Pipeline whose config drove the detection. + pub pipeline_id: Uuid, + /// Account the detection is attributed to (the user who started it, or the + /// pipeline's creator for a system-initiated detection). + pub account_id: Uuid, + /// Source document the detection analyzes. + pub input_file_id: Uuid, + /// Audit file (`file_kind = audit`) holding the encrypted analysis. `None` + /// until analysis writes it. + pub audit_file_id: Option, + /// How the detection was initiated. + pub trigger_type: PipelineTriggerType, + /// Current detection status. + pub status: DetectionStatus, + /// Detect idempotency key (dedupes retries). + pub idempotency_key: Option, + /// Non-encrypted metadata for filtering/display. + pub metadata: Json, + /// When a worker last claimed this detection. Acts as a lease: a redelivered + /// job whose claim is still fresh is skipped, while a stale claim (a worker + /// that died mid-analysis) can be re-claimed. `None` until first claimed. + pub claimed_at: Option, + /// When the detection started. + pub started_at: Timestamp, + /// When the detection completed analysis. + pub completed_at: Option, +} + +/// Data for creating a new workspace detection. +#[derive(Debug, Default, Clone, Insertable)] +#[diesel(table_name = workspace_detections)] +#[diesel(check_for_backend(diesel::pg::Pg))] +pub struct NewWorkspaceDetection { + /// Pipeline ID (required). + pub pipeline_id: Uuid, + /// Account the detection is attributed to (required). + pub account_id: Uuid, + /// Source document ID (required). + pub input_file_id: Uuid, + /// Audit file holding the encrypted analysis (set once analyzed). + pub audit_file_id: Option, + /// Trigger type. + pub trigger_type: Option, + /// Initial status. + pub status: Option, + /// Detect idempotency key. + pub idempotency_key: Option, + /// Non-encrypted metadata for filtering/display. + pub metadata: Option>, +} + +/// Data for updating a workspace detection. +#[derive(Debug, Clone, Default, AsChangeset)] +#[diesel(table_name = workspace_detections)] +#[diesel(check_for_backend(diesel::pg::Pg))] +pub struct UpdateWorkspaceDetection { + /// Detection status. + pub status: Option, + /// Audit file holding the encrypted analysis. + pub audit_file_id: Option>, + /// Non-encrypted metadata for filtering/display. + pub metadata: Option>, + /// When a worker last claimed this detection (lease timestamp). + pub claimed_at: Option>, + /// When the detection completed analysis. + pub completed_at: Option>, +} + +impl WorkspaceDetection { + /// Returns whether analysis is done and the detection is ready to redact. + pub fn is_complete(&self) -> bool { + self.status.is_complete() + } +} diff --git a/crates/nvisy-postgres/src/model/workspace_detection_job.rs b/crates/nvisy-postgres/src/model/workspace_detection_job.rs new file mode 100644 index 00000000..b8192f84 --- /dev/null +++ b/crates/nvisy-postgres/src/model/workspace_detection_job.rs @@ -0,0 +1,50 @@ +//! Transactional-outbox model for detection jobs. + +use diesel::prelude::*; +use jiff_diesel::Timestamp; +use uuid::Uuid; + +use crate::schema::workspace_detection_jobs; +use crate::types::OutboxStatus; + +/// A pending or processed detection-job outbox row: a serialized `DetectionJob` +/// awaiting (or past) publication to the detection work-queue. +/// +/// The `job` column is an opaque JSON blob to this layer — a serialized +/// server-side `DetectionJob` — so the ORM stays free of the job vocabulary; the +/// drainer decodes it and publishes it. +#[derive(Debug, Clone, Queryable, Selectable)] +#[diesel(table_name = workspace_detection_jobs)] +#[diesel(check_for_backend(diesel::pg::Pg))] +pub struct WorkspaceDetectionJob { + /// Unique outbox row identifier. + pub id: Uuid, + /// The detection this job analyzes. + pub detection_id: Uuid, + /// The serialized detection job. + pub job: serde_json::Value, + /// Processing state: pending, processed, or failed (dead-lettered). + pub status: OutboxStatus, + /// Number of publish attempts the drainer has made. + pub attempts: i32, + /// Earliest time the row may next be claimed; advanced by a backoff after + /// each failed attempt. + pub next_attempt_at: Timestamp, + /// When the job was queued. + pub created_at: Timestamp, + /// When a terminal (processed or failed) row was resolved by an operator; + /// `None` until then. A manual affordance for inspecting the outbox. + pub resolved_at: Option, +} + +/// A new detection-job outbox row, inserted in the same transaction as the +/// detection it queues. +#[derive(Debug, Clone, Insertable)] +#[diesel(table_name = workspace_detection_jobs)] +#[diesel(check_for_backend(diesel::pg::Pg))] +pub struct NewWorkspaceDetectionJob { + /// The detection this job analyzes. + pub detection_id: Uuid, + /// The serialized detection job. + pub job: serde_json::Value, +} diff --git a/crates/nvisy-postgres/src/model/workspace_pipeline_run_usage.rs b/crates/nvisy-postgres/src/model/workspace_detection_usage.rs similarity index 71% rename from crates/nvisy-postgres/src/model/workspace_pipeline_run_usage.rs rename to crates/nvisy-postgres/src/model/workspace_detection_usage.rs index 42d1dfd0..cca1b436 100644 --- a/crates/nvisy-postgres/src/model/workspace_pipeline_run_usage.rs +++ b/crates/nvisy-postgres/src/model/workspace_detection_usage.rs @@ -1,19 +1,19 @@ -//! Per-model inference usage for a pipeline run. +//! Per-model inference usage for a detection. use diesel::prelude::*; use uuid::Uuid; -use crate::schema::workspace_pipeline_run_usage; +use crate::schema::workspace_detection_usage; -/// One model's token usage within a run, as the provider reported it. +/// One model's token usage within a detection, as the provider reported it. #[derive(Debug, Clone, Queryable, Selectable)] -#[diesel(table_name = workspace_pipeline_run_usage)] +#[diesel(table_name = workspace_detection_usage)] #[diesel(check_for_backend(diesel::pg::Pg))] -pub struct WorkspacePipelineRunUsage { +pub struct WorkspaceDetectionUsage { /// Unique usage row identifier. pub id: Uuid, - /// The run this usage belongs to. - pub run_id: Uuid, + /// The detection this usage belongs to. + pub detection_id: Uuid, /// The model the recognizers used. pub model: String, /// The model version, if the provider reported one. @@ -29,13 +29,13 @@ pub struct WorkspacePipelineRunUsage { pub duration_ms: i64, } -/// Data for recording one model's usage on a run. +/// Data for recording one model's usage on a detection. #[derive(Debug, Clone, Insertable)] -#[diesel(table_name = workspace_pipeline_run_usage)] +#[diesel(table_name = workspace_detection_usage)] #[diesel(check_for_backend(diesel::pg::Pg))] -pub struct NewWorkspacePipelineRunUsage { - /// The run this usage belongs to. - pub run_id: Uuid, +pub struct NewWorkspaceDetectionUsage { + /// The detection this usage belongs to. + pub detection_id: Uuid, /// The model the recognizers used. pub model: String, /// The model version, if any. diff --git a/crates/nvisy-postgres/src/model/workspace_pipeline_run.rs b/crates/nvisy-postgres/src/model/workspace_pipeline_run.rs deleted file mode 100644 index d8d82558..00000000 --- a/crates/nvisy-postgres/src/model/workspace_pipeline_run.rs +++ /dev/null @@ -1,102 +0,0 @@ -//! Workspace pipeline run model for PostgreSQL database operations. - -use diesel::prelude::*; -use jiff_diesel::Timestamp; -use uuid::Uuid; - -use crate::schema::workspace_pipeline_runs; -use crate::types::{Json, PipelineRunStatus, PipelineTriggerType, RunMetadata}; - -/// A detect/redact run: one analysis of a file through a pipeline. -/// -/// Detect creates the run and stores the engine's `AnalyzedDocument` in the -/// object store, keeping its key here; the run then awaits reviewer -/// verification before redact fetches it back and consumes it. -#[derive(Debug, Clone, PartialEq, Queryable, Selectable)] -#[diesel(table_name = workspace_pipeline_runs)] -#[diesel(check_for_backend(diesel::pg::Pg))] -pub struct WorkspacePipelineRun { - /// Unique run identifier. - pub id: Uuid, - /// Pipeline whose config drove the run. - pub pipeline_id: Uuid, - /// Account the run is attributed to (the user who started it, or the - /// pipeline's creator for a system-initiated run). - pub account_id: Uuid, - /// Source document the run analyzes / redacts. - pub input_file_id: Uuid, - /// Audit file (`file_kind = audit`) holding the encrypted analysis between - /// detect and redact. `None` until analysis writes it. - pub audit_file_id: Option, - /// Redacted document (`file_kind = redacted`) produced by redact. `None` - /// until the run completes. - pub output_file_id: Option, - /// How the run was initiated. - pub trigger_type: PipelineTriggerType, - /// Current run status. - pub status: PipelineRunStatus, - /// Detect idempotency key (dedupes retries). - pub idempotency_key: Option, - /// Non-encrypted metadata for filtering/display. - pub metadata: Json, - /// When a worker last claimed this run for detection. Acts as a lease: a - /// redelivered job whose claim is still fresh is skipped, while a stale - /// claim (a worker that died mid-analysis) can be re-claimed. `None` until - /// first claimed. - pub claimed_at: Option, - /// When the run started. - pub started_at: Timestamp, - /// When the run completed. - pub completed_at: Option, -} - -/// Data for creating a new workspace pipeline run. -#[derive(Debug, Default, Clone, Insertable)] -#[diesel(table_name = workspace_pipeline_runs)] -#[diesel(check_for_backend(diesel::pg::Pg))] -pub struct NewWorkspacePipelineRun { - /// Pipeline ID (required). - pub pipeline_id: Uuid, - /// Account the run is attributed to (required). - pub account_id: Uuid, - /// Source document ID (required). - pub input_file_id: Uuid, - /// Audit file holding the encrypted analysis (set once analyzed). - pub audit_file_id: Option, - /// Redacted output file (set once completed). - pub output_file_id: Option, - /// Trigger type. - pub trigger_type: Option, - /// Initial status. - pub status: Option, - /// Detect idempotency key. - pub idempotency_key: Option, - /// Non-encrypted metadata for filtering/display. - pub metadata: Option>, -} - -/// Data for updating a workspace pipeline run. -#[derive(Debug, Clone, Default, AsChangeset)] -#[diesel(table_name = workspace_pipeline_runs)] -#[diesel(check_for_backend(diesel::pg::Pg))] -pub struct UpdateWorkspacePipelineRun { - /// Run status. - pub status: Option, - /// Audit file holding the encrypted analysis. - pub audit_file_id: Option>, - /// Redacted output file produced by redact. - pub output_file_id: Option>, - /// Non-encrypted metadata for filtering/display. - pub metadata: Option>, - /// When a worker last claimed this run for detection (lease timestamp). - pub claimed_at: Option>, - /// When the run completed. - pub completed_at: Option>, -} - -impl WorkspacePipelineRun { - /// Returns whether detection is done and the run awaits verification. - pub fn is_analyzed(&self) -> bool { - self.status.is_analyzed() - } -} diff --git a/crates/nvisy-postgres/src/model/workspace_redaction.rs b/crates/nvisy-postgres/src/model/workspace_redaction.rs new file mode 100644 index 00000000..1ef7c806 --- /dev/null +++ b/crates/nvisy-postgres/src/model/workspace_redaction.rs @@ -0,0 +1,47 @@ +//! Workspace redaction model for PostgreSQL database operations. + +use diesel::prelude::*; +use jiff_diesel::Timestamp; +use uuid::Uuid; + +use crate::schema::workspace_redactions; + +/// A redaction: one redact pass over a detection's analysis. +/// +/// A detection can be redacted many times — each redact request may carry a +/// different set of reviewer edits — so each redaction is its own row owning the +/// review audit it applied and the redacted document it produced. +#[derive(Debug, Clone, PartialEq, Queryable, Selectable)] +#[diesel(table_name = workspace_redactions)] +#[diesel(check_for_backend(diesel::pg::Pg))] +pub struct WorkspaceRedaction { + /// Unique redaction identifier. + pub id: Uuid, + /// Detection this redaction was produced from. + pub detection_id: Uuid, + /// Account that requested the redaction. + pub account_id: Uuid, + /// Review audit (`file_kind = review`) recording the applied edits and the + /// redaction outcome. `None` only if the file was later hard-deleted. + pub review_file_id: Option, + /// Redacted document (`file_kind = redacted`) this redaction produced. + /// `None` only if the file was later hard-deleted. + pub output_file_id: Option, + /// When the redaction was created. + pub created_at: Timestamp, +} + +/// Data for creating a new workspace redaction. +#[derive(Debug, Default, Clone, Insertable)] +#[diesel(table_name = workspace_redactions)] +#[diesel(check_for_backend(diesel::pg::Pg))] +pub struct NewWorkspaceRedaction { + /// Detection this redaction was produced from (required). + pub detection_id: Uuid, + /// Account that requested the redaction (required). + pub account_id: Uuid, + /// Review audit holding the applied edits and outcome. + pub review_file_id: Option, + /// Redacted output document this redaction produced. + pub output_file_id: Option, +} diff --git a/crates/nvisy-postgres/src/query/analytics.rs b/crates/nvisy-postgres/src/query/analytics.rs index c1333043..049ad8d7 100644 --- a/crates/nvisy-postgres/src/query/analytics.rs +++ b/crates/nvisy-postgres/src/query/analytics.rs @@ -1,13 +1,13 @@ -//! Workspace analytics: aggregate queries over a workspace's files and pipeline -//! runs, for the analytics endpoint. +//! Workspace analytics: aggregate queries over a workspace's files and +//! detections, for the analytics endpoint. //! //! Two entry points, each reading a consistent snapshot in one read-only -//! transaction: `snapshot` (storage, run health, token usage) and `runs_by_day` -//! (the daily time series). Both compose several grouped aggregates from private -//! `load_*` helpers. Rows come back sparse (one per group that has data, never -//! zero-filled here) — the handler maps them onto the full set of enum values, -//! filling absent groups with zero, so the response is stable regardless of what -//! data exists. +//! transaction: `snapshot` (storage, detection health, token usage) and +//! `detections_by_day` (the daily time series). Both compose several grouped +//! aggregates from private `load_*` helpers. Rows come back sparse (one per group +//! that has data, never zero-filled here) — the handler maps them onto the full +//! set of enum values, filling absent groups with zero, so the response is stable +//! regardless of what data exists. use std::future::Future; @@ -17,14 +17,15 @@ use diesel::sql_types::Timestamptz; use diesel_async::RunQueryDsl; use uuid::Uuid; -use crate::types::{FileKind, PipelineRunStatus}; +use crate::types::{DetectionStatus, FileKind}; use crate::{PgConnection, PgError, PgResult, schema}; -/// Per-day run counts and durations, as loaded from the grouped run query. +/// Per-day detection counts and durations, as loaded from the grouped detection +/// query. #[derive(Debug, Clone, Queryable)] -struct RunDayCounts { +struct DetectionDayCounts { day: jiff_diesel::Timestamp, - runs: i64, + detections: i64, // Conditional counts are `sum(CASE WHEN cond THEN 1 ELSE 0 END)`; Postgres // `sum` of bigint returns numeric, hence BigDecimal (nullable over no rows). // Converted to i64 when assembling the point. @@ -37,15 +38,15 @@ struct RunDayCounts { /// Per-day token totals, as loaded from the grouped usage query. #[derive(Debug, Clone, Queryable)] -struct RunDayTokens { +struct DetectionDayTokens { day: jiff_diesel::Timestamp, input: Option, output: Option, total: Option, } -/// A day's token totals converted to `i64`, keyed by day while merging the run -/// and usage series. +/// A day's token totals converted to `i64`, keyed by day while merging the +/// detection and usage series. #[derive(Debug, Clone, Copy, Default)] struct DayTokens { input: Option, @@ -53,28 +54,28 @@ struct DayTokens { total: Option, } -/// One day of pipeline-run activity for a workspace. Only days with at least one -/// run are produced; the caller fills the gaps to a dense series over its window. -/// `day` is the UTC day (midnight) the run counts fall in. +/// One day of detection activity for a workspace. Only days with at least one +/// detection are produced; the caller fills the gaps to a dense series over its +/// window. `day` is the UTC day (midnight) the detection counts fall in. #[derive(Debug, Clone)] -pub struct RunDayPoint { +pub struct DetectionDayPoint { /// The day (UTC midnight) this point covers. pub day: jiff::Timestamp, - /// Runs started on this day. - pub runs: i64, - /// Runs that reached a terminal state (`completed` or `failed`) this day. + /// Detections started on this day. + pub detections: i64, + /// Detections that reached a terminal state (`complete` or `failed`) this day. pub terminal: i64, - /// Runs that failed this day. + /// Detections that failed this day. pub failed: i64, - /// Mean duration (milliseconds) of runs that completed this day; null if none did. + /// Mean duration (milliseconds) of detections that completed this day; null if none did. pub avg_ms: Option, - /// 95th-percentile duration (milliseconds) of runs completed this day; null if none. + /// 95th-percentile duration (milliseconds) of detections completed this day; null if none. pub p95_ms: Option, - /// Input/prompt tokens across models used by this day's runs; null if none. + /// Input/prompt tokens across models used by this day's detections; null if none. pub input_tokens: Option, - /// Output/completion tokens across models used by this day's runs; null if none. + /// Output/completion tokens across models used by this day's detections; null if none. pub output_tokens: Option, - /// Reported total tokens across models used by this day's runs; null if none. + /// Reported total tokens across models used by this day's detections; null if none. pub total_tokens: Option, } @@ -88,14 +89,14 @@ struct UsageByModelRow { total_tokens: Option, } -/// Inference token totals for one model across a workspace's runs. Each field is -/// independently nullable because a provider may report only some of them (a -/// `total` that is not `input + output`, or no breakdown at all). +/// Inference token totals for one model across a workspace's detections. Each +/// field is independently nullable because a provider may report only some of +/// them (a `total` that is not `input + output`, or no breakdown at all). #[derive(Debug, Clone)] pub struct UsageByModel { /// The model these totals are for. pub model: String, - /// Summed input/prompt tokens, or `None` if no run reported them for this model. + /// Summed input/prompt tokens, or `None` if no detection reported them for this model. pub input_tokens: Option, /// Summed output/completion tokens, or `None` if none reported. pub output_tokens: Option, @@ -124,45 +125,45 @@ pub struct StorageByKind { pub total_bytes: i64, } -/// Run count for one `status` in a workspace. +/// Detection count for one `status` in a workspace. #[derive(Debug, Clone, Queryable)] -pub struct RunStatusCount { - /// The run status this row aggregates. - pub status: PipelineRunStatus, - /// Number of runs in this status. +pub struct DetectionStatusCount { + /// The detection status this row aggregates. + pub status: DetectionStatus, + /// Number of detections in this status. pub count: i64, } -/// Completed-run duration summary for a workspace, in milliseconds. Both are -/// `None` until at least one run has completed. +/// Completed-detection duration summary for a workspace, in milliseconds. Both +/// are `None` until at least one detection has completed. #[derive(Debug, Clone)] -pub struct RunDurations { - /// Mean wall-clock duration of completed runs. +pub struct DetectionDurations { + /// Mean wall-clock duration of completed detections. pub avg_ms: Option, - /// 95th-percentile duration of completed runs. + /// 95th-percentile duration of completed detections. pub p95_ms: Option, } -/// A workspace's point-in-time analytics: storage, run health, and token usage -/// read together so the parts agree. Produced by +/// A workspace's point-in-time analytics: storage, detection health, and token +/// usage read together so the parts agree. Produced by /// [`snapshot`](WorkspaceAnalyticsRepository::snapshot). #[derive(Debug, Clone)] pub struct AnalyticsSnapshot { /// Live-file counts and byte totals, one per `file_kind` present. pub storage: Vec, - /// Run counts, one per `status` present. - pub runs: Vec, - /// Completed-run duration summary. - pub durations: RunDurations, + /// Detection counts, one per `status` present. + pub detections: Vec, + /// Completed-detection duration summary. + pub durations: DetectionDurations, /// Inference token totals, one per model used. pub usage: Vec, } /// Read-only aggregate queries backing the workspace analytics endpoint. pub trait WorkspaceAnalyticsRepository { - /// A workspace's point-in-time analytics — storage by kind, run counts by - /// status, completed-run durations, and per-model token usage — read in one - /// read-only, repeatable-read transaction so the parts reflect a single + /// A workspace's point-in-time analytics — storage by kind, detection counts + /// by status, completed-detection durations, and per-model token usage — read + /// in one read-only, repeatable-read transaction so the parts reflect a single /// snapshot. Each breakdown lists only the groups that have data; the caller /// zero-fills the rest. fn snapshot( @@ -170,66 +171,67 @@ pub trait WorkspaceAnalyticsRepository { workspace_id: Uuid, ) -> impl Future> + Send; - /// Daily pipeline-run activity for a workspace over `[from, to)` (UTC day + /// Daily detection activity for a workspace over `[from, to)` (UTC day /// boundaries; `to` exclusive). Returns one row per day that has at least one - /// run — the series is sparse — so the caller gap-fills the window to a dense - /// series (see `RunTimeSeries::from_window`). The window is bucketed by - /// `date_trunc('day', started_at)`; runs are scoped through their live - /// pipeline. The caller is responsible for bounding the window. + /// detection — the series is sparse — so the caller gap-fills the window to a + /// dense series (see `DetectionTimeSeries::from_window`). The window is + /// bucketed by `date_trunc('day', started_at)`; detections are scoped through + /// their live pipeline. The caller is responsible for bounding the window. /// - /// Run counts, durations, and token totals are read in one transaction so the - /// two grouped statements observe the same snapshot. - fn runs_by_day( + /// Detection counts, durations, and token totals are read in one transaction + /// so the two grouped statements observe the same snapshot. + fn detections_by_day( &mut self, workspace_id: Uuid, from: jiff::Timestamp, to: jiff::Timestamp, - ) -> impl Future>> + Send; + ) -> impl Future>> + Send; } impl WorkspaceAnalyticsRepository for PgConnection { async fn snapshot(&mut self, workspace_id: Uuid) -> PgResult { // The four reads run in one read-only, repeatable-read transaction so the - // snapshot is internally consistent: a run cannot appear in the status - // counts while its tokens are missing from the usage totals because a - // write landed between two of the reads. + // snapshot is internally consistent: a detection cannot appear in the + // status counts while its tokens are missing from the usage totals because + // a write landed between two of the reads. self.build_transaction() .read_only() .repeatable_read() .run(async |conn| { Ok(AnalyticsSnapshot { storage: load_storage_by_kind(conn, workspace_id).await?, - runs: load_runs_by_status(conn, workspace_id).await?, - durations: load_run_durations(conn, workspace_id).await?, + detections: load_detections_by_status(conn, workspace_id).await?, + durations: load_detection_durations(conn, workspace_id).await?, usage: load_usage_by_model(conn, workspace_id).await?, }) }) .await } - async fn runs_by_day( + async fn detections_by_day( &mut self, workspace_id: Uuid, from: jiff::Timestamp, to: jiff::Timestamp, - ) -> PgResult> { + ) -> PgResult> { use std::collections::BTreeMap; let from = jiff_diesel::Timestamp::from(from); let to = jiff_diesel::Timestamp::from(to); - // Read the run counts/durations and the token totals in one read-only, - // repeatable-read transaction so both grouped statements observe the same - // snapshot; otherwise a run written between them could show in one series - // but not the other. - let (run_rows, token_rows): (Vec, Vec) = self + // Read the detection counts/durations and the token totals in one + // read-only, repeatable-read transaction so both grouped statements + // observe the same snapshot; otherwise a detection written between them + // could show in one series but not the other. + let (detection_rows, token_rows): (Vec, Vec) = self .build_transaction() .read_only() .repeatable_read() .run(async |conn| { - let run_rows = load_run_day_counts(conn, workspace_id, from, to).await?; - let token_rows = load_run_day_tokens(conn, workspace_id, from, to).await?; - Ok::<_, PgError>((run_rows, token_rows)) + let detection_rows = + load_detection_day_counts(conn, workspace_id, from, to).await?; + let token_rows = load_detection_day_tokens(conn, workspace_id, from, to).await?; + Ok::<_, PgError>((detection_rows, token_rows)) }) .await?; @@ -248,14 +250,14 @@ impl WorkspaceAnalyticsRepository for PgConnection { ); } - let points = run_rows + let points = detection_rows .into_iter() .map(|row| { let day: jiff::Timestamp = row.day.into(); let t = tokens.remove(&day).unwrap_or_default(); - RunDayPoint { + DetectionDayPoint { day, - runs: row.runs, + detections: row.detections, terminal: to_i64(row.terminal).unwrap_or(0), failed: to_i64(row.failed).unwrap_or(0), avg_ms: row.avg_ms, @@ -304,36 +306,39 @@ async fn load_storage_by_kind( .collect()) } -/// Run count per `status`, scoped through the live pipeline. Only statuses with -/// a run appear. -async fn load_runs_by_status( +/// Detection count per `status`, scoped through the live pipeline. Only statuses +/// with a detection appear. +async fn load_detections_by_status( conn: &mut PgConnection, workspace_id: Uuid, -) -> PgResult> { +) -> PgResult> { use diesel::dsl::count_star; - use schema::workspace_pipeline_runs::dsl as runs; + use schema::workspace_detections::dsl as detections; use schema::workspace_pipelines::dsl as pipelines; - use schema::{workspace_pipeline_runs, workspace_pipelines}; + use schema::{workspace_detections, workspace_pipelines}; - workspace_pipeline_runs::table + workspace_detections::table .inner_join(workspace_pipelines::table) .filter(pipelines::workspace_id.eq(workspace_id)) .filter(pipelines::deleted_at.is_null()) - .group_by(runs::status) - .select((runs::status, count_star())) + .group_by(detections::status) + .select((detections::status, count_star())) .load(conn) .await .map_err(PgError::from) } /// Mean and 95th-percentile duration (milliseconds) of the workspace's completed -/// runs. Both `None` when no run has completed. -async fn load_run_durations(conn: &mut PgConnection, workspace_id: Uuid) -> PgResult { +/// detections. Both `None` when no detection has completed. +async fn load_detection_durations( + conn: &mut PgConnection, + workspace_id: Uuid, +) -> PgResult { use diesel::dsl::sql; use diesel::sql_types::{BigInt, Nullable}; - use schema::workspace_pipeline_runs::dsl as runs; + use schema::workspace_detections::dsl as detections; use schema::workspace_pipelines::dsl as pipelines; - use schema::{workspace_pipeline_runs, workspace_pipelines}; + use schema::{workspace_detections, workspace_pipelines}; // Duration in milliseconds: the interval's epoch-seconds are scaled by 1000 // and rounded to a bigint in SQL, so the value crosses the boundary already in @@ -341,45 +346,48 @@ async fn load_run_durations(conn: &mut PgConnection, workspace_id: Uuid) -> PgRe // with no Diesel builtin) both return NULL over no rows. Columns are // table-qualified so the join can never make them ambiguous. let avg_ms = sql::>( - "round(avg(EXTRACT(EPOCH FROM (workspace_pipeline_runs.completed_at \ - - workspace_pipeline_runs.started_at)) * 1000))::bigint", + "round(avg(EXTRACT(EPOCH FROM (workspace_detections.completed_at \ + - workspace_detections.started_at)) * 1000))::bigint", ); let p95_ms = sql::>( "round(percentile_cont(0.95) WITHIN GROUP \ - (ORDER BY EXTRACT(EPOCH FROM (workspace_pipeline_runs.completed_at \ - - workspace_pipeline_runs.started_at))) * 1000)::bigint", + (ORDER BY EXTRACT(EPOCH FROM (workspace_detections.completed_at \ + - workspace_detections.started_at))) * 1000)::bigint", ); - let (avg_ms, p95_ms): (Option, Option) = workspace_pipeline_runs::table + let (avg_ms, p95_ms): (Option, Option) = workspace_detections::table .inner_join(workspace_pipelines::table) .filter(pipelines::workspace_id.eq(workspace_id)) .filter(pipelines::deleted_at.is_null()) - .filter(runs::completed_at.is_not_null()) + // Durations describe successful analysis only. `completed_at` is stamped + // on failure too, so filter on the terminal `Complete` status, not merely + // on the timestamp being present. + .filter(detections::status.eq(DetectionStatus::Complete)) .select((avg_ms, p95_ms)) .first(conn) .await .map_err(PgError::from)?; - Ok(RunDurations { avg_ms, p95_ms }) + Ok(DetectionDurations { avg_ms, p95_ms }) } -/// Inference token totals per model across the workspace's runs, scoped through -/// the live pipeline. Only models actually used appear. Kept per-model rather -/// than summed to one total, since a run may mix models. +/// Inference token totals per model across the workspace's detections, scoped +/// through the live pipeline. Only models actually used appear. Kept per-model +/// rather than summed to one total, since a detection may mix models. async fn load_usage_by_model( conn: &mut PgConnection, workspace_id: Uuid, ) -> PgResult> { use bigdecimal::ToPrimitive; use diesel::dsl::sum; - use schema::workspace_pipeline_run_usage::dsl as usage; - use schema::workspace_pipeline_runs::dsl as runs; + use schema::workspace_detection_usage::dsl as usage; + use schema::workspace_detections::dsl as detections; use schema::workspace_pipelines::dsl as pipelines; - use schema::{workspace_pipeline_run_usage, workspace_pipeline_runs, workspace_pipelines}; + use schema::{workspace_detection_usage, workspace_detections, workspace_pipelines}; - let rows: Vec = workspace_pipeline_run_usage::table - .inner_join(workspace_pipeline_runs::table.on(runs::id.eq(usage::run_id))) - .inner_join(workspace_pipelines::table.on(pipelines::id.eq(runs::pipeline_id))) + let rows: Vec = workspace_detection_usage::table + .inner_join(workspace_detections::table.on(detections::id.eq(usage::detection_id))) + .inner_join(workspace_pipelines::table.on(pipelines::id.eq(detections::pipeline_id))) .filter(pipelines::workspace_id.eq(workspace_id)) .filter(pipelines::deleted_at.is_null()) .group_by(usage::model) @@ -405,29 +413,29 @@ async fn load_usage_by_model( .collect()) } -/// The day bucket a run's start falls in. Shared by both daily queries so the -/// truncation text cannot drift — they must group on the same expression for the -/// per-day merge to line up. The column is table-qualified so the join to +/// The day bucket a detection's start falls in. Shared by both daily queries so +/// the truncation text cannot drift — they must group on the same expression for +/// the per-day merge to line up. The column is table-qualified so the join to /// pipelines can never make it ambiguous. Returns a fresh fragment per call, as /// the builder consumes it in both `group_by` and `select`. -fn run_day() -> diesel::expression::SqlLiteral { - diesel::dsl::sql::("date_trunc('day', workspace_pipeline_runs.started_at)") +fn detection_day() -> diesel::expression::SqlLiteral { + diesel::dsl::sql::("date_trunc('day', workspace_detections.started_at)") } -/// Per-day run counts and durations over `[from, to)`, scoped through the live -/// pipeline. Sparse: only days that have a run. Shared with the token query so -/// both observe the same snapshot inside one transaction. -async fn load_run_day_counts( +/// Per-day detection counts and durations over `[from, to)`, scoped through the +/// live pipeline. Sparse: only days that have a detection. Shared with the token +/// query so both observe the same snapshot inside one transaction. +async fn load_detection_day_counts( conn: &mut PgConnection, workspace_id: Uuid, from: jiff_diesel::Timestamp, to: jiff_diesel::Timestamp, -) -> PgResult> { +) -> PgResult> { use diesel::dsl::{case_when, count_star, sql, sum}; use diesel::sql_types::{BigInt, Nullable as SqlNullable}; - use schema::workspace_pipeline_runs::dsl as runs; + use schema::workspace_detections::dsl as detections; use schema::workspace_pipelines::dsl as pipelines; - use schema::{workspace_pipeline_runs, workspace_pipelines}; + use schema::{workspace_detections, workspace_pipelines}; // Durations in milliseconds (epoch-seconds scaled by 1000, rounded to bigint), // so the value crosses the boundary already in the API unit and type. The @@ -436,34 +444,38 @@ async fn load_run_day_counts( // Conditional counts are `sum(CASE WHEN cond THEN 1 ELSE 0 END)` since Diesel's // aggregate FILTER is not available on `count(*)`. Columns are table-qualified // so the join to pipelines can never make them ambiguous. - let avg_ms = sql::>( - "round(avg(EXTRACT(EPOCH FROM (workspace_pipeline_runs.completed_at \ - - workspace_pipeline_runs.started_at))) \ - FILTER (WHERE workspace_pipeline_runs.completed_at IS NOT NULL) * 1000)::bigint", - ); - let p95_ms = sql::>( + // Durations describe successful analysis only, so both aggregates filter on + // the terminal `complete` status: `completed_at` is stamped on failure too, so + // a presence check alone would fold failed detections into avg/p95. The FILTER + // clause is shared so the two fragments cannot drift apart. + const COMPLETE_FILTER: &str = " FILTER (WHERE workspace_detections.status = 'complete')"; + let avg_ms = sql::>(&format!( + "round(avg(EXTRACT(EPOCH FROM (workspace_detections.completed_at \ + - workspace_detections.started_at))){COMPLETE_FILTER} * 1000)::bigint" + )); + let p95_ms = sql::>(&format!( "round(percentile_cont(0.95) WITHIN GROUP \ - (ORDER BY EXTRACT(EPOCH FROM (workspace_pipeline_runs.completed_at \ - - workspace_pipeline_runs.started_at))) \ - FILTER (WHERE workspace_pipeline_runs.completed_at IS NOT NULL) * 1000)::bigint", - ); + (ORDER BY EXTRACT(EPOCH FROM (workspace_detections.completed_at \ + - workspace_detections.started_at))){COMPLETE_FILTER} * 1000)::bigint" + )); - workspace_pipeline_runs::table + workspace_detections::table .inner_join(workspace_pipelines::table) .filter(pipelines::workspace_id.eq(workspace_id)) .filter(pipelines::deleted_at.is_null()) - .filter(runs::started_at.ge(from)) - .filter(runs::started_at.lt(to)) - .group_by(run_day()) + .filter(detections::started_at.ge(from)) + .filter(detections::started_at.lt(to)) + .group_by(detection_day()) .select(( - run_day(), + detection_day(), count_star(), + sum(case_when::<_, _, BigInt>( + detections::status.eq_any(DetectionStatus::TERMINAL), + 1i64, + ) + .otherwise(0i64)), sum( - case_when::<_, _, BigInt>(runs::status.eq_any(PipelineRunStatus::OUTCOMES), 1i64) - .otherwise(0i64), - ), - sum( - case_when::<_, _, BigInt>(runs::status.eq(PipelineRunStatus::Failed), 1i64) + case_when::<_, _, BigInt>(detections::status.eq(DetectionStatus::Failed), 1i64) .otherwise(0i64), ), avg_ms, @@ -475,47 +487,47 @@ async fn load_run_day_counts( } /// Per-day inference token totals over `[from, to)`, scoped through the live -/// pipeline. Usage is per-model-per-run, so each token field is summed per run -/// first (a correlated subquery) before day-grouping, otherwise a run's multiple -/// model rows would multiply the day totals. Sparse: only days with a run (token -/// sums are null on days with no usage). -async fn load_run_day_tokens( +/// pipeline. Usage is per-model-per-detection, so each token field is summed per +/// detection first (a correlated subquery) before day-grouping, otherwise a +/// detection's multiple model rows would multiply the day totals. Sparse: only +/// days with a detection (token sums are null on days with no usage). +async fn load_detection_day_tokens( conn: &mut PgConnection, workspace_id: Uuid, from: jiff_diesel::Timestamp, to: jiff_diesel::Timestamp, -) -> PgResult> { +) -> PgResult> { use diesel::dsl::sum; - use schema::workspace_pipeline_run_usage::dsl as usage; - use schema::workspace_pipeline_runs::dsl as runs; + use schema::workspace_detection_usage::dsl as usage; + use schema::workspace_detections::dsl as detections; use schema::workspace_pipelines::dsl as pipelines; - use schema::{workspace_pipeline_run_usage, workspace_pipeline_runs, workspace_pipelines}; + use schema::{workspace_detection_usage, workspace_detections, workspace_pipelines}; - let per_run_input = workspace_pipeline_run_usage::table - .filter(usage::run_id.eq(runs::id)) + let per_detection_input = workspace_detection_usage::table + .filter(usage::detection_id.eq(detections::id)) .select(sum(usage::input_tokens)) .single_value(); - let per_run_output = workspace_pipeline_run_usage::table - .filter(usage::run_id.eq(runs::id)) + let per_detection_output = workspace_detection_usage::table + .filter(usage::detection_id.eq(detections::id)) .select(sum(usage::output_tokens)) .single_value(); - let per_run_total = workspace_pipeline_run_usage::table - .filter(usage::run_id.eq(runs::id)) + let per_detection_total = workspace_detection_usage::table + .filter(usage::detection_id.eq(detections::id)) .select(sum(usage::total_tokens)) .single_value(); - workspace_pipeline_runs::table + workspace_detections::table .inner_join(workspace_pipelines::table) .filter(pipelines::workspace_id.eq(workspace_id)) .filter(pipelines::deleted_at.is_null()) - .filter(runs::started_at.ge(from)) - .filter(runs::started_at.lt(to)) - .group_by(run_day()) + .filter(detections::started_at.ge(from)) + .filter(detections::started_at.lt(to)) + .group_by(detection_day()) .select(( - run_day(), - sum(per_run_input), - sum(per_run_output), - sum(per_run_total), + detection_day(), + sum(per_detection_input), + sum(per_detection_output), + sum(per_detection_total), )) .load(conn) .await diff --git a/crates/nvisy-postgres/src/query/mod.rs b/crates/nvisy-postgres/src/query/mod.rs index 4fbffaa8..809b7de9 100644 --- a/crates/nvisy-postgres/src/query/mod.rs +++ b/crates/nvisy-postgres/src/query/mod.rs @@ -27,20 +27,22 @@ mod workspace_activity; mod workspace_connection; mod workspace_connection_schedule; mod workspace_connection_sync; +mod workspace_detection; +mod workspace_detection_job; mod workspace_file; mod workspace_invite; mod workspace_member; mod workspace_pipeline; -mod workspace_pipeline_run; mod workspace_policy; +mod workspace_redaction; mod workspace_webhook; pub use account::AccountRepository; pub use account_api_token::AccountApiTokenRepository; pub use account_notification::AccountNotificationRepository; pub use analytics::{ - AnalyticsSnapshot, RunDayPoint, RunDurations, RunStatusCount, StorageByKind, UsageByModel, - WorkspaceAnalyticsRepository, + AnalyticsSnapshot, DetectionDayPoint, DetectionDurations, DetectionStatusCount, StorageByKind, + UsageByModel, WorkspaceAnalyticsRepository, }; pub use chat_message::{AppendSessionUpdate, ChatMessageRepository}; pub use chat_session::ChatSessionRepository; @@ -51,10 +53,12 @@ pub use workspace_activity::{ActivityFilter, WorkspaceActivityRepository}; pub use workspace_connection::{ScheduledConnection, WorkspaceConnectionRepository}; pub use workspace_connection_schedule::WorkspaceConnectionScheduleRepository; pub use workspace_connection_sync::WorkspaceConnectionSyncRepository; +pub use workspace_detection::{DetectionFiles, DetectionListRow, WorkspaceDetectionRepository}; +pub use workspace_detection_job::DetectionJobOutboxRepository; pub use workspace_file::{ExpiredFileRef, ImportedFileRef, WorkspaceFileRepository}; pub use workspace_invite::WorkspaceInviteRepository; pub use workspace_member::WorkspaceMemberRepository; pub use workspace_pipeline::WorkspacePipelineRepository; -pub use workspace_pipeline_run::{PipelineRunListRow, RunFiles, WorkspacePipelineRunRepository}; pub use workspace_policy::WorkspacePolicyRepository; +pub use workspace_redaction::WorkspaceRedactionRepository; pub use workspace_webhook::WorkspaceWebhookRepository; diff --git a/crates/nvisy-postgres/src/query/workspace_detection.rs b/crates/nvisy-postgres/src/query/workspace_detection.rs new file mode 100644 index 00000000..270d1130 --- /dev/null +++ b/crates/nvisy-postgres/src/query/workspace_detection.rs @@ -0,0 +1,635 @@ +//! Workspace detections repository for managing analysis instances. + +use std::future::Future; + +use diesel::prelude::*; +use diesel_async::RunQueryDsl; +use uuid::Uuid; + +use crate::model::{ + NewWorkspaceDetection, NewWorkspaceDetectionUsage, UpdateWorkspaceDetection, + WorkspaceDetection, WorkspacePipeline, +}; +use crate::types::{ + AccountRefRow, CursorPage, CursorPagination, DetectionFilter, DetectionStatus, Handle, +}; +use crate::{PgConnection, PgError, PgResult, schema}; + +/// Resolved display name of a detection's input file. +/// +/// `None` when the file has been removed (e.g. by retention). Redacted outputs +/// belong to redactions, not the detection, so they are not resolved here. +#[derive(Debug, Default, Clone)] +pub struct DetectionFiles { + /// Display name of the input document the detection analyzes. + pub input: Option, +} + +/// One row of a detection listing: the detection plus the context a response +/// needs to render it without follow-up lookups — the triggering account, the +/// owning pipeline's slug, and the input file's display name (`None` if the file +/// was removed). +#[derive(Debug, Clone)] +pub struct DetectionListRow { + /// The detection. + pub detection: WorkspaceDetection, + /// The account that triggered the detection. + pub account: AccountRefRow, + /// Slug of the detection's owning pipeline. + pub pipeline_slug: Handle, + /// Display name of the detection's input document, if still present. + pub input_file_name: Option, +} + +/// Repository for workspace detection database operations. +/// +/// Handles detection lifecycle management including creation, status updates, +/// completion tracking, and queries. +pub trait WorkspaceDetectionRepository { + /// Creates a new workspace detection record. + fn create_workspace_detection( + &mut self, + new_detection: NewWorkspaceDetection, + ) -> impl Future> + Send; + + /// Finds a detection by its opaque id, scoped to a workspace, returning the + /// detection and its owning pipeline. + /// + /// The detection is addressed by its own id (behind `/detections/{detectionId}`); + /// scoping through the owning pipeline keeps it workspace-bounded and hides + /// detections of soft-deleted pipelines. + fn find_workspace_detection_by_id( + &mut self, + workspace_id: Uuid, + detection_id: Uuid, + ) -> impl Future>> + Send; + + /// Finds a detection by its `(pipeline, idempotency key)` pair, for detect + /// replay. + fn find_detection_by_idempotency_key( + &mut self, + pipeline_id: Uuid, + idempotency_key: &str, + ) -> impl Future>> + Send; + + /// Lists a specific pipeline's detections with cursor pagination. `filter` + /// narrows by status and/or file (its `pipeline_id` is ignored — the listing + /// is already pipeline-scoped). + fn cursor_list_pipeline_detections( + &mut self, + pipeline_id: Uuid, + pagination: CursorPagination, + filter: &DetectionFilter, + ) -> impl Future>> + Send; + + /// Lists all detections across a workspace's pipelines with cursor + /// pagination. + /// + /// Detections carry no workspace reference of their own, so this joins through + /// the owning pipeline and filters on its workspace. `filter` narrows by + /// status, file, and/or owning pipeline; use [`cursor_list_pipeline_detections`] + /// for a single pipeline. + /// + /// [`cursor_list_pipeline_detections`]: Self::cursor_list_pipeline_detections + fn cursor_list_workspace_detections( + &mut self, + workspace_id: Uuid, + pagination: CursorPagination, + filter: &DetectionFilter, + ) -> impl Future>> + Send; + + /// Atomically claims a detection, transitioning it to `Executing`. + /// + /// Succeeds (returning the claimed detection) only when it is still `Pending`, + /// or is `Executing` but its previous claim has gone stale (`claimed_at` + /// older than `stale_before` — a worker that died mid-analysis). A detection + /// already executing under a fresh claim, or past the detect phase, yields + /// `None` so a redelivered job skips it instead of analyzing twice. + /// + /// The claim stamps `claimed_at = now()`, so the lease renews on each + /// (re)claim. Callers pass `stale_before = now - lease` computed against the + /// same clock the DB uses closely enough for a lease measured in minutes. + fn claim_detection( + &mut self, + detection_id: Uuid, + stale_before: jiff::Timestamp, + ) -> impl Future>> + Send; + + /// Resolves the display name of a detection's input file. + /// + /// An indexed lookup by id; a file removed (e.g. by retention) yields `None`. + /// Used to name a single detection's input file in its response without + /// threading a join through the shared detection lookup. + fn detection_file_names( + &mut self, + workspace_id: Uuid, + detection: &WorkspaceDetection, + ) -> impl Future> + Send; + + /// Transitions a detection to `Complete` only while the caller still holds + /// its claim — the detection is still `Executing` and its `claimed_at` matches + /// the value stamped when the caller claimed it. Returns `true` on success, + /// `false` if the claim has gone stale (another worker re-claimed the + /// detection after the lease expired), so the caller can abort without + /// stamping over the new owner's work. `updates` carries the analyze results + /// (audit file, metadata); status and the claim guard are applied here. + fn finalize_detection( + &mut self, + detection_id: Uuid, + claimed_at: jiff::Timestamp, + updates: UpdateWorkspaceDetection, + ) -> impl Future> + Send; + + /// Transitions a detection to `Failed` only while the caller still holds its + /// claim — the detection is still `Executing` and its `claimed_at` matches the + /// value stamped when the caller claimed it. Returns `true` on success, + /// `false` if the claim has gone stale (another worker re-claimed the + /// detection). Mirrors [`finalize_detection`](Self::finalize_detection) for + /// the failure path, so a worker whose lease expired mid-analysis cannot fail + /// a detection another worker now owns. `updates` carries the failure reason + /// and completion time; status and the claim guard are applied here. + fn fail_detection( + &mut self, + detection_id: Uuid, + claimed_at: jiff::Timestamp, + updates: UpdateWorkspaceDetection, + ) -> impl Future> + Send; + + /// Transitions a detection to `Failed` only while it is still `Pending` — no + /// worker has claimed it. For the enqueue-failure path in the create handler: + /// if enqueue reported an error but the job was in fact delivered, a worker + /// may already have claimed the detection (moving it to `Executing`), and this + /// guard makes the handler's failure a no-op so it never clobbers the outcome + /// the worker will drive. Returns `true` if it failed the detection, `false` + /// if it was no longer `Pending`. Forces `status` and `completed_at`. + fn fail_pending_detection( + &mut self, + detection_id: Uuid, + updates: UpdateWorkspaceDetection, + ) -> impl Future> + Send; + + /// Records a detection's per-model inference usage. A no-op for an empty slice + /// (a deterministic detection spends no tokens). Inserted once, at analyze + /// time. + fn record_detection_usage( + &mut self, + usage: &[NewWorkspaceDetectionUsage], + ) -> impl Future> + Send; +} + +impl WorkspaceDetectionRepository for PgConnection { + async fn create_workspace_detection( + &mut self, + new_detection: NewWorkspaceDetection, + ) -> PgResult { + use schema::workspace_detections; + + let detection = diesel::insert_into(workspace_detections::table) + .values(&new_detection) + .returning(WorkspaceDetection::as_returning()) + .get_result(self) + .await + .map_err(PgError::from)?; + + Ok(detection) + } + + async fn find_workspace_detection_by_id( + &mut self, + workspace_id: Uuid, + detection_id: Uuid, + ) -> PgResult> { + use schema::workspace_detections::dsl as detections; + use schema::{workspace_detections, workspace_pipelines}; + + // Detections carry no workspace column; scope through the owning pipeline + // so the id resolves only within its workspace, and only while that + // pipeline is live (a soft-deleted pipeline hides its detections). The + // pipeline is returned alongside so callers need no second lookup. + let detection = workspace_detections::table + .inner_join(workspace_pipelines::table) + .filter(detections::id.eq(detection_id)) + .filter(workspace_pipelines::workspace_id.eq(workspace_id)) + .filter(workspace_pipelines::deleted_at.is_null()) + .select(( + WorkspaceDetection::as_select(), + WorkspacePipeline::as_select(), + )) + .first(self) + .await + .optional() + .map_err(PgError::from)?; + + Ok(detection) + } + + async fn find_detection_by_idempotency_key( + &mut self, + pipeline_id: Uuid, + idempotency_key: &str, + ) -> PgResult> { + use schema::workspace_detections::{self, dsl}; + + let detection = workspace_detections::table + .filter(dsl::pipeline_id.eq(pipeline_id)) + .filter(dsl::idempotency_key.eq(idempotency_key)) + .select(WorkspaceDetection::as_select()) + .first(self) + .await + .optional() + .map_err(PgError::from)?; + + Ok(detection) + } + + async fn cursor_list_pipeline_detections( + &mut self, + pipeline_id: Uuid, + pagination: CursorPagination, + filter: &DetectionFilter, + ) -> PgResult> { + use schema::workspace_detections::dsl; + use schema::{accounts, workspace_detections, workspace_files, workspace_pipelines}; + + // One scoped builder for both the count and the page, so a future filter + // cannot be added to one and forgotten on the other. The listing is + // already scoped to one pipeline, so `filter.pipeline_id` is not applied. + // Join the owning pipeline (for its slug) and the input file (to name the + // detection's analyzed document) so a row is self-contained; the file is + // LEFT-joined so one removed by retention yields a null name. + let scoped = || { + let mut query = workspace_detections::table + .inner_join(accounts::table) + .inner_join(workspace_pipelines::table) + .left_join(workspace_files::table.on(dsl::input_file_id.eq(workspace_files::id))) + .filter(dsl::pipeline_id.eq(pipeline_id)) + .into_boxed(); + if let Some(status) = filter.status { + query = query.filter(dsl::status.eq(status)); + } + if let Some(file_id) = filter.input_file_id { + query = query.filter(dsl::input_file_id.eq(file_id)); + } + if let Some(account_id) = filter.account_id { + query = query.filter(dsl::account_id.eq(account_id)); + } + if let Some(trigger_type) = filter.trigger_type { + query = query.filter(dsl::trigger_type.eq(trigger_type)); + } + query + }; + + let total = if pagination.include_count { + Some( + scoped() + .count() + .get_result::(self) + .await + .map_err(PgError::from)?, + ) + } else { + None + }; + + let query = scoped(); + let limit = pagination.fetch_limit(); + let selection = ( + WorkspaceDetection::as_select(), + ( + accounts::username, + accounts::display_name, + accounts::avatar_url, + ), + workspace_pipelines::slug, + workspace_files::display_name.nullable(), + ); + + let rows: Vec<(WorkspaceDetection, AccountRefRow, Handle, Option)> = + if let Some(cursor) = &pagination.after { + let cursor_time = jiff_diesel::Timestamp::from(cursor.timestamp); + + query + .filter( + dsl::started_at + .lt(&cursor_time) + .or(dsl::started_at.eq(&cursor_time).and(dsl::id.lt(cursor.id))), + ) + .select(selection) + .order((dsl::started_at.desc(), dsl::id.desc())) + .limit(limit) + .load(self) + .await + .map_err(PgError::from)? + } else { + query + .select(selection) + .order((dsl::started_at.desc(), dsl::id.desc())) + .limit(limit) + .load(self) + .await + .map_err(PgError::from)? + }; + + let items = rows + .into_iter() + .map( + |(detection, account, pipeline_slug, input_file_name)| DetectionListRow { + detection, + account, + pipeline_slug, + input_file_name, + }, + ) + .collect(); + + Ok(CursorPage::new(items, total, pagination.limit, |row| { + (row.detection.started_at.into(), row.detection.id) + })) + } + + async fn cursor_list_workspace_detections( + &mut self, + workspace_id: Uuid, + pagination: CursorPagination, + filter: &DetectionFilter, + ) -> PgResult> { + use schema::accounts::dsl as accounts; + use schema::workspace_detections::dsl as detections; + use schema::workspace_files::dsl as files; + use schema::workspace_pipelines::dsl as pipelines; + + // Detections have no workspace column; scope them through the owning + // pipeline. The owning pipeline's slug, the triggering account, and the + // input file's name are selected alongside each detection so the + // cross-pipeline response can name its pipeline, trigger, and analyzed + // document without a per-row lookup. The input file is LEFT-joined so a + // file removed by retention yields a null name rather than dropping the + // detection. + let scoped = || { + let mut query = detections::workspace_detections + .inner_join(pipelines::workspace_pipelines) + .inner_join(accounts::accounts) + .left_join(files::workspace_files.on(detections::input_file_id.eq(files::id))) + .filter(pipelines::workspace_id.eq(workspace_id)) + .into_boxed(); + if let Some(status) = filter.status { + query = query.filter(detections::status.eq(status)); + } + if let Some(file_id) = filter.input_file_id { + query = query.filter(detections::input_file_id.eq(file_id)); + } + if let Some(pipeline_id) = filter.pipeline_id { + query = query.filter(detections::pipeline_id.eq(pipeline_id)); + } + if let Some(account_id) = filter.account_id { + query = query.filter(detections::account_id.eq(account_id)); + } + if let Some(trigger_type) = filter.trigger_type { + query = query.filter(detections::trigger_type.eq(trigger_type)); + } + query + }; + + let total = if pagination.include_count { + Some( + scoped() + .count() + .get_result::(self) + .await + .map_err(PgError::from)?, + ) + } else { + None + }; + + let limit = pagination.fetch_limit(); + let selection = ( + WorkspaceDetection::as_select(), + pipelines::slug, + ( + accounts::username, + accounts::display_name, + accounts::avatar_url, + ), + files::display_name.nullable(), + ); + + let rows: Vec<(WorkspaceDetection, Handle, AccountRefRow, Option)> = + if let Some(cursor) = &pagination.after { + let cursor_time = jiff_diesel::Timestamp::from(cursor.timestamp); + + scoped() + .filter( + detections::started_at + .lt(&cursor_time) + .or(detections::started_at + .eq(&cursor_time) + .and(detections::id.lt(cursor.id))), + ) + .select(selection) + .order((detections::started_at.desc(), detections::id.desc())) + .limit(limit) + .load(self) + .await + .map_err(PgError::from)? + } else { + scoped() + .select(selection) + .order((detections::started_at.desc(), detections::id.desc())) + .limit(limit) + .load(self) + .await + .map_err(PgError::from)? + }; + + let items = rows + .into_iter() + .map( + |(detection, pipeline_slug, account, input_file_name)| DetectionListRow { + detection, + account, + pipeline_slug, + input_file_name, + }, + ) + .collect(); + + Ok(CursorPage::new(items, total, pagination.limit, |row| { + (row.detection.started_at.into(), row.detection.id) + })) + } + + async fn claim_detection( + &mut self, + detection_id: Uuid, + stale_before: jiff::Timestamp, + ) -> PgResult> { + use schema::workspace_detections::{self, dsl}; + + let stale_before = jiff_diesel::Timestamp::from(stale_before); + + // Claim only if still pending, or executing under a claim that has gone + // stale (a dead worker). The WHERE clause makes the transition atomic: + // two concurrent deliveries race on the same row and exactly one flips + // it to `executing`; the loser matches no row and gets `None`. + let claimed = diesel::update( + workspace_detections::table + .filter(dsl::id.eq(detection_id)) + .filter( + dsl::status.eq(DetectionStatus::Pending).or(dsl::status + .eq(DetectionStatus::Executing) + .and(dsl::claimed_at.lt(stale_before))), + ), + ) + .set(( + dsl::status.eq(DetectionStatus::Executing), + dsl::claimed_at.eq(diesel::dsl::now), + )) + .returning(WorkspaceDetection::as_returning()) + .get_result(self) + .await + .optional() + .map_err(PgError::from)?; + + Ok(claimed) + } + + async fn detection_file_names( + &mut self, + workspace_id: Uuid, + detection: &WorkspaceDetection, + ) -> PgResult { + use schema::workspace_files::{self, dsl}; + + // Select the display name only, scoped to the workspace and excluding + // soft-deleted files, so a file removed by retention resolves to `None`. + async fn name_of( + conn: &mut PgConnection, + workspace_id: Uuid, + file_id: Uuid, + ) -> PgResult> { + workspace_files::table + .filter(dsl::id.eq(file_id)) + .filter(dsl::workspace_id.eq(workspace_id)) + .filter(dsl::deleted_at.is_null()) + .select(dsl::display_name) + .first::(conn) + .await + .optional() + .map_err(PgError::from) + } + + let input = name_of(self, workspace_id, detection.input_file_id).await?; + + Ok(DetectionFiles { input }) + } + + async fn finalize_detection( + &mut self, + detection_id: Uuid, + claimed_at: jiff::Timestamp, + mut updates: UpdateWorkspaceDetection, + ) -> PgResult { + use schema::workspace_detections::{self, dsl}; + + // Force the terminal transition and stamp the terminal time here, so every + // terminal outcome records `completed_at` regardless of the caller; the + // guard makes it a no-op unless we still own the claim. + updates.status = Some(DetectionStatus::Complete); + updates.completed_at = Some(Some(jiff::Timestamp::now().into())); + let claimed_at = jiff_diesel::Timestamp::from(claimed_at); + + // Guard on the claim we hold: same detection, still `Executing`, and the + // exact `claimed_at` our claim stamped. A worker that re-claimed a stale + // detection renews `claimed_at`, so a lost claim matches no row and + // returns false. + let updated = diesel::update( + workspace_detections::table + .filter(dsl::id.eq(detection_id)) + .filter(dsl::status.eq(DetectionStatus::Executing)) + .filter(dsl::claimed_at.eq(claimed_at)), + ) + .set(&updates) + .execute(self) + .await + .map_err(PgError::from)?; + + Ok(updated == 1) + } + + async fn fail_detection( + &mut self, + detection_id: Uuid, + claimed_at: jiff::Timestamp, + mut updates: UpdateWorkspaceDetection, + ) -> PgResult { + use schema::workspace_detections::{self, dsl}; + + // Force the terminal transition and stamp the terminal time here, mirroring + // `finalize_detection`, so both terminal outcomes record `completed_at`. + updates.status = Some(DetectionStatus::Failed); + updates.completed_at = Some(Some(jiff::Timestamp::now().into())); + let claimed_at = jiff_diesel::Timestamp::from(claimed_at); + + // Same claim guard as the complete finalize: only our still-live claim + // (detection `Executing`, `claimed_at` unchanged) may fail the detection. + let updated = diesel::update( + workspace_detections::table + .filter(dsl::id.eq(detection_id)) + .filter(dsl::status.eq(DetectionStatus::Executing)) + .filter(dsl::claimed_at.eq(claimed_at)), + ) + .set(&updates) + .execute(self) + .await + .map_err(PgError::from)?; + + Ok(updated == 1) + } + + async fn fail_pending_detection( + &mut self, + detection_id: Uuid, + mut updates: UpdateWorkspaceDetection, + ) -> PgResult { + use schema::workspace_detections::{self, dsl}; + + // Force the terminal transition and stamp the terminal time, as the other + // finalize methods do. + updates.status = Some(DetectionStatus::Failed); + updates.completed_at = Some(Some(jiff::Timestamp::now().into())); + + // Guard on `Pending`: once a worker claims the detection (moving it to + // `Executing`), this matches no row and the worker owns the outcome. + let updated = diesel::update( + workspace_detections::table + .filter(dsl::id.eq(detection_id)) + .filter(dsl::status.eq(DetectionStatus::Pending)), + ) + .set(&updates) + .execute(self) + .await + .map_err(PgError::from)?; + + Ok(updated == 1) + } + + async fn record_detection_usage( + &mut self, + usage: &[NewWorkspaceDetectionUsage], + ) -> PgResult<()> { + use schema::workspace_detection_usage; + + if usage.is_empty() { + return Ok(()); + } + + diesel::insert_into(workspace_detection_usage::table) + .values(usage) + .execute(self) + .await + .map_err(PgError::from)?; + + Ok(()) + } +} diff --git a/crates/nvisy-postgres/src/query/workspace_detection_job.rs b/crates/nvisy-postgres/src/query/workspace_detection_job.rs new file mode 100644 index 00000000..174e4d90 --- /dev/null +++ b/crates/nvisy-postgres/src/query/workspace_detection_job.rs @@ -0,0 +1,148 @@ +//! Detection-job outbox repository: the write side (insert in the create-detection +//! transaction) and the drainer side (claim a due batch, mark processed, defer or +//! dead-letter a failure). +//! +//! The drainer runs the claim and the subsequent `mark_*`/`defer_*` inside one +//! transaction per batch (see the detection-job drainer), so the `FOR UPDATE SKIP +//! LOCKED` locks are held from claim through completion: no other drainer takes +//! the same rows, and a row's state transition commits atomically with its +//! publication. + +use std::future::Future; + +use diesel::prelude::*; +use diesel::sql_types::{BigInt, Timestamptz}; +use diesel_async::RunQueryDsl; +use uuid::Uuid; + +use crate::model::{NewWorkspaceDetectionJob, WorkspaceDetectionJob}; +use crate::types::OutboxStatus; +use crate::{PgConnection, PgError, PgResult, schema}; + +/// Read and write operations on the detection-job outbox. +pub trait DetectionJobOutboxRepository { + /// Inserts one job outbox row. Called in the same transaction as the + /// detection it queues, so the two commit atomically. + fn insert_detection_job( + &mut self, + row: NewWorkspaceDetectionJob, + ) -> impl Future> + Send; + + /// Claims up to `limit` due pending rows for publication, oldest first. + /// + /// Due means unprocessed, not dead-lettered, and past its `next_attempt_at`, + /// so a row deferred by a backoff is skipped until its time arrives. Locks the + /// claimed rows with `FOR UPDATE SKIP LOCKED` so concurrent drainers take + /// disjoint batches without blocking each other; the lock is held for the + /// caller's transaction. Must run inside that transaction. + fn claim_detection_job_batch( + &mut self, + limit: i64, + ) -> impl Future>> + Send; + + /// Marks a row processed (its job durably published), taking it out of the + /// pending set. Runs in the drainer's batch transaction. + fn mark_detection_job_processed( + &mut self, + id: Uuid, + ) -> impl Future> + Send; + + /// Records a failed attempt: increments `attempts` and defers the next attempt + /// to `now() + backoff_secs` (computed by the database clock, so a drainer's + /// wall-clock skew cannot mis-schedule it), leaving the row pending for a + /// later retry. Runs in the drainer's batch transaction. + fn defer_detection_job_attempt( + &mut self, + id: Uuid, + backoff_secs: i64, + ) -> impl Future> + Send; + + /// Dead-letters a row: increments `attempts` and marks it `Failed`, taking it + /// out of the pending set so a job that can never publish stops consuming + /// drain cycles. The row is retained for inspection. Runs in the drainer's + /// batch transaction. + fn mark_detection_job_failed(&mut self, id: Uuid) -> impl Future> + Send; +} + +impl DetectionJobOutboxRepository for PgConnection { + async fn insert_detection_job( + &mut self, + row: NewWorkspaceDetectionJob, + ) -> PgResult { + use schema::workspace_detection_jobs; + + diesel::insert_into(workspace_detection_jobs::table) + .values(&row) + .returning(WorkspaceDetectionJob::as_returning()) + .get_result(self) + .await + .map_err(PgError::from) + } + + async fn claim_detection_job_batch( + &mut self, + limit: i64, + ) -> PgResult> { + use schema::workspace_detection_jobs::{self, dsl}; + + workspace_detection_jobs::table + .filter(dsl::status.eq(OutboxStatus::Pending)) + .filter(dsl::next_attempt_at.le(diesel::dsl::now)) + .order((dsl::next_attempt_at.asc(), dsl::created_at.asc())) + .limit(limit) + .select(WorkspaceDetectionJob::as_select()) + .for_update() + .skip_locked() + .load(self) + .await + .map_err(PgError::from) + } + + async fn mark_detection_job_processed(&mut self, id: Uuid) -> PgResult<()> { + use schema::workspace_detection_jobs::{self, dsl}; + + diesel::update(workspace_detection_jobs::table.filter(dsl::id.eq(id))) + .set(( + dsl::status.eq(OutboxStatus::Processed), + dsl::attempts.eq(dsl::attempts + 1), + )) + .execute(self) + .await + .map_err(PgError::from)?; + Ok(()) + } + + async fn defer_detection_job_attempt(&mut self, id: Uuid, backoff_secs: i64) -> PgResult<()> { + use schema::workspace_detection_jobs::{self, dsl}; + + // The row stays `Pending`; only its attempt count and next-due time move. + // `now() + (backoff_secs * interval '1 second')` schedules the next attempt + // by the database clock, not the drainer's. + let next_attempt_at = diesel::dsl::sql::("now() + (") + .bind::(backoff_secs) + .sql(" * interval '1 second')"); + diesel::update(workspace_detection_jobs::table.filter(dsl::id.eq(id))) + .set(( + dsl::attempts.eq(dsl::attempts + 1), + dsl::next_attempt_at.eq(next_attempt_at), + )) + .execute(self) + .await + .map_err(PgError::from)?; + Ok(()) + } + + async fn mark_detection_job_failed(&mut self, id: Uuid) -> PgResult<()> { + use schema::workspace_detection_jobs::{self, dsl}; + + diesel::update(workspace_detection_jobs::table.filter(dsl::id.eq(id))) + .set(( + dsl::status.eq(OutboxStatus::Failed), + dsl::attempts.eq(dsl::attempts + 1), + )) + .execute(self) + .await + .map_err(PgError::from)?; + Ok(()) + } +} diff --git a/crates/nvisy-postgres/src/query/workspace_file.rs b/crates/nvisy-postgres/src/query/workspace_file.rs index c32f2247..c4c57ede 100644 --- a/crates/nvisy-postgres/src/query/workspace_file.rs +++ b/crates/nvisy-postgres/src/query/workspace_file.rs @@ -11,8 +11,8 @@ use uuid::Uuid; use crate::model::{NewWorkspaceFile, NewWorkspaceFileImport, UpdateWorkspaceFile, WorkspaceFile}; use crate::query::search::ilike_contains; use crate::types::{ - AccountRefRow, CursorPage, CursorPagination, FileFilter, FileKind, FileSortBy, FileSortField, - OffsetPagination, PipelineRunStatus, SortOrder, WithAccountRef, + AccountRefRow, CursorPage, CursorPagination, DetectionStatus, FileFilter, FileKind, FileSortBy, + FileSortField, OffsetPagination, SortOrder, WithAccountRef, }; use crate::{PgConnection, PgError, PgResult, schema}; @@ -135,8 +135,8 @@ pub trait WorkspaceFileRepository { ) -> impl Future> + Send; /// Recomputes `expires_at` for live files of `kind` produced by a specific - /// pipeline's runs (redacted outputs via `output_file_id`, audit blobs via - /// `audit_file_id`), returning the number updated. Used to backfill when a + /// pipeline's detections and redactions (detection audits, redaction outputs, + /// and review audits), returning the number updated. Used to backfill when a /// pipeline's own retention override changes, without touching other /// pipelines' files. `None` clears the expiry. fn backfill_pipeline_files_expiry( @@ -362,28 +362,26 @@ impl WorkspaceFileRepository for PgConnection { async fn files_due_for_expiry(&mut self, limit: i64) -> PgResult> { use diesel::dsl::{exists, not, now}; - use schema::workspace_pipeline_runs::dsl as runs; - use schema::{workspace_files, workspace_pipeline_runs}; - - // A run that has not finished (running or awaiting redaction) still needs - // its input document and audit blob, so those files are held back from - // expiry until the run reaches a terminal state. Otherwise an in-flight - // detect/redact could lose its source or analysis mid-flight and get - // stuck. Produced outputs are not protected — they only exist once a run - // has completed. - let active_run_holds_file = exists( - workspace_pipeline_runs::table.filter( - runs::status - .eq_any([ - PipelineRunStatus::Queued, - PipelineRunStatus::Analyzing, - PipelineRunStatus::Analyzed, - ]) - .and( - runs::input_file_id - .eq(workspace_files::id) - .or(runs::audit_file_id.eq(workspace_files::id.nullable())), - ), + use schema::workspace_detections::dsl as detections; + use schema::{workspace_detections, workspace_files}; + + // A detection that is still analyzing (pending or executing) needs its + // input document and audit blob, so those files are held back from expiry + // until analysis reaches a terminal state — otherwise an in-flight detect + // could lose its source or analysis mid-flight and get stuck. A `Complete` + // detection is NOT held: it is terminal (redaction never changes its + // status), so holding it would pin the input and audit forever and defeat + // retention. Re-redaction of a complete detection is bounded by those + // files' own `expires_at` — retention itself decides how long they remain + // redactable. Redaction outputs are not protected here — they belong to + // redactions, not the detection. + let active_detection_holds_file = exists( + workspace_detections::table.filter( + detections::status.eq_any(DetectionStatus::IN_PROGRESS).and( + detections::input_file_id + .eq(workspace_files::id) + .or(detections::audit_file_id.eq(workspace_files::id.nullable())), + ), ), ); @@ -391,7 +389,7 @@ impl WorkspaceFileRepository for PgConnection { .filter(workspace_files::expires_at.is_not_null()) .filter(workspace_files::expires_at.lt(now)) .filter(workspace_files::deleted_at.is_null()) - .filter(not(active_run_holds_file)) + .filter(not(active_detection_holds_file)) .select(( workspace_files::id, workspace_files::storage_path, @@ -466,26 +464,38 @@ impl WorkspaceFileRepository for PgConnection { kind: FileKind, expires_at: Option, ) -> PgResult { - use schema::workspace_pipeline_runs::dsl as runs; - use schema::{workspace_files, workspace_pipeline_runs}; + use schema::workspace_detections::dsl as detections; + use schema::workspace_redactions::dsl as redactions; + use schema::{workspace_detections, workspace_files, workspace_redactions}; let expires_at = expires_at.map(jiff_diesel::Timestamp::from); - // Collect the ids of files this pipeline's runs produced for `kind`: - // redacted files are the runs' outputs, audit files their audit blobs. - // Any other kind is not pipeline-produced, so there is nothing to do. + // Collect the ids of files this pipeline produced for `kind`. Audit blobs + // belong to the pipeline's detections; redacted outputs and review blobs + // belong to the redactions of those detections (joined back to the + // pipeline through the detection). Any other kind is not pipeline-produced, + // so there is nothing to do. let file_ids: Vec = match kind { - FileKind::Redacted => workspace_pipeline_runs::table - .filter(runs::pipeline_id.eq(pipeline_id)) - .filter(runs::output_file_id.is_not_null()) - .select(runs::output_file_id.assume_not_null()) + FileKind::Audit => workspace_detections::table + .filter(detections::pipeline_id.eq(pipeline_id)) + .filter(detections::audit_file_id.is_not_null()) + .select(detections::audit_file_id.assume_not_null()) + .load(self) + .await + .map_err(PgError::from)?, + FileKind::Redacted => workspace_redactions::table + .inner_join(workspace_detections::table) + .filter(detections::pipeline_id.eq(pipeline_id)) + .filter(redactions::output_file_id.is_not_null()) + .select(redactions::output_file_id.assume_not_null()) .load(self) .await .map_err(PgError::from)?, - FileKind::Audit => workspace_pipeline_runs::table - .filter(runs::pipeline_id.eq(pipeline_id)) - .filter(runs::audit_file_id.is_not_null()) - .select(runs::audit_file_id.assume_not_null()) + FileKind::Review => workspace_redactions::table + .inner_join(workspace_detections::table) + .filter(detections::pipeline_id.eq(pipeline_id)) + .filter(redactions::review_file_id.is_not_null()) + .select(redactions::review_file_id.assume_not_null()) .load(self) .await .map_err(PgError::from)?, diff --git a/crates/nvisy-postgres/src/query/workspace_pipeline_run.rs b/crates/nvisy-postgres/src/query/workspace_pipeline_run.rs deleted file mode 100644 index 45c6595f..00000000 --- a/crates/nvisy-postgres/src/query/workspace_pipeline_run.rs +++ /dev/null @@ -1,625 +0,0 @@ -//! Workspace pipeline runs repository for managing pipeline execution instances. - -use std::future::Future; - -use diesel::prelude::*; -use diesel_async::RunQueryDsl; -use uuid::Uuid; - -use crate::model::{ - NewWorkspacePipelineRun, NewWorkspacePipelineRunUsage, UpdateWorkspacePipelineRun, - WorkspacePipeline, WorkspacePipelineRun, -}; -use crate::types::{ - AccountRefRow, CursorPage, CursorPagination, Handle, PipelineRunStatus, RunFilter, -}; -use crate::{PgConnection, PgError, PgResult, schema}; - -/// Resolved display names of a run's input and output files. -/// -/// Each is `None` when the run has no such file yet (no output before redaction) -/// or the file has been removed (e.g. by retention). -#[derive(Debug, Default, Clone)] -pub struct RunFiles { - /// Display name of the input document the run analyzes. - pub input: Option, - /// Display name of the redacted output, once the run has produced one. - pub output: Option, -} - -/// One row of a pipeline-run listing: the run plus the context a response needs -/// to render it without follow-up lookups — the triggering account, the owning -/// pipeline's slug, and the input file's display name (`None` if the file was -/// removed). -#[derive(Debug, Clone)] -pub struct PipelineRunListRow { - /// The run. - pub run: WorkspacePipelineRun, - /// The account that triggered the run. - pub account: AccountRefRow, - /// Slug of the run's owning pipeline. - pub pipeline_slug: Handle, - /// Display name of the run's input document, if still present. - pub input_file_name: Option, -} - -/// Repository for workspace pipeline run database operations. -/// -/// Handles pipeline run lifecycle management including creation, status updates, -/// completion tracking, and queries. -pub trait WorkspacePipelineRunRepository { - /// Creates a new workspace pipeline run record. - fn create_workspace_pipeline_run( - &mut self, - new_run: NewWorkspacePipelineRun, - ) -> impl Future> + Send; - - /// Finds a run by its opaque id, scoped to a workspace, returning the run - /// and its owning pipeline. - /// - /// The run is addressed by its own id (behind `/runs/{runId}`); scoping - /// through the owning pipeline keeps it workspace-bounded and hides runs of - /// soft-deleted pipelines. - fn find_workspace_run_by_id( - &mut self, - workspace_id: Uuid, - run_id: Uuid, - ) -> impl Future>> + Send; - - /// Finds a run by its `(pipeline, idempotency key)` pair, for detect replay. - fn find_pipeline_run_by_idempotency_key( - &mut self, - pipeline_id: Uuid, - idempotency_key: &str, - ) -> impl Future>> + Send; - - /// Lists a specific pipeline's runs with cursor pagination. `filter` narrows - /// by status and/or file (its `pipeline_id` is ignored — the listing is - /// already pipeline-scoped). - fn cursor_list_workspace_pipeline_runs( - &mut self, - pipeline_id: Uuid, - pagination: CursorPagination, - filter: &RunFilter, - ) -> impl Future>> + Send; - - /// Lists all runs across a workspace's pipelines with cursor pagination. - /// - /// Runs carry no workspace reference of their own, so this joins through the - /// owning pipeline and filters on its workspace. `filter` narrows by status, - /// file, and/or owning pipeline; use [`cursor_list_workspace_pipeline_runs`] - /// for a single pipeline. - /// - /// [`cursor_list_workspace_pipeline_runs`]: Self::cursor_list_workspace_pipeline_runs - fn cursor_list_workspace_runs( - &mut self, - workspace_id: Uuid, - pagination: CursorPagination, - filter: &RunFilter, - ) -> impl Future>> + Send; - - /// Atomically claims a run for detection, transitioning it to `Analyzing`. - /// - /// Succeeds (returning the claimed run) only when the run is still `Queued`, - /// or is `Analyzing` but its previous claim has gone stale (`claimed_at` - /// older than `stale_before` — a worker that died mid-analysis). A run - /// already being analyzed under a fresh claim, or past the detect phase, - /// yields `None` so a redelivered job skips it instead of analyzing twice. - /// - /// The claim stamps `claimed_at = now()`, so the lease renews on each - /// (re)claim. Callers pass `stale_before = now - lease` computed against the - /// same clock the DB uses closely enough for a lease measured in minutes. - fn claim_run_for_detection( - &mut self, - run_id: Uuid, - stale_before: jiff::Timestamp, - ) -> impl Future>> + Send; - - /// Resolves the display names of a run's input and output files. - /// - /// Two indexed lookups by id (the output only when the run has produced one); - /// a file removed (e.g. by retention) yields `None` for that name. Used to - /// name a single run's files in its response without threading a join - /// through the shared run lookup. - fn run_file_names( - &mut self, - workspace_id: Uuid, - run: &WorkspacePipelineRun, - ) -> impl Future> + Send; - - /// Updates a workspace pipeline run with new data. - fn update_workspace_pipeline_run( - &mut self, - run_id: Uuid, - updates: UpdateWorkspacePipelineRun, - ) -> impl Future> + Send; - - /// Transitions a run to `Analyzed` only while the caller still holds its - /// claim — the run is still `Analyzing` and its `claimed_at` matches the value - /// stamped when the caller claimed it. Returns `true` on success, `false` if - /// the claim has gone stale (another worker re-claimed the run after the lease - /// expired), so the caller can abort without stamping over the new owner's - /// work. `updates` carries the analyze results (audit file, metadata); status - /// and the claim guard are applied here. - fn finalize_analyzed_run( - &mut self, - run_id: Uuid, - claimed_at: jiff::Timestamp, - updates: UpdateWorkspacePipelineRun, - ) -> impl Future> + Send; - - /// Transitions a run to `Failed` only while the caller still holds its claim - /// — the run is still `Analyzing` and its `claimed_at` matches the value - /// stamped when the caller claimed it. Returns `true` on success, `false` if - /// the claim has gone stale (another worker re-claimed the run). Mirrors - /// [`finalize_analyzed_run`](Self::finalize_analyzed_run) for the failure path, - /// so a worker whose lease expired mid-analysis cannot fail a run another - /// worker now owns. `updates` carries the failure reason and completion time; - /// status and the claim guard are applied here. - fn finalize_failed_run( - &mut self, - run_id: Uuid, - claimed_at: jiff::Timestamp, - updates: UpdateWorkspacePipelineRun, - ) -> impl Future> + Send; - - /// Records a run's per-model inference usage. A no-op for an empty slice - /// (a deterministic run spends no tokens). Inserted once, at analyze time. - fn record_run_usage( - &mut self, - usage: &[NewWorkspacePipelineRunUsage], - ) -> impl Future> + Send; -} - -impl WorkspacePipelineRunRepository for PgConnection { - async fn create_workspace_pipeline_run( - &mut self, - new_run: NewWorkspacePipelineRun, - ) -> PgResult { - use schema::workspace_pipeline_runs; - - let run = diesel::insert_into(workspace_pipeline_runs::table) - .values(&new_run) - .returning(WorkspacePipelineRun::as_returning()) - .get_result(self) - .await - .map_err(PgError::from)?; - - Ok(run) - } - - async fn find_workspace_run_by_id( - &mut self, - workspace_id: Uuid, - run_id: Uuid, - ) -> PgResult> { - use schema::workspace_pipeline_runs::dsl as runs; - use schema::{workspace_pipeline_runs, workspace_pipelines}; - - // Runs carry no workspace column; scope through the owning pipeline so - // the id resolves only within its workspace, and only while that - // pipeline is live (a soft-deleted pipeline hides its runs). The - // pipeline is returned alongside so callers need no second lookup. - let run = workspace_pipeline_runs::table - .inner_join(workspace_pipelines::table) - .filter(runs::id.eq(run_id)) - .filter(workspace_pipelines::workspace_id.eq(workspace_id)) - .filter(workspace_pipelines::deleted_at.is_null()) - .select(( - WorkspacePipelineRun::as_select(), - WorkspacePipeline::as_select(), - )) - .first(self) - .await - .optional() - .map_err(PgError::from)?; - - Ok(run) - } - - async fn find_pipeline_run_by_idempotency_key( - &mut self, - pipeline_id: Uuid, - idempotency_key: &str, - ) -> PgResult> { - use schema::workspace_pipeline_runs::{self, dsl}; - - let run = workspace_pipeline_runs::table - .filter(dsl::pipeline_id.eq(pipeline_id)) - .filter(dsl::idempotency_key.eq(idempotency_key)) - .select(WorkspacePipelineRun::as_select()) - .first(self) - .await - .optional() - .map_err(PgError::from)?; - - Ok(run) - } - - async fn cursor_list_workspace_pipeline_runs( - &mut self, - pipeline_id: Uuid, - pagination: CursorPagination, - filter: &RunFilter, - ) -> PgResult> { - use schema::workspace_pipeline_runs::dsl; - use schema::{accounts, workspace_files, workspace_pipeline_runs, workspace_pipelines}; - - // Build base query with filters. The listing is already scoped to one - // pipeline, so `filter.pipeline_id` is not applied here. - let mut base_query = workspace_pipeline_runs::table - .filter(dsl::pipeline_id.eq(pipeline_id)) - .into_boxed(); - - if let Some(status) = filter.status { - base_query = base_query.filter(dsl::status.eq(status)); - } - if let Some(file_id) = filter.input_file_id { - base_query = base_query.filter(dsl::input_file_id.eq(file_id)); - } - if let Some(account_id) = filter.account_id { - base_query = base_query.filter(dsl::account_id.eq(account_id)); - } - if let Some(trigger_type) = filter.trigger_type { - base_query = base_query.filter(dsl::trigger_type.eq(trigger_type)); - } - - let total = if pagination.include_count { - Some( - base_query - .count() - .get_result::(self) - .await - .map_err(PgError::from)?, - ) - } else { - None - }; - - // Rebuild query for fetching items. Join the owning pipeline (for its - // slug) and the input file (to name the run's analyzed document) so a - // row is self-contained; a LEFT JOIN on the file tolerates one removed - // by retention, yielding a null name. - let mut query = workspace_pipeline_runs::table - .inner_join(accounts::table) - .inner_join(workspace_pipelines::table) - .left_join(workspace_files::table.on(dsl::input_file_id.eq(workspace_files::id))) - .filter(dsl::pipeline_id.eq(pipeline_id)) - .into_boxed(); - - if let Some(status) = filter.status { - query = query.filter(dsl::status.eq(status)); - } - if let Some(file_id) = filter.input_file_id { - query = query.filter(dsl::input_file_id.eq(file_id)); - } - if let Some(account_id) = filter.account_id { - query = query.filter(dsl::account_id.eq(account_id)); - } - if let Some(trigger_type) = filter.trigger_type { - query = query.filter(dsl::trigger_type.eq(trigger_type)); - } - - let limit = pagination.fetch_limit(); - let selection = ( - WorkspacePipelineRun::as_select(), - ( - accounts::username, - accounts::display_name, - accounts::avatar_url, - ), - workspace_pipelines::slug, - workspace_files::display_name.nullable(), - ); - - let rows: Vec<(WorkspacePipelineRun, AccountRefRow, Handle, Option)> = - if let Some(cursor) = &pagination.after { - let cursor_time = jiff_diesel::Timestamp::from(cursor.timestamp); - - query - .filter( - dsl::started_at - .lt(&cursor_time) - .or(dsl::started_at.eq(&cursor_time).and(dsl::id.lt(cursor.id))), - ) - .select(selection) - .order((dsl::started_at.desc(), dsl::id.desc())) - .limit(limit) - .load(self) - .await - .map_err(PgError::from)? - } else { - query - .select(selection) - .order((dsl::started_at.desc(), dsl::id.desc())) - .limit(limit) - .load(self) - .await - .map_err(PgError::from)? - }; - - let items = rows - .into_iter() - .map( - |(run, account, pipeline_slug, input_file_name)| PipelineRunListRow { - run, - account, - pipeline_slug, - input_file_name, - }, - ) - .collect(); - - Ok(CursorPage::new(items, total, pagination.limit, |row| { - (row.run.started_at.into(), row.run.id) - })) - } - - async fn cursor_list_workspace_runs( - &mut self, - workspace_id: Uuid, - pagination: CursorPagination, - filter: &RunFilter, - ) -> PgResult> { - use schema::accounts::dsl as accounts; - use schema::workspace_files::dsl as files; - use schema::workspace_pipeline_runs::dsl as runs; - use schema::workspace_pipelines::dsl as pipelines; - - // Runs have no workspace column; scope them through the owning pipeline. - // The owning pipeline's slug, the triggering account, and the input - // file's name are selected alongside each run so the cross-pipeline - // response can name its pipeline, trigger, and analyzed document without - // a per-row lookup. The input file is LEFT-joined so a file removed by - // retention yields a null name rather than dropping the run. - let scoped = || { - let mut query = runs::workspace_pipeline_runs - .inner_join(pipelines::workspace_pipelines) - .inner_join(accounts::accounts) - .left_join(files::workspace_files.on(runs::input_file_id.eq(files::id))) - .filter(pipelines::workspace_id.eq(workspace_id)) - .into_boxed(); - if let Some(status) = filter.status { - query = query.filter(runs::status.eq(status)); - } - if let Some(file_id) = filter.input_file_id { - query = query.filter(runs::input_file_id.eq(file_id)); - } - if let Some(pipeline_id) = filter.pipeline_id { - query = query.filter(runs::pipeline_id.eq(pipeline_id)); - } - if let Some(account_id) = filter.account_id { - query = query.filter(runs::account_id.eq(account_id)); - } - if let Some(trigger_type) = filter.trigger_type { - query = query.filter(runs::trigger_type.eq(trigger_type)); - } - query - }; - - let total = if pagination.include_count { - Some( - scoped() - .count() - .get_result::(self) - .await - .map_err(PgError::from)?, - ) - } else { - None - }; - - let limit = pagination.fetch_limit(); - let selection = ( - WorkspacePipelineRun::as_select(), - pipelines::slug, - ( - accounts::username, - accounts::display_name, - accounts::avatar_url, - ), - files::display_name.nullable(), - ); - - let rows: Vec<(WorkspacePipelineRun, Handle, AccountRefRow, Option)> = - if let Some(cursor) = &pagination.after { - let cursor_time = jiff_diesel::Timestamp::from(cursor.timestamp); - - scoped() - .filter( - runs::started_at.lt(&cursor_time).or(runs::started_at - .eq(&cursor_time) - .and(runs::id.lt(cursor.id))), - ) - .select(selection) - .order((runs::started_at.desc(), runs::id.desc())) - .limit(limit) - .load(self) - .await - .map_err(PgError::from)? - } else { - scoped() - .select(selection) - .order((runs::started_at.desc(), runs::id.desc())) - .limit(limit) - .load(self) - .await - .map_err(PgError::from)? - }; - - let items = rows - .into_iter() - .map( - |(run, pipeline_slug, account, input_file_name)| PipelineRunListRow { - run, - account, - pipeline_slug, - input_file_name, - }, - ) - .collect(); - - Ok(CursorPage::new(items, total, pagination.limit, |row| { - (row.run.started_at.into(), row.run.id) - })) - } - - async fn claim_run_for_detection( - &mut self, - run_id: Uuid, - stale_before: jiff::Timestamp, - ) -> PgResult> { - use schema::workspace_pipeline_runs::{self, dsl}; - - let stale_before = jiff_diesel::Timestamp::from(stale_before); - - // Claim only if still queued, or analyzing under a claim that has gone - // stale (a dead worker). The WHERE clause makes the transition atomic: - // two concurrent deliveries race on the same row and exactly one flips - // it to `analyzing`; the loser matches no row and gets `None`. - let claimed = diesel::update( - workspace_pipeline_runs::table - .filter(dsl::id.eq(run_id)) - .filter( - dsl::status.eq(PipelineRunStatus::Queued).or(dsl::status - .eq(PipelineRunStatus::Analyzing) - .and(dsl::claimed_at.lt(stale_before))), - ), - ) - .set(( - dsl::status.eq(PipelineRunStatus::Analyzing), - dsl::claimed_at.eq(diesel::dsl::now), - )) - .returning(WorkspacePipelineRun::as_returning()) - .get_result(self) - .await - .optional() - .map_err(PgError::from)?; - - Ok(claimed) - } - - async fn run_file_names( - &mut self, - workspace_id: Uuid, - run: &WorkspacePipelineRun, - ) -> PgResult { - use schema::workspace_files::{self, dsl}; - - // Select the display name only, scoped to the workspace and excluding - // soft-deleted files, so a file removed by retention resolves to `None`. - async fn name_of( - conn: &mut PgConnection, - workspace_id: Uuid, - file_id: Uuid, - ) -> PgResult> { - workspace_files::table - .filter(dsl::id.eq(file_id)) - .filter(dsl::workspace_id.eq(workspace_id)) - .filter(dsl::deleted_at.is_null()) - .select(dsl::display_name) - .first::(conn) - .await - .optional() - .map_err(PgError::from) - } - - let input = name_of(self, workspace_id, run.input_file_id).await?; - let output = match run.output_file_id { - Some(output_file_id) => name_of(self, workspace_id, output_file_id).await?, - None => None, - }; - - Ok(RunFiles { input, output }) - } - - async fn update_workspace_pipeline_run( - &mut self, - run_id: Uuid, - updates: UpdateWorkspacePipelineRun, - ) -> PgResult { - use schema::workspace_pipeline_runs::{self, dsl}; - - let run = diesel::update(workspace_pipeline_runs::table.filter(dsl::id.eq(run_id))) - .set(&updates) - .returning(WorkspacePipelineRun::as_returning()) - .get_result(self) - .await - .map_err(PgError::from)?; - - Ok(run) - } - - async fn finalize_analyzed_run( - &mut self, - run_id: Uuid, - claimed_at: jiff::Timestamp, - mut updates: UpdateWorkspacePipelineRun, - ) -> PgResult { - use schema::workspace_pipeline_runs::{self, dsl}; - - // Force the terminal transition here; the guard makes it a no-op unless we - // still own the claim. - updates.status = Some(PipelineRunStatus::Analyzed); - let claimed_at = jiff_diesel::Timestamp::from(claimed_at); - - // Guard on the claim we hold: same run, still `Analyzing`, and the exact - // `claimed_at` our claim stamped. A worker that re-claimed a stale run - // renews `claimed_at`, so a lost claim matches no row and returns false. - let updated = diesel::update( - workspace_pipeline_runs::table - .filter(dsl::id.eq(run_id)) - .filter(dsl::status.eq(PipelineRunStatus::Analyzing)) - .filter(dsl::claimed_at.eq(claimed_at)), - ) - .set(&updates) - .execute(self) - .await - .map_err(PgError::from)?; - - Ok(updated == 1) - } - - async fn finalize_failed_run( - &mut self, - run_id: Uuid, - claimed_at: jiff::Timestamp, - mut updates: UpdateWorkspacePipelineRun, - ) -> PgResult { - use schema::workspace_pipeline_runs::{self, dsl}; - - updates.status = Some(PipelineRunStatus::Failed); - let claimed_at = jiff_diesel::Timestamp::from(claimed_at); - - // Same claim guard as the analyzed finalize: only our still-live claim - // (run `Analyzing`, `claimed_at` unchanged) may fail the run. - let updated = diesel::update( - workspace_pipeline_runs::table - .filter(dsl::id.eq(run_id)) - .filter(dsl::status.eq(PipelineRunStatus::Analyzing)) - .filter(dsl::claimed_at.eq(claimed_at)), - ) - .set(&updates) - .execute(self) - .await - .map_err(PgError::from)?; - - Ok(updated == 1) - } - - async fn record_run_usage(&mut self, usage: &[NewWorkspacePipelineRunUsage]) -> PgResult<()> { - use schema::workspace_pipeline_run_usage; - - if usage.is_empty() { - return Ok(()); - } - - diesel::insert_into(workspace_pipeline_run_usage::table) - .values(usage) - .execute(self) - .await - .map_err(PgError::from)?; - - Ok(()) - } -} diff --git a/crates/nvisy-postgres/src/query/workspace_redaction.rs b/crates/nvisy-postgres/src/query/workspace_redaction.rs new file mode 100644 index 00000000..2ea9ea79 --- /dev/null +++ b/crates/nvisy-postgres/src/query/workspace_redaction.rs @@ -0,0 +1,140 @@ +//! Workspace redactions repository for managing redaction instances. + +use std::future::Future; + +use diesel::prelude::*; +use diesel_async::RunQueryDsl; +use uuid::Uuid; + +use crate::model::{NewWorkspaceRedaction, WorkspaceRedaction}; +use crate::types::{CursorPage, CursorPagination}; +use crate::{PgConnection, PgError, PgResult, schema}; + +/// Repository for workspace redaction database operations. +/// +/// A redaction is one redact pass over a detection's analysis; a detection can +/// have many. Each redaction owns the review audit it applied and the redacted +/// document it produced. +pub trait WorkspaceRedactionRepository { + /// Creates a new workspace redaction record. + fn create_redaction( + &mut self, + new_redaction: NewWorkspaceRedaction, + ) -> impl Future> + Send; + + /// Finds a redaction by its id, scoped to a workspace. + /// + /// A [`RedactionId`](crate::types::RedactionId) is globally unique, so a + /// redaction is addressable by id alone; this resolves it only within the + /// given workspace by joining through its detection's pipeline. + fn find_redaction_in_workspace( + &mut self, + workspace_id: Uuid, + redaction_id: Uuid, + ) -> impl Future>> + Send; + + /// Lists a detection's redactions with cursor pagination, newest first. + fn cursor_list_detection_redactions( + &mut self, + detection_id: Uuid, + pagination: CursorPagination, + ) -> impl Future>> + Send; +} + +impl WorkspaceRedactionRepository for PgConnection { + async fn create_redaction( + &mut self, + new_redaction: NewWorkspaceRedaction, + ) -> PgResult { + use schema::workspace_redactions; + + let redaction = diesel::insert_into(workspace_redactions::table) + .values(&new_redaction) + .returning(WorkspaceRedaction::as_returning()) + .get_result(self) + .await + .map_err(PgError::from)?; + + Ok(redaction) + } + + async fn find_redaction_in_workspace( + &mut self, + workspace_id: Uuid, + redaction_id: Uuid, + ) -> PgResult> { + use schema::workspace_redactions::dsl as redactions; + use schema::{workspace_detections, workspace_pipelines, workspace_redactions}; + + // Redactions carry no workspace column; scope through the detection's + // pipeline so the id resolves only within its workspace, and only while + // that pipeline is live (a soft-deleted pipeline hides its redactions). + let redaction = workspace_redactions::table + .inner_join(workspace_detections::table.inner_join(workspace_pipelines::table)) + .filter(redactions::id.eq(redaction_id)) + .filter(workspace_pipelines::workspace_id.eq(workspace_id)) + .filter(workspace_pipelines::deleted_at.is_null()) + .select(WorkspaceRedaction::as_select()) + .first(self) + .await + .optional() + .map_err(PgError::from)?; + + Ok(redaction) + } + + async fn cursor_list_detection_redactions( + &mut self, + detection_id: Uuid, + pagination: CursorPagination, + ) -> PgResult> { + use schema::workspace_redactions::{self, dsl}; + + let base_query = workspace_redactions::table.filter(dsl::detection_id.eq(detection_id)); + + let total = if pagination.include_count { + Some( + base_query + .count() + .get_result::(self) + .await + .map_err(PgError::from)?, + ) + } else { + None + }; + + let limit = pagination.fetch_limit(); + + let items: Vec = if let Some(cursor) = &pagination.after { + let cursor_time = jiff_diesel::Timestamp::from(cursor.timestamp); + + workspace_redactions::table + .filter(dsl::detection_id.eq(detection_id)) + .filter( + dsl::created_at + .lt(&cursor_time) + .or(dsl::created_at.eq(&cursor_time).and(dsl::id.lt(cursor.id))), + ) + .select(WorkspaceRedaction::as_select()) + .order((dsl::created_at.desc(), dsl::id.desc())) + .limit(limit) + .load(self) + .await + .map_err(PgError::from)? + } else { + workspace_redactions::table + .filter(dsl::detection_id.eq(detection_id)) + .select(WorkspaceRedaction::as_select()) + .order((dsl::created_at.desc(), dsl::id.desc())) + .limit(limit) + .load(self) + .await + .map_err(PgError::from)? + }; + + Ok(CursorPage::new(items, total, pagination.limit, |row| { + (row.created_at.into(), row.id) + })) + } +} diff --git a/crates/nvisy-postgres/src/schema.rs b/crates/nvisy-postgres/src/schema.rs index 32d462b3..ae81a1c8 100644 --- a/crates/nvisy-postgres/src/schema.rs +++ b/crates/nvisy-postgres/src/schema.rs @@ -13,6 +13,10 @@ pub mod sql_types { #[diesel(postgres_type(name = "chat_role"))] pub struct ChatRole; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "detection_status"))] + pub struct DetectionStatus; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] #[diesel(postgres_type(name = "file_kind"))] pub struct FileKind; @@ -29,10 +33,6 @@ pub mod sql_types { #[diesel(postgres_type(name = "outbox_status"))] pub struct OutboxStatus; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "pipeline_run_status"))] - pub struct PipelineRunStatus; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] #[diesel(postgres_type(name = "pipeline_status"))] pub struct PipelineStatus; @@ -247,6 +247,58 @@ diesel::table! { } } +diesel::table! { + use diesel::sql_types::*; + use super::sql_types::OutboxStatus; + + workspace_detection_jobs (id) { + id -> Uuid, + detection_id -> Uuid, + job -> Jsonb, + status -> OutboxStatus, + attempts -> Int4, + next_attempt_at -> Timestamptz, + created_at -> Timestamptz, + resolved_at -> Nullable, + } +} + +diesel::table! { + use diesel::sql_types::*; + + workspace_detection_usage (id) { + id -> Uuid, + detection_id -> Uuid, + model -> Text, + version -> Nullable, + input_tokens -> Nullable, + output_tokens -> Nullable, + total_tokens -> Nullable, + duration_ms -> Int8, + } +} + +diesel::table! { + use diesel::sql_types::*; + use super::sql_types::PipelineTriggerType; + use super::sql_types::DetectionStatus; + + workspace_detections (id) { + id -> Uuid, + pipeline_id -> Uuid, + account_id -> Uuid, + input_file_id -> Uuid, + audit_file_id -> Nullable, + trigger_type -> PipelineTriggerType, + status -> DetectionStatus, + idempotency_key -> Nullable, + metadata -> Jsonb, + claimed_at -> Nullable, + started_at -> Timestamptz, + completed_at -> Nullable, + } +} + diesel::table! { use diesel::sql_types::*; @@ -334,43 +386,6 @@ diesel::table! { } } -diesel::table! { - use diesel::sql_types::*; - - workspace_pipeline_run_usage (id) { - id -> Uuid, - run_id -> Uuid, - model -> Text, - version -> Nullable, - input_tokens -> Nullable, - output_tokens -> Nullable, - total_tokens -> Nullable, - duration_ms -> Int8, - } -} - -diesel::table! { - use diesel::sql_types::*; - use super::sql_types::PipelineTriggerType; - use super::sql_types::PipelineRunStatus; - - workspace_pipeline_runs (id) { - id -> Uuid, - pipeline_id -> Uuid, - account_id -> Uuid, - input_file_id -> Uuid, - audit_file_id -> Nullable, - output_file_id -> Nullable, - trigger_type -> PipelineTriggerType, - status -> PipelineRunStatus, - idempotency_key -> Nullable, - metadata -> Jsonb, - claimed_at -> Nullable, - started_at -> Timestamptz, - completed_at -> Nullable, - } -} - diesel::table! { use diesel::sql_types::*; use super::sql_types::PipelineStatus; @@ -409,6 +424,19 @@ diesel::table! { } } +diesel::table! { + use diesel::sql_types::*; + + workspace_redactions (id) { + id -> Uuid, + detection_id -> Uuid, + account_id -> Uuid, + review_file_id -> Nullable, + output_file_id -> Nullable, + created_at -> Timestamptz, + } +} + diesel::table! { use diesel::sql_types::*; use super::sql_types::WebhookEvent; @@ -465,6 +493,10 @@ diesel::joinable!(workspace_connection_syncs -> accounts (account_id)); diesel::joinable!(workspace_connection_syncs -> workspace_connection_schedule (connection_id)); diesel::joinable!(workspace_connections -> accounts (account_id)); diesel::joinable!(workspace_connections -> workspaces (workspace_id)); +diesel::joinable!(workspace_detection_jobs -> workspace_detections (detection_id)); +diesel::joinable!(workspace_detection_usage -> workspace_detections (detection_id)); +diesel::joinable!(workspace_detections -> accounts (account_id)); +diesel::joinable!(workspace_detections -> workspace_pipelines (pipeline_id)); diesel::joinable!(workspace_file_imports -> workspace_connections (connection_id)); diesel::joinable!(workspace_file_imports -> workspace_files (file_id)); diesel::joinable!(workspace_files -> accounts (account_id)); @@ -472,13 +504,12 @@ diesel::joinable!(workspace_files -> workspaces (workspace_id)); diesel::joinable!(workspace_invites -> workspaces (workspace_id)); diesel::joinable!(workspace_members -> workspaces (workspace_id)); diesel::joinable!(workspace_pipeline_policies -> workspaces (workspace_id)); -diesel::joinable!(workspace_pipeline_run_usage -> workspace_pipeline_runs (run_id)); -diesel::joinable!(workspace_pipeline_runs -> accounts (account_id)); -diesel::joinable!(workspace_pipeline_runs -> workspace_pipelines (pipeline_id)); diesel::joinable!(workspace_pipelines -> accounts (account_id)); diesel::joinable!(workspace_pipelines -> workspaces (workspace_id)); diesel::joinable!(workspace_policies -> accounts (account_id)); diesel::joinable!(workspace_policies -> workspaces (workspace_id)); +diesel::joinable!(workspace_redactions -> accounts (account_id)); +diesel::joinable!(workspace_redactions -> workspace_detections (detection_id)); diesel::joinable!(workspace_webhooks -> accounts (created_by)); diesel::joinable!(workspace_webhooks -> workspaces (workspace_id)); diesel::joinable!(workspaces -> accounts (created_by)); @@ -494,15 +525,17 @@ diesel::allow_tables_to_appear_in_same_query!( workspace_connection_schedule, workspace_connection_syncs, workspace_connections, + workspace_detection_jobs, + workspace_detection_usage, + workspace_detections, workspace_file_imports, workspace_files, workspace_invites, workspace_members, workspace_pipeline_policies, - workspace_pipeline_run_usage, - workspace_pipeline_runs, workspace_pipelines, workspace_policies, + workspace_redactions, workspace_webhooks, workspaces, ); diff --git a/crates/nvisy-postgres/src/types/constraint/detections.rs b/crates/nvisy-postgres/src/types/constraint/detections.rs new file mode 100644 index 00000000..1e90ea74 --- /dev/null +++ b/crates/nvisy-postgres/src/types/constraint/detections.rs @@ -0,0 +1,14 @@ +//! Detections table constraint violations. + +use strum::EnumString; + +/// Detections table constraint violations. +#[derive(Debug, Clone, Copy, Eq, PartialEq, EnumString)] +pub enum WorkspaceDetectionConstraints { + #[strum(serialize = "workspace_detections_metadata_size")] + MetadataSize, + #[strum(serialize = "workspace_detections_idempotency_key_length")] + IdempotencyKeyLength, + #[strum(serialize = "workspace_detections_idempotency_idx")] + IdempotencyUnique, +} diff --git a/crates/nvisy-postgres/src/types/constraint/mod.rs b/crates/nvisy-postgres/src/types/constraint/mod.rs index dbfbed56..255bcca9 100644 --- a/crates/nvisy-postgres/src/types/constraint/mod.rs +++ b/crates/nvisy-postgres/src/types/constraint/mod.rs @@ -22,9 +22,9 @@ mod workspaces; // File-related constraint modules mod files; -// Pipeline-related constraint modules +// Detection / pipeline-related constraint modules +mod detections; mod pipeline_references; -mod pipeline_runs; mod pipelines; mod workspace_connection_syncs; @@ -36,9 +36,9 @@ pub use self::account_notifications::AccountNotificationConstraints; pub use self::accounts::AccountConstraints; pub use self::chat_messages::ChatMessageConstraints; pub use self::chat_sessions::ChatSessionConstraints; +pub use self::detections::WorkspaceDetectionConstraints; pub use self::files::WorkspaceFileConstraints; pub use self::pipeline_references::WorkspacePipelineReferenceConstraints; -pub use self::pipeline_runs::WorkspacePipelineRunConstraints; pub use self::pipelines::WorkspacePipelineConstraints; pub use self::workspace_activities::WorkspaceActivitiesConstraints; pub use self::workspace_connection_syncs::WorkspaceConnectionSyncConstraints; @@ -75,9 +75,9 @@ pub enum ConstraintViolation { // File-related constraints WorkspaceFile(WorkspaceFileConstraints), - // Pipeline-related constraints + // Detection / pipeline-related constraints WorkspacePipeline(WorkspacePipelineConstraints), - WorkspacePipelineRun(WorkspacePipelineRunConstraints), + WorkspaceDetection(WorkspaceDetectionConstraints), WorkspacePipelineReference(WorkspacePipelineReferenceConstraints), WorkspaceConnection(WorkspaceConnectionConstraints), WorkspaceConnectionSync(WorkspaceConnectionSyncConstraints), @@ -132,7 +132,7 @@ impl ConstraintViolation { WorkspaceWebhook, WorkspaceFile, WorkspacePipeline, - WorkspacePipelineRun, + WorkspaceDetection, WorkspacePipelineReference, WorkspaceConnection, WorkspaceConnectionSync, diff --git a/crates/nvisy-postgres/src/types/constraint/pipeline_runs.rs b/crates/nvisy-postgres/src/types/constraint/pipeline_runs.rs deleted file mode 100644 index 6775b56e..00000000 --- a/crates/nvisy-postgres/src/types/constraint/pipeline_runs.rs +++ /dev/null @@ -1,14 +0,0 @@ -//! Pipeline runs table constraint violations. - -use strum::EnumString; - -/// Pipeline runs table constraint violations. -#[derive(Debug, Clone, Copy, Eq, PartialEq, EnumString)] -pub enum WorkspacePipelineRunConstraints { - #[strum(serialize = "workspace_pipeline_runs_metadata_size")] - MetadataSize, - #[strum(serialize = "workspace_pipeline_runs_idempotency_key_length")] - IdempotencyKeyLength, - #[strum(serialize = "workspace_pipeline_runs_idempotency_idx")] - IdempotencyUnique, -} diff --git a/crates/nvisy-postgres/src/types/enums/activity_type.rs b/crates/nvisy-postgres/src/types/enums/activity_type.rs index a46cbc69..01079880 100644 --- a/crates/nvisy-postgres/src/types/enums/activity_type.rs +++ b/crates/nvisy-postgres/src/types/enums/activity_type.rs @@ -171,29 +171,29 @@ pub enum ActivityType { #[strum(serialize = "pipeline.deleted")] PipelineDeleted, - /// Pipeline run was started - #[db_rename = "pipeline.run.started"] - #[serde(rename = "pipeline.run.started")] - #[strum(serialize = "pipeline.run.started")] - PipelineRunStarted, - - /// Pipeline run finished detection - #[db_rename = "pipeline.run.analyzed"] - #[serde(rename = "pipeline.run.analyzed")] - #[strum(serialize = "pipeline.run.analyzed")] - PipelineRunAnalyzed, - - /// Pipeline run completed - #[db_rename = "pipeline.run.completed"] - #[serde(rename = "pipeline.run.completed")] - #[strum(serialize = "pipeline.run.completed")] - PipelineRunCompleted, - - /// Pipeline run failed - #[db_rename = "pipeline.run.failed"] - #[serde(rename = "pipeline.run.failed")] - #[strum(serialize = "pipeline.run.failed")] - PipelineRunFailed, + /// Detection was started + #[db_rename = "pipeline.detection.started"] + #[serde(rename = "pipeline.detection.started")] + #[strum(serialize = "pipeline.detection.started")] + DetectionStarted, + + /// Detection finished analysis + #[db_rename = "pipeline.detection.completed"] + #[serde(rename = "pipeline.detection.completed")] + #[strum(serialize = "pipeline.detection.completed")] + DetectionCompleted, + + /// Detection failed + #[db_rename = "pipeline.detection.failed"] + #[serde(rename = "pipeline.detection.failed")] + #[strum(serialize = "pipeline.detection.failed")] + DetectionFailed, + + /// Redaction was created + #[db_rename = "pipeline.redaction.created"] + #[serde(rename = "pipeline.redaction.created")] + #[strum(serialize = "pipeline.redaction.created")] + RedactionCreated, // Policy activities /// Policy was created @@ -217,15 +217,16 @@ pub enum ActivityType { impl ActivityType { /// The canonical dotted tag for this type, e.g. `file.created` or - /// `pipeline.run.completed` — the same string used on the wire and in the DB, - /// from the variant's `strum(serialize)`. + /// `pipeline.redaction.created` — the same string used on the wire and in the + /// DB, from the variant's `strum(serialize)`. pub fn as_tag(self) -> &'static str { self.into() } /// The object half of the tag: everything before the final segment, e.g. - /// `file` for `file.created`, `pipeline.run` for `pipeline.run.completed`, - /// `connection.sync` for `connection.sync.failed`. + /// `file` for `file.created`, `pipeline.redaction` for + /// `pipeline.redaction.created`, `connection.sync` for + /// `connection.sync.failed`. pub fn object_type(self) -> &'static str { let tag = self.as_tag(); match tag.rsplit_once('.') { @@ -235,7 +236,7 @@ impl ActivityType { } /// The action half of the tag: the final segment, e.g. `created` for - /// `file.created`, `completed` for `pipeline.run.completed`. + /// `file.created`, `created` for `pipeline.redaction.created`. pub fn action_type(self) -> &'static str { let tag = self.as_tag(); match tag.rsplit_once('.') { @@ -279,13 +280,10 @@ mod tests { assert_eq!(ActivityType::FileCreated.action_type(), "created"); // Three-part tags: object is everything before the final segment. assert_eq!( - ActivityType::PipelineRunCompleted.object_type(), - "pipeline.run" - ); - assert_eq!( - ActivityType::PipelineRunCompleted.action_type(), - "completed" + ActivityType::RedactionCreated.object_type(), + "pipeline.redaction" ); + assert_eq!(ActivityType::RedactionCreated.action_type(), "created"); assert_eq!( ActivityType::ConnectionSyncFailed.object_type(), "connection.sync" diff --git a/crates/nvisy-postgres/src/types/enums/detection_status.rs b/crates/nvisy-postgres/src/types/enums/detection_status.rs new file mode 100644 index 00000000..db90a05b --- /dev/null +++ b/crates/nvisy-postgres/src/types/enums/detection_status.rs @@ -0,0 +1,64 @@ +//! Detection status enumeration indicating the execution state of a detection. + +use diesel_derive_enum::DbEnum; +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumIter, EnumString}; + +/// The execution status of a detection (one analysis pass of a file). +/// +/// Corresponds to the `DETECTION_STATUS` PostgreSQL enum. A detection is +/// `Pending` (enqueued, no worker yet), then `Executing` (a worker is actively +/// analyzing), then settles into `Complete` (analysis done, ready to redact) or +/// `Failed`. Redaction is a separate, repeatable action over a complete +/// detection and does not change this status. +#[derive(Debug, Default, Clone, Copy, Eq, PartialEq)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Serialize, Deserialize, DbEnum, Display, EnumIter, EnumString)] +#[ExistingTypePath = "crate::schema::sql_types::DetectionStatus"] +pub enum DetectionStatus { + /// Enqueued for detection; no worker has picked it up yet. + #[db_rename = "pending"] + #[serde(rename = "pending")] + #[default] + Pending, + + /// A worker is actively analyzing the document. + #[db_rename = "executing"] + #[serde(rename = "executing")] + Executing, + + /// Analysis done; the detection is ready to redact. + #[db_rename = "complete"] + #[serde(rename = "complete")] + Complete, + + /// Detection failed with an error. + #[db_rename = "failed"] + #[serde(rename = "failed")] + Failed, +} + +impl DetectionStatus { + /// In-progress statuses: a detection is still analyzing — enqueued or + /// running — so its input and audit files must not expire yet. `Complete` is + /// excluded (it is terminal), so holding it would pin those files forever. + pub const IN_PROGRESS: [DetectionStatus; 2] = + [DetectionStatus::Pending, DetectionStatus::Executing]; + /// Terminal statuses: a detection reached one of these iff it either finished + /// analysis or failed, and its status will not change again. This is the + /// correct basis for an error rate (`failed / (complete + failed)`). + pub const TERMINAL: [DetectionStatus; 2] = [DetectionStatus::Complete, DetectionStatus::Failed]; + + /// Returns whether analysis is done and the detection is ready to redact. + #[inline] + pub fn is_complete(self) -> bool { + matches!(self, DetectionStatus::Complete) + } + + /// Returns whether the detection has not finished analysis yet (pending or + /// executing). + #[inline] + pub fn is_detecting(self) -> bool { + matches!(self, DetectionStatus::Pending | DetectionStatus::Executing) + } +} diff --git a/crates/nvisy-postgres/src/types/enums/file_kind.rs b/crates/nvisy-postgres/src/types/enums/file_kind.rs index 489d3abb..0a86dd9e 100644 --- a/crates/nvisy-postgres/src/types/enums/file_kind.rs +++ b/crates/nvisy-postgres/src/types/enums/file_kind.rs @@ -26,10 +26,16 @@ pub enum FileKind { #[serde(rename = "redacted")] Redacted, - /// Engine analysis blob, not shown in file lists. + /// Engine detection-analysis blob, not shown in file lists. #[db_rename = "audit"] #[serde(rename = "audit")] Audit, + + /// Engine analysis after reviewer edits and redaction (a redaction's review + /// audit), not shown in file lists. + #[db_rename = "review"] + #[serde(rename = "review")] + Review, } impl FileKind { diff --git a/crates/nvisy-postgres/src/types/enums/mod.rs b/crates/nvisy-postgres/src/types/enums/mod.rs index 6e080aa2..1e4e7e57 100644 --- a/crates/nvisy-postgres/src/types/enums/mod.rs +++ b/crates/nvisy-postgres/src/types/enums/mod.rs @@ -29,19 +29,19 @@ pub mod workspace_role; // File-related enumerations pub mod file_kind; -// Pipeline-related enumerations -pub mod pipeline_run_status; +// Detection / pipeline-related enumerations +pub mod detection_status; pub mod pipeline_status; pub mod pipeline_trigger_type; pub use activity_type::ActivityType; pub use api_token_type::ApiTokenType; pub use chat_role::ChatRole; +pub use detection_status::DetectionStatus; pub use file_kind::FileKind; pub use invite_status::InviteStatus; pub use notification_event::NotificationEvent; pub use outbox_status::OutboxStatus; -pub use pipeline_run_status::PipelineRunStatus; pub use pipeline_status::PipelineStatus; pub use pipeline_trigger_type::PipelineTriggerType; pub use provider_type::ProviderType; diff --git a/crates/nvisy-postgres/src/types/enums/notification_event.rs b/crates/nvisy-postgres/src/types/enums/notification_event.rs index 2e3d3399..6a40a81b 100644 --- a/crates/nvisy-postgres/src/types/enums/notification_event.rs +++ b/crates/nvisy-postgres/src/types/enums/notification_event.rs @@ -7,7 +7,8 @@ use strum::{Display, EnumIter, EnumString}; /// Defines the type of notification event sent to a user. /// /// This enumeration corresponds to the `NOTIFICATION_EVENT` PostgreSQL enum and -/// is used for member, connection-sync, pipeline-run, and system notifications. +/// is used for member, connection-sync, detection, redaction, and system +/// notifications. /// The values mirror the [`WebhookEvent`](super::WebhookEvent) naming for the /// events the two channels share. #[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] @@ -37,19 +38,19 @@ pub enum NotificationEvent { #[serde(rename = "connection.sync.failed")] ConnectionSyncFailed, - // Pipeline run events - /// A pipeline run finished detection and is awaiting review - #[db_rename = "pipeline.run.analyzed"] - #[serde(rename = "pipeline.run.analyzed")] - PipelineRunAnalyzed, - - /// A pipeline run completed (redaction produced) - #[db_rename = "pipeline.run.completed"] - #[serde(rename = "pipeline.run.completed")] - PipelineRunCompleted, - - /// A pipeline run failed - #[db_rename = "pipeline.run.failed"] - #[serde(rename = "pipeline.run.failed")] - PipelineRunFailed, + // Detection / redaction events + /// A detection finished analysis and is ready to redact + #[db_rename = "pipeline.detection.completed"] + #[serde(rename = "pipeline.detection.completed")] + DetectionCompleted, + + /// A redaction was created (redacted output produced) + #[db_rename = "pipeline.redaction.created"] + #[serde(rename = "pipeline.redaction.created")] + RedactionCreated, + + /// A detection failed + #[db_rename = "pipeline.detection.failed"] + #[serde(rename = "pipeline.detection.failed")] + DetectionFailed, } diff --git a/crates/nvisy-postgres/src/types/enums/pipeline_run_status.rs b/crates/nvisy-postgres/src/types/enums/pipeline_run_status.rs deleted file mode 100644 index 2a29072e..00000000 --- a/crates/nvisy-postgres/src/types/enums/pipeline_run_status.rs +++ /dev/null @@ -1,74 +0,0 @@ -//! Pipeline run status enumeration indicating the execution state of a pipeline run. - -use diesel_derive_enum::DbEnum; -use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIter, EnumString}; - -/// Defines the execution status of a pipeline run. -/// -/// This enumeration corresponds to the `PIPELINE_RUN_STATUS` PostgreSQL enum and is used -/// to track the current state of a pipeline execution. -/// -/// The detect phase has two states: `Queued` (the run is enqueued but no worker -/// has begun) and `Analyzing` (a worker is actively analyzing). They settle into -/// `Analyzed` (detection done, awaiting review), then `Completed` after redaction. -#[derive(Debug, Default, Clone, Copy, Eq, PartialEq)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[derive(Serialize, Deserialize, DbEnum, Display, EnumIter, EnumString)] -#[ExistingTypePath = "crate::schema::sql_types::PipelineRunStatus"] -pub enum PipelineRunStatus { - /// Enqueued for detection; no worker has picked it up yet - #[db_rename = "queued"] - #[serde(rename = "queued")] - #[default] - Queued, - - /// A worker is actively analyzing the document - #[db_rename = "analyzing"] - #[serde(rename = "analyzing")] - Analyzing, - - /// Detection done; awaiting reviewer verification - #[db_rename = "analyzed"] - #[serde(rename = "analyzed")] - Analyzed, - - /// Redaction applied; run finished - #[db_rename = "completed"] - #[serde(rename = "completed")] - Completed, - - /// Run failed with error - #[db_rename = "failed"] - #[serde(rename = "failed")] - Failed, - - /// Run was cancelled by user - #[db_rename = "cancelled"] - #[serde(rename = "cancelled")] - Cancelled, -} - -impl PipelineRunStatus { - /// Statuses that carry a success/failure outcome: a run reached one of these - /// iff it either completed redaction or failed. `Cancelled` is excluded — a - /// cancelled run has no outcome — so this is the correct basis for an error - /// rate (`failed / (completed + failed)`). - pub const OUTCOMES: [PipelineRunStatus; 2] = - [PipelineRunStatus::Completed, PipelineRunStatus::Failed]; - - /// Returns whether detection is done and the run awaits verification. - #[inline] - pub fn is_analyzed(self) -> bool { - matches!(self, PipelineRunStatus::Analyzed) - } - - /// Returns whether detection is still pending (queued or analyzing). - #[inline] - pub fn is_detecting(self) -> bool { - matches!( - self, - PipelineRunStatus::Queued | PipelineRunStatus::Analyzing - ) - } -} diff --git a/crates/nvisy-postgres/src/types/enums/webhook_event.rs b/crates/nvisy-postgres/src/types/enums/webhook_event.rs index 6d092dae..c04f0a58 100644 --- a/crates/nvisy-postgres/src/types/enums/webhook_event.rs +++ b/crates/nvisy-postgres/src/types/enums/webhook_event.rs @@ -107,29 +107,29 @@ pub enum WebhookEvent { #[strum(serialize = "pipeline.deleted")] PipelineDeleted, - /// A pipeline run started - #[db_rename = "pipeline.run.started"] - #[serde(rename = "pipeline.run.started")] - #[strum(serialize = "pipeline.run.started")] - PipelineRunStarted, - - /// A pipeline run's detection finished (findings ready for review) - #[db_rename = "pipeline.run.analyzed"] - #[serde(rename = "pipeline.run.analyzed")] - #[strum(serialize = "pipeline.run.analyzed")] - PipelineRunAnalyzed, - - /// A pipeline run finished successfully - #[db_rename = "pipeline.run.completed"] - #[serde(rename = "pipeline.run.completed")] - #[strum(serialize = "pipeline.run.completed")] - PipelineRunCompleted, - - /// A pipeline run failed - #[db_rename = "pipeline.run.failed"] - #[serde(rename = "pipeline.run.failed")] - #[strum(serialize = "pipeline.run.failed")] - PipelineRunFailed, + /// A detection started + #[db_rename = "pipeline.detection.started"] + #[serde(rename = "pipeline.detection.started")] + #[strum(serialize = "pipeline.detection.started")] + DetectionStarted, + + /// A detection's analysis finished (findings ready to redact) + #[db_rename = "pipeline.detection.completed"] + #[serde(rename = "pipeline.detection.completed")] + #[strum(serialize = "pipeline.detection.completed")] + DetectionCompleted, + + /// A detection failed + #[db_rename = "pipeline.detection.failed"] + #[serde(rename = "pipeline.detection.failed")] + #[strum(serialize = "pipeline.detection.failed")] + DetectionFailed, + + /// A redaction was created + #[db_rename = "pipeline.redaction.created"] + #[serde(rename = "pipeline.redaction.created")] + #[strum(serialize = "pipeline.redaction.created")] + RedactionCreated, // Policy events /// A policy was created @@ -170,10 +170,10 @@ impl WebhookEvent { WebhookEvent::PipelineCreated | WebhookEvent::PipelineUpdated | WebhookEvent::PipelineDeleted - | WebhookEvent::PipelineRunStarted - | WebhookEvent::PipelineRunAnalyzed - | WebhookEvent::PipelineRunCompleted - | WebhookEvent::PipelineRunFailed => "pipeline", + | WebhookEvent::DetectionStarted + | WebhookEvent::DetectionCompleted + | WebhookEvent::DetectionFailed + | WebhookEvent::RedactionCreated => "pipeline", WebhookEvent::PolicyCreated | WebhookEvent::PolicyUpdated | WebhookEvent::PolicyDeleted => "policy", @@ -183,7 +183,7 @@ impl WebhookEvent { /// Returns the event as a subject string for NATS routing. /// /// The event name is already a dotted, NATS-legal subject (e.g. - /// `file.created`, `pipeline.run.completed`), so this is the event's own + /// `file.created`, `pipeline.redaction.created`), so this is the event's own /// string representation (from its `strum(serialize)`). pub fn as_subject(&self) -> &'static str { self.into() diff --git a/crates/nvisy-postgres/src/types/filtering/runs.rs b/crates/nvisy-postgres/src/types/filtering/detections.rs similarity index 74% rename from crates/nvisy-postgres/src/types/filtering/runs.rs rename to crates/nvisy-postgres/src/types/filtering/detections.rs index 99ecb196..48eea538 100644 --- a/crates/nvisy-postgres/src/types/filtering/runs.rs +++ b/crates/nvisy-postgres/src/types/filtering/detections.rs @@ -1,11 +1,11 @@ -//! Filtering options for pipeline run queries. +//! Filtering options for detection queries. use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::types::{PipelineRunStatus, PipelineTriggerType}; +use crate::types::{DetectionStatus, PipelineTriggerType}; -/// Filter options for pipeline runs. +/// Filter options for detections. /// /// Each field narrows the result when set; unset fields impose no constraint. /// The owning pipeline (single-pipeline listing) and workspace scope are applied @@ -13,40 +13,40 @@ use crate::types::{PipelineRunStatus, PipelineTriggerType}; #[derive(Debug, Default, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] -pub struct RunFilter { - /// Filter by run status. +pub struct DetectionFilter { + /// Filter by detection status. #[serde(skip_serializing_if = "Option::is_none")] - pub status: Option, - /// Filter by the source file the run analyzes. + pub status: Option, + /// Filter by the source file the detection analyzes. #[serde(skip_serializing_if = "Option::is_none")] pub input_file_id: Option, /// Filter by the owning pipeline. Ignored by the single-pipeline listing /// (already scoped to one pipeline); used by the workspace-wide listing. #[serde(skip_serializing_if = "Option::is_none")] pub pipeline_id: Option, - /// Filter by the account that triggered the run. + /// Filter by the account that triggered the detection. #[serde(skip_serializing_if = "Option::is_none")] pub account_id: Option, - /// Filter by how the run was initiated (user vs system). + /// Filter by how the detection was initiated (user vs system). #[serde(skip_serializing_if = "Option::is_none")] pub trigger_type: Option, } -impl RunFilter { +impl DetectionFilter { /// Creates a new empty filter. #[inline] pub fn new() -> Self { Self::default() } - /// Filters by run status. + /// Filters by detection status. #[inline] - pub fn with_status(mut self, status: PipelineRunStatus) -> Self { + pub fn with_status(mut self, status: DetectionStatus) -> Self { self.status = Some(status); self } - /// Filters by the source file the run analyzes. + /// Filters by the source file the detection analyzes. #[inline] pub fn with_input_file_id(mut self, input_file_id: Uuid) -> Self { self.input_file_id = Some(input_file_id); @@ -60,14 +60,14 @@ impl RunFilter { self } - /// Filters by the account that triggered the run. + /// Filters by the account that triggered the detection. #[inline] pub fn with_account_id(mut self, account_id: Uuid) -> Self { self.account_id = Some(account_id); self } - /// Filters by how the run was initiated. + /// Filters by how the detection was initiated. #[inline] pub fn with_trigger_type(mut self, trigger_type: PipelineTriggerType) -> Self { self.trigger_type = Some(trigger_type); diff --git a/crates/nvisy-postgres/src/types/filtering/mod.rs b/crates/nvisy-postgres/src/types/filtering/mod.rs index af8d668d..106ee3b8 100644 --- a/crates/nvisy-postgres/src/types/filtering/mod.rs +++ b/crates/nvisy-postgres/src/types/filtering/mod.rs @@ -1,11 +1,11 @@ //! Filtering options for database queries. +mod detections; mod files; mod invites; mod members; -mod runs; +pub use detections::DetectionFilter; pub use files::FileFilter; pub use invites::InviteFilter; pub use members::MemberFilter; -pub use runs::RunFilter; diff --git a/crates/nvisy-postgres/src/types/json/activity_params.rs b/crates/nvisy-postgres/src/types/json/activity_params.rs index cfebf954..f04472c0 100644 --- a/crates/nvisy-postgres/src/types/json/activity_params.rs +++ b/crates/nvisy-postgres/src/types/json/activity_params.rs @@ -9,7 +9,9 @@ use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::types::{ActivityType, ConnectionId, Handle, RunId, WebhookEvent, WebhookId}; +use crate::types::{ + ActivityType, ConnectionId, DetectionId, Handle, RedactionId, WebhookEvent, WebhookId, +}; /// Params of a workspace-scoped activity (`workspace.*`). #[derive(Debug, Clone, Serialize, Deserialize)] @@ -84,15 +86,26 @@ pub struct PipelineActivityParams { pub pipeline_slug: Handle, } -/// Params of a pipeline-run activity (`pipeline.run.*`). +/// Params of a detection activity (`pipeline.detection.*`). #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] -pub struct PipelineRunActivityParams { +pub struct DetectionActivityParams { /// Slug of the owning pipeline. pub pipeline_slug: Handle, - /// Id of the run. - pub run_id: RunId, + /// Id of the detection. + pub detection_id: DetectionId, +} + +/// Params of a redaction activity (`pipeline.redaction.*`). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub struct RedactionActivityParams { + /// Slug of the owning pipeline. + pub pipeline_slug: Handle, + /// Id of the redaction. + pub redaction_id: RedactionId, } /// Params of a policy activity (`policy.*`). @@ -198,18 +211,18 @@ pub enum ActivityPayload { /// A pipeline was deleted. #[serde(rename = "pipeline.deleted")] PipelineDeleted(PipelineActivityParams), - /// A pipeline run was started. - #[serde(rename = "pipeline.run.started")] - PipelineRunStarted(PipelineRunActivityParams), - /// A pipeline run finished detection. - #[serde(rename = "pipeline.run.analyzed")] - PipelineRunAnalyzed(PipelineRunActivityParams), - /// A pipeline run completed. - #[serde(rename = "pipeline.run.completed")] - PipelineRunCompleted(PipelineRunActivityParams), - /// A pipeline run failed. - #[serde(rename = "pipeline.run.failed")] - PipelineRunFailed(PipelineRunActivityParams), + /// A detection was started. + #[serde(rename = "pipeline.detection.started")] + DetectionStarted(DetectionActivityParams), + /// A detection finished analysis. + #[serde(rename = "pipeline.detection.completed")] + DetectionCompleted(DetectionActivityParams), + /// A detection failed. + #[serde(rename = "pipeline.detection.failed")] + DetectionFailed(DetectionActivityParams), + /// A redaction was created. + #[serde(rename = "pipeline.redaction.created")] + RedactionCreated(RedactionActivityParams), /// A policy was created. #[serde(rename = "policy.created")] @@ -252,10 +265,10 @@ impl ActivityPayload { ActivityPayload::PipelineCreated(_) => ActivityType::PipelineCreated, ActivityPayload::PipelineUpdated(_) => ActivityType::PipelineUpdated, ActivityPayload::PipelineDeleted(_) => ActivityType::PipelineDeleted, - ActivityPayload::PipelineRunStarted(_) => ActivityType::PipelineRunStarted, - ActivityPayload::PipelineRunAnalyzed(_) => ActivityType::PipelineRunAnalyzed, - ActivityPayload::PipelineRunCompleted(_) => ActivityType::PipelineRunCompleted, - ActivityPayload::PipelineRunFailed(_) => ActivityType::PipelineRunFailed, + ActivityPayload::DetectionStarted(_) => ActivityType::DetectionStarted, + ActivityPayload::DetectionCompleted(_) => ActivityType::DetectionCompleted, + ActivityPayload::DetectionFailed(_) => ActivityType::DetectionFailed, + ActivityPayload::RedactionCreated(_) => ActivityType::RedactionCreated, ActivityPayload::PolicyCreated(_) => ActivityType::PolicyCreated, ActivityPayload::PolicyUpdated(_) => ActivityType::PolicyUpdated, ActivityPayload::PolicyDeleted(_) => ActivityType::PolicyDeleted, @@ -294,10 +307,10 @@ impl ActivityPayload { ActivityPayload::PipelineCreated(_) => W::PipelineCreated, ActivityPayload::PipelineUpdated(_) => W::PipelineUpdated, ActivityPayload::PipelineDeleted(_) => W::PipelineDeleted, - ActivityPayload::PipelineRunStarted(_) => W::PipelineRunStarted, - ActivityPayload::PipelineRunAnalyzed(_) => W::PipelineRunAnalyzed, - ActivityPayload::PipelineRunCompleted(_) => W::PipelineRunCompleted, - ActivityPayload::PipelineRunFailed(_) => W::PipelineRunFailed, + ActivityPayload::DetectionStarted(_) => W::DetectionStarted, + ActivityPayload::DetectionCompleted(_) => W::DetectionCompleted, + ActivityPayload::DetectionFailed(_) => W::DetectionFailed, + ActivityPayload::RedactionCreated(_) => W::RedactionCreated, ActivityPayload::PolicyCreated(_) => W::PolicyCreated, ActivityPayload::PolicyUpdated(_) => W::PolicyUpdated, ActivityPayload::PolicyDeleted(_) => W::PolicyDeleted, @@ -333,10 +346,11 @@ impl ActivityPayload { | ActivityPayload::FileUpdated(p) | ActivityPayload::FileDeleted(p) => Some(p.file_id.to_string()), - ActivityPayload::PipelineRunStarted(p) - | ActivityPayload::PipelineRunAnalyzed(p) - | ActivityPayload::PipelineRunCompleted(p) - | ActivityPayload::PipelineRunFailed(p) => Some(p.run_id.to_string()), + ActivityPayload::DetectionStarted(p) + | ActivityPayload::DetectionCompleted(p) + | ActivityPayload::DetectionFailed(p) => Some(p.detection_id.to_string()), + + ActivityPayload::RedactionCreated(p) => Some(p.redaction_id.to_string()), ActivityPayload::PolicyCreated(p) | ActivityPayload::PolicyUpdated(p) @@ -380,10 +394,11 @@ impl ActivityPayload { | ActivityPayload::PipelineUpdated(p) | ActivityPayload::PipelineDeleted(p) => Some(p.pipeline_slug.to_string()), - ActivityPayload::PipelineRunStarted(p) - | ActivityPayload::PipelineRunAnalyzed(p) - | ActivityPayload::PipelineRunCompleted(p) - | ActivityPayload::PipelineRunFailed(p) => Some(p.pipeline_slug.to_string()), + ActivityPayload::DetectionStarted(p) + | ActivityPayload::DetectionCompleted(p) + | ActivityPayload::DetectionFailed(p) => Some(p.pipeline_slug.to_string()), + + ActivityPayload::RedactionCreated(p) => Some(p.pipeline_slug.to_string()), ActivityPayload::ConnectionCreated(p) | ActivityPayload::ConnectionUpdated(p) diff --git a/crates/nvisy-postgres/src/types/json/pipeline_run_metadata.rs b/crates/nvisy-postgres/src/types/json/detection_metadata.rs similarity index 70% rename from crates/nvisy-postgres/src/types/json/pipeline_run_metadata.rs rename to crates/nvisy-postgres/src/types/json/detection_metadata.rs index 747e2810..388cd2b7 100644 --- a/crates/nvisy-postgres/src/types/json/pipeline_run_metadata.rs +++ b/crates/nvisy-postgres/src/types/json/detection_metadata.rs @@ -1,4 +1,4 @@ -//! Structured metadata for the `workspace_pipeline_runs.metadata` JSONB column. +//! Structured metadata for the `workspace_detections.metadata` JSONB column. //! //! A column the server itself populates has a known shape, so it is typed rather //! than left as free-form JSON. Read with `Json::or_default` so an absent or @@ -6,21 +6,21 @@ use serde::{Deserialize, Serialize}; -/// Structured metadata for a pipeline run. +/// Structured metadata for a detection. #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase", default)] -pub struct RunMetadata { - /// Free-form labels attached to the run. +pub struct DetectionMetadata { + /// Free-form labels attached to the detection. #[serde(skip_serializing_if = "Option::is_none")] pub tags: Option>, - /// Failure reason recorded when the run failed. + /// Failure reason recorded when the detection failed. #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, /// The engine's full per-recognizer usage report (durations, per-model token /// counts), stored opaquely for drill-down. Per-model token totals for - /// aggregation live in the `workspace_pipeline_run_usage` table; this keeps - /// the detail. Absent when the run produced no usage. + /// aggregation live in the `workspace_detection_usage` table; this keeps the + /// detail. Absent when the detection produced no usage. #[serde(skip_serializing_if = "Option::is_none")] #[cfg_attr(feature = "schema", schemars(with = "Option"))] pub usage: Option, diff --git a/crates/nvisy-postgres/src/types/json/mod.rs b/crates/nvisy-postgres/src/types/json/mod.rs index fa81afb5..f4ef1128 100644 --- a/crates/nvisy-postgres/src/types/json/mod.rs +++ b/crates/nvisy-postgres/src/types/json/mod.rs @@ -5,9 +5,9 @@ //! settings / retention value types. mod activity_params; +mod detection_metadata; mod notification_params; mod pipeline_metadata; -mod pipeline_run_metadata; mod retention; mod typed_json; mod webhook_headers; @@ -15,17 +15,17 @@ mod workspace_metadata; mod workspace_settings; pub use activity_params::{ - ActivityPayload, ConnectionActivityParams, FileActivityParams, InviteActivityParams, - MemberActivityParams, PipelineActivityParams, PipelineRunActivityParams, PolicyActivityParams, - WebhookActivityParams, WorkspaceActivityParams, + ActivityPayload, ConnectionActivityParams, DetectionActivityParams, FileActivityParams, + InviteActivityParams, MemberActivityParams, PipelineActivityParams, PolicyActivityParams, + RedactionActivityParams, WebhookActivityParams, WorkspaceActivityParams, }; +pub use detection_metadata::DetectionMetadata; pub use notification_params::{ - ConnectionSyncCompletedParams, ConnectionSyncFailedParams, MemberInvitedParams, - MemberJoinedParams, NotificationPayload, PipelineRunAnalyzedParams, PipelineRunCompletedParams, - PipelineRunFailedParams, + ConnectionSyncCompletedParams, ConnectionSyncFailedParams, DetectionCompletedParams, + DetectionFailedParams, MemberInvitedParams, MemberJoinedParams, NotificationPayload, + RedactionCreatedParams, }; pub use pipeline_metadata::{PipelineMetadata, RetentionOverride}; -pub use pipeline_run_metadata::RunMetadata; pub use retention::{Retention, RetentionScope, RetentionSettings}; pub use typed_json::Json; pub use webhook_headers::{InvalidHeader, WebhookHeaders}; diff --git a/crates/nvisy-postgres/src/types/json/notification_params.rs b/crates/nvisy-postgres/src/types/json/notification_params.rs index d8f9414f..482ce3e6 100644 --- a/crates/nvisy-postgres/src/types/json/notification_params.rs +++ b/crates/nvisy-postgres/src/types/json/notification_params.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; use super::Json; -use crate::types::{ConnectionId, Handle, NotificationEvent, RunId}; +use crate::types::{ConnectionId, DetectionId, Handle, NotificationEvent, RedactionId}; /// Params of a `member.invited` notification. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -61,13 +61,13 @@ pub struct ConnectionSyncFailedParams { pub error: Option, } -/// Params of a `pipeline.run.analyzed` notification. +/// Params of a `pipeline.detection.completed` notification. #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] -pub struct PipelineRunAnalyzedParams { - /// Id of the run. - pub run_id: RunId, +pub struct DetectionCompletedParams { + /// Id of the detection. + pub detection_id: DetectionId, /// Slug of the owning pipeline. pub pipeline_slug: Handle, /// Display name of the analyzed file, if known. @@ -75,27 +75,29 @@ pub struct PipelineRunAnalyzedParams { pub input_file_name: Option, } -/// Params of a `pipeline.run.completed` notification. +/// Params of a `pipeline.redaction.created` notification. #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] -pub struct PipelineRunCompletedParams { - /// Id of the run. - pub run_id: RunId, +pub struct RedactionCreatedParams { + /// Id of the redaction. + pub redaction_id: RedactionId, + /// Id of the detection the redaction was produced from. + pub detection_id: DetectionId, /// Slug of the owning pipeline. pub pipeline_slug: Handle, - /// Display name of the analyzed file, if known. + /// Display name of the redacted file, if known. #[serde(skip_serializing_if = "Option::is_none")] pub input_file_name: Option, } -/// Params of a `pipeline.run.failed` notification. +/// Params of a `pipeline.detection.failed` notification. #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] -pub struct PipelineRunFailedParams { - /// Id of the run. - pub run_id: RunId, +pub struct DetectionFailedParams { + /// Id of the detection. + pub detection_id: DetectionId, /// Slug of the owning pipeline. pub pipeline_slug: Handle, /// Display name of the analyzed file, if known. @@ -132,17 +134,17 @@ pub enum NotificationPayload { #[serde(rename = "connection.sync.failed")] ConnectionSyncFailed(ConnectionSyncFailedParams), - /// A pipeline run finished detection and is awaiting review. - #[serde(rename = "pipeline.run.analyzed")] - PipelineRunAnalyzed(PipelineRunAnalyzedParams), + /// A detection finished analysis and is ready to redact. + #[serde(rename = "pipeline.detection.completed")] + DetectionCompleted(DetectionCompletedParams), - /// A pipeline run completed (redaction produced). - #[serde(rename = "pipeline.run.completed")] - PipelineRunCompleted(PipelineRunCompletedParams), + /// A redaction was created (redacted output produced). + #[serde(rename = "pipeline.redaction.created")] + RedactionCreated(RedactionCreatedParams), - /// A pipeline run failed. - #[serde(rename = "pipeline.run.failed")] - PipelineRunFailed(PipelineRunFailedParams), + /// A detection failed. + #[serde(rename = "pipeline.detection.failed")] + DetectionFailed(DetectionFailedParams), } impl NotificationPayload { @@ -155,9 +157,9 @@ impl NotificationPayload { NotificationEvent::ConnectionSyncCompleted } NotificationPayload::ConnectionSyncFailed(_) => NotificationEvent::ConnectionSyncFailed, - NotificationPayload::PipelineRunAnalyzed(_) => NotificationEvent::PipelineRunAnalyzed, - NotificationPayload::PipelineRunCompleted(_) => NotificationEvent::PipelineRunCompleted, - NotificationPayload::PipelineRunFailed(_) => NotificationEvent::PipelineRunFailed, + NotificationPayload::DetectionCompleted(_) => NotificationEvent::DetectionCompleted, + NotificationPayload::RedactionCreated(_) => NotificationEvent::RedactionCreated, + NotificationPayload::DetectionFailed(_) => NotificationEvent::DetectionFailed, } } @@ -178,13 +180,13 @@ mod tests { use uuid::Uuid; use super::*; - use crate::types::RunId; + use crate::types::DetectionId; #[test] fn serializes_as_a_type_data_envelope_and_round_trips() { - let run_id = RunId::from_uuid(Uuid::now_v7()); - let payload = NotificationPayload::PipelineRunAnalyzed(PipelineRunAnalyzedParams { - run_id, + let detection_id = DetectionId::from_uuid(Uuid::now_v7()); + let payload = NotificationPayload::DetectionCompleted(DetectionCompletedParams { + detection_id, pipeline_slug: Handle::from_str("redact-invoices").unwrap(), input_file_name: Some("invoice.pdf".to_owned()), }); @@ -192,15 +194,15 @@ mod tests { let value = serde_json::to_value(&payload).unwrap(); // The durable wire shape: a `type` tag and a nested `data` object, matching // the activity payload and outbox event. Stored rows depend on it. - assert_eq!(value["type"], "pipeline.run.analyzed"); - assert_eq!(value["data"]["runId"], run_id.to_string()); + assert_eq!(value["type"], "pipeline.detection.completed"); + assert_eq!(value["data"]["detectionId"], detection_id.to_string()); assert_eq!(value["data"]["pipelineSlug"], "redact-invoices"); assert!( - value.get("runId").is_none(), + value.get("detectionId").is_none(), "params must nest under `data`" ); let decoded: NotificationPayload = serde_json::from_value(value).unwrap(); - assert_eq!(decoded.event(), NotificationEvent::PipelineRunAnalyzed); + assert_eq!(decoded.event(), NotificationEvent::DetectionCompleted); } } diff --git a/crates/nvisy-postgres/src/types/mod.rs b/crates/nvisy-postgres/src/types/mod.rs index 6177cadd..29a80ad8 100644 --- a/crates/nvisy-postgres/src/types/mod.rs +++ b/crates/nvisy-postgres/src/types/mod.rs @@ -16,29 +16,31 @@ pub use constraint::{ AccountApiTokenConstraints, AccountConstraints, AccountNotificationConstraints, ChatMessageConstraints, ChatSessionConstraints, ConstraintViolation, WorkspaceActivitiesConstraints, WorkspaceConnectionConstraints, - WorkspaceConnectionSyncConstraints, WorkspaceConstraints, WorkspaceFileConstraints, - WorkspaceInviteConstraints, WorkspaceMemberConstraints, WorkspacePipelineConstraints, - WorkspacePipelineReferenceConstraints, WorkspacePipelineRunConstraints, + WorkspaceConnectionSyncConstraints, WorkspaceConstraints, WorkspaceDetectionConstraints, + WorkspaceFileConstraints, WorkspaceInviteConstraints, WorkspaceMemberConstraints, + WorkspacePipelineConstraints, WorkspacePipelineReferenceConstraints, WorkspacePolicyConstraints, WorkspaceWebhookConstraints, }; pub use enums::{ - ActivityType, ApiTokenType, ChatRole, FileKind, InviteStatus, NotificationEvent, OutboxStatus, - PipelineRunStatus, PipelineStatus, PipelineTriggerType, ProviderType, SyncDeletionPolicy, - SyncMode, SyncStatus, SyncTriggerType, WebhookEvent, WebhookStatus, WorkspaceRole, + ActivityType, ApiTokenType, ChatRole, DetectionStatus, FileKind, InviteStatus, + NotificationEvent, OutboxStatus, PipelineStatus, PipelineTriggerType, ProviderType, + SyncDeletionPolicy, SyncMode, SyncStatus, SyncTriggerType, WebhookEvent, WebhookStatus, + WorkspaceRole, }; -pub use filtering::{FileFilter, InviteFilter, MemberFilter, RunFilter}; +pub use filtering::{DetectionFilter, FileFilter, InviteFilter, MemberFilter}; pub use handle::{HANDLE_MAX_LENGTH, HANDLE_MIN_LENGTH, Handle, HandleError}; pub use json::{ ActivityPayload, ConnectionActivityParams, ConnectionSyncCompletedParams, - ConnectionSyncFailedParams, FileActivityParams, InvalidHeader, InviteActivityParams, Json, - MemberActivityParams, MemberInvitedParams, MemberJoinedParams, NotificationPayload, OcrPolicy, - PipelineActivityParams, PipelineMetadata, PipelineRunActivityParams, PipelineRunAnalyzedParams, - PipelineRunCompletedParams, PipelineRunFailedParams, PolicyActivityParams, Retention, - RetentionOverride, RetentionScope, RetentionSettings, RunMetadata, WebhookActivityParams, - WebhookHeaders, WorkspaceActivityParams, WorkspaceMetadata, WorkspaceSettings, + ConnectionSyncFailedParams, DetectionActivityParams, DetectionCompletedParams, + DetectionFailedParams, DetectionMetadata, FileActivityParams, InvalidHeader, + InviteActivityParams, Json, MemberActivityParams, MemberInvitedParams, MemberJoinedParams, + NotificationPayload, OcrPolicy, PipelineActivityParams, PipelineMetadata, PolicyActivityParams, + RedactionActivityParams, RedactionCreatedParams, Retention, RetentionOverride, RetentionScope, + RetentionSettings, WebhookActivityParams, WebhookHeaders, WorkspaceActivityParams, + WorkspaceMetadata, WorkspaceSettings, }; pub use pagination::{Cursor, CursorPage, CursorPagination, OffsetPage, OffsetPagination}; -pub use prefixed_id::{ConnectionId, PrefixedIdError, RunId, WebhookId}; +pub use prefixed_id::{ConnectionId, DetectionId, PrefixedIdError, RedactionId, WebhookId}; pub use sorting::{ FileSortBy, FileSortField, InviteSortBy, InviteSortField, MemberSortBy, MemberSortField, SortBy, SortOrder, diff --git a/crates/nvisy-postgres/src/types/prefixed_id.rs b/crates/nvisy-postgres/src/types/prefixed_id.rs index effb46c1..d31df71a 100644 --- a/crates/nvisy-postgres/src/types/prefixed_id.rs +++ b/crates/nvisy-postgres/src/types/prefixed_id.rs @@ -137,8 +137,13 @@ prefixed_id! { } prefixed_id! { - /// Opaque identifier for a pipeline run (`run_`). - RunId, "run" + /// Opaque identifier for a detection (`detection_`). + DetectionId, "detection" +} + +prefixed_id! { + /// Opaque identifier for a redaction (`redaction_`). + RedactionId, "redaction" } #[cfg(test)] diff --git a/crates/nvisy-server/src/handler/analytics.rs b/crates/nvisy-server/src/handler/analytics.rs index 79b2f4f5..c57990f0 100644 --- a/crates/nvisy-server/src/handler/analytics.rs +++ b/crates/nvisy-server/src/handler/analytics.rs @@ -1,5 +1,5 @@ //! Workspace analytics handler: aggregate metrics over a workspace's files and -//! pipeline runs, for a dashboard. +//! detections, for a dashboard. use aide::axum::ApiRouter; use aide::transform::TransformOperation; @@ -10,7 +10,7 @@ use nvisy_postgres::query::WorkspaceAnalyticsRepository; use crate::extract::{AuthProvider, AuthState, Json, Permission, Query, WorkspaceContext}; use crate::handler::request::DateWindow; -use crate::handler::response::{ErrorResponse, RunTimeSeries, WorkspaceAnalytics}; +use crate::handler::response::{DetectionTimeSeries, ErrorResponse, WorkspaceAnalytics}; use crate::handler::{Result, ServiceState}; /// Tracing target for workspace analytics operations. @@ -47,7 +47,7 @@ async fn get_analytics( fn get_analytics_docs(op: TransformOperation) -> TransformOperation { op.summary("Workspace analytics") .description( - "Returns aggregate analytics for a workspace: stored-file totals with a per-kind breakdown, pipeline-run health (status mix, error rate, and durations), and inference token usage (workspace totals plus a per-model breakdown). Breakdowns list every kind/status, zero-filled, in a stable order.", + "Returns aggregate analytics for a workspace: stored-file totals with a per-kind breakdown, detection health (status mix, error rate, and durations), and inference token usage (workspace totals plus a per-model breakdown). Breakdowns list every kind/status, zero-filled, in a stable order.", ) .response::<200, Json>() .response::<401, Json>() @@ -55,7 +55,7 @@ fn get_analytics_docs(op: TransformOperation) -> TransformOperation { .response::<404, Json>() } -/// Returns a workspace's daily run activity over a date window. +/// Returns a workspace's daily detection activity over a date window. #[tracing::instrument( skip_all, fields( @@ -63,13 +63,13 @@ fn get_analytics_docs(op: TransformOperation) -> TransformOperation { workspace_id = %workspace.id, ) )] -async fn get_run_timeseries( +async fn get_detection_timeseries( State(pg_client): State, AuthState(auth_state): AuthState, WorkspaceContext(workspace): WorkspaceContext, Query(window): Query, -) -> Result<(StatusCode, Json)> { - tracing::debug!(target: TRACING_TARGET, "Computing run time series"); +) -> Result<(StatusCode, Json)> { + tracing::debug!(target: TRACING_TARGET, "Computing detection time series"); let window = window.resolve()?; @@ -80,23 +80,23 @@ async fn get_run_timeseries( .await?; let points = conn - .runs_by_day( + .detections_by_day( workspace.id, window.from_timestamp()?, window.to_timestamp()?, ) .await?; - let series = RunTimeSeries::from_window(window.from, window.to, points); + let series = DetectionTimeSeries::from_window(window.from, window.to, points); Ok((StatusCode::OK, Json(series))) } -fn get_run_timeseries_docs(op: TransformOperation) -> TransformOperation { - op.summary("Workspace run time series") +fn get_detection_timeseries_docs(op: TransformOperation) -> TransformOperation { + op.summary("Workspace detection time series") .description( - "Returns a workspace's daily pipeline-run activity over a date window: runs per day, plus each day's error rate and durations. Every day in the window is present (quiet days report runs: 0), so the series plots as a continuous line or a contribution-style calendar. The window is `from`/`to` (inclusive, YYYY-MM-DD); it defaults to the last 30 days and is capped at 366 days.", + "Returns a workspace's daily detection activity over a date window: detections per day, plus each day's error rate and durations. Every day in the window is present (quiet days report detections: 0), so the series plots as a continuous line or a contribution-style calendar. The window is `from`/`to` (inclusive, YYYY-MM-DD); it defaults to the last 30 days and is capped at 366 days.", ) - .response::<200, Json>() + .response::<200, Json>() .response::<400, Json>() .response::<401, Json>() .response::<403, Json>() @@ -113,8 +113,8 @@ pub fn routes() -> ApiRouter { get_with(get_analytics, get_analytics_docs), ) .api_route( - "/workspaces/{workspaceSlug}/analytics/runs/timeseries/", - get_with(get_run_timeseries, get_run_timeseries_docs), + "/workspaces/{workspaceSlug}/analytics/detections/timeseries/", + get_with(get_detection_timeseries, get_detection_timeseries_docs), ) .with_path_items(|item| item.tag("Analytics")) } diff --git a/crates/nvisy-server/src/handler/pipeline_audits.rs b/crates/nvisy-server/src/handler/detection_audits.rs similarity index 63% rename from crates/nvisy-server/src/handler/pipeline_audits.rs rename to crates/nvisy-server/src/handler/detection_audits.rs index 9ce0854c..a9e23790 100644 --- a/crates/nvisy-server/src/handler/pipeline_audits.rs +++ b/crates/nvisy-server/src/handler/detection_audits.rs @@ -1,9 +1,9 @@ -//! Pipeline-run audit handlers: read and export a run's analysis. +//! Detection audit handlers: read and export a detection's analysis. //! -//! Once a run is analyzed, its `Audit` (the decrypted map of detected findings) -//! can be reviewed inline or downloaded as JSON or a zip of CSV tables. The run -//! lifecycle itself (create, list, redact) lives in -//! [`pipeline_runs`](super::pipeline_runs). +//! Once a detection is complete, its `Audit` (the decrypted map of detected +//! findings) can be reviewed inline or downloaded as JSON or a zip of CSV tables. +//! The detection lifecycle itself (create, list, redact) lives in +//! [`detections`](super::detections). use aide::axum::ApiRouter; use aide::axum::routing::get_with; @@ -15,38 +15,38 @@ use elide_pipeline::Audit; use elide_pipeline::export::{ExportCsv, ExportJson}; use nvisy_postgres::PgClient; -use super::pipeline_runs::find_pipeline_run; +use super::detections::find_detection; use crate::extract::{AuthProvider, AuthState, Json, Path, Permission, Query, WorkspaceContext}; -use crate::handler::request::{ExportFormat, ExportQuery, PipelineRunPathParams}; +use crate::handler::request::{DetectionPathParams, ExportFormat, ExportQuery}; use crate::handler::response::ErrorResponse; use crate::handler::utility::{DownloadResponseExt, attachment_headers}; use crate::handler::{Error, ErrorKind, Result}; use crate::service::{EngineService, RunBlobStore, ServiceState}; -/// Tracing target for pipeline audit operations. -const TRACING_TARGET: &str = "nvisy_server::handler::audits"; +/// Tracing target for detection audit operations. +const TRACING_TARGET: &str = "nvisy_server::handler::detection_audits"; -/// Returns the run's analysis (the detected findings) for review. +/// Returns the detection's analysis (the detected findings) for review. /// /// Fetches and decrypts the engine's `Audit` from the audit bucket. Available -/// once the run is analyzed. Requires `ViewPipelines`. +/// once the detection is complete. Requires `ViewPipelines`. #[tracing::instrument( skip_all, fields( account_id = %auth_state.account_id, workspace_id = %workspace.id, - run_id = %path_params.run_id, + detection_id = %path_params.detection_id, ) )] -async fn get_pipeline_run_analysis( +async fn get_detection_analysis( State(pg_client): State, State(blob): State, State(engine): State, AuthState(auth_state): AuthState, WorkspaceContext(workspace): WorkspaceContext, - Path(path_params): Path, + Path(path_params): Path, ) -> Result<(StatusCode, Json)> { - tracing::debug!(target: TRACING_TARGET, "Getting pipeline run analysis"); + tracing::debug!(target: TRACING_TARGET, "Getting detection analysis"); let mut conn = pg_client.get_connection().await?; @@ -54,21 +54,23 @@ async fn get_pipeline_run_analysis( .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) .await?; - let (run, _pipeline) = - find_pipeline_run(&mut conn, workspace.id, path_params.run_id.as_uuid()).await?; + let (detection, _pipeline) = + find_detection(&mut conn, workspace.id, path_params.detection_id.as_uuid()).await?; let analyzed = blob - .load_analyzed_document(&mut conn, &engine, workspace.id, &run) + .load_analyzed_document(&mut conn, &engine, workspace.id, &detection) .await?; - tracing::debug!(target: TRACING_TARGET, "Pipeline run analysis retrieved"); + tracing::debug!(target: TRACING_TARGET, "Detection analysis retrieved"); Ok((StatusCode::OK, Json(analyzed))) } -fn get_pipeline_run_analysis_docs(op: TransformOperation) -> TransformOperation { - op.summary("Get run detections") - .description("Returns the run's detected findings (the analyzed document) for review.") +fn get_detection_analysis_docs(op: TransformOperation) -> TransformOperation { + op.summary("Get detection findings") + .description( + "Returns the detection's detected findings (the analyzed document) for review.", + ) .response::<200, Json>() .response::<401, Json>() .response::<403, Json>() @@ -76,7 +78,7 @@ fn get_pipeline_run_analysis_docs(op: TransformOperation) -> TransformOperation .response::<409, Json>() } -/// Downloads a run's audit as a file, in the requested `format`. +/// Downloads a detection's audit as a file, in the requested `format`. /// /// `json` yields a pretty-printed JSON file with the full structure — body, /// parts, context, and each entity's provenance chain. `csv` (the default) @@ -87,30 +89,30 @@ fn get_pipeline_run_analysis_docs(op: TransformOperation) -> TransformOperation fields( account_id = %auth_state.account_id, workspace_id = %workspace.id, - run_id = %path_params.run_id, + detection_id = %path_params.detection_id, format = ?query.format, ) )] -async fn download_pipeline_run_audit( +async fn download_detection_audit( State(pg_client): State, State(blob): State, State(engine): State, AuthState(auth_state): AuthState, WorkspaceContext(workspace): WorkspaceContext, - Path(path_params): Path, + Path(path_params): Path, Query(query): Query, ) -> Result<(StatusCode, HeaderMap, Body)> { - tracing::debug!(target: TRACING_TARGET, "Downloading pipeline run audit"); + tracing::debug!(target: TRACING_TARGET, "Downloading detection audit"); let mut conn = pg_client.get_connection().await?; auth_state .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) .await?; - let (run, _pipeline) = - find_pipeline_run(&mut conn, workspace.id, path_params.run_id.as_uuid()).await?; + let (detection, _pipeline) = + find_detection(&mut conn, workspace.id, path_params.detection_id.as_uuid()).await?; let audit = blob - .load_analyzed_document(&mut conn, &engine, workspace.id, &run) + .load_analyzed_document(&mut conn, &engine, workspace.id, &detection) .await?; let (content_type, filename, body) = match query.format { @@ -121,13 +123,17 @@ async fn download_pipeline_run_audit( .with_message("Failed to export audit as JSON") .with_context(err.to_string()) })?; - ("application/json", format!("audit-{}.json", run.id), buffer) + ( + "application/json", + format!("audit-{}.json", detection.id), + buffer, + ) } ExportFormat::Csv => { let archive = build_audit_csv_zip(&audit)?; ( "application/zip", - format!("audit-{}.csv.zip", run.id), + format!("audit-{}.csv.zip", detection.id), archive, ) } @@ -139,18 +145,18 @@ async fn download_pipeline_run_audit( body.len() as u64, ); - tracing::debug!(target: TRACING_TARGET, "Pipeline run audit exported"); + tracing::debug!(target: TRACING_TARGET, "Detection audit exported"); Ok((StatusCode::OK, headers, Body::from(body))) } -fn download_pipeline_run_audit_docs(op: TransformOperation) -> TransformOperation { - op.summary("Download run audit") +fn download_detection_audit_docs(op: TransformOperation) -> TransformOperation { + op.summary("Download detection audit") .description( - "Downloads the run's audit as a file. `format` is `csv` (default) — a zip of \ + "Downloads the detection's audit as a file. `format` is `csv` (default) — a zip of \ entities.csv, provenance.csv, and reviews.csv — or `json`, a pretty-printed JSON file.", ) .download_response( - "The run's exported audit.", + "The detection's exported audit.", &["application/zip", "application/json"], ) .response::<401, Json>() @@ -177,19 +183,16 @@ fn archive_error(error: impl std::fmt::Display) -> Error<'static> { .with_context(error.to_string()) } -/// Builds the pipeline-run audit routes. +/// Builds the detection audit routes. pub fn routes() -> ApiRouter { ApiRouter::new() .api_route( - "/workspaces/{workspaceSlug}/runs/{runId}/detections/", - get_with(get_pipeline_run_analysis, get_pipeline_run_analysis_docs), + "/workspaces/{workspaceSlug}/detections/{detectionId}/analysis/", + get_with(get_detection_analysis, get_detection_analysis_docs), ) .api_route( - "/workspaces/{workspaceSlug}/runs/{runId}/audit", - get_with( - download_pipeline_run_audit, - download_pipeline_run_audit_docs, - ), + "/workspaces/{workspaceSlug}/detections/{detectionId}/audit/", + get_with(download_detection_audit, download_detection_audit_docs), ) - .with_path_items(|item| item.tag("Pipeline Runs")) + .with_path_items(|item| item.tag("Detections")) } diff --git a/crates/nvisy-server/src/handler/detections.rs b/crates/nvisy-server/src/handler/detections.rs new file mode 100644 index 00000000..850cb51a --- /dev/null +++ b/crates/nvisy-server/src/handler/detections.rs @@ -0,0 +1,799 @@ +//! Detection handlers: detect, review, and redact. +//! +//! A detection is one analysis of a file through a pipeline. Detect creates the +//! detection and stores the findings; once it is complete a redaction consumes +//! the findings (with optional reviewer edits) and produces a redacted file. A +//! detection can be redacted many times. + +use aide::axum::ApiRouter; +use aide::transform::TransformOperation; +use async_stream::stream; +use axum::extract::State; +use axum::http::StatusCode; +use axum::response::sse::Event; +use futures::StreamExt; +use nvisy_postgres::model::{ + NewWorkspaceDetection, NewWorkspaceDetectionJob, NewWorkspaceRedaction, WorkspaceDetection, + WorkspacePipeline, +}; +use nvisy_postgres::query::{ + DetectionFiles, DetectionJobOutboxRepository, PipelineReferenceRepository, + WorkspaceDetectionRepository, WorkspaceFileRepository, WorkspacePipelineRepository, + WorkspaceRedactionRepository, +}; +use nvisy_postgres::types::DetectionStatus; +use nvisy_postgres::{AsyncConnection, PgClient, PgConn}; +use uuid::Uuid; + +use crate::extract::{ + AuthProvider, AuthState, IdempotencyKey, Json, Path, Permission, Query, SecurityContext, + ValidateJson, WorkspaceContext, +}; +use crate::handler::request::{ + CreateDetection, CursorPagination, DetectionPathParams, PipelineDefinition, + PipelineDetectionsQuery, PipelinePathParams, RedactDetection, WorkspaceDetectionsQuery, +}; +use crate::handler::response::{Detection, DetectionsPage, ErrorResponse, RedactionResult}; +use crate::handler::utility::{SseResponse, resolve_account_ref}; +use crate::handler::{Error, ErrorKind, Result}; +use crate::service::{ + CryptoService, DetectionJob, DetectionQueue, DetectionRef, DetectionStatusEvent, EngineService, + EventEmitter, EventOrigin, RunBlobStore, ServiceState, WorkspaceEvent, resolve_policies, +}; + +/// Tracing target for detection operations. +const TRACING_TARGET: &str = "nvisy_server::handler::detections"; + +/// Starts a detection: analyzes a file with the pipeline's configuration. +/// +/// Returns the detection holding the findings for review. A repeated request with +/// the same `Idempotency-Key` returns the existing detection instead of analyzing +/// again. Requires `RunPipelines` permission. +#[tracing::instrument( + skip_all, + fields( + account_id = %auth_state.account_id, + workspace_id = %workspace.id, + pipeline_slug = %path_params.pipeline_slug, + ) +)] +async fn create_detection( + State(pg_client): State, + State(detection): State, + AuthState(auth_state): AuthState, + WorkspaceContext(workspace): WorkspaceContext, + Path(path_params): Path, + IdempotencyKey(idempotency_key): IdempotencyKey, + security: SecurityContext, + ValidateJson(request): ValidateJson, +) -> Result<(StatusCode, Json)> { + tracing::debug!(target: TRACING_TARGET, "Starting detection"); + + let mut conn = pg_client.get_connection().await?; + + auth_state + .authorize_workspace(&mut conn, workspace.id, Permission::RunPipelines) + .await?; + + let pipeline = find_pipeline(&mut conn, workspace.id, &path_params.pipeline_slug).await?; + + // Only an enabled pipeline runs: a draft (still being configured) or a + // disabled (paused) pipeline is rejected. + if !pipeline.status.is_enabled() { + return Err(ErrorKind::Conflict + .with_message("Pipeline is not enabled") + .with_resource("pipeline")); + } + + // Idempotent replay: a repeated key returns the detection created the first + // time, attributed to whoever originally triggered it (not the current + // caller). + if let Some(key) = &idempotency_key + && let Some(existing) = conn + .find_detection_by_idempotency_key(pipeline.id, key) + .await? + { + tracing::debug!(target: TRACING_TARGET, "Replaying detection for idempotency key"); + let trigger = resolve_account_ref(&mut conn, existing.account_id).await?; + let files = conn.detection_file_names(workspace.id, &existing).await?; + return Ok(( + StatusCode::OK, + Json(Detection::from_model( + existing, + pipeline.slug.clone(), + workspace.slug.clone(), + trigger, + files, + )), + )); + } + + // Validate synchronously so a bad request fails fast (4xx) rather than as a + // detection that immediately fails in the worker. + let file = conn + .find_file_in_workspace(pipeline.workspace_id, request.file_id) + .await? + .ok_or_else(|| Error::not_found("file"))?; + + if conn.list_pipeline_policy_ids(pipeline.id).await?.is_empty() { + return Err(ErrorKind::BadRequest + .with_message("Pipeline has no policies; attach at least one before running") + .with_resource("pipeline")); + } + + // Decode the pipeline definition now so an undecodable definition fails the + // request synchronously (400) instead of returning 202 and failing later in + // the worker. The decoded value is rebuilt in the worker from the same + // stored bytes; this is validation only. + let _validated = PipelineDefinition::from_parts(pipeline.definition.clone(), Vec::new()) + .map_err(|err| { + ErrorKind::BadRequest + .with_message("Pipeline definition is invalid") + .with_resource("pipeline") + .with_context(err.to_string()) + })?; + + // Create the detection (its id is the engine correlation id). The response + // returns immediately; the client learns the findings are ready via the + // detection's status (SSE at `.../events` or a re-read). + let new_detection = NewWorkspaceDetection { + pipeline_id: pipeline.id, + input_file_id: file.id, + account_id: auth_state.account_id, + status: Some(DetectionStatus::Pending), + idempotency_key: idempotency_key.clone(), + ..Default::default() + }; + + // Create the detection, record its start event, and queue its analysis in one + // transaction, so all three commit or roll back together. The job goes onto + // the outbox (not published inline) so the detection is never lost to a + // publish that failed after the row committed, nor marked failed for a publish + // that in fact went through: the drainer relays the outbox row to the + // work-queue, and the worker's claim dedups an at-least-once redelivery. + let scope = request.scope; + let detection_row = conn + .transaction(async |conn| { + let detection_row = conn.create_workspace_detection(new_detection).await?; + conn.emit_event( + EventOrigin { + workspace_id: workspace.id, + account_id: auth_state.account_id, + security: &security, + }, + WorkspaceEvent::DetectionStarted(DetectionRef { + detection_id: detection_row.id, + pipeline_slug: pipeline.slug.clone(), + }), + ) + .await?; + let job = DetectionJob { + workspace_id: workspace.id, + detection_id: detection_row.id, + scope, + }; + conn.insert_detection_job(NewWorkspaceDetectionJob { + detection_id: detection_row.id, + job: serde_json::to_value(&job).map_err(|err| { + ErrorKind::InternalServerError + .with_message("Failed to encode detection job") + .with_context(err.to_string()) + })?, + }) + .await?; + Ok::<_, Error>(detection_row) + }) + .await?; + + // Best-effort UI hint; the detection row is authoritative. + detection + .broadcast_status(detection_row.id, DetectionStatus::Pending) + .await; + + tracing::info!(target: TRACING_TARGET, detection_id = %detection_row.id, "Detection queued"); + + let trigger = resolve_account_ref(&mut conn, detection_row.account_id).await?; + + // The detection was just created from this file. + let files = DetectionFiles { + input: Some(file.display_name), + }; + + Ok(( + StatusCode::ACCEPTED, + Json(Detection::from_model( + detection_row, + pipeline.slug, + workspace.slug, + trigger, + files, + )), + )) +} + +fn create_detection_docs(op: TransformOperation) -> TransformOperation { + op.summary("Start a detection") + .description( + "Starts analysis for a file and returns 202 with the detection in the \ + `pending` state; the analysis runs in the background. Watch the \ + detection's status via the SSE stream at \ + `.../detections/{detectionId}/events` (or re-read the detection) and \ + fetch the findings from `.../detections/{detectionId}/analysis/` once \ + it reaches `complete`. A repeated Idempotency-Key returns the existing \ + detection.", + ) + .response::<202, Json>() + .response::<200, Json>() + .response::<400, Json>() + .response::<401, Json>() + .response::<403, Json>() + .response::<404, Json>() + .response::<409, Json>() +} + +/// Lists detections for a specific pipeline. +#[tracing::instrument( + skip_all, + fields( + account_id = %auth_state.account_id, + workspace_id = %workspace.id, + pipeline_slug = %path_params.pipeline_slug, + ) +)] +async fn list_pipeline_detections( + State(pg_client): State, + AuthState(auth_state): AuthState, + WorkspaceContext(workspace): WorkspaceContext, + Path(path_params): Path, + Query(pagination): Query, + Query(query): Query, +) -> Result<(StatusCode, Json)> { + tracing::debug!(target: TRACING_TARGET, "Listing pipeline detections"); + + let mut conn = pg_client.get_connection().await?; + + auth_state + .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) + .await?; + + let pipeline = find_pipeline(&mut conn, workspace.id, &path_params.pipeline_slug).await?; + + let page = conn + .cursor_list_pipeline_detections(pipeline.id, pagination.into(), &query.into()) + .await?; + + tracing::debug!( + target: TRACING_TARGET, + detection_count = page.items.len(), + "Pipeline detections listed" + ); + + let response = DetectionsPage::from_cursor_page(page, |row| { + Detection::from_model( + row.detection, + row.pipeline_slug, + workspace.slug.clone(), + row.account.into(), + DetectionFiles { + input: row.input_file_name, + }, + ) + }); + + Ok((StatusCode::OK, Json(response))) +} + +fn list_pipeline_detections_docs(op: TransformOperation) -> TransformOperation { + op.summary("List pipeline detections") + .description( + "Returns detections for a specific pipeline, most recent first, with \ + optional status, file, trigger-account, and trigger-type filters.", + ) + .response::<200, Json>() + .response::<401, Json>() + .response::<403, Json>() + .response::<404, Json>() +} + +/// Lists all detections across the workspace's pipelines. +/// +/// Aggregates detections from every pipeline in the workspace, most recent first, +/// with optional status and pipeline filters. Requires `ViewPipelines`. +#[tracing::instrument( + skip_all, + fields( + account_id = %auth_state.account_id, + workspace_id = %workspace.id, + ) +)] +async fn list_workspace_detections( + State(pg_client): State, + AuthState(auth_state): AuthState, + WorkspaceContext(workspace): WorkspaceContext, + Query(pagination): Query, + Query(query): Query, +) -> Result<(StatusCode, Json)> { + tracing::debug!(target: TRACING_TARGET, "Listing workspace detections"); + + let mut conn = pg_client.get_connection().await?; + + auth_state + .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) + .await?; + + let page = conn + .cursor_list_workspace_detections(workspace.id, pagination.into(), &query.into()) + .await?; + + tracing::debug!( + target: TRACING_TARGET, + detection_count = page.items.len(), + "Workspace detections listed" + ); + + Ok(( + StatusCode::OK, + Json(DetectionsPage::from_cursor_page(page, |row| { + Detection::from_model( + row.detection, + row.pipeline_slug, + workspace.slug.clone(), + row.account.into(), + DetectionFiles { + input: row.input_file_name, + }, + ) + })), + )) +} + +fn list_workspace_detections_docs(op: TransformOperation) -> TransformOperation { + op.summary("List workspace detections") + .description( + "Returns all detections across the workspace, most recent first, \ + with optional status, file, pipeline, trigger-account, and \ + trigger-type filters.", + ) + .response::<200, Json>() + .response::<401, Json>() + .response::<403, Json>() + .response::<404, Json>() +} + +/// Gets a specific detection. +#[tracing::instrument( + skip_all, + fields( + account_id = %auth_state.account_id, + workspace_id = %workspace.id, + detection_id = %path_params.detection_id, + ) +)] +async fn get_detection( + State(pg_client): State, + AuthState(auth_state): AuthState, + WorkspaceContext(workspace): WorkspaceContext, + Path(path_params): Path, +) -> Result<(StatusCode, Json)> { + tracing::debug!(target: TRACING_TARGET, "Getting detection"); + + let mut conn = pg_client.get_connection().await?; + + auth_state + .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) + .await?; + + let (detection, pipeline) = + find_detection(&mut conn, workspace.id, path_params.detection_id.as_uuid()).await?; + + let trigger = resolve_account_ref(&mut conn, detection.account_id).await?; + let files = conn.detection_file_names(workspace.id, &detection).await?; + + tracing::debug!(target: TRACING_TARGET, "Detection retrieved"); + + Ok(( + StatusCode::OK, + Json(Detection::from_model( + detection, + pipeline.slug, + workspace.slug, + trigger, + files, + )), + )) +} + +fn get_detection_docs(op: TransformOperation) -> TransformOperation { + op.summary("Get detection") + .description("Returns the detection and its status for review.") + .response::<200, Json>() + .response::<401, Json>() + .response::<403, Json>() + .response::<404, Json>() +} + +/// Streams a detection's status changes as Server-Sent Events until it settles. +/// +/// Emits one `status` event with the detection's current status immediately (so a +/// client that connects after analysis already finished still learns the state), +/// then forwards each status change. The stream ends once the detection leaves +/// the detecting phase (`pending`/`executing`) — i.e. analysis has produced +/// `complete`, or the detection `failed`. +/// +/// Live status changes arrive over a best-effort core-NATS broadcast; if none +/// arrives within a short interval the authoritative detection row is re-read from +/// the database, so a dropped broadcast never leaves the stream hanging. +/// +/// Authenticated like every other route (Bearer); browsers should consume it via +/// a `fetch` stream rather than the native `EventSource`, which cannot send an +/// `Authorization` header. +#[tracing::instrument( + skip_all, + fields( + account_id = %auth_state.account_id, + workspace_id = %workspace.id, + detection_id = %path_params.detection_id, + ) +)] +async fn stream_detection_events( + State(pg_client): State, + State(detection): State, + AuthState(auth_state): AuthState, + WorkspaceContext(workspace): WorkspaceContext, + Path(path_params): Path, +) -> Result> { + tracing::debug!(target: TRACING_TARGET, "Opening detection status stream"); + + let detection_id = path_params.detection_id.as_uuid(); + let mut conn = pg_client.get_connection().await?; + auth_state + .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) + .await?; + + // Subscribe BEFORE reading the current status: core-NATS broadcasts are not + // replayed, so a terminal status published between the read and the + // subscription going live would otherwise be lost and the stream would hang. + let mut updates = detection.subscribe_status(detection_id).await?; + + // Confirm the detection exists (and is workspace-scoped) so a bad id 404s here + // rather than opening an empty stream. + let (detection_row, _pipeline) = find_detection(&mut conn, workspace.id, detection_id).await?; + let current = detection_row.status; + drop(conn); + + let stream = stream! { + // Emit the current status first: covers the race where analysis settled + // before the subscription was live (no live event would ever arrive). + yield status_event(&DetectionStatusEvent { detection_id, status: current }); + if !current.is_detecting() { + return; + } + + loop { + match tokio::time::timeout(STATUS_POLL_INTERVAL, updates.next()).await { + // A live broadcast arrived; forward it and stop once it settles. + Ok(Some(event)) => { + let settled = !event.status.is_detecting(); + yield status_event(&event); + if settled { + break; + } + } + // The subscription ended; fall back to the DB so the client still + // learns the final status. + Ok(None) => { + if let Some(status) = reread_detection_status(&pg_client, workspace.id, detection_id).await { + yield status_event(&DetectionStatusEvent { detection_id, status }); + } + break; + } + // No broadcast within the interval: re-read the authoritative + // detection row. This recovers a dropped best-effort broadcast + // (core NATS is at-most-once) instead of hanging on keep-alive. + Err(_) => { + if let Some(status) = reread_detection_status(&pg_client, workspace.id, detection_id).await { + yield status_event(&DetectionStatusEvent { detection_id, status }); + if !status.is_detecting() { + break; + } + } + } + } + } + }; + + Ok(SseResponse::new(stream)) +} + +/// How long the status stream waits for a live broadcast before re-reading the +/// authoritative detection row from the database (the fallback for a dropped +/// best-effort broadcast). +const STATUS_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_secs(15); + +/// Re-reads a detection's current status from the database, returning `None` if +/// the detection can no longer be read (missing, or a transient error — the next +/// poll retries). +async fn reread_detection_status( + pg_client: &PgClient, + workspace_id: Uuid, + detection_id: Uuid, +) -> Option { + let mut conn = pg_client.get_connection().await.ok()?; + match find_detection(&mut conn, workspace_id, detection_id).await { + Ok((detection, _pipeline)) => Some(detection.status), + Err(err) => { + tracing::debug!(target: TRACING_TARGET, error = %err, %detection_id, "Failed to re-read detection status"); + None + } + } +} + +/// OpenAPI documentation for the detection status SSE stream. +fn stream_detection_events_docs(op: TransformOperation) -> TransformOperation { + op.summary("Stream detection status") + .description( + "Opens a Server-Sent Events stream of the detection's status changes. \ + Emits the current status immediately, then each transition, and \ + ends once the detection settles (complete or failed). Each event's \ + `data` is a `DetectionStatusEvent` (see the response schema). \ + Authenticate with a Bearer token via a `fetch`-based client; the \ + native `EventSource` cannot send an `Authorization` header.", + ) + .response::<401, Json>() + .response::<403, Json>() + .response::<404, Json>() +} + +/// Builds a `status` SSE event carrying the detection's status change. +fn status_event(event: &DetectionStatusEvent) -> Event { + Event::default() + .event("status") + .json_data(event) + .unwrap_or_else(|_| Event::default().event("status")) +} + +/// Redacts a detection using its findings, storing the result. +/// +/// Applies the pipeline's policies to the detection's stored analysis, stores the +/// redacted bytes as a new file, and emits a redaction event. Requires +/// `RunPipelines` permission. A detection can be redacted more than once. +#[tracing::instrument( + skip_all, + fields( + account_id = %auth_state.account_id, + workspace_id = %workspace.id, + detection_id = %path_params.detection_id, + ) +)] +async fn redact_detection( + State(pg_client): State, + State(blob): State, + State(crypto): State, + State(engine): State, + AuthState(auth_state): AuthState, + WorkspaceContext(workspace): WorkspaceContext, + Path(path_params): Path, + security: SecurityContext, + Json(request): Json, +) -> Result<(StatusCode, Json)> { + tracing::debug!(target: TRACING_TARGET, "Redacting detection"); + + let mut conn = pg_client.get_connection().await?; + + auth_state + .authorize_workspace(&mut conn, workspace.id, Permission::RunPipelines) + .await?; + + let (detection, pipeline) = + find_detection(&mut conn, workspace.id, path_params.detection_id.as_uuid()).await?; + + // A detection can only be redacted once its analysis is complete. + if !detection.is_complete() { + return Err(ErrorKind::Conflict + .with_message("Detection is not ready to redact") + .with_resource("detection")); + } + + // The source document is normally held back from retention while the + // detection is unfinished (see files_due_for_expiry), so this is reachable + // only if the input was explicitly deleted; surface a message that names the + // cause. + let file = conn + .find_file_in_workspace(workspace.id, detection.input_file_id) + .await? + .ok_or_else(|| { + ErrorKind::Conflict + .with_message("The detection's source document is no longer available") + .with_resource("detection") + })?; + + // The stored detection analysis is loaded into a working audit and never + // mutated on disk: reviewer edits and the redaction outcome land on this + // clone, which is persisted as the redaction's own review audit, leaving the + // detection analysis immutable and re-redactable. + let mut reviewed = blob + .load_analyzed_document(&mut conn, &engine, workspace.id, &detection) + .await?; + let policies = resolve_policies(&mut conn, &crypto, workspace.id, pipeline.id).await?; + + // Layer the reviewer's edits onto the working audit's report before + // redaction. Validation is report-relative (an unknown target or a + // self-contradiction → 400, via `EditError`'s `From` impl) so a reviewer is + // never told a decision took effect when the document says otherwise. + if let Some(edits) = &request.edits { + edits.validate(&reviewed.report)?; + edits.apply(&mut reviewed.report); + } + + let document = blob.build_document(&file, detection.id).await?; + + // No per-request key: the server does not yet drive keyed operators + // (HMAC/encrypt), whose `KeyConfig` would be supplied here. The codec params + // and document context are read back from the audit, recorded at detect time. + let redacted = engine + .anonymize(document, &policies, &mut reviewed, None) + .await?; + + // Stage both produced objects (redacted document + review audit) outside the + // transaction — object writes are not transactional — then commit their file + // rows and the redaction row together. On rollback the staged objects are + // reclaimed so no orphaned bytes accrue. + let retention = workspace.settings.or_default().retention; + let staged_output = blob + .stage_redacted_file( + &file, + &pipeline, + &retention, + auth_state.account_id, + redacted.bytes, + ) + .await?; + // Staging the review audit after the output means a failure here would strand + // the already-written output object (no row to reclaim it); discard it first. + let staged_review = match blob + .stage_review_audit(&pipeline, &retention, auth_state.account_id, &reviewed) + .await + { + Ok(staged) => staged, + Err(err) => { + blob.discard_staged_object(&staged_output).await.ok(); + return Err(err); + } + }; + + let redaction = conn + .transaction(async |conn| { + let output_file = conn.create_workspace_file(staged_output.clone()).await?; + let review_file = conn.create_workspace_file(staged_review.clone()).await?; + let redaction = conn + .create_redaction(NewWorkspaceRedaction { + detection_id: detection.id, + account_id: auth_state.account_id, + review_file_id: Some(review_file.id), + output_file_id: Some(output_file.id), + }) + .await?; + conn.emit_event( + EventOrigin { + workspace_id: workspace.id, + account_id: auth_state.account_id, + security: &security, + }, + WorkspaceEvent::RedactionCreated { + detection: DetectionRef { + detection_id: detection.id, + pipeline_slug: pipeline.slug.clone(), + }, + redaction_id: redaction.id, + input_file_name: Some(file.display_name.clone()), + notify: detection.account_id, + }, + ) + .await?; + Ok::<_, Error>(redaction) + }) + .await; + + let redaction = match redaction { + Ok(redaction) => redaction, + Err(err) => { + // The rows rolled back, so their staged objects are orphans: reclaim + // both (best effort — a failure only leaves them for a later sweep). + blob.discard_staged_object(&staged_output).await.ok(); + blob.discard_staged_object(&staged_review).await.ok(); + return Err(err); + } + }; + + tracing::info!( + target: TRACING_TARGET, + detection_id = %detection.id, + redaction_id = %redaction.id, + "Detection redacted" + ); + + let requested_by = resolve_account_ref(&mut conn, redaction.account_id).await?; + + Ok(( + StatusCode::CREATED, + Json(RedactionResult::from_model( + redaction, + workspace.slug, + requested_by, + )), + )) +} + +fn redact_detection_docs(op: TransformOperation) -> TransformOperation { + op.summary("Redact a detection") + .description( + "Applies the pipeline's policies to the detection's stored analysis — with any \ + reviewer `edits` layered on first (suppress a false positive, retag a detection, or \ + add one the analysis missed) — and produces a new redaction: a redacted document \ + plus a review audit recording what was redacted. A detection can be redacted more \ + than once. An edit targeting a detection not in the analysis, or a set that \ + contradicts itself, is rejected (400).", + ) + .response::<201, Json>() + .response::<400, Json>() + .response::<401, Json>() + .response::<403, Json>() + .response::<404, Json>() + .response::<409, Json>() +} + +/// Finds a pipeline within a workspace by slug or returns NotFound. +async fn find_pipeline( + conn: &mut PgConn, + workspace_id: Uuid, + pipeline_slug: &str, +) -> Result { + conn.find_pipeline_in_workspace_by_slug(workspace_id, pipeline_slug) + .await? + .map(|wc| wc.item) + .ok_or_else(|| Error::not_found("pipeline")) +} + +/// Resolves a detection by its opaque id within a workspace, returning the +/// detection and its owning pipeline (for the response's pipeline slug). The +/// lookup is workspace-scoped through the owning pipeline. +pub(super) async fn find_detection( + conn: &mut PgConn, + workspace_id: Uuid, + detection_id: Uuid, +) -> Result<(WorkspaceDetection, WorkspacePipeline)> { + conn.find_workspace_detection_by_id(workspace_id, detection_id) + .await? + .ok_or_else(|| Error::not_found("detection")) +} + +/// Returns a [`Router`] with all detection routes. +/// +/// [`Router`]: axum::routing::Router +pub fn routes() -> ApiRouter { + use aide::axum::routing::*; + + ApiRouter::new() + .api_route( + "/workspaces/{workspaceSlug}/pipelines/detections/", + get_with(list_workspace_detections, list_workspace_detections_docs), + ) + .api_route( + "/workspaces/{workspaceSlug}/pipelines/{pipelineSlug}/detections/", + post_with(create_detection, create_detection_docs) + .get_with(list_pipeline_detections, list_pipeline_detections_docs), + ) + .api_route( + "/workspaces/{workspaceSlug}/detections/{detectionId}/", + get_with(get_detection, get_detection_docs), + ) + .api_route( + "/workspaces/{workspaceSlug}/detections/{detectionId}/events/", + get_with(stream_detection_events, stream_detection_events_docs), + ) + .api_route( + "/workspaces/{workspaceSlug}/detections/{detectionId}/redactions/", + post_with(redact_detection, redact_detection_docs), + ) + .with_path_items(|item| item.tag("Detections")) +} diff --git a/crates/nvisy-server/src/handler/error/engine_error.rs b/crates/nvisy-server/src/handler/error/engine_error.rs index 0f53dbd8..4baacdad 100644 --- a/crates/nvisy-server/src/handler/error/engine_error.rs +++ b/crates/nvisy-server/src/handler/error/engine_error.rs @@ -7,6 +7,7 @@ //! own message travels along as context. use elide_pipeline::ErrorKind as EngineErrorKind; +use elide_pipeline::entity::EditError; use super::http_error::{Error as HttpError, ErrorKind}; @@ -22,3 +23,15 @@ impl<'a> From for HttpError<'a> { } } } + +impl<'a> From for HttpError<'a> { + /// A reviewer edit set that does not apply to the analysis is always a client + /// error: both an unknown target (a stale or wrong-modality entity id) and a + /// self-contradiction (two edits deciding one entity differently) are the + /// caller's edits being wrong, so they map to a 400. + fn from(error: EditError) -> Self { + ErrorKind::BadRequest + .with_message("A reviewer edit does not apply to this analysis") + .with_context(error.to_string()) + } +} diff --git a/crates/nvisy-server/src/handler/error/pg_error.rs b/crates/nvisy-server/src/handler/error/pg_error.rs index 62958722..36f7665b 100644 --- a/crates/nvisy-server/src/handler/error/pg_error.rs +++ b/crates/nvisy-server/src/handler/error/pg_error.rs @@ -29,7 +29,7 @@ impl From for Error<'static> { ConstraintViolation::WorkspaceWebhook(c) => c.into(), ConstraintViolation::WorkspaceFile(c) => c.into(), ConstraintViolation::WorkspacePipeline(c) => c.into(), - ConstraintViolation::WorkspacePipelineRun(c) => c.into(), + ConstraintViolation::WorkspaceDetection(c) => c.into(), ConstraintViolation::WorkspacePipelineReference(c) => c.into(), ConstraintViolation::WorkspaceConnection(c) => c.into(), ConstraintViolation::WorkspaceConnectionSync(c) => c.into(), diff --git a/crates/nvisy-server/src/handler/error/pg_pipeline.rs b/crates/nvisy-server/src/handler/error/pg_pipeline.rs index d2f3c079..4d6fe8e9 100644 --- a/crates/nvisy-server/src/handler/error/pg_pipeline.rs +++ b/crates/nvisy-server/src/handler/error/pg_pipeline.rs @@ -2,8 +2,8 @@ use nvisy_postgres::types::{ WorkspaceConnectionConstraints, WorkspaceConnectionSyncConstraints, - WorkspacePipelineConstraints, WorkspacePipelineReferenceConstraints, - WorkspacePipelineRunConstraints, WorkspacePolicyConstraints, + WorkspaceDetectionConstraints, WorkspacePipelineConstraints, + WorkspacePipelineReferenceConstraints, WorkspacePolicyConstraints, }; use crate::handler::{Error, ErrorKind}; @@ -36,19 +36,19 @@ impl From for Error<'static> { } } -impl From for Error<'static> { - fn from(c: WorkspacePipelineRunConstraints) -> Self { +impl From for Error<'static> { + fn from(c: WorkspaceDetectionConstraints) -> Self { let error = match c { - WorkspacePipelineRunConstraints::MetadataSize => ErrorKind::BadRequest - .with_message("Pipeline run metadata size exceeds maximum limit"), - WorkspacePipelineRunConstraints::IdempotencyKeyLength => ErrorKind::BadRequest + WorkspaceDetectionConstraints::MetadataSize => ErrorKind::BadRequest + .with_message("Detection metadata size exceeds maximum limit"), + WorkspaceDetectionConstraints::IdempotencyKeyLength => ErrorKind::BadRequest .with_message("Idempotency key must be 1 to 255 characters"), - WorkspacePipelineRunConstraints::IdempotencyUnique => ErrorKind::Conflict - .with_message("A run with this idempotency key already exists"), + WorkspaceDetectionConstraints::IdempotencyUnique => ErrorKind::Conflict + .with_message("A detection with this idempotency key already exists"), }; - error.with_resource("pipeline_run") + error.with_resource("detection") } } diff --git a/crates/nvisy-server/src/handler/mod.rs b/crates/nvisy-server/src/handler/mod.rs index a16ef83f..f367ed90 100644 --- a/crates/nvisy-server/src/handler/mod.rs +++ b/crates/nvisy-server/src/handler/mod.rs @@ -12,16 +12,17 @@ mod catalog; mod chat; mod connection_syncs; mod connections; +mod detection_audits; +mod detections; mod error; mod files; mod invites; mod members; mod monitors; mod notifications; -mod pipeline_audits; -mod pipeline_runs; mod pipelines; mod policies; +mod redactions; pub mod request; pub mod response; mod tokens; @@ -89,8 +90,9 @@ fn private_routes( .merge(connection_syncs::routes()) .merge(files::routes()) .merge(pipelines::routes()) - .merge(pipeline_runs::routes()) - .merge(pipeline_audits::routes()) + .merge(detections::routes()) + .merge(detection_audits::routes()) + .merge(redactions::routes()) .merge(policies::routes()) .merge(catalog::routes()); diff --git a/crates/nvisy-server/src/handler/pipeline_runs.rs b/crates/nvisy-server/src/handler/pipeline_runs.rs deleted file mode 100644 index 68123708..00000000 --- a/crates/nvisy-server/src/handler/pipeline_runs.rs +++ /dev/null @@ -1,763 +0,0 @@ -//! Pipeline run handlers: detect, review, and redact. -//! -//! A run is one analysis of a file through a pipeline. Detect creates the run -//! and stores the findings; the run then awaits reviewer verification before -//! redact consumes the verified findings and produces a redacted file. - -use aide::axum::ApiRouter; -use aide::transform::TransformOperation; -use async_stream::stream; -use axum::extract::State; -use axum::http::StatusCode; -use axum::response::sse::Event; -use futures::StreamExt; -use nvisy_postgres::model::{ - NewWorkspacePipelineRun, UpdateWorkspacePipelineRun, WorkspacePipeline, WorkspacePipelineRun, -}; -use nvisy_postgres::query::{ - PipelineReferenceRepository, RunFiles, WorkspaceFileRepository, WorkspacePipelineRepository, - WorkspacePipelineRunRepository, -}; -use nvisy_postgres::types::PipelineRunStatus; -use nvisy_postgres::{AsyncConnection, PgClient, PgConn}; -use uuid::Uuid; - -use crate::extract::{ - AuthProvider, AuthState, IdempotencyKey, Json, Path, Permission, Query, SecurityContext, - ValidateJson, WorkspaceContext, -}; -use crate::handler::request::{ - CreatePipelineRun, CursorPagination, PipelineDefinition, PipelinePathParams, - PipelineRunPathParams, PipelineRunsQuery, WorkspaceRunsQuery, -}; -use crate::handler::response::{ErrorResponse, PipelineRun, PipelineRunsPage}; -use crate::handler::utility::{SseResponse, resolve_account_ref}; -use crate::handler::{Error, ErrorKind, Result}; -use crate::service::{ - CryptoService, DetectionJob, DetectionQueue, EngineService, EventEmitter, EventOrigin, FailRun, - PipelineRunRef, RunBlobStore, RunStatusEvent, ServiceState, WorkspaceEvent, fail_run, - resolve_policies, -}; - -/// Tracing target for pipeline run operations. -const TRACING_TARGET: &str = "nvisy_server::handler::runs"; - -/// Starts a run: analyzes a file with the pipeline's configuration (detect). -/// -/// Returns the run holding the findings for review. A repeated request with the -/// same `Idempotency-Key` returns the existing run instead of analyzing again. -/// Requires `RunPipelines` permission. -#[tracing::instrument( - skip_all, - fields( - account_id = %auth_state.account_id, - workspace_id = %workspace.id, - pipeline_slug = %path_params.pipeline_slug, - ) -)] -async fn create_pipeline_run( - State(pg_client): State, - State(detection): State, - AuthState(auth_state): AuthState, - WorkspaceContext(workspace): WorkspaceContext, - Path(path_params): Path, - IdempotencyKey(idempotency_key): IdempotencyKey, - security: SecurityContext, - ValidateJson(request): ValidateJson, -) -> Result<(StatusCode, Json)> { - tracing::debug!(target: TRACING_TARGET, "Starting pipeline run (detect)"); - - let mut conn = pg_client.get_connection().await?; - - auth_state - .authorize_workspace(&mut conn, workspace.id, Permission::RunPipelines) - .await?; - - let pipeline = find_pipeline(&mut conn, workspace.id, &path_params.pipeline_slug).await?; - - // Only an enabled pipeline runs: a draft (still being configured) or a - // disabled (paused) pipeline is rejected. - if !pipeline.status.is_enabled() { - return Err(ErrorKind::Conflict - .with_message("Pipeline is not enabled") - .with_resource("pipeline")); - } - - // Idempotent replay: a repeated key returns the run created the first time, - // attributed to whoever originally triggered it (not the current caller). - if let Some(key) = &idempotency_key - && let Some(existing) = conn - .find_pipeline_run_by_idempotency_key(pipeline.id, key) - .await? - { - tracing::debug!(target: TRACING_TARGET, "Replaying run for idempotency key"); - let trigger = resolve_account_ref(&mut conn, existing.account_id).await?; - let files = conn.run_file_names(workspace.id, &existing).await?; - return Ok(( - StatusCode::OK, - Json(PipelineRun::from_model( - existing, - pipeline.slug.clone(), - workspace.slug.clone(), - trigger, - files, - )), - )); - } - - // Validate synchronously so a bad request fails fast (4xx) rather than as a - // run that immediately fails in the worker. - let file = conn - .find_file_in_workspace(pipeline.workspace_id, request.file_id) - .await? - .ok_or_else(|| Error::not_found("file"))?; - - if conn.list_pipeline_policy_ids(pipeline.id).await?.is_empty() { - return Err(ErrorKind::BadRequest - .with_message("Pipeline has no policies; attach at least one before running") - .with_resource("pipeline")); - } - - // Decode the pipeline definition now so an undecodable definition fails the - // request synchronously (400) instead of returning 202 and failing later in - // the worker. The decoded value is rebuilt in the worker from the same - // stored bytes; this is validation only. - let _validated = PipelineDefinition::from_parts(pipeline.definition.clone(), Vec::new()) - .map_err(|err| { - ErrorKind::BadRequest - .with_message("Pipeline definition is invalid") - .with_resource("pipeline") - .with_context(err.to_string()) - })?; - - // Create the run (its id is the engine correlation id) and enqueue detection - // for the worker. The response returns immediately; the client learns the - // findings are ready via the run's status (SSE at `.../events` or a re-read). - let new_run = NewWorkspacePipelineRun { - pipeline_id: pipeline.id, - input_file_id: file.id, - account_id: auth_state.account_id, - status: Some(PipelineRunStatus::Queued), - idempotency_key: idempotency_key.clone(), - ..Default::default() - }; - - // Create the run and record its start event in one transaction, so the event - // is never lost, nor recorded for a run that rolled back. - let run = conn - .transaction(async |conn| { - let run = conn.create_workspace_pipeline_run(new_run).await?; - conn.emit_event( - EventOrigin { - workspace_id: workspace.id, - account_id: auth_state.account_id, - security: &security, - }, - WorkspaceEvent::PipelineRunStarted(PipelineRunRef { - run_id: run.id, - pipeline_slug: pipeline.slug.clone(), - }), - ) - .await?; - Ok::<_, Error>(run) - }) - .await?; - - let job = DetectionJob { - workspace_id: workspace.id, - run_id: run.id, - scope: request.scope, - }; - if let Err(err) = detection.enqueue(job).await { - // Enqueue failed, so the worker will never pick this run up: fail it now - // rather than leaving it stuck in `Queued`. - fail_run( - &mut conn, - &detection, - FailRun { - workspace_id: workspace.id, - run_id: run.id, - pipeline_slug: pipeline.slug.clone(), - triggered_by: auth_state.account_id, - reason: "Failed to enqueue detection", - claim: None, - }, - ) - .await; - return Err(err); - } - - detection - .broadcast_status(run.id, PipelineRunStatus::Queued) - .await; - - tracing::info!(target: TRACING_TARGET, run_id = %run.id, "Pipeline run enqueued for detection"); - - let trigger = resolve_account_ref(&mut conn, run.account_id).await?; - - // The run was just created from this file and has no output yet. - let files = RunFiles { - input: Some(file.display_name), - output: None, - }; - - Ok(( - StatusCode::ACCEPTED, - Json(PipelineRun::from_model( - run, - pipeline.slug, - workspace.slug, - trigger, - files, - )), - )) -} - -fn create_pipeline_run_docs(op: TransformOperation) -> TransformOperation { - op.summary("Start a run (detect)") - .description( - "Starts detection for a file and returns 202 with the run in the \ - `running` state; the analysis runs in the background. Watch the run's \ - status via the SSE stream at `.../runs/{runId}/events` (or re-read the \ - run) and fetch the findings from `.../runs/{runId}/detections/` once it \ - reaches `analyzed`. A repeated Idempotency-Key returns the existing run.", - ) - .response::<202, Json>() - .response::<200, Json>() - .response::<400, Json>() - .response::<401, Json>() - .response::<403, Json>() - .response::<404, Json>() - .response::<409, Json>() -} - -/// Lists runs for a specific pipeline. -#[tracing::instrument( - skip_all, - fields( - account_id = %auth_state.account_id, - workspace_id = %workspace.id, - pipeline_slug = %path_params.pipeline_slug, - ) -)] -async fn list_pipeline_runs( - State(pg_client): State, - AuthState(auth_state): AuthState, - WorkspaceContext(workspace): WorkspaceContext, - Path(path_params): Path, - Query(pagination): Query, - Query(query): Query, -) -> Result<(StatusCode, Json)> { - tracing::debug!(target: TRACING_TARGET, "Listing pipeline runs"); - - let mut conn = pg_client.get_connection().await?; - - auth_state - .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) - .await?; - - let pipeline = find_pipeline(&mut conn, workspace.id, &path_params.pipeline_slug).await?; - - let page = conn - .cursor_list_workspace_pipeline_runs(pipeline.id, pagination.into(), &query.into()) - .await?; - - tracing::debug!( - target: TRACING_TARGET, - run_count = page.items.len(), - "Pipeline runs listed" - ); - - let response = PipelineRunsPage::from_cursor_page(page, |row| { - PipelineRun::from_model( - row.run, - row.pipeline_slug, - workspace.slug.clone(), - row.account.into(), - RunFiles { - input: row.input_file_name, - output: None, - }, - ) - }); - - Ok((StatusCode::OK, Json(response))) -} - -fn list_pipeline_runs_docs(op: TransformOperation) -> TransformOperation { - op.summary("List pipeline runs") - .description( - "Returns runs for a specific pipeline, most recent first, with \ - optional status, file, trigger-account, and trigger-type filters.", - ) - .response::<200, Json>() - .response::<401, Json>() - .response::<403, Json>() - .response::<404, Json>() -} - -/// Lists all runs across the workspace's pipelines. -/// -/// Aggregates runs from every pipeline in the workspace, most recent first, -/// with optional status and pipeline filters. Requires `ViewPipelines`. -#[tracing::instrument( - skip_all, - fields( - account_id = %auth_state.account_id, - workspace_id = %workspace.id, - ) -)] -async fn list_workspace_runs( - State(pg_client): State, - AuthState(auth_state): AuthState, - WorkspaceContext(workspace): WorkspaceContext, - Query(pagination): Query, - Query(query): Query, -) -> Result<(StatusCode, Json)> { - tracing::debug!(target: TRACING_TARGET, "Listing workspace runs"); - - let mut conn = pg_client.get_connection().await?; - - auth_state - .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) - .await?; - - let page = conn - .cursor_list_workspace_runs(workspace.id, pagination.into(), &query.into()) - .await?; - - tracing::debug!( - target: TRACING_TARGET, - run_count = page.items.len(), - "Workspace runs listed" - ); - - Ok(( - StatusCode::OK, - Json(PipelineRunsPage::from_cursor_page(page, |row| { - PipelineRun::from_model( - row.run, - row.pipeline_slug, - workspace.slug.clone(), - row.account.into(), - RunFiles { - input: row.input_file_name, - output: None, - }, - ) - })), - )) -} - -fn list_workspace_runs_docs(op: TransformOperation) -> TransformOperation { - op.summary("List workspace runs") - .description( - "Returns all pipeline runs across the workspace, most recent first, \ - with optional status, file, pipeline, trigger-account, and \ - trigger-type filters.", - ) - .response::<200, Json>() - .response::<401, Json>() - .response::<403, Json>() - .response::<404, Json>() -} - -/// Gets a specific pipeline run. -#[tracing::instrument( - skip_all, - fields( - account_id = %auth_state.account_id, - workspace_id = %workspace.id, - run_id = %path_params.run_id, - ) -)] -async fn get_pipeline_run( - State(pg_client): State, - AuthState(auth_state): AuthState, - WorkspaceContext(workspace): WorkspaceContext, - Path(path_params): Path, -) -> Result<(StatusCode, Json)> { - tracing::debug!(target: TRACING_TARGET, "Getting pipeline run"); - - let mut conn = pg_client.get_connection().await?; - - auth_state - .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) - .await?; - - let (run, pipeline) = - find_pipeline_run(&mut conn, workspace.id, path_params.run_id.as_uuid()).await?; - - let trigger = resolve_account_ref(&mut conn, run.account_id).await?; - let files = conn.run_file_names(workspace.id, &run).await?; - - tracing::debug!(target: TRACING_TARGET, "Pipeline run retrieved"); - - Ok(( - StatusCode::OK, - Json(PipelineRun::from_model( - run, - pipeline.slug, - workspace.slug, - trigger, - files, - )), - )) -} - -fn get_pipeline_run_docs(op: TransformOperation) -> TransformOperation { - op.summary("Get pipeline run") - .description("Returns the run and its status for review.") - .response::<200, Json>() - .response::<401, Json>() - .response::<403, Json>() - .response::<404, Json>() -} - -/// Streams a run's status changes as Server-Sent Events until detection settles. -/// -/// Emits one `status` event with the run's current status immediately (so a -/// client that connects after detection already finished still learns the -/// state), then forwards each status change. The stream ends once the run leaves -/// the detecting phase (`queued`/`analyzing`) — i.e. detection has produced -/// `analyzed`, or the run `failed`/`cancelled`. -/// -/// Live status changes arrive over a best-effort core-NATS broadcast; if none -/// arrives within a short interval the authoritative run row is re-read from the -/// database, so a dropped broadcast never leaves the stream hanging. -/// -/// Authenticated like every other route (Bearer); browsers should consume it via -/// a `fetch` stream rather than the native `EventSource`, which cannot send an -/// `Authorization` header. -#[tracing::instrument( - skip_all, - fields( - account_id = %auth_state.account_id, - workspace_id = %workspace.id, - run_id = %path_params.run_id, - ) -)] -async fn stream_pipeline_run_events( - State(pg_client): State, - State(detection): State, - AuthState(auth_state): AuthState, - WorkspaceContext(workspace): WorkspaceContext, - Path(path_params): Path, -) -> Result> { - tracing::debug!(target: TRACING_TARGET, "Opening run status stream"); - - let run_id = path_params.run_id.as_uuid(); - let mut conn = pg_client.get_connection().await?; - auth_state - .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) - .await?; - - // Subscribe BEFORE reading the current status: core-NATS broadcasts are not - // replayed, so a terminal status published between the read and the - // subscription going live would otherwise be lost and the stream would hang. - let mut updates = detection.subscribe_status(run_id).await?; - - // Confirm the run exists (and is workspace-scoped) so a bad id 404s here - // rather than opening an empty stream. - let (run, _pipeline) = find_pipeline_run(&mut conn, workspace.id, run_id).await?; - let current = run.status; - drop(conn); - - let stream = stream! { - // Emit the current status first: covers the race where detection settled - // before the subscription was live (no live event would ever arrive). - yield status_event(&RunStatusEvent { run_id, status: current }); - if !current.is_detecting() { - return; - } - - loop { - match tokio::time::timeout(STATUS_POLL_INTERVAL, updates.next()).await { - // A live broadcast arrived; forward it and stop once detection settles. - Ok(Some(event)) => { - let settled = !event.status.is_detecting(); - yield status_event(&event); - if settled { - break; - } - } - // The subscription ended; fall back to the DB so the client still - // learns the final status. - Ok(None) => { - if let Some(status) = reread_run_status(&pg_client, workspace.id, run_id).await { - yield status_event(&RunStatusEvent { run_id, status }); - } - break; - } - // No broadcast within the interval: re-read the authoritative run - // row. This recovers a dropped best-effort broadcast (core NATS is - // at-most-once) instead of hanging on keep-alive forever. - Err(_) => { - if let Some(status) = reread_run_status(&pg_client, workspace.id, run_id).await { - yield status_event(&RunStatusEvent { run_id, status }); - if !status.is_detecting() { - break; - } - } - } - } - } - }; - - Ok(SseResponse::new(stream)) -} - -/// How long the status stream waits for a live broadcast before re-reading the -/// authoritative run row from the database (the fallback for a dropped -/// best-effort broadcast). -const STATUS_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_secs(15); - -/// Re-reads a run's current status from the database, returning `None` if the -/// run can no longer be read (missing, or a transient error — the next poll -/// retries). -async fn reread_run_status( - pg_client: &PgClient, - workspace_id: Uuid, - run_id: Uuid, -) -> Option { - let mut conn = pg_client.get_connection().await.ok()?; - match find_pipeline_run(&mut conn, workspace_id, run_id).await { - Ok((run, _pipeline)) => Some(run.status), - Err(err) => { - tracing::debug!(target: TRACING_TARGET, error = %err, %run_id, "Failed to re-read run status"); - None - } - } -} - -/// OpenAPI documentation for the run status SSE stream. -fn stream_pipeline_run_events_docs(op: TransformOperation) -> TransformOperation { - op.summary("Stream pipeline run status") - .description( - "Opens a Server-Sent Events stream of the run's status changes. \ - Emits the current status immediately, then each transition, and \ - ends once the run settles (analyzed, failed, or cancelled). Each \ - event's `data` is a `RunStatusEvent` (see the response schema). \ - Authenticate with a Bearer token via a `fetch`-based client; the \ - native `EventSource` cannot send an `Authorization` header.", - ) - .response::<401, Json>() - .response::<403, Json>() - .response::<404, Json>() -} - -/// Builds a `status` SSE event carrying the run's status change. -fn status_event(event: &RunStatusEvent) -> Event { - Event::default() - .event("status") - .json_data(event) - .unwrap_or_else(|_| Event::default().event("status")) -} - -/// Redacts a run using the reviewer-verified findings, storing the result. -/// -/// Consumes the analyzed run (which must be awaiting review), applies the -/// pipeline's policies to the verified findings, stores the redacted bytes as a -/// new file, and completes the run. Requires `RunPipelines` permission. -#[tracing::instrument( - skip_all, - fields( - account_id = %auth_state.account_id, - workspace_id = %workspace.id, - run_id = %path_params.run_id, - ) -)] -async fn redact_pipeline_run( - State(pg_client): State, - State(blob): State, - State(crypto): State, - State(engine): State, - AuthState(auth_state): AuthState, - WorkspaceContext(workspace): WorkspaceContext, - Path(path_params): Path, - security: SecurityContext, -) -> Result<(StatusCode, Json)> { - tracing::debug!(target: TRACING_TARGET, "Redacting pipeline run"); - - let mut conn = pg_client.get_connection().await?; - - auth_state - .authorize_workspace(&mut conn, workspace.id, Permission::RunPipelines) - .await?; - - let (run, pipeline) = - find_pipeline_run(&mut conn, workspace.id, path_params.run_id.as_uuid()).await?; - - // A run can only be redacted once, after detection. - if !run.is_analyzed() { - return Err(ErrorKind::Conflict - .with_message("Run is not awaiting redaction") - .with_resource("pipeline_run")); - } - - // The source document is normally held back from retention while the run is - // unfinished (see files_due_for_expiry), so this is reachable only if the - // input was explicitly deleted; surface a message that names the cause. - let file = conn - .find_file_in_workspace(workspace.id, run.input_file_id) - .await? - .ok_or_else(|| { - ErrorKind::Conflict - .with_message("The run's source document is no longer available") - .with_resource("pipeline_run") - })?; - - // The stored analysis is the source of truth for what gets redacted. It - // carries the scope (label catalog) analyze resolved, so redaction compiles - // against the same vocabulary without re-deriving it. - let mut analyzed = blob - .load_analyzed_document(&mut conn, &engine, workspace.id, &run) - .await?; - let policies = resolve_policies(&mut conn, &crypto, workspace.id, pipeline.id).await?; - let document = blob.build_document(&file, run.id).await?; - - // No per-request key: the server does not yet drive keyed operators - // (HMAC/encrypt), whose `KeyConfig` would be supplied here. The codec params - // and document context are read back from the audit, recorded at detect time. - let redacted = engine - .anonymize(document, &policies, &mut analyzed, None) - .await?; - - // Store the redacted bytes as a new workspace file and link it to the run. - let workspace_settings = workspace.settings.or_default().retention; - let output_file = blob - .store_redacted_file( - &mut conn, - &file, - &pipeline, - &workspace_settings, - auth_state.account_id, - redacted.bytes, - ) - .await?; - - // Complete the run and record its completion event in one transaction, so the - // event is never lost, nor recorded for an update that rolled back. - let run = conn - .transaction(async |conn| { - let run = conn - .update_workspace_pipeline_run( - run.id, - UpdateWorkspacePipelineRun { - status: Some(PipelineRunStatus::Completed), - output_file_id: Some(Some(output_file.id)), - completed_at: Some(Some(jiff::Timestamp::now().into())), - ..Default::default() - }, - ) - .await?; - conn.emit_event( - EventOrigin { - workspace_id: workspace.id, - account_id: auth_state.account_id, - security: &security, - }, - WorkspaceEvent::PipelineRunCompleted { - run: PipelineRunRef { - run_id: run.id, - pipeline_slug: pipeline.slug.clone(), - }, - input_file_name: Some(file.display_name.clone()), - notify: run.account_id, - }, - ) - .await?; - Ok::<_, Error>(run) - }) - .await?; - - tracing::info!( - target: TRACING_TARGET, - run_id = %run.id, - output_file_id = %output_file.id, - "Pipeline run redacted" - ); - - let trigger = resolve_account_ref(&mut conn, run.account_id).await?; - let files = conn.run_file_names(workspace.id, &run).await?; - - Ok(( - StatusCode::OK, - Json(PipelineRun::from_model( - run, - pipeline.slug, - workspace.slug, - trigger, - files, - )), - )) -} - -fn redact_pipeline_run_docs(op: TransformOperation) -> TransformOperation { - op.summary("Redact a run") - .description( - "Applies the pipeline's policies to the run's stored analysis, stores \ - the redacted file, and completes the run.", - ) - .response::<200, Json>() - .response::<401, Json>() - .response::<403, Json>() - .response::<404, Json>() - .response::<409, Json>() -} - -/// Finds a pipeline within a workspace by slug or returns NotFound. -async fn find_pipeline( - conn: &mut PgConn, - workspace_id: Uuid, - pipeline_slug: &str, -) -> Result { - conn.find_pipeline_in_workspace_by_slug(workspace_id, pipeline_slug) - .await? - .map(|wc| wc.item) - .ok_or_else(|| Error::not_found("pipeline")) -} - -/// Resolves a run by its opaque id within a workspace, returning the run and its -/// owning pipeline (for the response's pipeline slug). The lookup is -/// workspace-scoped through the owning pipeline. -pub(super) async fn find_pipeline_run( - conn: &mut PgConn, - workspace_id: Uuid, - run_id: Uuid, -) -> Result<(WorkspacePipelineRun, WorkspacePipeline)> { - conn.find_workspace_run_by_id(workspace_id, run_id) - .await? - .ok_or_else(|| Error::not_found("pipeline_run")) -} - -/// Returns a [`Router`] with all pipeline run routes. -/// -/// [`Router`]: axum::routing::Router -pub fn routes() -> ApiRouter { - use aide::axum::routing::*; - - ApiRouter::new() - .api_route( - "/workspaces/{workspaceSlug}/pipelines/runs/", - get_with(list_workspace_runs, list_workspace_runs_docs), - ) - .api_route( - "/workspaces/{workspaceSlug}/pipelines/{pipelineSlug}/runs/", - post_with(create_pipeline_run, create_pipeline_run_docs) - .get_with(list_pipeline_runs, list_pipeline_runs_docs), - ) - .api_route( - "/workspaces/{workspaceSlug}/runs/{runId}/", - get_with(get_pipeline_run, get_pipeline_run_docs), - ) - .api_route( - "/workspaces/{workspaceSlug}/runs/{runId}/events", - get_with(stream_pipeline_run_events, stream_pipeline_run_events_docs), - ) - .api_route( - "/workspaces/{workspaceSlug}/runs/{runId}/redactions/", - post_with(redact_pipeline_run, redact_pipeline_run_docs), - ) - .with_path_items(|item| item.tag("Pipeline Runs")) -} diff --git a/crates/nvisy-server/src/handler/pipelines.rs b/crates/nvisy-server/src/handler/pipelines.rs index bd5d1794..5c7d26a7 100644 --- a/crates/nvisy-server/src/handler/pipelines.rs +++ b/crates/nvisy-server/src/handler/pipelines.rs @@ -261,6 +261,9 @@ async fn update_pipeline( let backfill = retention_override.map(|over| { let workspace_retention = workspace.settings.or_default().retention; let now = jiff::Timestamp::now(); + let audit_logs_expiry = workspace_retention + .resolve(RetentionScope::AuditLogs, Some(&over)) + .expires_at(now); [ ( FileKind::Redacted, @@ -268,12 +271,9 @@ async fn update_pipeline( .resolve(RetentionScope::RedactedDocuments, Some(&over)) .expires_at(now), ), - ( - FileKind::Audit, - workspace_retention - .resolve(RetentionScope::AuditLogs, Some(&over)) - .expires_at(now), - ), + (FileKind::Audit, audit_logs_expiry), + // Review audits share the audit-logs scope with detection audits. + (FileKind::Review, audit_logs_expiry), ] }); diff --git a/crates/nvisy-server/src/handler/redactions.rs b/crates/nvisy-server/src/handler/redactions.rs new file mode 100644 index 00000000..41880e4a --- /dev/null +++ b/crates/nvisy-server/src/handler/redactions.rs @@ -0,0 +1,170 @@ +//! Redaction handlers: list a detection's redactions and read a redaction's +//! review audit. +//! +//! A redaction is produced by `POST /detections/{detectionId}/redactions/` (in +//! [`detections`](super::detections)); these endpoints read them back. + +use aide::axum::ApiRouter; +use aide::axum::routing::get_with; +use aide::transform::TransformOperation; +use axum::extract::State; +use axum::http::StatusCode; +use elide_pipeline::Audit; +use nvisy_postgres::query::WorkspaceRedactionRepository; +use nvisy_postgres::{PgClient, PgConn}; +use uuid::Uuid; + +use super::detections::find_detection; +use crate::extract::{AuthProvider, AuthState, Json, Path, Permission, Query, WorkspaceContext}; +use crate::handler::request::{CursorPagination, DetectionPathParams, RedactionPathParams}; +use crate::handler::response::{ErrorResponse, RedactionResult, RedactionsPage}; +use crate::handler::utility::resolve_account_ref; +use crate::handler::{ErrorKind, Result, ServiceState}; +use crate::service::{EngineService, RunBlobStore}; + +/// Tracing target for redaction operations. +const TRACING_TARGET: &str = "nvisy_server::handler::redactions"; + +/// Lists a detection's redactions, most recent first, cursor-paginated. +#[tracing::instrument( + skip_all, + fields( + account_id = %auth_state.account_id, + workspace_id = %workspace.id, + detection_id = %path_params.detection_id, + ) +)] +async fn list_detection_redactions( + State(pg_client): State, + AuthState(auth_state): AuthState, + WorkspaceContext(workspace): WorkspaceContext, + Path(path_params): Path, + Query(pagination): Query, +) -> Result<(StatusCode, Json)> { + tracing::debug!(target: TRACING_TARGET, "Listing detection redactions"); + + let mut conn = pg_client.get_connection().await?; + + auth_state + .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) + .await?; + + // Confirm the detection exists in this workspace (404 otherwise) before + // listing its redactions. + let (detection, _pipeline) = + find_detection(&mut conn, workspace.id, path_params.detection_id.as_uuid()).await?; + + let page = conn + .cursor_list_detection_redactions(detection.id, pagination.into()) + .await?; + + // Resolve the requesting account per row. A detection's redactions are few + // (one per manual redact request), so a per-row lookup is acceptable here. + let mut items = Vec::with_capacity(page.items.len()); + for redaction in page.items { + let requested_by = resolve_account_ref(&mut conn, redaction.account_id).await?; + items.push(RedactionResult::from_model( + redaction, + workspace.slug.clone(), + requested_by, + )); + } + let response = RedactionsPage::new(items, page.total, page.next_cursor); + + Ok((StatusCode::OK, Json(response))) +} + +fn list_detection_redactions_docs(op: TransformOperation) -> TransformOperation { + op.summary("List detection redactions") + .description( + "Returns a detection's redactions, most recent first, cursor-paginated. Each \ + redaction is one redact pass with its own reviewer edits, output document, and \ + review audit.", + ) + .response::<200, Json>() + .response::<401, Json>() + .response::<403, Json>() + .response::<404, Json>() +} + +/// Returns a redaction's review audit: the analysis with the reviewer's edits +/// applied and the per-entity redaction outcome recorded. +#[tracing::instrument( + skip_all, + fields( + account_id = %auth_state.account_id, + workspace_id = %workspace.id, + redaction_id = %path_params.redaction_id, + ) +)] +async fn get_redaction_review( + State(pg_client): State, + State(blob): State, + State(engine): State, + AuthState(auth_state): AuthState, + WorkspaceContext(workspace): WorkspaceContext, + Path(path_params): Path, +) -> Result<(StatusCode, Json)> { + tracing::debug!(target: TRACING_TARGET, "Getting redaction review audit"); + + let mut conn = pg_client.get_connection().await?; + + auth_state + .authorize_workspace(&mut conn, workspace.id, Permission::ViewPipelines) + .await?; + + let redaction = + find_redaction(&mut conn, workspace.id, path_params.redaction_id.as_uuid()).await?; + + let review = blob + .load_review_audit(&mut conn, &engine, workspace.id, redaction.review_file_id) + .await?; + + Ok((StatusCode::OK, Json(review))) +} + +fn get_redaction_review_docs(op: TransformOperation) -> TransformOperation { + op.summary("Get redaction review") + .description( + "Returns the redaction's review audit: the detection analysis with the reviewer's \ + edits applied and the per-entity redaction outcome recorded.", + ) + .response::<200, Json>() + .response::<401, Json>() + .response::<403, Json>() + .response::<404, Json>() + .response::<409, Json>() +} + +/// Loads a redaction scoped to the workspace, mapping a missing one to a 404. +/// +/// A [`RedactionId`](nvisy_postgres::types::RedactionId) is globally unique, so +/// the redaction is addressed by id alone and resolved within the workspace via +/// its detection's pipeline. +async fn find_redaction( + conn: &mut PgConn, + workspace_id: Uuid, + redaction_id: Uuid, +) -> Result { + conn.find_redaction_in_workspace(workspace_id, redaction_id) + .await? + .ok_or_else(|| { + ErrorKind::NotFound + .with_message("Redaction not found") + .with_resource("redaction") + }) +} + +/// Builds the redaction routes. +pub fn routes() -> ApiRouter { + ApiRouter::new() + .api_route( + "/workspaces/{workspaceSlug}/detections/{detectionId}/redactions/", + get_with(list_detection_redactions, list_detection_redactions_docs), + ) + .api_route( + "/workspaces/{workspaceSlug}/redactions/{redactionId}/review", + get_with(get_redaction_review, get_redaction_review_docs), + ) + .with_path_items(|item| item.tag("Redactions")) +} diff --git a/crates/nvisy-server/src/handler/request/detections.rs b/crates/nvisy-server/src/handler/request/detections.rs new file mode 100644 index 00000000..bd18a52e --- /dev/null +++ b/crates/nvisy-server/src/handler/request/detections.rs @@ -0,0 +1,102 @@ +//! Detection request types (detect and redact). + +use elide_pipeline::DocumentContext; +use elide_pipeline::entity::EditSet; +use nvisy_postgres::types::{DetectionFilter, DetectionStatus, PipelineTriggerType}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; +use validator::Validate; + +/// Query parameters for listing detections across a workspace. +/// +/// Every field is an optional filter; unset fields impose no constraint. +#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct WorkspaceDetectionsQuery { + /// Filter by detection status. + pub status: Option, + /// Filter by the source file the detection analyzes. + pub file_id: Option, + /// Filter by the owning pipeline. + pub pipeline_id: Option, + /// Filter by the account that triggered the detection. + pub triggered_by: Option, + /// Filter by how the detection was initiated (user vs system). + pub trigger_type: Option, +} + +impl From for DetectionFilter { + fn from(query: WorkspaceDetectionsQuery) -> Self { + DetectionFilter { + status: query.status, + input_file_id: query.file_id, + pipeline_id: query.pipeline_id, + account_id: query.triggered_by, + trigger_type: query.trigger_type, + } + } +} + +/// Query parameters for listing a single pipeline's detections. +/// +/// The pipeline is fixed by the route, so it narrows only by status, file, +/// trigger account, and trigger type. +#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct PipelineDetectionsQuery { + /// Filter by detection status. + pub status: Option, + /// Filter by the source file the detection analyzes. + pub file_id: Option, + /// Filter by the account that triggered the detection. + pub triggered_by: Option, + /// Filter by how the detection was initiated (user vs system). + pub trigger_type: Option, +} + +impl From for DetectionFilter { + fn from(query: PipelineDetectionsQuery) -> Self { + DetectionFilter { + status: query.status, + input_file_id: query.file_id, + pipeline_id: None, + account_id: query.triggered_by, + trigger_type: query.trigger_type, + } + } +} + +/// Request payload to start a detection over a file. +/// +/// Analyzes the file with the pipeline's configuration and returns the +/// detection, which holds the findings for review before redaction. +#[must_use] +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Validate)] +#[serde(rename_all = "camelCase")] +pub struct CreateDetection { + /// The file to analyze. + pub file_id: Uuid, + /// Per-document scope (languages, jurisdictions, document labels). + /// + /// Overrides the pipeline's `defaultScope` when present; absent falls back to + /// the pipeline default. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scope: Option, +} + +/// Request payload to redact a detection. +/// +/// The reviewer's edits layer over the detection's analysis before redaction: +/// suppress a false positive, retag a detection, or add one the analysis missed. +/// Omit `edits` to redact with the policy decisions exactly as detected. Each +/// redact request produces a new redaction. +#[must_use] +#[derive(Debug, Clone, Default, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct RedactDetection { + /// Reviewer edits to apply before redaction, grouped by modality. Omit to + /// redact with the policy decisions exactly as detected. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub edits: Option, +} diff --git a/crates/nvisy-server/src/handler/request/mod.rs b/crates/nvisy-server/src/handler/request/mod.rs index 8a4e16cd..b9e63363 100644 --- a/crates/nvisy-server/src/handler/request/mod.rs +++ b/crates/nvisy-server/src/handler/request/mod.rs @@ -6,13 +6,13 @@ mod authentications; mod chat; mod connection_syncs; mod connections; +mod detections; mod exports; mod files; mod invites; mod members; mod paginations; mod paths; -mod pipeline_runs; mod pipelines; mod policies; mod tokens; @@ -27,13 +27,13 @@ pub use authentications::*; pub use chat::*; pub use connection_syncs::*; pub use connections::*; +pub use detections::*; pub use exports::*; pub use files::*; pub use invites::*; pub use members::*; pub use paginations::*; pub use paths::*; -pub use pipeline_runs::*; pub use pipelines::*; pub use policies::*; pub use tokens::*; diff --git a/crates/nvisy-server/src/handler/request/paths.rs b/crates/nvisy-server/src/handler/request/paths.rs index ed49cc43..5c0f70a3 100644 --- a/crates/nvisy-server/src/handler/request/paths.rs +++ b/crates/nvisy-server/src/handler/request/paths.rs @@ -1,6 +1,6 @@ //! Path parameter types for HTTP handlers. -use nvisy_postgres::types::{Handle, RunId, WebhookId}; +use nvisy_postgres::types::{DetectionId, Handle, RedactionId, WebhookId}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -83,13 +83,25 @@ pub struct PipelinePathParams { pub pipeline_slug: String, } -/// Path parameters for pipeline run operations. +/// Path parameters for detection operations. #[must_use] #[derive(Debug, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] -pub struct PipelineRunPathParams { - /// Opaque identifier of the run. - pub run_id: RunId, +pub struct DetectionPathParams { + /// Opaque identifier of the detection. + pub detection_id: DetectionId, +} + +/// Path parameters for a redaction. +/// +/// The redaction id is globally unique, so a redaction is addressed by id alone +/// and resolved within the workspace by the query. +#[must_use] +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct RedactionPathParams { + /// Opaque identifier of the redaction. + pub redaction_id: RedactionId, } /// Path parameters for notification operations. diff --git a/crates/nvisy-server/src/handler/request/pipeline_runs.rs b/crates/nvisy-server/src/handler/request/pipeline_runs.rs deleted file mode 100644 index ec823bc2..00000000 --- a/crates/nvisy-server/src/handler/request/pipeline_runs.rs +++ /dev/null @@ -1,85 +0,0 @@ -//! Pipeline run request types (detect). - -use elide_pipeline::DocumentContext; -use nvisy_postgres::types::{PipelineRunStatus, PipelineTriggerType, RunFilter}; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use uuid::Uuid; -use validator::Validate; - -/// Query parameters for listing runs across a workspace. -/// -/// Every field is an optional filter; unset fields impose no constraint. -#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub struct WorkspaceRunsQuery { - /// Filter by run status. - pub status: Option, - /// Filter by the source file the run analyzes. - pub file_id: Option, - /// Filter by the owning pipeline. - pub pipeline_id: Option, - /// Filter by the account that triggered the run. - pub triggered_by: Option, - /// Filter by how the run was initiated (user vs system). - pub trigger_type: Option, -} - -impl From for RunFilter { - fn from(query: WorkspaceRunsQuery) -> Self { - RunFilter { - status: query.status, - input_file_id: query.file_id, - pipeline_id: query.pipeline_id, - account_id: query.triggered_by, - trigger_type: query.trigger_type, - } - } -} - -/// Query parameters for listing a single pipeline's runs. -/// -/// The pipeline is fixed by the route, so it narrows only by status, file, -/// trigger account, and trigger type. -#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub struct PipelineRunsQuery { - /// Filter by run status. - pub status: Option, - /// Filter by the source file the run analyzes. - pub file_id: Option, - /// Filter by the account that triggered the run. - pub triggered_by: Option, - /// Filter by how the run was initiated (user vs system). - pub trigger_type: Option, -} - -impl From for RunFilter { - fn from(query: PipelineRunsQuery) -> Self { - RunFilter { - status: query.status, - input_file_id: query.file_id, - pipeline_id: None, - account_id: query.triggered_by, - trigger_type: query.trigger_type, - } - } -} - -/// Request payload to start a run (detect) over a file. -/// -/// Analyzes the file with the pipeline's configuration and returns the run, -/// which holds the findings for review before redaction. -#[must_use] -#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Validate)] -#[serde(rename_all = "camelCase")] -pub struct CreatePipelineRun { - /// The file to analyze. - pub file_id: Uuid, - /// Per-document scope (languages, jurisdictions, document labels). - /// - /// Overrides the pipeline's `defaultScope` when present; absent falls back to - /// the pipeline default. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub scope: Option, -} diff --git a/crates/nvisy-server/src/handler/response/analytics.rs b/crates/nvisy-server/src/handler/response/analytics.rs index 4b481901..5280e90f 100644 --- a/crates/nvisy-server/src/handler/response/analytics.rs +++ b/crates/nvisy-server/src/handler/response/analytics.rs @@ -8,26 +8,26 @@ use std::collections::BTreeMap; use jiff::ToSpan; use jiff::civil::Date; -use nvisy_postgres::query::{AnalyticsSnapshot, RunDayPoint}; -use nvisy_postgres::types::{FileKind, PipelineRunStatus}; +use nvisy_postgres::query::{AnalyticsSnapshot, DetectionDayPoint}; +use nvisy_postgres::types::{DetectionStatus, FileKind}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use strum::IntoEnumIterator; -/// Aggregate analytics for a workspace: what it stores, how its runs fare, and +/// Aggregate analytics for a workspace: what it stores, how its detections fare, and /// the inference tokens they spent. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct WorkspaceAnalytics { /// Stored-file totals and their per-kind breakdown. pub storage: StorageAnalytics, - /// Pipeline-run health: volume, status mix, and durations. - pub runs: RunAnalytics, + /// Detection health: volume, status mix, and durations. + pub detections: DetectionAnalytics, /// Inference token usage: workspace totals and a per-model breakdown. pub usage: UsageAnalytics, } -/// Inference token usage across a workspace's runs. +/// Inference token usage across a workspace's detections. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct UsageAnalytics { @@ -42,7 +42,7 @@ pub struct UsageAnalytics { pub by_model: Vec, } -/// One model's token usage across a workspace's runs. +/// One model's token usage across a workspace's detections. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct ModelUsageEntry { @@ -81,35 +81,36 @@ pub struct StorageKindEntry { pub total_bytes: i64, } -/// Pipeline-run health for a workspace. +/// Detection health for a workspace. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] -pub struct RunAnalytics { - /// Total number of runs. +pub struct DetectionAnalytics { + /// Total number of detections. pub total: i64, - /// Per-status breakdown, one entry per run status (zero-filled), in a stable - /// order. - pub by_status: Vec, - /// Failed / (completed + failed). Omitted when no run has reached a terminal - /// state (genuinely no signal, not zero). + /// Per-status breakdown, one entry per detection status (zero-filled), in a + /// stable order. + pub by_status: Vec, + /// Failed / (completed + failed). Omitted when no detection has reached a + /// terminal state (genuinely no signal, not zero). #[serde(skip_serializing_if = "Option::is_none")] pub error_rate: Option, - /// Mean completed-run duration in milliseconds; omitted until a run completes. + /// Mean completed-detection duration in milliseconds; omitted until a + /// detection completes. #[serde(skip_serializing_if = "Option::is_none")] pub avg_duration_ms: Option, - /// 95th-percentile completed-run duration in milliseconds; omitted until a run - /// completes. + /// 95th-percentile completed-detection duration in milliseconds; omitted until + /// a detection completes. #[serde(skip_serializing_if = "Option::is_none")] pub p95_duration_ms: Option, } -/// One status's share of a workspace's runs. +/// One status's share of a workspace's detections. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] -pub struct RunStatusEntry { - /// The run status. - pub status: PipelineRunStatus, - /// Number of runs in this status. +pub struct DetectionStatusEntry { + /// The detection status. + pub status: DetectionStatus, + /// Number of detections in this status. pub count: i64, } @@ -119,7 +120,7 @@ impl WorkspaceAnalytics { pub fn from_snapshot(snapshot: AnalyticsSnapshot) -> Self { let AnalyticsSnapshot { storage, - runs, + detections, durations, usage, } = snapshot; @@ -136,10 +137,10 @@ impl WorkspaceAnalytics { let total_bytes = by_kind.iter().map(|e| e.total_bytes).sum(); let file_count = by_kind.iter().map(|e| e.file_count).sum(); - let by_status: Vec = PipelineRunStatus::iter() - .map(|status| RunStatusEntry { + let by_status: Vec = DetectionStatus::iter() + .map(|status| DetectionStatusEntry { status, - count: runs + count: detections .iter() .find(|r| r.status == status) .map_or(0, |r| r.count), @@ -147,8 +148,8 @@ impl WorkspaceAnalytics { .collect(); let total = by_status.iter().map(|e| e.count).sum(); - let completed = count_of(&by_status, PipelineRunStatus::Completed); - let failed = count_of(&by_status, PipelineRunStatus::Failed); + let completed = count_of(&by_status, DetectionStatus::Complete); + let failed = count_of(&by_status, DetectionStatus::Failed); let terminal = completed + failed; let error_rate = (terminal > 0).then(|| failed as f64 / terminal as f64); @@ -174,7 +175,7 @@ impl WorkspaceAnalytics { file_count, by_kind, }, - runs: RunAnalytics { + detections: DetectionAnalytics { total, by_status, error_rate, @@ -186,34 +187,35 @@ impl WorkspaceAnalytics { } } -/// A workspace's daily run activity over a window: one point per day, dense -/// (quiet days included with `runs: 0`), ready to plot as a continuous series. +/// A workspace's daily detection activity over a window: one point per day, dense +/// (quiet days included with `detections: 0`), ready to plot as a continuous +/// series. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] -pub struct RunTimeSeries { +pub struct DetectionTimeSeries { /// One entry per day in the requested window, oldest first. - pub points: Vec, + pub points: Vec, } -/// A single day of run activity. +/// A single day of detection activity. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] -pub struct RunDayEntry { +pub struct DetectionDayEntry { /// The day (`YYYY-MM-DD`, UTC). pub date: Date, - /// Runs started this day (`0` on a quiet day). - pub runs: i64, - /// Failed / (completed + failed) for this day; omitted when no run reached a - /// terminal state that day. + /// Detections started this day (`0` on a quiet day). + pub detections: i64, + /// Failed / (completed + failed) for this day; omitted when no detection + /// reached a terminal state that day. #[serde(skip_serializing_if = "Option::is_none")] pub error_rate: Option, - /// Mean completed-run duration (milliseconds) this day; omitted if none completed. + /// Mean completed-detection duration (milliseconds) this day; omitted if none completed. #[serde(skip_serializing_if = "Option::is_none")] pub avg_duration_ms: Option, - /// 95th-percentile completed-run duration (milliseconds) this day; omitted if none. + /// 95th-percentile completed-detection duration (milliseconds) this day; omitted if none. #[serde(skip_serializing_if = "Option::is_none")] pub p95_duration_ms: Option, - /// Input/prompt tokens spent by this day's runs; omitted when none used a model. + /// Input/prompt tokens spent by this day's detections; omitted when none used a model. #[serde(skip_serializing_if = "Option::is_none")] pub input_tokens: Option, /// Output/completion tokens spent this day; omitted when none used a model. @@ -224,15 +226,16 @@ pub struct RunDayEntry { pub total_tokens: Option, } -impl RunTimeSeries { +impl DetectionTimeSeries { /// Builds a dense daily series over `[from, to]` from the sparse per-day rows /// the query returns. Every day in the window is emitted in order; a day with - /// no runs reports `runs: 0` and omits the rate/duration/token fields. The - /// query only produces days that had a run, so gap-filling happens here. - pub fn from_window(from: Date, to: Date, points: Vec) -> Self { + /// no detections reports `detections: 0` and omits the rate/duration/token + /// fields. The query only produces days that had a detection, so gap-filling + /// happens here. + pub fn from_window(from: Date, to: Date, points: Vec) -> Self { // Index the sparse rows by their UTC day for O(1) lookup while walking the // window. - let mut by_day: BTreeMap = BTreeMap::new(); + let mut by_day: BTreeMap = BTreeMap::new(); for p in points { let date = p.day.to_zoned(jiff::tz::TimeZone::UTC).date(); by_day.insert(date, p); @@ -242,9 +245,9 @@ impl RunTimeSeries { let mut date = from; while date <= to { days.push(match by_day.remove(&date) { - Some(p) => RunDayEntry { + Some(p) => DetectionDayEntry { date, - runs: p.runs, + detections: p.detections, error_rate: (p.terminal > 0).then(|| p.failed as f64 / p.terminal as f64), avg_duration_ms: p.avg_ms, p95_duration_ms: p.p95_ms, @@ -252,9 +255,9 @@ impl RunTimeSeries { output_tokens: p.output_tokens, total_tokens: p.total_tokens, }, - None => RunDayEntry { + None => DetectionDayEntry { date, - runs: 0, + detections: 0, error_rate: None, avg_duration_ms: None, p95_duration_ms: None, @@ -274,7 +277,7 @@ impl RunTimeSeries { } /// Looks up a status's count in the assembled breakdown. -fn count_of(by_status: &[RunStatusEntry], status: PipelineRunStatus) -> i64 { +fn count_of(by_status: &[DetectionStatusEntry], status: DetectionStatus) -> i64 { by_status .iter() .find(|e| e.status == status) @@ -283,7 +286,9 @@ fn count_of(by_status: &[RunStatusEntry], status: PipelineRunStatus) -> i64 { #[cfg(test)] mod tests { - use nvisy_postgres::query::{RunDurations, RunStatusCount, StorageByKind, UsageByModel}; + use nvisy_postgres::query::{ + DetectionDurations, DetectionStatusCount, StorageByKind, UsageByModel, + }; use super::*; @@ -301,17 +306,17 @@ mod tests { total_bytes: 50, }, ]; - let runs = vec![ - RunStatusCount { - status: PipelineRunStatus::Completed, + let detections = vec![ + DetectionStatusCount { + status: DetectionStatus::Complete, count: 3, }, - RunStatusCount { - status: PipelineRunStatus::Failed, + DetectionStatusCount { + status: DetectionStatus::Failed, count: 1, }, ]; - let durations = RunDurations { + let durations = DetectionDurations { avg_ms: Some(30_000), p95_ms: Some(56_000), }; @@ -332,15 +337,18 @@ mod tests { ]; let a = WorkspaceAnalytics::from_snapshot(AnalyticsSnapshot { storage, - runs, + detections, durations, usage, }); - // Every FileKind / PipelineRunStatus is present (zero-filled), so the + // Every FileKind / DetectionStatus is present (zero-filled), so the // breakdown lengths equal the enum sizes. assert_eq!(a.storage.by_kind.len(), FileKind::iter().count()); - assert_eq!(a.runs.by_status.len(), PipelineRunStatus::iter().count()); + assert_eq!( + a.detections.by_status.len(), + DetectionStatus::iter().count() + ); // Absent audit kind is zero, not missing. let audit = a .storage @@ -353,11 +361,11 @@ mod tests { // Totals sum the breakdown. assert_eq!(a.storage.total_bytes, 350); assert_eq!(a.storage.file_count, 3); - assert_eq!(a.runs.total, 4); + assert_eq!(a.detections.total, 4); // error_rate = failed / (completed + failed) = 1 / 4. - assert_eq!(a.runs.error_rate, Some(0.25)); - assert_eq!(a.runs.avg_duration_ms, Some(30_000)); + assert_eq!(a.detections.error_rate, Some(0.25)); + assert_eq!(a.detections.avg_duration_ms, Some(30_000)); // Usage: per-model entries preserved, workspace totals summed with // never-reported fields treated as 0 (not conflated across fields). @@ -377,10 +385,10 @@ mod tests { .timestamp() }; // Only the active day is returned by the query (sparse); Jan 6 and 7 have - // no runs and must be gap-filled by from_window over the Jan 5..7 window. - let sparse = vec![RunDayPoint { + // no detections and must be gap-filled by from_window over the Jan 5..7 window. + let sparse = vec![DetectionDayPoint { day: day_ts("2026-01-05"), - runs: 3, + detections: 3, terminal: 3, failed: 1, avg_ms: Some(20_000), @@ -390,7 +398,8 @@ mod tests { total_tokens: None, }]; - let series = RunTimeSeries::from_window(date("2026-01-05"), date("2026-01-07"), sparse); + let series = + DetectionTimeSeries::from_window(date("2026-01-05"), date("2026-01-07"), sparse); // Dense: every day in the window present, in order. assert_eq!(series.points.len(), 3); @@ -399,42 +408,42 @@ mod tests { // Active day: derived error rate + tokens carried through. let d5 = &series.points[0]; - assert_eq!(d5.runs, 3); + assert_eq!(d5.detections, 3); assert_eq!(d5.error_rate, Some(1.0 / 3.0)); assert_eq!(d5.avg_duration_ms, Some(20_000)); assert_eq!(d5.input_tokens, Some(1000)); assert_eq!(d5.total_tokens, None); - // Gap-filled days: runs 0, everything else omitted. + // Gap-filled days: detections 0, everything else omitted. let d6 = &series.points[1]; - assert_eq!(d6.runs, 0); + assert_eq!(d6.detections, 0); assert_eq!(d6.error_rate, None); assert_eq!(d6.avg_duration_ms, None); assert_eq!(d6.input_tokens, None); } #[test] - fn error_rate_is_none_with_no_terminal_runs() { - // Only active runs, no files, no completed durations. - let runs = vec![RunStatusCount { - status: PipelineRunStatus::Queued, + fn error_rate_is_none_with_no_terminal_detections() { + // Only in-progress detections, no files, no completed durations. + let detections = vec![DetectionStatusCount { + status: DetectionStatus::Pending, count: 5, }]; let a = WorkspaceAnalytics::from_snapshot(AnalyticsSnapshot { storage: vec![], - runs, - durations: RunDurations { + detections, + durations: DetectionDurations { avg_ms: None, p95_ms: None, }, usage: Vec::new(), }); - assert_eq!(a.runs.error_rate, None); - assert_eq!(a.runs.avg_duration_ms, None); + assert_eq!(a.detections.error_rate, None); + assert_eq!(a.detections.avg_duration_ms, None); assert_eq!(a.storage.total_bytes, 0); // Still every kind/status present, all zero except queued. assert_eq!(a.storage.by_kind.len(), FileKind::iter().count()); - assert_eq!(a.runs.total, 5); + assert_eq!(a.detections.total, 5); } } diff --git a/crates/nvisy-server/src/handler/response/detections.rs b/crates/nvisy-server/src/handler/response/detections.rs new file mode 100644 index 00000000..8b45da4d --- /dev/null +++ b/crates/nvisy-server/src/handler/response/detections.rs @@ -0,0 +1,91 @@ +//! Detection response types. + +use jiff::Timestamp; +use nvisy_postgres::model::WorkspaceDetection as DetectionModel; +use nvisy_postgres::query::DetectionFiles; +use nvisy_postgres::types::{ + DetectionId, DetectionMetadata, DetectionStatus, Handle, PipelineTriggerType, +}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use super::{AccountRef, Page}; + +/// Response type for a detection. +/// +/// A detection is addressed by its own opaque id; the owning pipeline and +/// workspace slugs are carried for context. Redacted outputs are not here — a +/// detection produces many redactions, each fetched from its `redactions` +/// endpoint. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct Detection { + /// Opaque identifier of the detection. + pub id: DetectionId, + /// Handle of the pipeline this detection belongs to. + pub pipeline_slug: Handle, + /// Handle of the workspace this detection belongs to. + pub workspace_slug: Handle, + /// Source document this detection analyzes. + pub input_file_id: Uuid, + /// Display name of the source document, for showing the detection without a + /// separate file lookup. `None` if the file was removed (e.g. by retention). + #[serde(skip_serializing_if = "Option::is_none")] + pub input_file_name: Option, + /// Account that triggered the detection. + pub triggered_by: AccountRef, + /// How the detection was triggered. + pub trigger_type: PipelineTriggerType, + /// Current detection status. + /// + /// The detections are available to fetch from the detection's `analysis` + /// endpoint once this reaches `complete`. + pub status: DetectionStatus, + /// Human-readable failure reason, present only when the detection `failed`. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// Non-encrypted metadata for filtering/display. + pub metadata: DetectionMetadata, + /// When the detection started. + pub started_at: Timestamp, + /// When the detection completed analysis. + #[serde(skip_serializing_if = "Option::is_none")] + pub completed_at: Option, +} + +/// Paginated response for detections. +pub type DetectionsPage = Page; + +impl Detection { + /// Creates a detection response from the database model, the slugs of its + /// owning pipeline and workspace, the triggering account, and the resolved + /// input file display name. + pub fn from_model( + detection: DetectionModel, + pipeline_slug: Handle, + workspace_slug: Handle, + triggered_by: AccountRef, + files: DetectionFiles, + ) -> Self { + // Surface the failure reason (written to metadata.error by the worker / + // enqueue-failure path) as a dedicated field for a failed detection. + let metadata = detection.metadata.or_default(); + let error = metadata.error.clone(); + + Self { + id: DetectionId::from_uuid(detection.id), + pipeline_slug, + workspace_slug, + input_file_id: detection.input_file_id, + input_file_name: files.input, + triggered_by, + trigger_type: detection.trigger_type, + status: detection.status, + error, + metadata, + started_at: detection.started_at.into(), + completed_at: detection.completed_at.map(Into::into), + } + } +} diff --git a/crates/nvisy-server/src/handler/response/mod.rs b/crates/nvisy-server/src/handler/response/mod.rs index 30f9c83b..053e8729 100644 --- a/crates/nvisy-server/src/handler/response/mod.rs +++ b/crates/nvisy-server/src/handler/response/mod.rs @@ -13,15 +13,16 @@ mod catalog; mod chat; mod connection_syncs; mod connections; +mod detections; mod errors; mod files; mod invites; mod members; mod monitors; mod notifications; -mod pipeline_runs; mod pipelines; mod policies; +mod redactions; mod tokens; mod webhooks; mod workspaces; @@ -35,15 +36,16 @@ pub use catalog::*; pub use chat::*; pub use connection_syncs::*; pub use connections::*; +pub use detections::*; pub use errors::*; pub use files::*; pub use invites::*; pub use members::*; pub use monitors::*; pub use notifications::*; -pub use pipeline_runs::*; pub use pipelines::*; pub use policies::*; +pub use redactions::*; pub use tokens::*; pub use webhooks::*; pub use workspaces::*; diff --git a/crates/nvisy-server/src/handler/response/pipeline_runs.rs b/crates/nvisy-server/src/handler/response/pipeline_runs.rs deleted file mode 100644 index 8e591867..00000000 --- a/crates/nvisy-server/src/handler/response/pipeline_runs.rs +++ /dev/null @@ -1,95 +0,0 @@ -//! Pipeline run response types. - -use jiff::Timestamp; -use nvisy_postgres::model::WorkspacePipelineRun as PipelineRunModel; -use nvisy_postgres::query::RunFiles; -use nvisy_postgres::types::{Handle, PipelineRunStatus, PipelineTriggerType, RunId, RunMetadata}; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use uuid::Uuid; - -use super::{AccountRef, Page}; - -/// Response type for a pipeline run. -/// -/// A run is addressed by its own opaque id; the owning pipeline and workspace -/// slugs are carried for context. -#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub struct PipelineRun { - /// Opaque identifier of the run. - pub id: RunId, - /// Handle of the pipeline this run belongs to. - pub pipeline_slug: Handle, - /// Handle of the workspace this run belongs to. - pub workspace_slug: Handle, - /// Source document this run analyzes / redacts. - pub input_file_id: Uuid, - /// Display name of the source document, for showing the run without a - /// separate file lookup. `None` if the file was removed (e.g. by retention). - #[serde(skip_serializing_if = "Option::is_none")] - pub input_file_name: Option, - /// Redacted document produced by the run, once it completes. - #[serde(skip_serializing_if = "Option::is_none")] - pub output_file_id: Option, - /// Display name of the redacted output, once the run completes. - #[serde(skip_serializing_if = "Option::is_none")] - pub output_file_name: Option, - /// Account that triggered the run. - pub triggered_by: AccountRef, - /// How the run was triggered. - pub trigger_type: PipelineTriggerType, - /// Current run status. - /// - /// The detections are available to fetch from the run's `detections` - /// endpoint once this reaches `analyzed`. - pub status: PipelineRunStatus, - /// Human-readable failure reason, present only when the run `failed`. - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, - /// Non-encrypted metadata for filtering/display. - pub metadata: RunMetadata, - /// When the run started. - pub started_at: Timestamp, - /// When the run completed. - #[serde(skip_serializing_if = "Option::is_none")] - pub completed_at: Option, -} - -/// Paginated response for pipeline runs. -pub type PipelineRunsPage = Page; - -impl PipelineRun { - /// Creates a pipeline run response from the database model, the slugs of its - /// owning pipeline and workspace, the triggering account, and the resolved - /// input/output file display names. - pub fn from_model( - run: PipelineRunModel, - pipeline_slug: Handle, - workspace_slug: Handle, - triggered_by: AccountRef, - files: RunFiles, - ) -> Self { - // Surface the failure reason (written to metadata.error by the worker / - // enqueue-failure path) as a dedicated field for a failed run. - let metadata = run.metadata.or_default(); - let error = metadata.error.clone(); - - Self { - id: RunId::from_uuid(run.id), - pipeline_slug, - workspace_slug, - input_file_id: run.input_file_id, - input_file_name: files.input, - output_file_id: run.output_file_id, - output_file_name: files.output, - triggered_by, - trigger_type: run.trigger_type, - status: run.status, - error, - metadata, - started_at: run.started_at.into(), - completed_at: run.completed_at.map(Into::into), - } - } -} diff --git a/crates/nvisy-server/src/handler/response/redactions.rs b/crates/nvisy-server/src/handler/response/redactions.rs new file mode 100644 index 00000000..be26d66a --- /dev/null +++ b/crates/nvisy-server/src/handler/response/redactions.rs @@ -0,0 +1,61 @@ +//! Redaction response types. + +use jiff::Timestamp; +use nvisy_postgres::model::WorkspaceRedaction as RedactionModel; +use nvisy_postgres::types::{DetectionId, Handle, RedactionId}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use super::{AccountRef, Page}; + +/// Response type for a redaction. +/// +/// A redaction is one redact pass over a detection, produced with a specific set +/// of reviewer edits. It owns the redacted output document (downloadable through +/// the normal file endpoints) and a review audit recording what was redacted and +/// why (fetched from the redaction's `review` endpoint). +/// +/// Named `RedactionResult` rather than `Redaction` because the engine's audit +/// schema already carries a `Redaction` (an audit event), and the two must not +/// collide in the generated OpenAPI. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct RedactionResult { + /// Opaque identifier of the redaction. + pub id: RedactionId, + /// The detection this redaction was produced from. + pub detection_id: DetectionId, + /// Handle of the workspace this redaction belongs to. + pub workspace_slug: Handle, + /// Redacted output document this redaction produced. `None` only if the file + /// was removed (e.g. by retention). + #[serde(skip_serializing_if = "Option::is_none")] + pub output_file_id: Option, + /// Account that requested the redaction. + pub requested_by: AccountRef, + /// When the redaction was created. + pub created_at: Timestamp, +} + +/// Paginated response for redactions. +pub type RedactionsPage = Page; + +impl RedactionResult { + /// Creates a redaction response from the database model, the owning + /// workspace slug, and the requesting account. + pub fn from_model( + redaction: RedactionModel, + workspace_slug: Handle, + requested_by: AccountRef, + ) -> Self { + Self { + id: RedactionId::from_uuid(redaction.id), + detection_id: DetectionId::from_uuid(redaction.detection_id), + workspace_slug, + output_file_id: redaction.output_file_id, + requested_by, + created_at: redaction.created_at.into(), + } + } +} diff --git a/crates/nvisy-server/src/handler/workspaces.rs b/crates/nvisy-server/src/handler/workspaces.rs index 2191f21d..9d207bc5 100644 --- a/crates/nvisy-server/src/handler/workspaces.rs +++ b/crates/nvisy-server/src/handler/workspaces.rs @@ -51,6 +51,9 @@ async fn backfill_retention( (RetentionScope::OriginalDocuments, FileKind::Original), (RetentionScope::RedactedDocuments, FileKind::Redacted), (RetentionScope::AuditLogs, FileKind::Audit), + // Review audits share the audit-logs scope with detection audits; a + // redaction stages them under `AuditLogs`, so they backfill under it too. + (RetentionScope::AuditLogs, FileKind::Review), ] { let expires_at = retention.get(scope).expires_at(now); conn.backfill_files_expiry(workspace_id, kind, expires_at) diff --git a/crates/nvisy-server/src/middleware/specification.rs b/crates/nvisy-server/src/middleware/specification.rs index e20b9fb8..2e58bf12 100644 --- a/crates/nvisy-server/src/middleware/specification.rs +++ b/crates/nvisy-server/src/middleware/specification.rs @@ -281,8 +281,13 @@ fn api_docs(api: TransformOpenApi) -> TransformOpenApi { ..Default::default() }) .tag(Tag { - name: "Pipeline Runs".into(), - description: Some("Pipeline run execution and review".into()), + name: "Detections".into(), + description: Some("Detection analysis and redaction".into()), + ..Default::default() + }) + .tag(Tag { + name: "Redactions".into(), + description: Some("Redactions produced from a detection, with reviewer edits".into()), ..Default::default() }) .tag(Tag { diff --git a/crates/nvisy-server/src/service/detection/drainer.rs b/crates/nvisy-server/src/service/detection/drainer.rs new file mode 100644 index 00000000..f096ec49 --- /dev/null +++ b/crates/nvisy-server/src/service/detection/drainer.rs @@ -0,0 +1,260 @@ +//! Detection-job outbox drainer. +//! +//! Publishes each pending detection-job outbox row onto the detection work-queue, +//! so an analysis queued transactionally with its detection reaches the worker +//! even if the process crashes between the commit and the publish. This is the +//! relay half of the transactional outbox: the create-detection handler writes +//! the row in the detection's transaction, and this drains it to NATS. + +use std::time::Duration; + +use nvisy_postgres::AsyncConnection; +use nvisy_postgres::model::{UpdateWorkspaceDetection, WorkspaceDetectionJob}; +use nvisy_postgres::query::{DetectionJobOutboxRepository, WorkspaceDetectionRepository}; +use nvisy_postgres::types::{DetectionMetadata, DetectionStatus, Json}; +use tokio_util::sync::CancellationToken; + +use super::job::DetectionJob; +use super::service::DetectionQueue; +use crate::handler::{Error, Result}; +use crate::service::{Infra, Worker}; + +/// Tracing target for the detection-job drainer. +const TRACING_TARGET: &str = "nvisy_server::service::detection::drainer"; + +/// How often the drainer polls for due jobs. Short, since it is the enqueue +/// latency between creating a detection and the worker picking it up. +const TICK_INTERVAL: Duration = Duration::from_secs(5); + +/// Maximum jobs drained per tick, bounding the work (and lock hold) per pass. +const DRAIN_BATCH: i64 = 100; + +/// Base unit of the retry backoff (seconds): a failed row's next attempt is +/// deferred by `RETRY_BACKOFF_BASE_SECS * attempts` (linear), capped at +/// [`RETRY_BACKOFF_MAX_SECS`]. +const RETRY_BACKOFF_BASE_SECS: i64 = 30; + +/// Ceiling on the retry backoff (seconds), so a long-failing row still retries +/// periodically rather than backing off unboundedly. +const RETRY_BACKOFF_MAX_SECS: i64 = 60 * 60; + +/// How many failed attempts a row gets before the drainer dead-letters it, so a +/// job that can never publish (e.g. an undecodable payload) stops consuming drain +/// cycles instead of retrying forever. +const MAX_ATTEMPTS: i32 = 10; + +/// The failure reason recorded on a detection whose job the drainer gave up +/// publishing, so the detection's terminal state explains why it never analyzed. +const DEAD_LETTER_REASON: &str = "Detection could not be queued for analysis"; + +/// Cap on a single publish, so a slow or unavailable NATS server cannot hold the +/// batch transaction's row locks open indefinitely. A publish that exceeds this is +/// treated as a failed attempt (deferred with a backoff), releasing the locks. +const PUBLISH_TIMEOUT: Duration = Duration::from_secs(5); + +/// Drains the detection-job outbox, publishing each pending job to the work-queue. +pub struct DetectionOutboxDrainer { + infra: Infra, + queue: DetectionQueue, +} + +/// The tally of one [`drain_batch`](DetectionOutboxDrainer::drain_batch) pass: of +/// the rows claimed, how many published, how many were deferred for a later retry, +/// and how many were dead-lettered. +struct DrainPass { + claimed: usize, + processed: usize, + deferred: usize, + dead_lettered: usize, +} + +impl Worker for DetectionOutboxDrainer { + type Output = Result<()>; + + fn name(&self) -> &'static str { + "detection_outbox_drainer" + } + + async fn run(&self, cancel: CancellationToken) -> Result<()> { + tracing::info!(target: TRACING_TARGET, "Starting detection-job drainer"); + + let mut ticker = tokio::time::interval(TICK_INTERVAL); + loop { + tokio::select! { + _ = cancel.cancelled() => break, + _ = ticker.tick() => self.tick(&cancel).await, + } + } + + tracing::info!(target: TRACING_TARGET, "Detection-job drainer stopped"); + Ok(()) + } +} + +impl DetectionOutboxDrainer { + /// Creates a new [`DetectionOutboxDrainer`]. + pub fn new(infra: Infra) -> Self { + Self { + queue: DetectionQueue::new(infra.clone()), + infra, + } + } + + /// One drain pass: claim and publish batches until a short page signals the + /// due set is drained, or until cancellation is requested. + /// + /// The cancellation check between batches keeps shutdown prompt even under a + /// sustained backlog. + async fn tick(&self, cancel: &CancellationToken) { + loop { + if cancel.is_cancelled() { + break; + } + match self.drain_batch().await { + Ok(pass) => { + if pass.deferred > 0 || pass.dead_lettered > 0 { + tracing::warn!( + target: TRACING_TARGET, + claimed = pass.claimed, + processed = pass.processed, + deferred = pass.deferred, + dead_lettered = pass.dead_lettered, + "Detection-job drain pass had failing jobs", + ); + } else if pass.claimed > 0 { + tracing::debug!(target: TRACING_TARGET, processed = pass.processed, "Detection-job drain pass published jobs"); + } + if pass.claimed < DRAIN_BATCH as usize { + break; + } + } + Err(err) => { + tracing::error!(target: TRACING_TARGET, error = %err, "Detection-job drain pass failed"); + break; + } + } + } + } + + /// Drains one batch: claims due rows and publishes each to the work-queue, all + /// in one transaction. Returns the [`DrainPass`] tally. + /// + /// The transaction holds the claim's `FOR UPDATE SKIP LOCKED` locks through + /// completion, so the claim and each row's state transition commit atomically + /// and no other drainer takes the same rows. The publish runs inside the + /// transaction and gates `mark_processed`: this is at-least-once (a crash after + /// publish but before commit re-publishes on the next pass), which the worker's + /// claim-based dedup absorbs — the same detection is only ever analyzed once. + async fn drain_batch(&self) -> Result { + let mut conn = self.infra.postgres.get_connection().await?; + + let outcome = conn + .transaction(async |conn| { + let batch = conn.claim_detection_job_batch(DRAIN_BATCH).await?; + let mut pass = DrainPass { + claimed: batch.len(), + processed: 0, + deferred: 0, + dead_lettered: 0, + }; + // Detections failed by a dead-lettered job, so their `Failed` + // status can be broadcast after the transaction commits. + let mut dead_lettered = Vec::new(); + + for row in batch { + match self.publish(&row).await { + Ok(()) => { + conn.mark_detection_job_processed(row.id).await?; + pass.processed += 1; + } + // `attempts` counts prior failures; this attempt makes it + // `attempts + 1`. Once that reaches the cap, dead-letter the + // row instead of deferring it forever. The job never + // published, so the worker will never drive its detection to + // a terminal state: fail the detection here too (same + // transaction) so it does not hang `Pending` forever. The + // guard on `Pending` makes it a no-op in the unlikely case a + // worker already claimed the detection. + Err(()) if row.attempts + 1 >= MAX_ATTEMPTS => { + tracing::error!(target: TRACING_TARGET, id = %row.id, detection_id = %row.detection_id, attempts = row.attempts + 1, "Dead-lettering detection job after too many failed attempts; failing the detection"); + conn.mark_detection_job_failed(row.id).await?; + let metadata = DetectionMetadata { + error: Some(DEAD_LETTER_REASON.to_owned()), + ..Default::default() + }; + // Only broadcast `Failed` if this actually transitioned + // the detection. A publish timeout does not prove the job + // never reached a worker, so a worker may already own the + // detection (no longer `Pending`); the guard then no-ops + // and we must not announce a false terminal status. + let failed = conn + .fail_pending_detection( + row.detection_id, + UpdateWorkspaceDetection { + metadata: Some(Json::encode(&metadata)), + ..Default::default() + }, + ) + .await?; + if failed { + dead_lettered.push(row.detection_id); + } + pass.dead_lettered += 1; + } + Err(()) => { + conn.defer_detection_job_attempt(row.id, retry_backoff(row.attempts)) + .await?; + pass.deferred += 1; + } + } + } + + Ok::<_, Error>((pass, dead_lettered)) + }) + .await?; + + // Announce the terminal status of each detection this pass actually failed + // (only those the guarded transition committed as `Failed` are collected), + // after its transaction commits. Best-effort: the detection row is + // authoritative, so a dropped broadcast is recoverable by a re-read. + let (pass, dead_lettered) = outcome; + for detection_id in dead_lettered { + self.queue + .broadcast_status(detection_id, DetectionStatus::Failed) + .await; + } + + Ok(pass) + } + + /// Decodes a row's job and publishes it to the work-queue. Returns `Err` if the + /// payload cannot decode or the publish fails, so the caller defers or + /// dead-letters it. + async fn publish(&self, row: &WorkspaceDetectionJob) -> std::result::Result<(), ()> { + let job = serde_json::from_value::(row.job.clone()).map_err(|err| { + tracing::error!(target: TRACING_TARGET, error = %err, id = %row.id, "Failed to decode detection job"); + })?; + // Bound the publish so a hung NATS cannot hold the batch transaction's + // locks open; a timeout is a failed attempt like any other. + match tokio::time::timeout(PUBLISH_TIMEOUT, self.queue.enqueue(job)).await { + Ok(Ok(())) => Ok(()), + Ok(Err(err)) => { + tracing::warn!(target: TRACING_TARGET, error = %err, id = %row.id, "Failed to publish detection job; deferring"); + Err(()) + } + Err(_elapsed) => { + tracing::warn!(target: TRACING_TARGET, id = %row.id, "Detection-job publish timed out; deferring"); + Err(()) + } + } + } +} + +/// The delay in seconds before a failed row's next attempt: linear in `attempts` +/// (the count before this failure), capped at [`RETRY_BACKOFF_MAX_SECS`]. +fn retry_backoff(attempts: i32) -> i64 { + let steps = i64::from(attempts.max(0)) + 1; + RETRY_BACKOFF_BASE_SECS + .saturating_mul(steps) + .min(RETRY_BACKOFF_MAX_SECS) +} diff --git a/crates/nvisy-server/src/service/detection/job.rs b/crates/nvisy-server/src/service/detection/job.rs index 5c03f5ef..a4ef570a 100644 --- a/crates/nvisy-server/src/service/detection/job.rs +++ b/crates/nvisy-server/src/service/detection/job.rs @@ -1,44 +1,45 @@ -//! Detection job and run-status event types. +//! Detection job and detection-status event types. use elide_pipeline::DocumentContext; -use nvisy_postgres::types::PipelineRunStatus; +use nvisy_postgres::types::DetectionStatus; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use uuid::Uuid; -/// A queued request to run detection for a pipeline run. +/// A queued request to run a detection. /// -/// Published to the `DetectionStream` work-queue by the create-run handler and -/// consumed by the [`DetectionWorker`](super::DetectionWorker). The worker -/// re-loads the run, pipeline, file, and policies from the ids; only the +/// Published to the `DetectionStream` work-queue by the create-detection handler +/// and consumed by the [`DetectionWorker`](super::DetectionWorker). The worker +/// re-loads the detection, pipeline, file, and policies from the ids; only the /// caller-supplied per-request scope, which is not otherwise persisted, travels /// on the job. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DetectionJob { - /// Workspace owning the run. + /// Workspace owning the detection. pub workspace_id: Uuid, - /// The run to analyze. - pub run_id: Uuid, + /// The detection to analyze. + pub detection_id: Uuid, /// Caller-supplied per-request scope override, if any. #[serde(default, skip_serializing_if = "Option::is_none")] pub scope: Option, } -/// A run's status change, broadcast on the core-NATS subject [`run_subject`]. +/// A detection's status change, broadcast on the core-NATS subject +/// [`detection_subject`]. /// -/// Fan-out to any watching SSE connections; the run row in Postgres remains the -/// source of truth, so a missed broadcast is recoverable by re-reading the run. +/// Fan-out to any watching SSE connections; the detection row in Postgres remains +/// the source of truth, so a missed broadcast is recoverable by re-reading it. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] -pub struct RunStatusEvent { - /// The run whose status changed. - pub run_id: Uuid, - /// The run's new status. - pub status: PipelineRunStatus, +pub struct DetectionStatusEvent { + /// The detection whose status changed. + pub detection_id: Uuid, + /// The detection's new status. + pub status: DetectionStatus, } -/// The core-NATS subject a run's status changes are broadcast on. +/// The core-NATS subject a detection's status changes are broadcast on. #[must_use] -pub fn run_subject(run_id: Uuid) -> String { - format!("pipeline.runs.{run_id}.status") +pub fn detection_subject(detection_id: Uuid) -> String { + format!("pipeline.detections.{detection_id}.status") } diff --git a/crates/nvisy-server/src/service/detection/mod.rs b/crates/nvisy-server/src/service/detection/mod.rs index 4c154608..e098d459 100644 --- a/crates/nvisy-server/src/service/detection/mod.rs +++ b/crates/nvisy-server/src/service/detection/mod.rs @@ -1,17 +1,18 @@ //! Async pipeline detection. //! -//! A pipeline run is created synchronously by the API, then its detection -//! (analyze) is enqueued to the `DetectionStream` work-queue and handled by the -//! [`DetectionWorker`] off the request thread. Status changes are broadcast on a -//! core-NATS subject (see [`run_subject`]) for SSE watchers and emitted as -//! webhook events. +//! A detection is created synchronously by the API, then its analysis is enqueued +//! to the `DetectionStream` work-queue and handled by the [`DetectionWorker`] off +//! the request thread. Status changes are broadcast on a core-NATS subject (see +//! [`detection_subject`]) for SSE watchers and emitted as webhook events. +mod drainer; mod job; mod service; mod support; mod worker; -pub use job::{DetectionJob, RunStatusEvent, run_subject}; +pub use drainer::DetectionOutboxDrainer; +pub use job::{DetectionJob, DetectionStatusEvent, detection_subject}; pub use service::DetectionQueue; -pub(crate) use support::{FailRun, fail_run, resolve_policies}; +pub(crate) use support::resolve_policies; pub use worker::DetectionWorker; diff --git a/crates/nvisy-server/src/service/detection/service.rs b/crates/nvisy-server/src/service/detection/service.rs index 361f675f..cacc34ad 100644 --- a/crates/nvisy-server/src/service/detection/service.rs +++ b/crates/nvisy-server/src/service/detection/service.rs @@ -1,19 +1,20 @@ //! Detection enqueue service. //! //! The request-side counterpart to the [`DetectionWorker`](super::DetectionWorker): -//! publishes a run's detection to the `DetectionStream` work-queue and broadcasts -//! its status on the run's core-NATS subject. Injected into the create-run -//! handler so the handler stays thin and the NATS wiring lives in one place. +//! publishes a detection's analysis to the `DetectionStream` work-queue and +//! broadcasts its status on the detection's core-NATS subject. Injected into the +//! create-detection handler so the handler stays thin and the NATS wiring lives in +//! one place. use nvisy_nats::stream::{BroadcastStream, DetectionStream}; -use nvisy_postgres::types::PipelineRunStatus; +use nvisy_postgres::types::DetectionStatus; use uuid::Uuid; -use super::job::{DetectionJob, RunStatusEvent, run_subject}; +use super::job::{DetectionJob, DetectionStatusEvent, detection_subject}; use crate::handler::Result; use crate::service::Infra; -/// Enqueues pipeline detection jobs and broadcasts run-status changes. +/// Enqueues detection jobs and broadcasts detection-status changes. /// /// Cheaply cloneable (holds the shared [`Infra`] clients, all `Arc`-backed). #[derive(Clone)] @@ -28,7 +29,8 @@ impl DetectionQueue { Self { infra } } - /// Enqueues a run's detection onto the work-queue for the worker to pick up. + /// Enqueues a detection's analysis onto the work-queue for the worker to pick + /// up. pub async fn enqueue(&self, job: DetectionJob) -> Result<()> { let publisher = self .infra @@ -39,32 +41,40 @@ impl DetectionQueue { Ok(()) } - /// Broadcasts a run's status change on its core-NATS subject (best-effort; - /// the run row is authoritative, so a dropped broadcast is recoverable). - pub async fn broadcast_status(&self, run_id: Uuid, status: PipelineRunStatus) { - let event = RunStatusEvent { run_id, status }; + /// Broadcasts a detection's status change on its core-NATS subject + /// (best-effort; the detection row is authoritative, so a dropped broadcast is + /// recoverable). + pub async fn broadcast_status(&self, detection_id: Uuid, status: DetectionStatus) { + let event = DetectionStatusEvent { + detection_id, + status, + }; if let Err(err) = self .infra .nats - .publish_broadcast(run_subject(run_id), &event) + .publish_broadcast(detection_subject(detection_id), &event) .await { tracing::debug!( target: "nvisy_server::service::detection", error = %err, - "Failed to broadcast run status", + "Failed to broadcast detection status", ); } } - /// Subscribes to a run's status broadcasts, yielding each [`RunStatusEvent`]. + /// Subscribes to a detection's status broadcasts, yielding each + /// [`DetectionStatusEvent`]. /// /// Used by the SSE endpoint to forward status changes to a watching client. - pub async fn subscribe_status(&self, run_id: Uuid) -> Result> { + pub async fn subscribe_status( + &self, + detection_id: Uuid, + ) -> Result> { let stream = self .infra .nats - .subscribe_broadcast::(run_subject(run_id)) + .subscribe_broadcast::(detection_subject(detection_id)) .await?; Ok(stream) } diff --git a/crates/nvisy-server/src/service/detection/support.rs b/crates/nvisy-server/src/service/detection/support.rs index c65abc2b..0fbc8b3b 100644 --- a/crates/nvisy-server/src/service/detection/support.rs +++ b/crates/nvisy-server/src/service/detection/support.rs @@ -1,45 +1,47 @@ -//! Shared detection helpers used by both the create-run handler and the worker. +//! Shared detection helpers used by both the create-detection handler and the +//! worker. use elide_pipeline::policy::PolicyDefinition; -use nvisy_postgres::model::{NewWorkspacePipelineRunUsage, UpdateWorkspacePipelineRun}; +use nvisy_postgres::model::{NewWorkspaceDetectionUsage, UpdateWorkspaceDetection}; use nvisy_postgres::query::{ - PipelineReferenceRepository, WorkspacePipelineRunRepository, WorkspacePolicyRepository, + PipelineReferenceRepository, WorkspaceDetectionRepository, WorkspacePolicyRepository, }; -use nvisy_postgres::types::{Handle, Json, PipelineRunStatus, RunMetadata}; +use nvisy_postgres::types::{DetectionMetadata, DetectionStatus, Handle, Json}; use uuid::Uuid; use super::service::DetectionQueue; use crate::extract::SecurityContext; use crate::handler::Result; -use crate::service::{CryptoService, EventEmitter, EventOrigin, PipelineRunRef, WorkspaceEvent}; +use crate::service::{CryptoService, DetectionRef, EventEmitter, EventOrigin, WorkspaceEvent}; /// Tracing target for shared detection operations. const TRACING_TARGET: &str = "nvisy_server::service::detection"; -/// A run's inference usage, extracted from the engine's [`Audit`]: the per-model -/// token rows for the usage table and the full report as JSON for the metadata -/// blob. Absent when the run used no model-based recognizers. +/// A detection's inference usage, extracted from the engine's [`Audit`]: the +/// per-model token rows for the usage table and the full report as JSON for the +/// metadata blob. Absent when the detection used no model-based recognizers. /// /// [`Audit`]: elide_pipeline::Audit -pub(crate) struct RunUsage { +pub(crate) struct DetectionUsage { /// One row per distinct model, tokens summed across the recognizers that /// shared it. Empty when no recognizer used a model. - pub per_model: Vec, - /// The full per-recognizer usage report, serialized for `RunMetadata.usage`. + pub per_model: Vec, + /// The full per-recognizer usage report, serialized for + /// `DetectionMetadata.usage`. pub report: serde_json::Value, } -/// Extracts a run's inference usage from the analysis, or `None` when the report -/// is empty (a purely deterministic run spends no tokens). +/// Extracts a detection's inference usage from the analysis, or `None` when the +/// report is empty (a purely deterministic detection spends no tokens). /// /// Recognizers are grouped by `(model, version)` and their token counts summed — /// `input`, `output`, and `total` independently, since a provider may report a /// `total` that is not `input + output` (cached/reasoning tokens). Durations are /// summed into milliseconds. The whole report is also kept as JSON for drill-down. -pub(crate) fn extract_run_usage( - run_id: Uuid, +pub(crate) fn extract_detection_usage( + detection_id: Uuid, analyzed: &elide_pipeline::Audit, -) -> Option { +) -> Option { use std::collections::BTreeMap; let usage = &analyzed.usage; @@ -80,8 +82,8 @@ pub(crate) fn extract_run_usage( let per_model = by_model .into_iter() - .map(|((model, version), acc)| NewWorkspacePipelineRunUsage { - run_id, + .map(|((model, version), acc)| NewWorkspaceDetectionUsage { + detection_id, model, version, input_tokens: acc.input.map(to_i64), @@ -93,7 +95,7 @@ pub(crate) fn extract_run_usage( let report = serde_json::to_value(usage).unwrap_or(serde_json::Value::Null); - Some(RunUsage { per_model, report }) + Some(DetectionUsage { per_model, report }) } /// Clamps an upstream `u64` token count into the `i64` column; token counts are @@ -102,86 +104,95 @@ fn to_i64(value: u64) -> i64 { value.try_into().unwrap_or(i64::MAX) } -/// Identifies the run to fail and how, so [`fail_run`] takes one bundle rather -/// than a long positional list. -pub(crate) struct FailRun<'a> { - /// The workspace the run belongs to. +/// Identifies the detection to fail and how, so [`fail_detection`] takes one +/// bundle rather than a long positional list. +pub(crate) struct FailDetection<'a> { + /// The workspace the detection belongs to. pub workspace_id: Uuid, - /// The run to fail. - pub run_id: Uuid, - /// Slug of the run's pipeline, for the emitted event. + /// The detection to fail. + pub detection_id: Uuid, + /// Slug of the detection's pipeline, for the emitted event. pub pipeline_slug: Handle, - /// The account that triggered the run (the failure's actor and notify target). + /// The account that triggered the detection (the failure's actor and notify + /// target). pub triggered_by: Uuid, - /// Human-readable failure reason, stored in the run's metadata. + /// Human-readable failure reason, stored in the detection's metadata. pub reason: &'a str, + /// The detection's current metadata, so the failure reason is layered onto it + /// rather than replacing recorded fields such as tags. + pub metadata: DetectionMetadata, /// The worker's claim timestamp, guarding the transition; `None` on the /// handler path, which has no claim to fence. pub claim: Option, } -/// Marks a run `Failed` (best effort), recording the reason in its metadata, -/// broadcasting the terminal status for SSE watchers, and emitting the -/// `PipelineRunFailed` event (activity log, webhook, and owner notification). +/// Marks a detection `Failed` (best effort), recording the reason in its +/// metadata, broadcasting the terminal status for SSE watchers, and emitting the +/// `DetectionFailed` event (activity log, webhook, and owner notification). /// -/// Shared by the create-run handler (enqueue failed) and the worker (analysis -/// failed) so a failure takes the same steps on every path. +/// Shared by the create-detection handler (enqueue failed) and the worker +/// (analysis failed) so a failure takes the same steps on every path. /// -/// `params.claim` fences the worker path: when `Some(claimed_at)`, the run is -/// failed only while that claim still holds (still `Analyzing`, `claimed_at` +/// `params.claim` fences the worker path: when `Some(claimed_at)`, the detection +/// is failed only while that claim still holds (still `Executing`, `claimed_at` /// unchanged), and the broadcast/event fire only if it did — so a worker whose -/// lease expired mid-analysis cannot fail, or announce the failure of, a run -/// another worker now owns. The handler passes `None`: it fails the run it just -/// created, with no claim to guard. -pub(crate) async fn fail_run( +/// lease expired mid-analysis cannot fail, or announce the failure of, a +/// detection another worker now owns. The handler passes `None`: it fails the +/// detection it just created, with no claim to guard. +pub(crate) async fn fail_detection( conn: &mut nvisy_postgres::PgConn, detection: &DetectionQueue, - params: FailRun<'_>, -) { - let FailRun { + params: FailDetection<'_>, +) -> FailOutcome { + let FailDetection { workspace_id, - run_id, + detection_id, pipeline_slug, triggered_by, reason, + mut metadata, claim, } = params; - let metadata = RunMetadata { - error: Some(reason.to_owned()), - ..Default::default() - }; - let update = UpdateWorkspacePipelineRun { - status: Some(PipelineRunStatus::Failed), + // Layer the failure reason onto the detection's existing metadata so recorded + // fields (e.g. reviewer tags) survive the failure write. + metadata.error = Some(reason.to_owned()); + // `status` and `completed_at` are forced by the finalize methods themselves + // (the terminal transition owns the terminal timestamp), so only the metadata + // is supplied here. + let update = UpdateWorkspaceDetection { metadata: Some(Json::encode(&metadata)), - completed_at: Some(Some(jiff::Timestamp::now().into())), ..Default::default() }; - match claim { + let persisted = match claim { // Worker path: guard on the claim. A stale claim fails nothing and stays - // silent — the new owner drives the run to its own outcome. - Some(claimed_at) => match conn.finalize_failed_run(run_id, claimed_at, update).await { - Ok(true) => {} - Ok(false) => { - tracing::warn!(target: TRACING_TARGET, %run_id, "Claim went stale before failure; another worker owns the run"); - return; - } - Err(err) => { - tracing::warn!(target: TRACING_TARGET, error = %err, %run_id, "Failed to mark run failed"); - return; - } - }, - // Handler path: no claim to guard. - None => { - if let Err(err) = conn.update_workspace_pipeline_run(run_id, update).await { - tracing::warn!(target: TRACING_TARGET, error = %err, %run_id, "Failed to mark run failed"); - } + // silent — the new owner drives the detection to its own outcome. + Some(claimed_at) => conn.fail_detection(detection_id, claimed_at, update).await, + // Handler path (enqueue failure): guard on `Pending` so this no-ops if a + // worker already claimed the detection — enqueue can report an error even + // when the job was delivered, and the worker then owns the outcome. + None => conn.fail_pending_detection(detection_id, update).await, + }; + match persisted { + Ok(true) => {} + // The guard didn't match: another owner (or an already-terminal detection) + // drives the outcome. Nothing to persist, announce, or retry. + Ok(false) => { + tracing::warn!(target: TRACING_TARGET, %detection_id, "Detection no longer owned at failure; another owner drives it"); + return FailOutcome::NotOwned; + } + // The terminal write itself failed: the detection is still `Executing` + // with no queued job to reclaim it, so the caller must retry (redeliver) + // rather than treat the failure as handled. + Err(err) => { + tracing::warn!(target: TRACING_TARGET, error = %err, %detection_id, "Failed to persist detection failure; will retry"); + return FailOutcome::PersistFailed; } } detection - .broadcast_status(run_id, PipelineRunStatus::Failed) + .broadcast_status(detection_id, DetectionStatus::Failed) .await; if let Err(err) = conn @@ -191,9 +202,9 @@ pub(crate) async fn fail_run( account_id: triggered_by, security: &SecurityContext::default(), }, - WorkspaceEvent::PipelineRunFailed { - run: PipelineRunRef { - run_id, + WorkspaceEvent::DetectionFailed { + detection: DetectionRef { + detection_id, pipeline_slug, }, input_file_name: None, @@ -203,8 +214,26 @@ pub(crate) async fn fail_run( ) .await { - tracing::warn!(target: TRACING_TARGET, error = %err, %run_id, "Failed to record run-failed event"); + tracing::warn!(target: TRACING_TARGET, error = %err, %detection_id, "Failed to record detection-failed event"); } + + // The state transition is persisted; a lost event is recoverable from the + // outbox and does not change the outcome. + FailOutcome::Failed +} + +/// The result of attempting to fail a detection, so a caller driving a work +/// queue can decide whether to redeliver. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum FailOutcome { + /// The detection was transitioned to `Failed` and its failure announced. + Failed, + /// The failure was not applied because the detection is no longer owned by + /// the caller (another owner, or an already-terminal detection). No retry. + NotOwned, + /// The terminal write failed to persist, leaving the detection mid-flight; + /// the caller should redeliver so the stale lease is reclaimed. + PersistFailed, } /// Resolves a pipeline's live policy references into decrypted engine policies. diff --git a/crates/nvisy-server/src/service/detection/worker.rs b/crates/nvisy-server/src/service/detection/worker.rs index 58a68646..67a305f3 100644 --- a/crates/nvisy-server/src/service/detection/worker.rs +++ b/crates/nvisy-server/src/service/detection/worker.rs @@ -1,32 +1,35 @@ //! Pipeline detection worker. //! //! Consumes [`DetectionJob`]s from the `DetectionStream` work-queue and runs a -//! run's detection (analyze) in the background: builds the document, analyzes it -//! with the pipeline's policies, stores the encrypted audit, and marks the run -//! `Analyzed` (or `Failed`). Each terminal transition is broadcast on the run's -//! core-NATS status subject (for SSE watchers) and emitted as a webhook event. +//! detection's analysis in the background: builds the document, analyzes it with +//! the pipeline's policies, stores the encrypted audit, and marks the detection +//! `Complete` (or `Failed`). Each terminal transition is broadcast on the +//! detection's core-NATS status subject (for SSE watchers) and emitted as a +//! webhook event. use std::time::Duration; use elide_pipeline::RasterMode; use nvisy_nats::stream::DetectionStream; -use nvisy_postgres::model::{UpdateWorkspacePipelineRun, WorkspacePipeline, WorkspacePipelineRun}; +use nvisy_postgres::model::{UpdateWorkspaceDetection, WorkspaceDetection, WorkspacePipeline}; use nvisy_postgres::query::{ - EventOutboxRepository, WorkspaceFileRepository, WorkspacePipelineRunRepository, + EventOutboxRepository, WorkspaceDetectionRepository, WorkspaceFileRepository, WorkspaceRepository, }; -use nvisy_postgres::types::{Json, OcrPolicy, PipelineRunStatus, WorkspaceSettings}; +use nvisy_postgres::types::{DetectionStatus, Json, OcrPolicy, WorkspaceSettings}; use nvisy_postgres::{AsyncConnection, DieselError, PgConn, PgError}; use tokio_util::sync::CancellationToken; use super::job::DetectionJob; use super::service::DetectionQueue; -use super::support::{FailRun, extract_run_usage, fail_run, resolve_policies}; +use super::support::{ + FailDetection, FailOutcome, extract_detection_usage, fail_detection, resolve_policies, +}; use crate::extract::SecurityContext; use crate::handler::request::PipelineDefinition; use crate::handler::{ErrorKind, Result}; use crate::service::{ - EngineService, EventOrigin, Infra, PipelineRunRef, RunBlobStore, Worker, WorkspaceEvent, + DetectionRef, EngineService, EventOrigin, Infra, RunBlobStore, Worker, WorkspaceEvent, event_outbox_row, }; @@ -34,9 +37,10 @@ use crate::service::{ const TRACING_TARGET: &str = "nvisy_server::worker::detection"; /// How long a detection claim stays valid before another delivery may re-claim -/// the run. Set above `DetectionStream::ACK_WAIT` (15 min) so a slow-but-healthy -/// worker whose job is redelivered keeps its claim; only a run whose worker died -/// (no progress past the lease) is re-claimed and re-analyzed. +/// the detection. Set above `DetectionStream::ACK_WAIT` (15 min) so a +/// slow-but-healthy worker whose job is redelivered keeps its claim; only a +/// detection whose worker died (no progress past the lease) is re-claimed and +/// re-analyzed. const DETECTION_LEASE: Duration = Duration::from_secs(30 * 60); /// Background worker that runs pipeline detection off the request thread. @@ -90,11 +94,11 @@ impl DetectionWorker { /// Consumes detection jobs until cancelled. /// /// At-least-once with an explicit claim: a job is acked once it reaches a - /// terminal outcome (analyzed, or marked failed), and nacked for redelivery - /// on a transient error (a DB/pool blip before the run could even be - /// claimed), so a run is never silently stranded in a non-terminal state. - /// The claim (`claim_run_for_detection`) makes redelivery idempotent: a run - /// already being analyzed under a fresh lease is skipped. + /// terminal outcome (complete, or marked failed), and nacked for redelivery + /// on a transient error (a DB/pool blip before the detection could even be + /// claimed), so a detection is never silently stranded in a non-terminal + /// state. The claim (`claim_detection`) makes redelivery idempotent: a + /// detection already being analyzed under a fresh lease is skipped. async fn run_inner(&self, cancel: CancellationToken) -> Result<()> { let subscriber = self .infra @@ -136,72 +140,73 @@ impl DetectionWorker { Ok(()) } - /// Runs one detection job: claims the run, analyzes, and records the result. + /// Runs one detection job: claims the detection, analyzes, and records the + /// result. /// /// Returns [`JobOutcome::Retry`] when the job should be redelivered (a - /// transient error before the run was claimed), and [`JobOutcome::Done`] - /// when it reached a terminal outcome or is safe to drop (missing run, + /// transient error before the detection was claimed), and [`JobOutcome::Done`] + /// when it reached a terminal outcome or is safe to drop (missing detection, /// already claimed, already settled). - #[tracing::instrument(skip_all, fields(run_id = %job.run_id, workspace_id = %job.workspace_id))] + #[tracing::instrument(skip_all, fields(detection_id = %job.detection_id, workspace_id = %job.workspace_id))] async fn run_job(&self, job: DetectionJob) -> JobOutcome { let mut conn = match self.infra.postgres.get_connection().await { Ok(conn) => conn, Err(err) => { - // No connection: the run is still queued. Redeliver so it is not - // stranded in a non-terminal state. + // No connection: the detection is still pending. Redeliver so it + // is not stranded in a non-terminal state. tracing::error!(target: TRACING_TARGET, error = %err, "Failed to get connection for detection job"); return JobOutcome::Retry; } }; - let (run, pipeline) = match conn - .find_workspace_run_by_id(job.workspace_id, job.run_id) + let (detection, pipeline) = match conn + .find_workspace_detection_by_id(job.workspace_id, job.detection_id) .await { Ok(Some(pair)) => pair, Ok(None) => { - tracing::warn!(target: TRACING_TARGET, "Detection job for a missing run; dropping"); + tracing::warn!(target: TRACING_TARGET, "Detection job for a missing detection; dropping"); return JobOutcome::Done; } Err(err) => { - // Transient load error: redeliver rather than strand the run. - tracing::error!(target: TRACING_TARGET, error = %err, "Failed to load run for detection job"); + // Transient load error: redeliver rather than strand the detection. + tracing::error!(target: TRACING_TARGET, error = %err, "Failed to load detection for detection job"); return JobOutcome::Retry; } }; - // Nothing to do for a run already past the queued/analyzing phase. - if !run.status.is_detecting() { - tracing::debug!(target: TRACING_TARGET, status = %run.status, "Run is not detecting; dropping"); + // Nothing to do for a detection already past the pending/executing phase. + if !detection.status.is_detecting() { + tracing::debug!(target: TRACING_TARGET, status = %detection.status, "Detection is not detecting; dropping"); return JobOutcome::Done; } - // Atomically claim the run (queued -> analyzing). A redelivery whose - // claim is still fresh matches no row and is skipped, so a slow job is - // never analyzed twice; only a run whose worker died (stale lease) is - // re-claimed. + // Atomically claim the detection (pending -> executing). A redelivery + // whose claim is still fresh matches no row and is skipped, so a slow job + // is never analyzed twice; only a detection whose worker died (stale + // lease) is re-claimed. let stale_before = jiff::Timestamp::now() - DETECTION_LEASE; - let claimed = match conn.claim_run_for_detection(run.id, stale_before).await { + let claimed = match conn.claim_detection(detection.id, stale_before).await { Ok(Some(claimed)) => claimed, Ok(None) => { - tracing::debug!(target: TRACING_TARGET, "Run already claimed by another worker; skipping"); + tracing::debug!(target: TRACING_TARGET, "Detection already claimed by another worker; skipping"); return JobOutcome::Done; } Err(err) => { - tracing::error!(target: TRACING_TARGET, error = %err, "Failed to claim run for detection"); + tracing::error!(target: TRACING_TARGET, error = %err, "Failed to claim detection"); return JobOutcome::Retry; } }; // The claim stamped `claimed_at`; it fences the finalize (and a failure) // against a concurrent re-claim if this analysis outlives the lease. let Some(claim_token) = claimed.claimed_at else { - tracing::error!(target: TRACING_TARGET, "Claimed run has no claim timestamp; skipping"); + tracing::error!(target: TRACING_TARGET, "Claimed detection has no claim timestamp; skipping"); return JobOutcome::Done; }; let claim_token: jiff::Timestamp = claim_token.into(); self.detection - .broadcast_status(run.id, PipelineRunStatus::Analyzing) + .broadcast_status(detection.id, DetectionStatus::Executing) .await; if let Err(err) = self @@ -209,19 +214,27 @@ impl DetectionWorker { .await { tracing::warn!(target: TRACING_TARGET, error = %err, "Detection failed"); - fail_run( + let outcome = fail_detection( &mut conn, &self.detection, - FailRun { + FailDetection { workspace_id: job.workspace_id, - run_id: run.id, + detection_id: detection.id, pipeline_slug: pipeline.slug.clone(), - triggered_by: run.account_id, + triggered_by: detection.account_id, reason: &err.to_string(), + metadata: detection.metadata.or_default(), claim: Some(claim_token), }, ) .await; + // If the failure state could not be persisted, the detection is left + // `Executing` with no queued job to reclaim its lease: redeliver so a + // later attempt drives it to a terminal state rather than ack'ing a + // detection that will hang. + if outcome == FailOutcome::PersistFailed { + return JobOutcome::Retry; + } } JobOutcome::Done } @@ -229,7 +242,7 @@ impl DetectionWorker { /// Best-effort reclaim of a staged audit object whose file row did not /// commit. A failure only defers cleanup, so it is logged, never propagated. async fn discard_staged_audit(&self, staged: &nvisy_postgres::model::NewWorkspaceFile) { - if let Err(err) = self.blob.discard_staged_audit(staged).await { + if let Err(err) = self.blob.discard_staged_object(staged).await { tracing::warn!( target: TRACING_TARGET, error = %err, @@ -239,12 +252,12 @@ impl DetectionWorker { } } - /// Performs the analysis and records the run as `Analyzed`. + /// Performs the analysis and records the detection as `Complete`. async fn detect( &self, conn: &mut PgConn, job: &DetectionJob, - run: &WorkspacePipelineRun, + detection: &WorkspaceDetection, pipeline: &WorkspacePipeline, claim_token: jiff::Timestamp, ) -> Result<()> { @@ -253,7 +266,7 @@ impl DetectionWorker { .await? .ok_or_else(|| ErrorKind::NotFound.with_message("Workspace not found"))?; let file = conn - .find_file_in_workspace(job.workspace_id, run.input_file_id) + .find_file_in_workspace(job.workspace_id, detection.input_file_id) .await? .ok_or_else(|| ErrorKind::NotFound.with_message("Input file not found"))?; @@ -270,7 +283,7 @@ impl DetectionWorker { self.engine .request_context(&definition, job.scope.clone(), raster_mode_of(&settings)); - let document = self.blob.build_document(&file, run.id).await?; + let document = self.blob.build_document(&file, detection.id).await?; let policies = resolve_policies(conn, &self.infra.crypto, job.workspace_id, pipeline.id).await?; @@ -283,51 +296,58 @@ impl DetectionWorker { let analyzed = self.engine.analyze(document, &policies, &request).await?; // Write the (non-transactional) audit object first, then commit its file - // row together with the run's usage and status in one transaction below. + // row together with the detection's usage and status in one transaction + // below. let audit_file = self .blob - .stage_analyzed_document(pipeline, &settings.retention, run.account_id, &analyzed) + .stage_analyzed_document( + pipeline, + &settings.retention, + detection.account_id, + &analyzed, + ) .await?; // Record inference usage: per-model token rows into the usage table (the // usage aggregation surface) and the full per-recognizer report into - // metadata for drill-down. Absent for a purely deterministic run. The - // report is layered onto the run's existing metadata so tags and any - // recorded error survive the write. - let usage = extract_run_usage(run.id, &analyzed); + // metadata for drill-down. Absent for a purely deterministic detection. + // The report is layered onto the detection's existing metadata so tags and + // any recorded error survive the write. + let usage = extract_detection_usage(detection.id, &analyzed); let metadata = usage.as_ref().map(|u| { - let mut current = run.metadata.or_default(); + let mut current = detection.metadata.or_default(); current.usage = Some(u.report.clone()); Json::encode(¤t) }); - // Persist the audit file row, per-model usage, and the run's transition to - // `Analyzed` atomically: a partial failure would otherwise strand usage - // rows or an audit pointer on a run still marked `Analyzing`. The finalize - // is fenced on our claim; if it went stale (another worker re-claimed the - // run past the lease), the whole transaction rolls back so we do not stamp - // over the new owner's work or leak usage/audit rows for a run we lost. - // Kept to reclaim the just-staged object if the transaction does not - // commit: on rollback its `workspace_files` row never lands, so the - // row-driven reaper could never find the object otherwise. - // Build the outbox row here so the finalize transaction is `PgError`-typed - // for its rollback sentinel, and insert it alongside the finalize so the - // `Analyzed` event commits atomically with the run. - let analyzed_event = WorkspaceEvent::PipelineRunAnalyzed { - run: PipelineRunRef { - run_id: run.id, + // Persist the audit file row, per-model usage, and the detection's + // transition to `Complete` atomically: a partial failure would otherwise + // strand usage rows or an audit pointer on a detection still marked + // `Executing`. The finalize is fenced on our claim; if it went stale + // (another worker re-claimed the detection past the lease), the whole + // transaction rolls back so we do not stamp over the new owner's work or + // leak usage/audit rows for a detection we lost. Kept to reclaim the + // just-staged object if the transaction does not commit: on rollback its + // `workspace_files` row never lands, so the row-driven reaper could never + // find the object otherwise. Build the outbox row here so the finalize + // transaction is `PgError`-typed for its rollback sentinel, and insert it + // alongside the finalize so the `Complete` event commits atomically with + // the detection. + let completed_event = WorkspaceEvent::DetectionCompleted { + detection: DetectionRef { + detection_id: detection.id, pipeline_slug: pipeline.slug.clone(), }, input_file_name: Some(file.display_name.clone()), - notify: run.account_id, + notify: detection.account_id, }; let outbox_row = event_outbox_row( EventOrigin { workspace_id: job.workspace_id, - account_id: run.account_id, + account_id: detection.account_id, security: &SecurityContext::default(), }, - &analyzed_event, + &completed_event, )?; let staged_audit = audit_file.clone(); @@ -335,13 +355,13 @@ impl DetectionWorker { .transaction(async |conn| { let audit_file_id = conn.create_workspace_file(audit_file).await?.id; if let Some(usage) = &usage { - conn.record_run_usage(&usage.per_model).await?; + conn.record_detection_usage(&usage.per_model).await?; } let finalized = conn - .finalize_analyzed_run( - run.id, + .finalize_detection( + detection.id, claim_token, - UpdateWorkspacePipelineRun { + UpdateWorkspaceDetection { audit_file_id: Some(Some(audit_file_id)), metadata, ..Default::default() @@ -349,9 +369,9 @@ impl DetectionWorker { ) .await?; if !finalized { - // Abort the audit-file and usage inserts: the run is no longer - // ours to finalize. `RollbackTransaction` unwinds the writes - // without being a real error; it is matched below. + // Abort the audit-file and usage inserts: the detection is no + // longer ours to finalize. `RollbackTransaction` unwinds the + // writes without being a real error; it is matched below. return Err(PgError::Query(DieselError::RollbackTransaction)); } conn.insert_event_outbox(outbox_row).await?; @@ -363,7 +383,7 @@ impl DetectionWorker { Ok(()) => {} Err(PgError::Query(DieselError::RollbackTransaction)) => { self.discard_staged_audit(&staged_audit).await; - tracing::warn!(target: TRACING_TARGET, run_id = %run.id, "Claim went stale before finalize; another worker owns the run"); + tracing::warn!(target: TRACING_TARGET, detection_id = %detection.id, "Claim went stale before finalize; another worker owns the detection"); return Ok(()); } Err(err) => { @@ -374,9 +394,9 @@ impl DetectionWorker { } } - tracing::info!(target: TRACING_TARGET, run_id = %run.id, "Run analyzed"); + tracing::info!(target: TRACING_TARGET, detection_id = %detection.id, "Detection complete"); self.detection - .broadcast_status(run.id, PipelineRunStatus::Analyzed) + .broadcast_status(detection.id, DetectionStatus::Complete) .await; Ok(()) @@ -388,12 +408,12 @@ impl DetectionWorker { enum JobOutcome { /// Reached a terminal outcome or is safe to drop; ack the message. Done, - /// Transient error before the run was claimed; nack for redelivery. + /// Transient error before the detection was claimed; nack for redelivery. Retry, } -/// Maps a workspace's OCR policy to the engine's per-run page-rasterisation -/// mode. +/// Maps a workspace's OCR policy to the engine's per-detection +/// page-rasterisation mode. fn raster_mode_of(settings: &WorkspaceSettings) -> RasterMode { match settings.ocr { OcrPolicy::Auto => RasterMode::Auto, diff --git a/crates/nvisy-server/src/service/event/drainer.rs b/crates/nvisy-server/src/service/event/drainer.rs index e361cade..9cbb7bd2 100644 --- a/crates/nvisy-server/src/service/event/drainer.rs +++ b/crates/nvisy-server/src/service/event/drainer.rs @@ -21,11 +21,11 @@ use nvisy_postgres::model::{EventOutbox, NewWorkspaceActivity}; use nvisy_postgres::query::{EventOutboxRepository, WorkspaceActivityRepository}; use nvisy_postgres::types::{ ActivityPayload, ConnectionActivityParams, ConnectionId, ConnectionSyncCompletedParams, - ConnectionSyncFailedParams, FileActivityParams, Handle, InviteActivityParams, Json, - MemberActivityParams, NotificationPayload, PipelineActivityParams, PipelineRunActivityParams, - PipelineRunAnalyzedParams, PipelineRunCompletedParams, PipelineRunFailedParams, - PolicyActivityParams, RunId, WebhookActivityParams, WebhookEvent, WebhookId, - WorkspaceActivityParams, + ConnectionSyncFailedParams, DetectionActivityParams, DetectionCompletedParams, + DetectionFailedParams, DetectionId, FileActivityParams, Handle, InviteActivityParams, Json, + MemberActivityParams, NotificationPayload, PipelineActivityParams, PolicyActivityParams, + RedactionActivityParams, RedactionCreatedParams, RedactionId, WebhookActivityParams, + WebhookEvent, WebhookId, WorkspaceActivityParams, }; use nvisy_postgres::{AsyncConnection, PgConn}; use serde_json::Value; @@ -34,7 +34,7 @@ use uuid::Uuid; use crate::handler::{Error, Result}; use crate::service::event::{ - ConnectionRef, FileRef, InviteRef, MemberRef, PipelineRunRef, PolicyRef, WebhookRef, + ConnectionRef, DetectionRef, FileRef, InviteRef, MemberRef, PolicyRef, WebhookRef, WorkspaceEvent, WorkspaceRef, }; use crate::service::{Infra, NotificationEmitter, WebhookEmitter, Worker}; @@ -326,9 +326,13 @@ fn activity_of(event: &WorkspaceEvent) -> ActivityPayload { let pipeline = |pipeline_slug: &Handle| PipelineActivityParams { pipeline_slug: pipeline_slug.clone(), }; - let run = |run: &PipelineRunRef| PipelineRunActivityParams { - pipeline_slug: run.pipeline_slug.clone(), - run_id: RunId::from_uuid(run.run_id), + let detection = |detection: &DetectionRef| DetectionActivityParams { + pipeline_slug: detection.pipeline_slug.clone(), + detection_id: DetectionId::from_uuid(detection.detection_id), + }; + let redaction = |detection: &DetectionRef, redaction_id: Uuid| RedactionActivityParams { + pipeline_slug: detection.pipeline_slug.clone(), + redaction_id: RedactionId::from_uuid(redaction_id), }; let policy = |p: &PolicyRef| PolicyActivityParams { policy_id: p.policy_id, @@ -374,10 +378,16 @@ fn activity_of(event: &WorkspaceEvent) -> ActivityPayload { E::PipelineCreated(p) => ActivityPayload::PipelineCreated(pipeline(&p.pipeline_slug)), E::PipelineUpdated(p) => ActivityPayload::PipelineUpdated(pipeline(&p.pipeline_slug)), E::PipelineDeleted(p) => ActivityPayload::PipelineDeleted(pipeline(&p.pipeline_slug)), - E::PipelineRunStarted(r) => ActivityPayload::PipelineRunStarted(run(r)), - E::PipelineRunAnalyzed { run: r, .. } => ActivityPayload::PipelineRunAnalyzed(run(r)), - E::PipelineRunCompleted { run: r, .. } => ActivityPayload::PipelineRunCompleted(run(r)), - E::PipelineRunFailed { run: r, .. } => ActivityPayload::PipelineRunFailed(run(r)), + E::DetectionStarted(d) => ActivityPayload::DetectionStarted(detection(d)), + E::DetectionCompleted { detection: d, .. } => { + ActivityPayload::DetectionCompleted(detection(d)) + } + E::DetectionFailed { detection: d, .. } => ActivityPayload::DetectionFailed(detection(d)), + E::RedactionCreated { + detection: d, + redaction_id, + .. + } => ActivityPayload::RedactionCreated(redaction(d, *redaction_id)), E::PolicyCreated(p) => ActivityPayload::PolicyCreated(policy(p)), E::PolicyUpdated(p) => ActivityPayload::PolicyUpdated(policy(p)), E::PolicyDeleted(p) => ActivityPayload::PolicyDeleted(policy(p)), @@ -418,10 +428,10 @@ fn webhook_of(event: &WorkspaceEvent) -> Option<(WebhookEvent, Option)> { E::PipelineCreated(..) => (WebhookEvent::PipelineCreated, None), E::PipelineUpdated(..) => (WebhookEvent::PipelineUpdated, None), E::PipelineDeleted(..) => (WebhookEvent::PipelineDeleted, None), - E::PipelineRunStarted(..) => (WebhookEvent::PipelineRunStarted, None), - E::PipelineRunAnalyzed { .. } => (WebhookEvent::PipelineRunAnalyzed, None), - E::PipelineRunCompleted { .. } => (WebhookEvent::PipelineRunCompleted, None), - E::PipelineRunFailed { .. } => (WebhookEvent::PipelineRunFailed, None), + E::DetectionStarted(..) => (WebhookEvent::DetectionStarted, None), + E::DetectionCompleted { .. } => (WebhookEvent::DetectionCompleted, None), + E::DetectionFailed { .. } => (WebhookEvent::DetectionFailed, None), + E::RedactionCreated { .. } => (WebhookEvent::RedactionCreated, None), E::PolicyCreated(..) => (WebhookEvent::PolicyCreated, None), E::PolicyUpdated(..) => (WebhookEvent::PolicyUpdated, None), E::PolicyDeleted(..) => (WebhookEvent::PolicyDeleted, None), @@ -474,40 +484,42 @@ fn notification_of(event: WorkspaceEvent) -> Option<(Uuid, NotificationPayload)> }), ) }), - E::PipelineRunAnalyzed { - run, + E::DetectionCompleted { + detection, input_file_name, notify, } => Some(( notify, - NotificationPayload::PipelineRunAnalyzed(PipelineRunAnalyzedParams { - run_id: RunId::from_uuid(run.run_id), - pipeline_slug: run.pipeline_slug, + NotificationPayload::DetectionCompleted(DetectionCompletedParams { + detection_id: DetectionId::from_uuid(detection.detection_id), + pipeline_slug: detection.pipeline_slug, input_file_name, }), )), - E::PipelineRunCompleted { - run, + E::RedactionCreated { + detection, + redaction_id, input_file_name, notify, } => Some(( notify, - NotificationPayload::PipelineRunCompleted(PipelineRunCompletedParams { - run_id: RunId::from_uuid(run.run_id), - pipeline_slug: run.pipeline_slug, + NotificationPayload::RedactionCreated(RedactionCreatedParams { + redaction_id: RedactionId::from_uuid(redaction_id), + detection_id: DetectionId::from_uuid(detection.detection_id), + pipeline_slug: detection.pipeline_slug, input_file_name, }), )), - E::PipelineRunFailed { - run, + E::DetectionFailed { + detection, input_file_name, error, notify, } => Some(( notify, - NotificationPayload::PipelineRunFailed(PipelineRunFailedParams { - run_id: RunId::from_uuid(run.run_id), - pipeline_slug: run.pipeline_slug, + NotificationPayload::DetectionFailed(DetectionFailedParams { + detection_id: DetectionId::from_uuid(detection.detection_id), + pipeline_slug: detection.pipeline_slug, input_file_name, error, }), @@ -537,7 +549,7 @@ fn notification_of(event: WorkspaceEvent) -> Option<(Uuid, NotificationPayload)> | E::PipelineCreated(_) | E::PipelineUpdated(_) | E::PipelineDeleted(_) - | E::PipelineRunStarted(_) + | E::DetectionStarted(_) | E::PolicyCreated(_) | E::PolicyUpdated(_) | E::PolicyDeleted(_) => None, @@ -565,10 +577,10 @@ fn resource_id_of(event: &WorkspaceEvent) -> Uuid { E::FileCreated { file, .. } => file.file_id, E::FileUpdated(f) | E::FileDeleted(f) => f.file_id, E::PipelineCreated(p) | E::PipelineUpdated(p) | E::PipelineDeleted(p) => p.pipeline_id, - E::PipelineRunStarted(r) => r.run_id, - E::PipelineRunAnalyzed { run, .. } - | E::PipelineRunCompleted { run, .. } - | E::PipelineRunFailed { run, .. } => run.run_id, + E::DetectionStarted(d) => d.detection_id, + E::DetectionCompleted { detection, .. } + | E::DetectionFailed { detection, .. } + | E::RedactionCreated { detection, .. } => detection.detection_id, E::PolicyCreated(p) | E::PolicyUpdated(p) | E::PolicyDeleted(p) => p.policy_id, } } diff --git a/crates/nvisy-server/src/service/event/mod.rs b/crates/nvisy-server/src/service/event/mod.rs index e12ce816..8921208a 100644 --- a/crates/nvisy-server/src/service/event/mod.rs +++ b/crates/nvisy-server/src/service/event/mod.rs @@ -19,8 +19,8 @@ use crate::extract::SecurityContext; pub use crate::service::event::drainer::EventOutboxDrainer; pub use crate::service::event::emitter::{EventEmitter, event_outbox_row}; pub use crate::service::event::workspace_event::{ - ConnectionRef, FileRef, InviteRef, MemberRef, PipelineRef, PipelineRunRef, PolicyRef, - WebhookRef, WorkspaceEvent, WorkspaceRef, + ConnectionRef, DetectionRef, FileRef, InviteRef, MemberRef, PipelineRef, PolicyRef, WebhookRef, + WorkspaceEvent, WorkspaceRef, }; /// Who raised an event and where. diff --git a/crates/nvisy-server/src/service/event/workspace_event.rs b/crates/nvisy-server/src/service/event/workspace_event.rs index e9ee6ef2..4932275a 100644 --- a/crates/nvisy-server/src/service/event/workspace_event.rs +++ b/crates/nvisy-server/src/service/event/workspace_event.rs @@ -99,29 +99,34 @@ pub enum WorkspaceEvent { #[serde(rename = "pipeline.deleted")] PipelineDeleted(PipelineRef), - // Pipeline runs - #[serde(rename = "pipeline.run.started")] - PipelineRunStarted(PipelineRunRef), - #[serde(rename = "pipeline.run.analyzed")] - PipelineRunAnalyzed { + // Detections + #[serde(rename = "pipeline.detection.started")] + DetectionStarted(DetectionRef), + #[serde(rename = "pipeline.detection.completed")] + DetectionCompleted { #[serde(flatten)] - run: PipelineRunRef, + detection: DetectionRef, input_file_name: Option, notify: Uuid, }, - #[serde(rename = "pipeline.run.completed")] - PipelineRunCompleted { + #[serde(rename = "pipeline.detection.failed")] + DetectionFailed { #[serde(flatten)] - run: PipelineRunRef, + detection: DetectionRef, input_file_name: Option, + error: Option, notify: Uuid, }, - #[serde(rename = "pipeline.run.failed")] - PipelineRunFailed { + + // Redactions + #[serde(rename = "pipeline.redaction.created")] + RedactionCreated { #[serde(flatten)] - run: PipelineRunRef, + detection: DetectionRef, + /// The redaction that was produced (its own id, distinct from the + /// detection's — a detection can produce many redactions). + redaction_id: Uuid, input_file_name: Option, - error: Option, notify: Uuid, }, @@ -186,10 +191,10 @@ pub struct PipelineRef { pub pipeline_slug: Handle, } -/// A pipeline run and its pipeline's slug. +/// A detection and its pipeline's slug. #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PipelineRunRef { - pub run_id: Uuid, +pub struct DetectionRef { + pub detection_id: Uuid, pub pipeline_slug: Handle, } diff --git a/crates/nvisy-server/src/service/mod.rs b/crates/nvisy-server/src/service/mod.rs index a32c6614..de5ff09e 100644 --- a/crates/nvisy-server/src/service/mod.rs +++ b/crates/nvisy-server/src/service/mod.rs @@ -33,15 +33,15 @@ pub use crate::service::chat::{ChatService, TurnLocation}; pub use crate::service::connection_config::ConnectionConfig; pub use crate::service::crypto::{CryptoConfig, CryptoService}; pub(crate) use crate::service::crypto::{CryptoError, HashingReader, Measurements}; +pub(crate) use crate::service::detection::resolve_policies; pub use crate::service::detection::{ - DetectionJob, DetectionQueue, DetectionWorker, RunStatusEvent, run_subject, + DetectionJob, DetectionOutboxDrainer, DetectionQueue, DetectionStatusEvent, DetectionWorker, + detection_subject, }; -pub(crate) use crate::service::detection::{FailRun, fail_run, resolve_policies}; pub use crate::service::engine::{EngineConfig, EngineService, UnknownFormatToken}; pub use crate::service::event::{ - ConnectionRef, EventEmitter, EventOrigin, EventOutboxDrainer, FileRef, InviteRef, MemberRef, - PipelineRef, PipelineRunRef, PolicyRef, WebhookRef, WorkspaceEvent, WorkspaceRef, - event_outbox_row, + ConnectionRef, DetectionRef, EventEmitter, EventOrigin, EventOutboxDrainer, FileRef, InviteRef, + MemberRef, PipelineRef, PolicyRef, WebhookRef, WorkspaceEvent, WorkspaceRef, event_outbox_row, }; pub use crate::service::external_object_store::ExternalObjectStore; pub use crate::service::file_reaper::FileReaper; @@ -174,6 +174,7 @@ impl ServiceState { )); workers.spawn(FileReaper::new(self.infra.clone())); workers.spawn(EventOutboxDrainer::new(self.infra.clone())); + workers.spawn(DetectionOutboxDrainer::new(self.infra.clone())); workers.spawn(DetectionWorker::new( self.infra.clone(), self.engine.clone(), diff --git a/crates/nvisy-server/src/service/run_blob_store.rs b/crates/nvisy-server/src/service/run_blob_store.rs index 11ab215d..75011d89 100644 --- a/crates/nvisy-server/src/service/run_blob_store.rs +++ b/crates/nvisy-server/src/service/run_blob_store.rs @@ -16,7 +16,7 @@ use elide_pipeline::{Audit, Engine}; use nvisy_nats::object::{AuditBucket, AuditKey, FileKey, FilesBucket, ObjectBucket}; use nvisy_postgres::PgConn; use nvisy_postgres::model::{ - NewWorkspaceFile, WorkspaceFile, WorkspacePipeline, WorkspacePipelineRun, + NewWorkspaceFile, WorkspaceDetection, WorkspaceFile, WorkspacePipeline, }; use nvisy_postgres::query::WorkspaceFileRepository; use nvisy_postgres::types::{FileKind, RetentionScope, RetentionSettings}; @@ -178,64 +178,6 @@ impl RunBlobStore { Ok(Document::new(bytes, file.file_extension.clone()).with_correlation_id(correlation_id)) } - /// Stores redacted bytes as a new workspace file (the run's output). - /// - /// The redacted file is a first-class file — a sibling of the source — so it - /// is downloadable through the normal file endpoints. - pub async fn store_redacted_file( - &self, - conn: &mut PgConn, - source: &WorkspaceFile, - pipeline: &WorkspacePipeline, - workspace_settings: &RetentionSettings, - account_id: Uuid, - bytes: Bytes, - ) -> Result { - // Record the plaintext size and hash before encrypting; storage holds - // only the ciphertext. - let plaintext_size = bytes.len() as i64; - let plaintext_hash = Sha256::digest(&bytes).to_vec(); - let ciphertext = self - .infra - .crypto - .encrypt(source.workspace_id, &bytes) - .map_err(|err| { - ErrorKind::InternalServerError - .with_message("Failed to encrypt redacted file") - .with_context(err.to_string()) - })?; - - let store = self.infra.nats.object_store::().await?; - let key = FileKey::generate(source.workspace_id); - store.put(&key, Cursor::new(ciphertext)).await?; - - // Retention expiry for the redacted-documents scope (workspace baseline, - // pipeline override if set). - let over = pipeline.metadata.or_default().retention; - let expires_at = workspace_settings - .resolve(RetentionScope::RedactedDocuments, over.as_ref()) - .expires_at(jiff::Timestamp::now()); - - let redacted_name = format!("{}.redacted", source.display_name); - let new_file = NewWorkspaceFile { - workspace_id: source.workspace_id, - account_id, - parent_id: Some(source.id), - display_name: Some(redacted_name), - original_filename: Some(source.original_filename.clone()), - file_extension: Some(source.file_extension.clone()), - file_kind: Some(FileKind::Redacted), - file_size_bytes: plaintext_size, - file_hash_sha256: plaintext_hash, - storage_path: key.to_string(), - storage_bucket: store.bucket().to_owned(), - expires_at: expires_at.map(Into::into), - ..Default::default() - }; - - Ok(conn.create_workspace_file(new_file).await?) - } - /// Encrypts the analysis, writes it to the audit bucket, and builds the /// `audit`-kind [`WorkspaceFile`] row that will point at it — but does not /// insert the row. @@ -250,7 +192,7 @@ impl RunBlobStore { /// inserted (with the run's other writes) atomically. A rollback therefore /// leaves at worst an orphan object in the bucket, never a file row that points /// at bytes that were never written; the caller reclaims that orphan via - /// [`discard_staged_audit`](Self::discard_staged_audit). + /// [`discard_staged_object`](Self::discard_staged_object). pub async fn stage_analyzed_document( &self, pipeline: &WorkspacePipeline, @@ -299,15 +241,17 @@ impl RunBlobStore { }) } - /// Deletes a staged audit object whose file row was never committed. + /// Deletes a staged object whose file row was never committed. /// - /// [`stage_analyzed_document`](Self::stage_analyzed_document) writes the object - /// before its `workspace_files` row; if the committing transaction rolls back, - /// the object has no row and the row-driven reaper can never find it. The - /// caller invokes this on that path so the orphan is removed immediately - /// instead of accumulating. Best effort: a failure here only leaves the object - /// for a later manual sweep, so callers log rather than propagate. - pub async fn discard_staged_audit(&self, staged: &NewWorkspaceFile) -> Result<()> { + /// The `stage_*` methods write an object before its `workspace_files` row; if + /// the committing transaction rolls back, the object has no row and the + /// row-driven reaper can never find it. The caller invokes this on that path + /// so the orphan is removed immediately instead of accumulating. It deletes + /// from whichever bucket the staged row names, so it reclaims a staged audit, + /// review audit, or redacted document alike. Best effort: a failure here only + /// leaves the object for a later manual sweep, so callers log rather than + /// propagate. + pub async fn discard_staged_object(&self, staged: &NewWorkspaceFile) -> Result<()> { self.delete_object(&staged.storage_bucket, &staged.storage_path) .await } @@ -319,30 +263,30 @@ impl RunBlobStore { /// entity group by modality name and only the engine's registry can map those /// back to concrete types. /// - /// Errors if the run was never analyzed (409) or its analysis has since been - /// deleted (404). + /// Errors if the detection never analyzed (409) or its analysis has since + /// been deleted (404). pub async fn load_analyzed_document( &self, conn: &mut PgConn, engine: &Engine, workspace_id: Uuid, - run: &WorkspacePipelineRun, + detection: &WorkspaceDetection, ) -> Result { - // A NULL reference means the run never produced an analysis; a reference - // to a now-deleted file means it did, but the analysis has been removed. - // These are distinct states, so they map to distinct responses. - let audit_file_id = run.audit_file_id.ok_or_else(|| { + // A NULL reference means the detection never produced an analysis; a + // reference to a now-deleted file means it did, but the analysis has been + // removed. These are distinct states, so they map to distinct responses. + let audit_file_id = detection.audit_file_id.ok_or_else(|| { ErrorKind::Conflict - .with_message("Run has no analysis yet") - .with_resource("pipeline_run") + .with_message("Detection has no analysis yet") + .with_resource("detection") })?; let audit_file = conn .find_file_in_workspace(workspace_id, audit_file_id) .await? .ok_or_else(|| { ErrorKind::NotFound - .with_message("The analysis for this run has been deleted") - .with_resource("pipeline_run") + .with_message("The analysis for this detection has been deleted") + .with_resource("detection") })?; let key = AuditKey::from_str(&audit_file.storage_path).map_err(|err| { ErrorKind::InternalServerError @@ -379,6 +323,185 @@ impl RunBlobStore { .with_context(err.to_string()) }) } + + /// Encrypts a redaction's review audit, writes it to the audit bucket, and + /// builds the `review`-kind [`WorkspaceFile`] row that will point at it — + /// without inserting the row. + /// + /// The review audit is the post-redaction [`Audit`]: the detection's analysis + /// with the reviewer's edits applied and the redaction outcome recorded per + /// entity. It is the redaction's counterpart to + /// [`stage_analyzed_document`](Self::stage_analyzed_document) — same staged + /// object-then-row protocol, reclaimed on rollback via + /// [`discard_staged_object`](Self::discard_staged_object) — but a distinct + /// file kind so it is never confused with the immutable detection audit. + pub async fn stage_review_audit( + &self, + pipeline: &WorkspacePipeline, + workspace_settings: &RetentionSettings, + account_id: Uuid, + reviewed: &Audit, + ) -> Result { + let workspace_id = pipeline.workspace_id; + let plaintext = serde_json::to_vec(reviewed).map_err(analysis_serde_error)?; + let hash = Sha256::digest(&plaintext).to_vec(); + let size = plaintext.len() as i64; + let ciphertext = self + .infra + .crypto + .encrypt(workspace_id, &plaintext) + .map_err(|err| { + ErrorKind::InternalServerError + .with_message("Failed to encrypt review audit") + .with_context(err.to_string()) + })?; + + let store = self.infra.nats.object_store::().await?; + let key = AuditKey::generate(workspace_id); + store.put(&key, Cursor::new(ciphertext)).await?; + + // A review audit shares the audit-logs retention scope with the detection + // audit (workspace baseline, pipeline override if set). + let over = pipeline.metadata.or_default().retention; + let expires_at = workspace_settings + .resolve(RetentionScope::AuditLogs, over.as_ref()) + .expires_at(jiff::Timestamp::now()); + + Ok(NewWorkspaceFile { + workspace_id, + account_id, + display_name: Some("review.audit".to_owned()), + original_filename: Some("review.audit".to_owned()), + file_extension: Some("json".to_owned()), + file_kind: Some(FileKind::Review), + file_size_bytes: size, + file_hash_sha256: hash, + storage_path: key.to_string(), + storage_bucket: store.bucket().to_owned(), + expires_at: expires_at.map(Into::into), + ..Default::default() + }) + } + + /// Encrypts redacted bytes, writes them to the files bucket, and builds the + /// `redacted`-kind [`WorkspaceFile`] row that will point at them — without + /// inserting the row. + /// + /// A redaction commits its output row, review-audit row, and redaction row + /// together in one transaction, so the output file is staged (object written, + /// row returned for the caller to insert) rather than inserted here, and is + /// reclaimed on rollback via + /// [`discard_staged_object`](Self::discard_staged_object). The redacted file + /// is a first-class file (a sibling of the source), downloadable + /// through the normal file endpoints. + pub async fn stage_redacted_file( + &self, + source: &WorkspaceFile, + pipeline: &WorkspacePipeline, + workspace_settings: &RetentionSettings, + account_id: Uuid, + bytes: Bytes, + ) -> Result { + let plaintext_size = bytes.len() as i64; + let plaintext_hash = Sha256::digest(&bytes).to_vec(); + let ciphertext = self + .infra + .crypto + .encrypt(source.workspace_id, &bytes) + .map_err(|err| { + ErrorKind::InternalServerError + .with_message("Failed to encrypt redacted file") + .with_context(err.to_string()) + })?; + + let store = self.infra.nats.object_store::().await?; + let key = FileKey::generate(source.workspace_id); + store.put(&key, Cursor::new(ciphertext)).await?; + + let over = pipeline.metadata.or_default().retention; + let expires_at = workspace_settings + .resolve(RetentionScope::RedactedDocuments, over.as_ref()) + .expires_at(jiff::Timestamp::now()); + + let redacted_name = redacted_display_name(&source.display_name, &source.file_extension); + Ok(NewWorkspaceFile { + workspace_id: source.workspace_id, + account_id, + parent_id: Some(source.id), + display_name: Some(redacted_name), + original_filename: Some(source.original_filename.clone()), + file_extension: Some(source.file_extension.clone()), + file_kind: Some(FileKind::Redacted), + file_size_bytes: plaintext_size, + file_hash_sha256: plaintext_hash, + storage_path: key.to_string(), + storage_bucket: store.bucket().to_owned(), + expires_at: expires_at.map(Into::into), + ..Default::default() + }) + } + + /// Fetches and decrypts a redaction's stored review [`Audit`] by its file id. + /// + /// The review counterpart to + /// [`load_analyzed_document`](Self::load_analyzed_document); the `engine` + /// rebuilds the report from its serialized form. Errors if the redaction has + /// no review audit (409) or it has since been deleted (404). + pub async fn load_review_audit( + &self, + conn: &mut PgConn, + engine: &Engine, + workspace_id: Uuid, + review_file_id: Option, + ) -> Result { + let review_file_id = review_file_id.ok_or_else(|| { + ErrorKind::Conflict + .with_message("Redaction has no review audit") + .with_resource("redaction") + })?; + let review_file = conn + .find_file_in_workspace(workspace_id, review_file_id) + .await? + .ok_or_else(|| { + ErrorKind::NotFound + .with_message("The review audit for this redaction has been deleted") + .with_resource("redaction") + })?; + let key = AuditKey::from_str(&review_file.storage_path).map_err(|err| { + ErrorKind::InternalServerError + .with_message("Invalid review audit storage key") + .with_context(err.to_string()) + })?; + + let store = self.infra.nats.object_store::().await?; + let data = store.get(&key).await?.ok_or_else(|| { + ErrorKind::InternalServerError.with_message("Review audit is missing from storage") + })?; + let mut reader = data.into_reader(); + let mut ciphertext = Vec::new(); + reader.read_to_end(&mut ciphertext).await.map_err(|err| { + ErrorKind::InternalServerError + .with_message("Failed to read review audit") + .with_context(err.to_string()) + })?; + + let plaintext = self + .infra + .crypto + .decrypt(workspace_id, &ciphertext) + .map_err(|err| { + ErrorKind::InternalServerError + .with_message("Failed to decrypt review audit") + .with_context(err.to_string()) + })?; + engine + .deserialize_audit(&mut serde_json::Deserializer::from_slice(&plaintext)) + .map_err(|err| { + ErrorKind::InternalServerError + .with_message("Failed to decode review audit") + .with_context(err.to_string()) + }) + } } /// Maps an analysis (de)serialization failure to an internal error. @@ -387,3 +510,79 @@ fn analysis_serde_error(error: serde_json::Error) -> Error<'static> { .with_message("Failed to process analysis") .with_context(error.to_string()) } + +/// Builds the redacted file's display name by inserting a `redacted` marker +/// before the extension: `report.pdf` becomes `report.redacted.pdf`. +/// +/// The stem is taken by stripping a trailing `.{extension}` (case-insensitive) +/// from the display name; a name that does not end in its own extension (or has +/// none) simply gains a `.redacted` suffix. +fn redacted_display_name(display_name: &str, extension: &str) -> String { + // Split off a trailing `.{extension}`, matched case-insensitively on both + // sides so an upper- or mixed-case extension (`Report.PDF`) still has the + // marker inserted before it, not appended after. + let suffix = format!(".{extension}"); + let stem = display_name + .len() + .checked_sub(suffix.len()) + .filter(|_| !extension.is_empty()) + .filter(|&at| display_name.is_char_boundary(at)) + .map(|at| display_name.split_at(at)) + .filter(|(_, tail)| tail.eq_ignore_ascii_case(&suffix)) + .map(|(stem, _)| stem); + match stem { + Some(stem) => format!("{stem}.redacted.{extension}"), + None => format!("{display_name}.redacted"), + } +} + +#[cfg(test)] +mod tests { + use super::redacted_display_name; + + #[test] + fn inserts_marker_before_the_extension() { + assert_eq!( + redacted_display_name("report.pdf", "pdf"), + "report.redacted.pdf" + ); + } + + #[test] + fn matches_the_extension_case_insensitively() { + // The name's extension case differs from the passed extension... + assert_eq!( + redacted_display_name("Report.PDF", "pdf"), + "Report.redacted.pdf" + ); + // ...and the passed extension itself may be upper- or mixed-case; the + // marker still lands before it, and the original extension text is kept. + assert_eq!( + redacted_display_name("Report.PDF", "PDF"), + "Report.redacted.PDF" + ); + assert_eq!( + redacted_display_name("report.pdf", "PDF"), + "report.redacted.PDF" + ); + } + + #[test] + fn preserves_a_multi_dot_stem() { + assert_eq!( + redacted_display_name("2026.q1.report.pdf", "pdf"), + "2026.q1.report.redacted.pdf" + ); + } + + #[test] + fn appends_when_the_name_lacks_its_extension() { + // A display name that does not end in `.{extension}` just gains the + // marker, so no extension is fabricated. + assert_eq!(redacted_display_name("report", "pdf"), "report.redacted"); + assert_eq!( + redacted_display_name("report.txt", "pdf"), + "report.txt.redacted" + ); + } +} diff --git a/migrations/2025-05-21-121132_notifications/up.sql b/migrations/2025-05-21-121132_notifications/up.sql index 3832be8b..a2c0c56e 100644 --- a/migrations/2025-05-21-121132_notifications/up.sql +++ b/migrations/2025-05-21-121132_notifications/up.sql @@ -1,18 +1,18 @@ -- Notifications: per-account notification inbox for member, connection-sync, --- and pipeline-run events. Account-scoped but a standalone feature; the client --- renders copy from the event type and its typed params. +-- detection, and redaction events. Account-scoped but a standalone feature; the +-- client renders copy from the event type and its typed params. -- Type of a notification event: what happened that the account is told about. CREATE TYPE NOTIFICATION_EVENT AS ENUM ( - 'member.invited', -- User was invited to a workspace - 'member.joined', -- A new member joined a workspace + 'member.invited', -- User was invited to a workspace + 'member.joined', -- A new member joined a workspace - 'connection.sync.completed', -- A connection sync completed - 'connection.sync.failed', -- A connection sync failed + 'connection.sync.completed', -- A connection sync completed + 'connection.sync.failed', -- A connection sync failed - 'pipeline.run.analyzed', -- A run finished detection, awaiting review - 'pipeline.run.completed', -- A run completed (redaction produced) - 'pipeline.run.failed' -- A run failed + 'pipeline.detection.completed', -- A detection finished analysis, ready to redact + 'pipeline.redaction.created', -- A redaction was created (redacted output produced) + 'pipeline.detection.failed' -- A detection failed ); COMMENT ON TYPE NOTIFICATION_EVENT IS 'Type of a notification event delivered to an account.'; diff --git a/migrations/2025-05-21-222840_workspaces/up.sql b/migrations/2025-05-21-222840_workspaces/up.sql index 72a5adf0..def8bae9 100644 --- a/migrations/2025-05-21-222840_workspaces/up.sql +++ b/migrations/2025-05-21-222840_workspaces/up.sql @@ -117,7 +117,7 @@ CREATE TABLE workspace_members ( notification_events_app NOTIFICATION_EVENT[] NOT NULL DEFAULT ARRAY[ 'member.invited', 'member.joined', 'connection.sync.completed', 'connection.sync.failed', - 'pipeline.run.analyzed', 'pipeline.run.completed', 'pipeline.run.failed' + 'pipeline.detection.completed', 'pipeline.redaction.created', 'pipeline.detection.failed' ]::NOTIFICATION_EVENT[], notification_events_email NOTIFICATION_EVENT[] NOT NULL DEFAULT '{}', diff --git a/migrations/2025-05-21-222841_activities/up.sql b/migrations/2025-05-21-222841_activities/up.sql index 67fd3c48..8df66a14 100644 --- a/migrations/2025-05-21-222841_activities/up.sql +++ b/migrations/2025-05-21-222841_activities/up.sql @@ -38,14 +38,14 @@ CREATE TYPE ACTIVITY_TYPE AS ENUM ( 'file.updated', 'file.deleted', - -- Pipeline activities + -- Pipeline, detection, and redaction activities 'pipeline.created', 'pipeline.updated', 'pipeline.deleted', - 'pipeline.run.started', - 'pipeline.run.analyzed', - 'pipeline.run.completed', - 'pipeline.run.failed', + 'pipeline.detection.started', + 'pipeline.detection.completed', + 'pipeline.detection.failed', + 'pipeline.redaction.created', -- Policy activities 'policy.created', diff --git a/migrations/2025-05-21-222842_webhooks/up.sql b/migrations/2025-05-21-222842_webhooks/up.sql index 1d1bc8c7..387e6189 100644 --- a/migrations/2025-05-21-222842_webhooks/up.sql +++ b/migrations/2025-05-21-222842_webhooks/up.sql @@ -32,14 +32,14 @@ CREATE TYPE WEBHOOK_EVENT AS ENUM ( 'connection.sync.completed', 'connection.sync.failed', - -- Pipeline events + -- Pipeline, detection, and redaction events 'pipeline.created', 'pipeline.updated', 'pipeline.deleted', - 'pipeline.run.started', - 'pipeline.run.analyzed', - 'pipeline.run.completed', - 'pipeline.run.failed', + 'pipeline.detection.started', + 'pipeline.detection.completed', + 'pipeline.detection.failed', + 'pipeline.redaction.created', -- Policy events 'policy.created', diff --git a/migrations/2026-08-21-045211_event_outbox/down.sql b/migrations/2025-05-21-222843_event_outbox/down.sql similarity index 100% rename from migrations/2026-08-21-045211_event_outbox/down.sql rename to migrations/2025-05-21-222843_event_outbox/down.sql diff --git a/migrations/2026-08-21-045211_event_outbox/up.sql b/migrations/2025-05-21-222843_event_outbox/up.sql similarity index 100% rename from migrations/2026-08-21-045211_event_outbox/up.sql rename to migrations/2025-05-21-222843_event_outbox/up.sql diff --git a/migrations/2025-05-27-011852_files/up.sql b/migrations/2025-05-27-011852_files/up.sql index 8feb3620..1c612eef 100644 --- a/migrations/2025-05-27-011852_files/up.sql +++ b/migrations/2025-05-27-011852_files/up.sql @@ -5,11 +5,12 @@ -- Role of a file: drives data-retention scope and whether it is user-facing. CREATE TYPE FILE_KIND AS ENUM ( 'original', -- Source document (uploaded or imported) - 'redacted', -- Redacted output produced by a pipeline - 'audit' -- Engine analysis blob (not shown in file lists) + 'redacted', -- Redacted output produced by a redaction + 'audit', -- Engine detection analysis blob (not shown in file lists) + 'review' -- Engine analysis after reviewer edits + redaction (not shown in file lists) ); -COMMENT ON TYPE FILE_KIND IS 'The role of a file: original document, redacted output, or audit blob.'; +COMMENT ON TYPE FILE_KIND IS 'The role of a file: original document, redacted output, detection audit, or review (redaction) audit.'; -- Workspace files table: one stored document, with version tracking and dedup. CREATE TABLE workspace_files ( diff --git a/migrations/2026-01-19-045014_pipelines/down.sql b/migrations/2026-01-19-045014_pipelines/down.sql deleted file mode 100644 index 8edca28a..00000000 --- a/migrations/2026-01-19-045014_pipelines/down.sql +++ /dev/null @@ -1,10 +0,0 @@ --- Revert the pipelines tables. --- Objects are dropped in reverse order of creation. - -DROP TABLE IF EXISTS workspace_pipeline_run_usage; -DROP TABLE IF EXISTS workspace_pipeline_runs; -DROP TABLE IF EXISTS workspace_pipelines; - -DROP TYPE IF EXISTS PIPELINE_TRIGGER_TYPE; -DROP TYPE IF EXISTS PIPELINE_RUN_STATUS; -DROP TYPE IF EXISTS PIPELINE_STATUS; diff --git a/migrations/2026-01-19-045014_pipelines/up.sql b/migrations/2026-01-19-045014_pipelines/up.sql deleted file mode 100644 index c36e3479..00000000 --- a/migrations/2026-01-19-045014_pipelines/up.sql +++ /dev/null @@ -1,277 +0,0 @@ --- Pipelines: redaction pipeline definitions, their runs, and produced files. --- A pipeline is a workspace-scoped detection/redaction config; a run is one --- pass of a file through that config. Policy references live in a join table --- declared alongside policies, not embedded here. - --- Lifecycle status of a pipeline definition. -CREATE TYPE PIPELINE_STATUS AS ENUM ( - 'draft', -- Pipeline is being configured - 'enabled', -- Pipeline is ready to run - 'disabled' -- Pipeline is turned off -); - -COMMENT ON TYPE PIPELINE_STATUS IS 'Lifecycle status of a pipeline definition: draft, enabled, or disabled.'; - --- Execution status of a pipeline run. -CREATE TYPE PIPELINE_RUN_STATUS AS ENUM ( - 'queued', -- Enqueued for detection; no worker has picked it up yet - 'analyzing', -- A worker is actively analyzing the document - 'analyzed', -- Detection done; awaiting reviewer verification - 'completed', -- Redaction applied; run finished - 'failed', -- Run failed with error - 'cancelled' -- Run was cancelled by user -); - -COMMENT ON TYPE PIPELINE_RUN_STATUS IS 'Execution status of a pipeline run.'; - --- How a pipeline run was initiated. -CREATE TYPE PIPELINE_TRIGGER_TYPE AS ENUM ( - 'user', -- Started directly by a user - 'system' -- Started automatically (e.g. a file upload auto-redacted) -); - -COMMENT ON TYPE PIPELINE_TRIGGER_TYPE IS 'How a pipeline run was initiated: by a user or by the system.'; - --- Pipeline definitions table: a workspace's detection/redaction configs. -CREATE TABLE workspace_pipelines ( - -- Primary identifier - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - - -- References - workspace_id UUID NOT NULL REFERENCES workspaces (id) ON DELETE CASCADE, - account_id UUID NOT NULL REFERENCES accounts (id) ON DELETE CASCADE, - - -- Composite key target for workspace-scoped foreign keys (join tables). - CONSTRAINT workspace_pipelines_workspace_id_id_key UNIQUE (workspace_id, id), - - -- URL identity, unique within the workspace (among live pipelines; enforced - -- by a partial index below so a slug frees up after soft deletion): lowercase - -- alphanumeric with single internal dashes, 3-32 characters. - slug TEXT NOT NULL, - CONSTRAINT workspace_pipelines_slug_length CHECK (length(slug) BETWEEN 3 AND 32), - CONSTRAINT workspace_pipelines_slug_format CHECK (slug ~ '^[a-z0-9]+(-[a-z0-9]+)*$'), - - -- Core attributes - display_name TEXT NOT NULL, - CONSTRAINT workspace_pipelines_display_name_length CHECK (length(trim(display_name)) BETWEEN 2 AND 128), - description TEXT DEFAULT NULL, - CONSTRAINT workspace_pipelines_description_length CHECK (description IS NULL OR length(description) <= 500), - status PIPELINE_STATUS NOT NULL DEFAULT 'draft', - - -- Engine detection + redaction config (nvisy_schema plan as JSON): - -- recognizers, enrichers, deduplication, label catalog, default scope. - -- Policy references are relational (workspace_pipeline_policies, declared - -- alongside policies), not embedded here. - definition JSONB NOT NULL, - CONSTRAINT workspace_pipelines_definition_size CHECK (length(definition::TEXT) BETWEEN 2 AND 1048576), - - -- Free-form metadata for filtering and display. - metadata JSONB NOT NULL DEFAULT '{}', - CONSTRAINT workspace_pipelines_metadata_size CHECK (length(metadata::TEXT) BETWEEN 2 AND 65536), - - -- Lifecycle timestamps - created_at TIMESTAMPTZ NOT NULL DEFAULT current_timestamp, - updated_at TIMESTAMPTZ NOT NULL DEFAULT current_timestamp, - deleted_at TIMESTAMPTZ DEFAULT NULL, - CONSTRAINT workspace_pipelines_updated_after_created CHECK (updated_at >= created_at), - CONSTRAINT workspace_pipelines_deleted_after_created CHECK (deleted_at IS NULL OR deleted_at >= created_at) -); - --- Maintain updated_at on every row modification. -SELECT setup_updated_at('workspace_pipelines'); - --- One live pipeline per slug within a workspace (slug frees up after deletion). -CREATE UNIQUE INDEX workspace_pipelines_slug_unique_idx - ON workspace_pipelines (workspace_id, slug) - WHERE deleted_at IS NULL; - --- Live pipelines of a workspace, newest first (the pipeline list). -CREATE INDEX workspace_pipelines_workspace_idx - ON workspace_pipelines (workspace_id, created_at DESC) - WHERE deleted_at IS NULL; - --- Live pipelines created by an account, newest first. -CREATE INDEX workspace_pipelines_account_idx - ON workspace_pipelines (account_id, created_at DESC) - WHERE deleted_at IS NULL; - --- Filter live pipelines by lifecycle status within a workspace. -CREATE INDEX workspace_pipelines_status_idx - ON workspace_pipelines (status, workspace_id) - WHERE deleted_at IS NULL; - --- Trigram search over live pipeline display names. -CREATE INDEX workspace_pipelines_display_name_trgm_idx - ON workspace_pipelines USING gin (display_name gin_trgm_ops) - WHERE deleted_at IS NULL; - -COMMENT ON TABLE workspace_pipelines IS 'Workspace-scoped redaction pipeline definitions.'; -COMMENT ON COLUMN workspace_pipelines.id IS 'Unique pipeline identifier'; -COMMENT ON COLUMN workspace_pipelines.workspace_id IS 'Workspace this pipeline belongs to'; -COMMENT ON COLUMN workspace_pipelines.account_id IS 'Account that created the pipeline'; -COMMENT ON COLUMN workspace_pipelines.slug IS 'URL identity, unique among live pipelines in the workspace'; -COMMENT ON COLUMN workspace_pipelines.display_name IS 'Pipeline display name (2-128 chars)'; -COMMENT ON COLUMN workspace_pipelines.description IS 'Pipeline description (up to 500 chars)'; -COMMENT ON COLUMN workspace_pipelines.status IS 'Pipeline lifecycle status'; -COMMENT ON COLUMN workspace_pipelines.definition IS 'Detection/redaction config (nvisy_schema plan as JSON)'; -COMMENT ON COLUMN workspace_pipelines.metadata IS 'Free-form metadata for filtering/display'; -COMMENT ON COLUMN workspace_pipelines.created_at IS 'Pipeline creation timestamp'; -COMMENT ON COLUMN workspace_pipelines.updated_at IS 'Last modification timestamp'; -COMMENT ON COLUMN workspace_pipelines.deleted_at IS 'Soft-deletion timestamp; NULL means live'; - --- Pipeline runs table: one pass of a file through a pipeline. -CREATE TABLE workspace_pipeline_runs ( - -- Primary identifier - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - - -- References - pipeline_id UUID NOT NULL REFERENCES workspace_pipelines (id) ON DELETE CASCADE, - account_id UUID NOT NULL REFERENCES accounts (id) ON DELETE CASCADE, - - -- The three files a run relates to, each a distinct role. The input is the - -- source document (required); the audit blob and redacted output are produced - -- by the run, so they are null until their phase completes. - -- input: the original document being analyzed/redacted. - -- audit: the engine's analysis (Audit), a `file_kind = audit` file held - -- between detect and redact; redact reads it as the source of truth. - -- output: the redacted document produced by redact. - -- A run is append-only audit history: its file references survive the files - -- themselves. Files are only ever soft-deleted (their objects purged), so - -- these ON DELETE actions fire only on a hard delete. The app never hard- - -- deletes an individual file; the one hard delete is a whole-workspace - -- teardown, which cascades files and runs away together. The input therefore - -- cascades — a workspace deletion that removes the source document should - -- remove its runs too. Produced files use ON DELETE SET NULL so that, in that - -- same teardown, they clear rather than cascade (redundant here, but the - -- correct action for a produced artifact). - input_file_id UUID NOT NULL REFERENCES workspace_files (id) ON DELETE CASCADE, - audit_file_id UUID DEFAULT NULL REFERENCES workspace_files (id) ON DELETE SET NULL, - output_file_id UUID DEFAULT NULL REFERENCES workspace_files (id) ON DELETE SET NULL, - - -- Run attributes - trigger_type PIPELINE_TRIGGER_TYPE NOT NULL DEFAULT 'user', - status PIPELINE_RUN_STATUS NOT NULL DEFAULT 'queued', - - -- Idempotency key from the initiating detect request; a repeat replays the - -- existing run instead of analyzing twice. - idempotency_key TEXT DEFAULT NULL, - CONSTRAINT workspace_pipeline_runs_idempotency_key_length CHECK (idempotency_key IS NULL OR length(idempotency_key) BETWEEN 1 AND 255), - - -- Non-encrypted metadata for filtering and display. The engine's full - -- per-recognizer usage report (durations, per-model token counts) is kept - -- here under `usage` for drill-down; per-model token totals for usage - -- aggregation live in the workspace_pipeline_run_usage table below. - metadata JSONB NOT NULL DEFAULT '{}', - CONSTRAINT workspace_pipeline_runs_metadata_size CHECK (length(metadata::TEXT) BETWEEN 2 AND 65536), - - -- Detection lease: when a worker last claimed this run. A redelivered job - -- whose claim is still fresh is skipped (no double-analyze); a stale claim - -- (a worker that died mid-analysis) can be re-claimed. Null until claimed. - claimed_at TIMESTAMPTZ DEFAULT NULL, - - -- Timing - started_at TIMESTAMPTZ NOT NULL DEFAULT current_timestamp, - completed_at TIMESTAMPTZ DEFAULT NULL, - CONSTRAINT workspace_pipeline_runs_completed_after_started CHECK (completed_at IS NULL OR completed_at >= started_at) -); - --- A pipeline's runs, newest first (the run list). -CREATE INDEX workspace_pipeline_runs_pipeline_idx - ON workspace_pipeline_runs (pipeline_id, started_at DESC); - --- Runs triggered by an account, newest first. -CREATE INDEX workspace_pipeline_runs_account_idx - ON workspace_pipeline_runs (account_id, started_at DESC); - --- In-flight runs by status (queue and review backlog). -CREATE INDEX workspace_pipeline_runs_status_idx - ON workspace_pipeline_runs (status, started_at DESC) - WHERE status IN ('queued', 'analyzing', 'analyzed'); - --- Runs analyzing a given input file, newest first. -CREATE INDEX workspace_pipeline_runs_input_file_idx - ON workspace_pipeline_runs (input_file_id, started_at DESC); - --- Idempotent detect: at most one run per (pipeline, idempotency key). -CREATE UNIQUE INDEX workspace_pipeline_runs_idempotency_idx - ON workspace_pipeline_runs (pipeline_id, idempotency_key) - WHERE idempotency_key IS NOT NULL; - --- A run is append-only history: its file references (input/audit/output) are --- kept even after those files are deleted, so the record of what a run analyzed --- and produced survives. The `ON DELETE SET NULL` FKs would fire only on a hard --- file delete, which never happens (files are soft-deleted); a reference to a --- soft-deleted file resolves to "gone" at read time, distinct from a NULL that --- means the run never had one. - -COMMENT ON TABLE workspace_pipeline_runs IS 'Detect/redact runs: one pass of a file through a pipeline.'; -COMMENT ON COLUMN workspace_pipeline_runs.id IS 'Unique run identifier'; -COMMENT ON COLUMN workspace_pipeline_runs.pipeline_id IS 'Pipeline whose config drove the run'; -COMMENT ON COLUMN workspace_pipeline_runs.account_id IS 'Account that triggered the run'; -COMMENT ON COLUMN workspace_pipeline_runs.input_file_id IS 'Source document the run analyzes / redacts'; -COMMENT ON COLUMN workspace_pipeline_runs.audit_file_id IS 'Audit file (file_kind=audit) holding the analysis between detect and redact'; -COMMENT ON COLUMN workspace_pipeline_runs.output_file_id IS 'Redacted document produced by redact; NULL until completed'; -COMMENT ON COLUMN workspace_pipeline_runs.trigger_type IS 'How the run was initiated'; -COMMENT ON COLUMN workspace_pipeline_runs.status IS 'Current run status'; -COMMENT ON COLUMN workspace_pipeline_runs.idempotency_key IS 'Detect idempotency key (dedupes retries)'; -COMMENT ON COLUMN workspace_pipeline_runs.metadata IS 'Non-encrypted metadata for filtering/display; holds the full per-recognizer usage report under `usage`'; -COMMENT ON COLUMN workspace_pipeline_runs.claimed_at IS 'Detection lease: when a worker last claimed this run'; -COMMENT ON COLUMN workspace_pipeline_runs.started_at IS 'When the run started'; -COMMENT ON COLUMN workspace_pipeline_runs.completed_at IS 'When the run completed; NULL while in flight'; - --- Per-model inference usage for a run: one row per distinct model a run's --- recognizers used. Token counts are aggregated across the recognizers that --- shared a model, letting usage analytics report tokens broken down by model --- (a run's summed tokens cannot, since a run may mix models). The full --- per-recognizer report is kept on the run (metadata.usage) for drill-down; --- this table is the aggregation surface. -CREATE TABLE workspace_pipeline_run_usage ( - -- Primary identifier - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - - -- The run this usage belongs to; usage is deleted with its run. - run_id UUID NOT NULL REFERENCES workspace_pipeline_runs (id) ON DELETE CASCADE, - - -- The model and its optional version. Identity is (run, model, version): - -- a run may use the same model at more than one version, and each is its own - -- row (uniqueness enforced by an index below that normalizes a NULL version). - model TEXT NOT NULL, - version TEXT DEFAULT NULL, - CONSTRAINT workspace_pipeline_run_usage_model_length CHECK (length(model) BETWEEN 1 AND 255), - CONSTRAINT workspace_pipeline_run_usage_version_length CHECK (version IS NULL OR length(version) BETWEEN 1 AND 255), - - -- Token counts as the provider reported them. Each is independently nullable: - -- `total` is NOT necessarily input + output (a provider may report only a - -- total, or a total that also counts cached/reasoning tokens), so all three - -- are carried faithfully and never derived from one another. - input_tokens BIGINT DEFAULT NULL, - output_tokens BIGINT DEFAULT NULL, - total_tokens BIGINT DEFAULT NULL, - CONSTRAINT workspace_pipeline_run_usage_input_non_negative CHECK (input_tokens IS NULL OR input_tokens >= 0), - CONSTRAINT workspace_pipeline_run_usage_output_non_negative CHECK (output_tokens IS NULL OR output_tokens >= 0), - CONSTRAINT workspace_pipeline_run_usage_total_non_negative CHECK (total_tokens IS NULL OR total_tokens >= 0), - - -- Wall-clock time this model's recognizers spent, in milliseconds. - duration_ms BIGINT NOT NULL DEFAULT 0, - CONSTRAINT workspace_pipeline_run_usage_duration_non_negative CHECK (duration_ms >= 0) -); - --- One row per (run, model, version); a NULL version is normalized so two --- unversioned rows for the same model collide instead of both being inserted. --- Leads with run_id, so it also serves per-run drill-down lookups. -CREATE UNIQUE INDEX workspace_pipeline_run_usage_run_model_version_key - ON workspace_pipeline_run_usage (run_id, model, COALESCE(version, '')); - --- Usage rollups by model across runs. -CREATE INDEX workspace_pipeline_run_usage_model_idx - ON workspace_pipeline_run_usage (model); - -COMMENT ON TABLE workspace_pipeline_run_usage IS 'Per-model inference token usage for a pipeline run.'; -COMMENT ON COLUMN workspace_pipeline_run_usage.id IS 'Unique usage row identifier'; -COMMENT ON COLUMN workspace_pipeline_run_usage.run_id IS 'Run this usage belongs to'; -COMMENT ON COLUMN workspace_pipeline_run_usage.model IS 'Model identifier the recognizers used'; -COMMENT ON COLUMN workspace_pipeline_run_usage.version IS 'Model version, if the provider reported one'; -COMMENT ON COLUMN workspace_pipeline_run_usage.input_tokens IS 'Input/prompt tokens for this model; NULL if not reported'; -COMMENT ON COLUMN workspace_pipeline_run_usage.output_tokens IS 'Output/completion tokens for this model; NULL if not reported'; -COMMENT ON COLUMN workspace_pipeline_run_usage.total_tokens IS 'Total tokens as reported (not necessarily input + output); NULL if not reported'; -COMMENT ON COLUMN workspace_pipeline_run_usage.duration_ms IS 'Wall-clock time this model spent, in milliseconds'; diff --git a/migrations/2026-01-19-045014_policies/down.sql b/migrations/2026-01-19-045014_policies/down.sql new file mode 100644 index 00000000..2d22722a --- /dev/null +++ b/migrations/2026-01-19-045014_policies/down.sql @@ -0,0 +1,2 @@ +-- Revert the policies table. +DROP TABLE IF EXISTS workspace_policies; diff --git a/migrations/2026-01-19-045015_policies/up.sql b/migrations/2026-01-19-045014_policies/up.sql similarity index 72% rename from migrations/2026-01-19-045015_policies/up.sql rename to migrations/2026-01-19-045014_policies/up.sql index 7f10fa5d..1dcb42f5 100644 --- a/migrations/2026-01-19-045015_policies/up.sql +++ b/migrations/2026-01-19-045014_policies/up.sql @@ -80,32 +80,3 @@ COMMENT ON COLUMN workspace_policies.metadata IS 'Metadata for filtering/display COMMENT ON COLUMN workspace_policies.created_at IS 'Creation timestamp'; COMMENT ON COLUMN workspace_policies.updated_at IS 'Last modification timestamp'; COMMENT ON COLUMN workspace_policies.deleted_at IS 'Soft-deletion timestamp; NULL means live'; - --- Pipeline -> policy join table: redaction policies a pipeline applies. --- Lives here (after both workspace_pipelines and workspace_policies exist); the --- shared workspace_id in both composite foreign keys enforces that a pipeline --- can only reference policies from its own workspace. Drops before its parent in --- down.sql. -CREATE TABLE workspace_pipeline_policies ( - -- References - workspace_id UUID NOT NULL REFERENCES workspaces (id) ON DELETE CASCADE, - pipeline_id UUID NOT NULL, - policy_id UUID NOT NULL, - - PRIMARY KEY (pipeline_id, policy_id), - - -- Composite foreign keys, both sharing workspace_id, so a pipeline can only - -- reference policies from its own workspace. - CONSTRAINT workspace_pipeline_policies_pipeline_fkey FOREIGN KEY (workspace_id, pipeline_id) - REFERENCES workspace_pipelines (workspace_id, id) ON DELETE CASCADE, - CONSTRAINT workspace_pipeline_policies_policy_fkey FOREIGN KEY (workspace_id, policy_id) - REFERENCES workspace_policies (workspace_id, id) ON DELETE CASCADE -); - --- All pipelines that apply a given policy (back the policy composite FK). -CREATE INDEX workspace_pipeline_policies_policy_idx ON workspace_pipeline_policies (policy_id); - -COMMENT ON TABLE workspace_pipeline_policies IS 'Policies a pipeline applies at redaction. CASCADE cleans up on hard delete.'; -COMMENT ON COLUMN workspace_pipeline_policies.workspace_id IS 'Workspace shared by the pipeline and policy'; -COMMENT ON COLUMN workspace_pipeline_policies.pipeline_id IS 'Pipeline that applies the policy'; -COMMENT ON COLUMN workspace_pipeline_policies.policy_id IS 'Policy applied by the pipeline'; diff --git a/migrations/2026-01-19-045015_pipelines/down.sql b/migrations/2026-01-19-045015_pipelines/down.sql new file mode 100644 index 00000000..aee1463a --- /dev/null +++ b/migrations/2026-01-19-045015_pipelines/down.sql @@ -0,0 +1,7 @@ +-- Revert the pipelines tables. +-- Objects are dropped in reverse order of creation. + +DROP TABLE IF EXISTS workspace_pipeline_policies; +DROP TABLE IF EXISTS workspace_pipelines; + +DROP TYPE IF EXISTS PIPELINE_STATUS; diff --git a/migrations/2026-01-19-045015_pipelines/up.sql b/migrations/2026-01-19-045015_pipelines/up.sql new file mode 100644 index 00000000..35312489 --- /dev/null +++ b/migrations/2026-01-19-045015_pipelines/up.sql @@ -0,0 +1,129 @@ +-- Pipelines: redaction pipeline definitions. A pipeline is a workspace-scoped +-- detection/redaction config that references workspace policies (via the join +-- table below) and drives the detections created from it. + +-- Lifecycle status of a pipeline definition. +CREATE TYPE PIPELINE_STATUS AS ENUM ( + 'draft', -- Pipeline is being configured + 'enabled', -- Pipeline is ready to run + 'disabled' -- Pipeline is turned off +); + +COMMENT ON TYPE PIPELINE_STATUS IS 'Lifecycle status of a pipeline definition: draft, enabled, or disabled.'; + +-- Pipeline definitions table: a workspace's detection/redaction configs. +CREATE TABLE workspace_pipelines ( + -- Primary identifier + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + -- References + workspace_id UUID NOT NULL REFERENCES workspaces (id) ON DELETE CASCADE, + account_id UUID NOT NULL REFERENCES accounts (id) ON DELETE CASCADE, + + -- Composite key target for workspace-scoped foreign keys (join tables). + CONSTRAINT workspace_pipelines_workspace_id_id_key UNIQUE (workspace_id, id), + + -- URL identity, unique within the workspace (among live pipelines; enforced + -- by a partial index below so a slug frees up after soft deletion): lowercase + -- alphanumeric with single internal dashes, 3-32 characters. + slug TEXT NOT NULL, + CONSTRAINT workspace_pipelines_slug_length CHECK (length(slug) BETWEEN 3 AND 32), + CONSTRAINT workspace_pipelines_slug_format CHECK (slug ~ '^[a-z0-9]+(-[a-z0-9]+)*$'), + + -- Core attributes + display_name TEXT NOT NULL, + CONSTRAINT workspace_pipelines_display_name_length CHECK (length(trim(display_name)) BETWEEN 2 AND 128), + description TEXT DEFAULT NULL, + CONSTRAINT workspace_pipelines_description_length CHECK (description IS NULL OR length(description) <= 500), + status PIPELINE_STATUS NOT NULL DEFAULT 'draft', + + -- Engine detection + redaction config (nvisy_schema plan as JSON): + -- recognizers, enrichers, deduplication, label catalog, default scope. + -- Policy references are relational (workspace_pipeline_policies, declared + -- alongside policies), not embedded here. + definition JSONB NOT NULL, + CONSTRAINT workspace_pipelines_definition_size CHECK (length(definition::TEXT) BETWEEN 2 AND 1048576), + + -- Free-form metadata for filtering and display. + metadata JSONB NOT NULL DEFAULT '{}', + CONSTRAINT workspace_pipelines_metadata_size CHECK (length(metadata::TEXT) BETWEEN 2 AND 65536), + + -- Lifecycle timestamps + created_at TIMESTAMPTZ NOT NULL DEFAULT current_timestamp, + updated_at TIMESTAMPTZ NOT NULL DEFAULT current_timestamp, + deleted_at TIMESTAMPTZ DEFAULT NULL, + CONSTRAINT workspace_pipelines_updated_after_created CHECK (updated_at >= created_at), + CONSTRAINT workspace_pipelines_deleted_after_created CHECK (deleted_at IS NULL OR deleted_at >= created_at) +); + +-- Maintain updated_at on every row modification. +SELECT setup_updated_at('workspace_pipelines'); + +-- One live pipeline per slug within a workspace (slug frees up after deletion). +CREATE UNIQUE INDEX workspace_pipelines_slug_unique_idx + ON workspace_pipelines (workspace_id, slug) + WHERE deleted_at IS NULL; + +-- Live pipelines of a workspace, newest first (the pipeline list). +CREATE INDEX workspace_pipelines_workspace_idx + ON workspace_pipelines (workspace_id, created_at DESC) + WHERE deleted_at IS NULL; + +-- Live pipelines created by an account, newest first. +CREATE INDEX workspace_pipelines_account_idx + ON workspace_pipelines (account_id, created_at DESC) + WHERE deleted_at IS NULL; + +-- Filter live pipelines by lifecycle status within a workspace. +CREATE INDEX workspace_pipelines_status_idx + ON workspace_pipelines (status, workspace_id) + WHERE deleted_at IS NULL; + +-- Trigram search over live pipeline display names. +CREATE INDEX workspace_pipelines_display_name_trgm_idx + ON workspace_pipelines USING gin (display_name gin_trgm_ops) + WHERE deleted_at IS NULL; + +COMMENT ON TABLE workspace_pipelines IS 'Workspace-scoped redaction pipeline definitions.'; +COMMENT ON COLUMN workspace_pipelines.id IS 'Unique pipeline identifier'; +COMMENT ON COLUMN workspace_pipelines.workspace_id IS 'Workspace this pipeline belongs to'; +COMMENT ON COLUMN workspace_pipelines.account_id IS 'Account that created the pipeline'; +COMMENT ON COLUMN workspace_pipelines.slug IS 'URL identity, unique among live pipelines in the workspace'; +COMMENT ON COLUMN workspace_pipelines.display_name IS 'Pipeline display name (2-128 chars)'; +COMMENT ON COLUMN workspace_pipelines.description IS 'Pipeline description (up to 500 chars)'; +COMMENT ON COLUMN workspace_pipelines.status IS 'Pipeline lifecycle status'; +COMMENT ON COLUMN workspace_pipelines.definition IS 'Detection/redaction config (nvisy_schema plan as JSON)'; +COMMENT ON COLUMN workspace_pipelines.metadata IS 'Free-form metadata for filtering/display'; +COMMENT ON COLUMN workspace_pipelines.created_at IS 'Pipeline creation timestamp'; +COMMENT ON COLUMN workspace_pipelines.updated_at IS 'Last modification timestamp'; +COMMENT ON COLUMN workspace_pipelines.deleted_at IS 'Soft-deletion timestamp; NULL means live'; + + +-- Pipeline -> policy join table: redaction policies a pipeline applies. +-- Lives here (after both workspace_pipelines and workspace_policies exist); the +-- shared workspace_id in both composite foreign keys enforces that a pipeline +-- can only reference policies from its own workspace. Drops before its parent in +-- down.sql. +CREATE TABLE workspace_pipeline_policies ( + -- References + workspace_id UUID NOT NULL REFERENCES workspaces (id) ON DELETE CASCADE, + pipeline_id UUID NOT NULL, + policy_id UUID NOT NULL, + + PRIMARY KEY (pipeline_id, policy_id), + + -- Composite foreign keys, both sharing workspace_id, so a pipeline can only + -- reference policies from its own workspace. + CONSTRAINT workspace_pipeline_policies_pipeline_fkey FOREIGN KEY (workspace_id, pipeline_id) + REFERENCES workspace_pipelines (workspace_id, id) ON DELETE CASCADE, + CONSTRAINT workspace_pipeline_policies_policy_fkey FOREIGN KEY (workspace_id, policy_id) + REFERENCES workspace_policies (workspace_id, id) ON DELETE CASCADE +); + +-- All pipelines that apply a given policy (back the policy composite FK). +CREATE INDEX workspace_pipeline_policies_policy_idx ON workspace_pipeline_policies (policy_id); + +COMMENT ON TABLE workspace_pipeline_policies IS 'Policies a pipeline applies at redaction. CASCADE cleans up on hard delete.'; +COMMENT ON COLUMN workspace_pipeline_policies.workspace_id IS 'Workspace shared by the pipeline and policy'; +COMMENT ON COLUMN workspace_pipeline_policies.pipeline_id IS 'Pipeline that applies the policy'; +COMMENT ON COLUMN workspace_pipeline_policies.policy_id IS 'Policy applied by the pipeline'; diff --git a/migrations/2026-01-19-045015_policies/down.sql b/migrations/2026-01-19-045015_policies/down.sql deleted file mode 100644 index 13919a95..00000000 --- a/migrations/2026-01-19-045015_policies/down.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Revert the policies tables. --- Objects are dropped in reverse order of creation. - --- workspace_pipeline_policies is the pipeline-to-policy join; drop it before --- its parent policy table. -DROP TABLE IF EXISTS workspace_pipeline_policies; -DROP TABLE IF EXISTS workspace_policies; diff --git a/migrations/2026-01-19-045016_detections/down.sql b/migrations/2026-01-19-045016_detections/down.sql new file mode 100644 index 00000000..1eee3a1d --- /dev/null +++ b/migrations/2026-01-19-045016_detections/down.sql @@ -0,0 +1,9 @@ +-- Revert the detections tables. +-- Objects are dropped in reverse order of creation. + +DROP TABLE IF EXISTS workspace_detection_jobs; +DROP TABLE IF EXISTS workspace_detection_usage; +DROP TABLE IF EXISTS workspace_detections; + +DROP TYPE IF EXISTS PIPELINE_TRIGGER_TYPE; +DROP TYPE IF EXISTS DETECTION_STATUS; diff --git a/migrations/2026-01-19-045016_detections/up.sql b/migrations/2026-01-19-045016_detections/up.sql new file mode 100644 index 00000000..3285d591 --- /dev/null +++ b/migrations/2026-01-19-045016_detections/up.sql @@ -0,0 +1,248 @@ +-- Detections: one analysis pass of a file through a pipeline, its per-model +-- usage, and the transactional outbox that queues each analysis. A detection +-- can produce many redactions (see the redactions migration). + +CREATE TYPE DETECTION_STATUS AS ENUM ( + 'pending', -- Enqueued for detection; no worker has picked it up yet + 'executing', -- A worker is actively analyzing the document + 'complete', -- Detection done; ready to redact + 'failed' -- Detection failed with error +); + +COMMENT ON TYPE DETECTION_STATUS IS 'Execution status of a detection: pending, executing, complete, or failed.'; + +-- How a detection was initiated. +CREATE TYPE PIPELINE_TRIGGER_TYPE AS ENUM ( + 'user', -- Started directly by a user + 'system' -- Started automatically (e.g. a file upload auto-redacted) +); + +COMMENT ON TYPE PIPELINE_TRIGGER_TYPE IS 'How a detection was initiated: by a user or by the system.'; + +-- Detections table: one analysis pass of a file through a pipeline. +CREATE TABLE workspace_detections ( + -- Primary identifier + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + -- References + pipeline_id UUID NOT NULL REFERENCES workspace_pipelines (id) ON DELETE CASCADE, + account_id UUID NOT NULL REFERENCES accounts (id) ON DELETE CASCADE, + + -- The two files a detection relates to, each a distinct role. The input is + -- the source document (required); the audit blob is produced by the analysis, + -- so it is null until the pass completes. Redacted outputs are not here — a + -- detection produces many redactions, each owning its own output (see the + -- workspace_redactions table below). + -- input: the original document being analyzed. + -- audit: the engine's analysis (Audit), a `file_kind = audit` file held + -- between detect and redact; redact reads it as the source of truth. + -- A detection is append-only audit history: its file references survive the + -- files themselves. Files are only ever soft-deleted (their objects purged), + -- so these ON DELETE actions fire only on a hard delete. The app never hard- + -- deletes an individual file; the one hard delete is a whole-workspace + -- teardown, which cascades files and detections away together. The input + -- therefore cascades — a workspace deletion that removes the source document + -- should remove its detections too. The produced audit uses ON DELETE SET + -- NULL so that, in that same teardown, it clears rather than cascades + -- (redundant here, but the correct action for a produced artifact). + input_file_id UUID NOT NULL REFERENCES workspace_files (id) ON DELETE CASCADE, + audit_file_id UUID DEFAULT NULL REFERENCES workspace_files (id) ON DELETE SET NULL, + + -- Detection attributes + trigger_type PIPELINE_TRIGGER_TYPE NOT NULL DEFAULT 'user', + status DETECTION_STATUS NOT NULL DEFAULT 'pending', + + -- Idempotency key from the initiating detect request; a repeat replays the + -- existing detection instead of analyzing twice. + idempotency_key TEXT DEFAULT NULL, + CONSTRAINT workspace_detections_idempotency_key_length CHECK (idempotency_key IS NULL OR length(idempotency_key) BETWEEN 1 AND 255), + + -- Non-encrypted metadata for filtering and display. The engine's full + -- per-recognizer usage report (durations, per-model token counts) is kept + -- here under `usage` for drill-down; per-model token totals for usage + -- aggregation live in the workspace_detection_usage table below. + metadata JSONB NOT NULL DEFAULT '{}', + CONSTRAINT workspace_detections_metadata_size CHECK (length(metadata::TEXT) BETWEEN 2 AND 65536), + + -- Detection lease: when a worker last claimed this detection. A redelivered + -- job whose claim is still fresh is skipped (no double-analyze); a stale claim + -- (a worker that died mid-analysis) can be re-claimed. Null until claimed. + claimed_at TIMESTAMPTZ DEFAULT NULL, + + -- Timing + started_at TIMESTAMPTZ NOT NULL DEFAULT current_timestamp, + completed_at TIMESTAMPTZ DEFAULT NULL, + CONSTRAINT workspace_detections_completed_after_started CHECK (completed_at IS NULL OR completed_at >= started_at) +); + +-- A pipeline's detections, newest first (the detection list). +CREATE INDEX workspace_detections_pipeline_idx + ON workspace_detections (pipeline_id, started_at DESC); + +-- Detections triggered by an account, newest first. +CREATE INDEX workspace_detections_account_idx + ON workspace_detections (account_id, started_at DESC); + +-- In-flight detections by status (queue and ready backlog). +CREATE INDEX workspace_detections_status_idx + ON workspace_detections (status, started_at DESC) + WHERE status IN ('pending', 'executing', 'complete'); + +-- Detections analyzing a given input file, newest first. +CREATE INDEX workspace_detections_input_file_idx + ON workspace_detections (input_file_id, started_at DESC); + +-- The file-expiry sweep's hold check matches a candidate file against a +-- detection's input OR audit file; the input side is covered above, this covers +-- the audit side. Partial, since most detections eventually carry an audit but +-- the column is NULL until analysis writes it. +CREATE INDEX workspace_detections_audit_file_idx + ON workspace_detections (audit_file_id) + WHERE audit_file_id IS NOT NULL; + +-- Idempotent detect: at most one detection per (pipeline, idempotency key). +CREATE UNIQUE INDEX workspace_detections_idempotency_idx + ON workspace_detections (pipeline_id, idempotency_key) + WHERE idempotency_key IS NOT NULL; + +-- A detection is append-only history: its file references (input/audit) are kept +-- even after those files are deleted, so the record of what it analyzed +-- survives. The `ON DELETE SET NULL` FK would fire only on a hard file delete, +-- which never happens (files are soft-deleted); a reference to a soft-deleted +-- file resolves to "gone" at read time, distinct from a NULL that means the +-- detection never had one. + +COMMENT ON TABLE workspace_detections IS 'Detections: one analysis pass of a file through a pipeline.'; +COMMENT ON COLUMN workspace_detections.id IS 'Unique detection identifier'; +COMMENT ON COLUMN workspace_detections.pipeline_id IS 'Pipeline whose config drove the detection'; +COMMENT ON COLUMN workspace_detections.account_id IS 'Account that triggered the detection'; +COMMENT ON COLUMN workspace_detections.input_file_id IS 'Source document the detection analyzes'; +COMMENT ON COLUMN workspace_detections.audit_file_id IS 'Audit file (file_kind=audit) holding the analysis between detect and redact'; +COMMENT ON COLUMN workspace_detections.trigger_type IS 'How the detection was initiated'; +COMMENT ON COLUMN workspace_detections.status IS 'Current detection status'; +COMMENT ON COLUMN workspace_detections.idempotency_key IS 'Detect idempotency key (dedupes retries)'; +COMMENT ON COLUMN workspace_detections.metadata IS 'Non-encrypted metadata for filtering/display; holds the full per-recognizer usage report under `usage`'; +COMMENT ON COLUMN workspace_detections.claimed_at IS 'Detection lease: when a worker last claimed this detection'; +COMMENT ON COLUMN workspace_detections.started_at IS 'When the detection started'; +COMMENT ON COLUMN workspace_detections.completed_at IS 'When the detection completed; NULL while in flight'; + +-- Per-model inference usage for a detection: one row per distinct model a +-- detection's recognizers used. Token counts are aggregated across the +-- recognizers that shared a model, letting usage analytics report tokens broken +-- down by model (a detection's summed tokens cannot, since it may mix models). +-- The full per-recognizer report is kept on the detection (metadata.usage) for +-- drill-down; this table is the aggregation surface. +CREATE TABLE workspace_detection_usage ( + -- Primary identifier + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + -- The detection this usage belongs to; usage is deleted with its detection. + detection_id UUID NOT NULL REFERENCES workspace_detections (id) ON DELETE CASCADE, + + -- The model and its optional version. Identity is (detection, model, + -- version): a detection may use the same model at more than one version, and + -- each is its own row (uniqueness enforced by an index below that normalizes + -- a NULL version). + model TEXT NOT NULL, + version TEXT DEFAULT NULL, + CONSTRAINT workspace_detection_usage_model_length CHECK (length(model) BETWEEN 1 AND 255), + CONSTRAINT workspace_detection_usage_version_length CHECK (version IS NULL OR length(version) BETWEEN 1 AND 255), + + -- Token counts as the provider reported them. Each is independently nullable: + -- `total` is NOT necessarily input + output (a provider may report only a + -- total, or a total that also counts cached/reasoning tokens), so all three + -- are carried faithfully and never derived from one another. + input_tokens BIGINT DEFAULT NULL, + output_tokens BIGINT DEFAULT NULL, + total_tokens BIGINT DEFAULT NULL, + CONSTRAINT workspace_detection_usage_input_non_negative CHECK (input_tokens IS NULL OR input_tokens >= 0), + CONSTRAINT workspace_detection_usage_output_non_negative CHECK (output_tokens IS NULL OR output_tokens >= 0), + CONSTRAINT workspace_detection_usage_total_non_negative CHECK (total_tokens IS NULL OR total_tokens >= 0), + + -- Wall-clock time this model's recognizers spent, in milliseconds. + duration_ms BIGINT NOT NULL DEFAULT 0, + CONSTRAINT workspace_detection_usage_duration_non_negative CHECK (duration_ms >= 0) +); + +-- One row per (detection, model, version); a NULL version is normalized so two +-- unversioned rows for the same model collide instead of both being inserted. +-- Leads with detection_id, so it also serves per-detection drill-down lookups. +CREATE UNIQUE INDEX workspace_detection_usage_detection_model_version_key + ON workspace_detection_usage (detection_id, model, COALESCE(version, '')); + +-- Usage rollups by model across detections. +CREATE INDEX workspace_detection_usage_model_idx + ON workspace_detection_usage (model); + +COMMENT ON TABLE workspace_detection_usage IS 'Per-model inference token usage for a detection.'; +COMMENT ON COLUMN workspace_detection_usage.id IS 'Unique usage row identifier'; +COMMENT ON COLUMN workspace_detection_usage.detection_id IS 'Detection this usage belongs to'; +COMMENT ON COLUMN workspace_detection_usage.model IS 'Model identifier the recognizers used'; +COMMENT ON COLUMN workspace_detection_usage.version IS 'Model version, if the provider reported one'; +COMMENT ON COLUMN workspace_detection_usage.input_tokens IS 'Input/prompt tokens for this model; NULL if not reported'; +COMMENT ON COLUMN workspace_detection_usage.output_tokens IS 'Output/completion tokens for this model; NULL if not reported'; +COMMENT ON COLUMN workspace_detection_usage.total_tokens IS 'Total tokens as reported (not necessarily input + output); NULL if not reported'; +COMMENT ON COLUMN workspace_detection_usage.duration_ms IS 'Wall-clock time this model spent, in milliseconds'; + +-- Detection-job outbox: the transactional-outbox queue for detection analysis. +-- Creating a detection inserts one row here in the same transaction as the +-- detection, so the two commit or roll back together; a background drainer then +-- publishes each pending row onto the detection NATS work-queue. This removes the +-- dual-write between the detection row and the queue: an analysis is never lost +-- to an enqueue that failed after the row committed, nor is a detection ever +-- marked failed for an enqueue that in fact went through. +CREATE TABLE workspace_detection_jobs ( + -- Primary identifier + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + -- The detection this job analyzes; the row is deleted with its detection. + detection_id UUID NOT NULL REFERENCES workspace_detections (id) ON DELETE CASCADE, + + -- The job. A serialized `DetectionJob`: the workspace, detection, and the + -- optional per-request scope the drainer publishes to the worker. + job JSONB NOT NULL, + CONSTRAINT workspace_detection_jobs_job_size CHECK (length(job::TEXT) BETWEEN 2 AND 16384), + + -- Drainer bookkeeping: the row's processing state, how many publish attempts + -- it has taken, and the earliest time it may next be claimed (advanced by a + -- backoff on each failed attempt so a failing row does not spin at the head of + -- the queue). + status OUTBOX_STATUS NOT NULL DEFAULT 'pending', + attempts INTEGER NOT NULL DEFAULT 0, + CONSTRAINT workspace_detection_jobs_attempts_non_negative CHECK (attempts >= 0), + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT current_timestamp, + + -- Lifecycle timestamps + created_at TIMESTAMPTZ NOT NULL DEFAULT current_timestamp, + + -- When a terminal row (processed or failed) was resolved by an operator; NULL + -- until then. A manual affordance for inspecting the outbox after the fact. + resolved_at TIMESTAMPTZ DEFAULT NULL, + CONSTRAINT workspace_detection_jobs_resolved_only_when_terminal + CHECK (resolved_at IS NULL OR status IN ('processed', 'failed')), + CONSTRAINT workspace_detection_jobs_resolved_after_created + CHECK (resolved_at IS NULL OR resolved_at >= created_at) +); + +-- The drainer's claim queue: pending rows ordered by due time then age, so a +-- batch claims the oldest due rows. Partial so it stays small as processed and +-- failed rows accumulate. +CREATE INDEX workspace_detection_jobs_pending_idx + ON workspace_detection_jobs (next_attempt_at, created_at) + WHERE status = 'pending'; + +-- Back the detection foreign key so a detection delete cascades without scanning +-- the whole outbox (Postgres does not index a referencing column automatically, +-- and the partial claim index above does not cover it). +CREATE INDEX workspace_detection_jobs_detection_idx + ON workspace_detection_jobs (detection_id); + +COMMENT ON TABLE workspace_detection_jobs IS 'Transactional outbox of detection jobs, drained to the detection NATS work-queue.'; +COMMENT ON COLUMN workspace_detection_jobs.id IS 'Unique outbox row identifier'; +COMMENT ON COLUMN workspace_detection_jobs.detection_id IS 'Detection this job analyzes'; +COMMENT ON COLUMN workspace_detection_jobs.job IS 'Serialized DetectionJob published to the worker (JSON, 2B-16KB)'; +COMMENT ON COLUMN workspace_detection_jobs.status IS 'Processing state: pending, processed, or failed (dead-lettered)'; +COMMENT ON COLUMN workspace_detection_jobs.attempts IS 'Number of publish attempts the drainer has made'; +COMMENT ON COLUMN workspace_detection_jobs.next_attempt_at IS 'Earliest time the row may next be claimed; advanced by a backoff after each failed attempt'; +COMMENT ON COLUMN workspace_detection_jobs.created_at IS 'Timestamp when the job was queued'; +COMMENT ON COLUMN workspace_detection_jobs.resolved_at IS 'When a terminal (processed or failed) row was resolved by an operator; NULL until then. A manual affordance for inspecting the outbox after the fact'; diff --git a/migrations/2026-01-19-045017_redactions/down.sql b/migrations/2026-01-19-045017_redactions/down.sql new file mode 100644 index 00000000..07532f98 --- /dev/null +++ b/migrations/2026-01-19-045017_redactions/down.sql @@ -0,0 +1,4 @@ +-- Revert the redactions table. +-- Objects are dropped in reverse order of creation. + +DROP TABLE IF EXISTS workspace_redactions; diff --git a/migrations/2026-01-19-045017_redactions/up.sql b/migrations/2026-01-19-045017_redactions/up.sql new file mode 100644 index 00000000..113bd14a --- /dev/null +++ b/migrations/2026-01-19-045017_redactions/up.sql @@ -0,0 +1,46 @@ +-- Redactions: one redact pass over a detection's analysis. A detection can be +-- redacted many times, so each redaction is its own row owning the review audit +-- it applied and the redacted document it produced. + +CREATE TABLE workspace_redactions ( + -- Primary identifier + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + -- The detection this redaction was produced from; redactions are deleted + -- with their detection. + detection_id UUID NOT NULL REFERENCES workspace_detections (id) ON DELETE CASCADE, + + -- Account that requested the redaction. + account_id UUID NOT NULL REFERENCES accounts (id) ON DELETE CASCADE, + + -- The two files a redaction produces. A redaction always yields both, so the + -- app sets them at creation; they are nullable only because `ON DELETE SET + -- NULL` clears a reference if its file is ever hard-deleted (for the same + -- append-only-history reasons as a detection — a soft-deleted file resolves to + -- "gone" at read time, distinct from a NULL that means the file was purged). + -- review: the engine's Audit after the reviewer edits were applied and + -- redaction ran (`file_kind = review`); the record of exactly what + -- was redacted and why. + -- output: the redacted document this redaction produced. + review_file_id UUID DEFAULT NULL REFERENCES workspace_files (id) ON DELETE SET NULL, + output_file_id UUID DEFAULT NULL REFERENCES workspace_files (id) ON DELETE SET NULL, + + -- Timing + created_at TIMESTAMPTZ NOT NULL DEFAULT current_timestamp +); + +-- A detection's redactions, newest first (the redaction list). +CREATE INDEX workspace_redactions_detection_idx + ON workspace_redactions (detection_id, created_at DESC); + +-- Redactions requested by an account, newest first. +CREATE INDEX workspace_redactions_account_idx + ON workspace_redactions (account_id, created_at DESC); + +COMMENT ON TABLE workspace_redactions IS 'Redactions: one redact pass over a detection, with its own reviewer edits, review audit, and output.'; +COMMENT ON COLUMN workspace_redactions.id IS 'Unique redaction identifier'; +COMMENT ON COLUMN workspace_redactions.detection_id IS 'Detection this redaction was produced from'; +COMMENT ON COLUMN workspace_redactions.account_id IS 'Account that requested the redaction'; +COMMENT ON COLUMN workspace_redactions.review_file_id IS 'Review audit (file_kind=review) recording the applied edits and redaction outcome'; +COMMENT ON COLUMN workspace_redactions.output_file_id IS 'Redacted document this redaction produced'; +COMMENT ON COLUMN workspace_redactions.created_at IS 'When the redaction was created';