diff --git a/crates/graphforge-api/src/lib.rs b/crates/graphforge-api/src/lib.rs index e2f96b2d1..36e29cf14 100644 --- a/crates/graphforge-api/src/lib.rs +++ b/crates/graphforge-api/src/lib.rs @@ -112,15 +112,24 @@ mod write_modes; pub use graphforge_storage::{ PortableV2Authenticity, PortableV2Compatibility, PortableV2Error, PortableV2ErrorCode, - PortableV2Integrity, PortableV2Limits, PortableV2Mode, PortableV2PackageClass, - PortableV2Representation, + PortableV2ExportLimits, PortableV2ExportPlan, PortableV2ExportProgress, + PortableV2ExportReceipt, PortableV2GraphSelector, PortableV2GraphSubsetMeta, + PortableV2Integrity, PortableV2Limits, PortableV2Mode, PortableV2OciAuthenticityPolicy, + PortableV2OciPhase, PortableV2OciProgress, PortableV2OciPullReceipt, PortableV2OciReference, + PortableV2OciSignatureMaterial, PortableV2OciSignatureState, PortableV2Output, + PortableV2PackageClass, PortableV2ParticipantId, PortableV2PropertyProjection, + PortableV2Representation, PortableV2SelectionEntry, PortableV2SelectionPlan, + PortableV2SelectionProfile, PortableV2SelectionReason, PortableV2SelectionRequest, + PortableV2SubsetClosure, PortableV2SubsetPlan, PortableV2SubsetRequest, }; pub use portable::{ PortableExportRequest, PortableExportResult, PortableImportRequest, PortableImportResult, - PortableSelection, PortableV2ImportRequest, PortableV2ImportResult, - PortableV2OciPublishFacadeRequest, PortableV2OciPullFacadeRequest, PortableVerifyRequest, - PortableVerifyResult, publish_portable_v2_oci, publish_portable_v2_oci_with_registry, - pull_portable_v2_oci, pull_portable_v2_oci_with_registry, verify_portable_v2, + PortableSelection, PortableV2ExportFacadeResult, PortableV2ExportRequest, + PortableV2ImportRequest, PortableV2ImportResult, PortableV2OciPublishFacadeRequest, + PortableV2OciPullFacadeRequest, PortableV2SelectionPreviewRequest, + PortableV2SubsetPreviewRequest, PortableVerifyRequest, PortableVerifyResult, + publish_portable_v2_oci, publish_portable_v2_oci_with_registry, pull_portable_v2_oci, + pull_portable_v2_oci_with_registry, verify_portable_v2, }; pub use repository::{ GitProvenance, InfraCapabilityCompatibility, InfraNotChecked, InfraPlan, InfraStaticValidity, diff --git a/crates/graphforge-api/src/portable.rs b/crates/graphforge-api/src/portable.rs index 1f97f676d..692c016ee 100644 --- a/crates/graphforge-api/src/portable.rs +++ b/crates/graphforge-api/src/portable.rs @@ -122,6 +122,72 @@ pub struct PortableVerifyRequest { /// Stable Rust-owned portable-v2 verification report. pub type PortableVerifyResult = graphforge_storage::PortableV2Report; +/// Portable-v2 selection preview request against a pinned generation. +#[derive(Clone, Debug)] +pub struct PortableV2SelectionPreviewRequest { + /// Current or named-checkpoint selection. + pub selection: PortableSelection, + /// Selection profile / custom identities. + pub request: graphforge_storage::PortableV2SelectionRequest, + /// Caller-selected finite resource limits. + pub limits: graphforge_storage::PortableV2Limits, +} + +/// Portable-v2 graph-subset preview request against a pinned generation. +#[derive(Clone, Debug)] +pub struct PortableV2SubsetPreviewRequest { + /// Current or named-checkpoint selection. + pub selection: PortableSelection, + /// Graph-subset selector and closure. + pub request: graphforge_storage::PortableV2SubsetRequest, + /// Caller-selected finite resource limits. + pub limits: graphforge_storage::PortableV2Limits, +} + +/// Portable-v2 export request (expanded directory or canonical bundle). +#[derive(Clone, Debug)] +pub struct PortableV2ExportRequest { + /// Current or named-checkpoint selection. + pub selection: PortableSelection, + /// Destination path (new file or directory). Existing paths are rejected. + pub output_path: PathBuf, + /// Expanded directory or canonical `.gfpb` bundle. + pub representation: graphforge_storage::PortableV2Output, + /// Component selection profile. Ignored when `subset` is `Some`. + pub profile: graphforge_storage::PortableV2SelectionProfile, + /// Optional graph/data subset. When set, exports `graph-data-subset`. + pub subset: Option, + /// Caller-selected finite planner and streaming limits. + pub limits: graphforge_storage::PortableV2Limits, +} + +/// Stable portable-v2 export receipt for bindings and CLI JSON. +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct PortableV2ExportFacadeResult { + /// Contract name. + pub contract: &'static str, + /// Source selector kind. + pub source: &'static str, + /// Named checkpoint, when selected. + pub checkpoint: Option, + /// Pinned source generation. + pub generation_uuid: Uuid, + /// Semantic package identity (`sha256:…`). + pub package_digest: String, + /// Representation-specific transport identity (`sha256:…`). + pub transport_digest: String, + /// Verified physical package entry count. + pub entry_count: usize, + /// Source payload bytes, excluding tags and manifest. + pub payload_bytes: u64, + /// Published representation token. + pub representation: &'static str, + /// Immutable content-free selection fingerprint. + pub selection_fingerprint: String, + /// Caller-selected output path. + pub output: PathBuf, +} + /// Stable export result. #[derive(Clone, Debug, Eq, PartialEq, Serialize)] pub struct PortableExportResult { @@ -237,27 +303,134 @@ pub fn pull_portable_v2_oci_with_registry( } impl GraphForge { - /// Export one pinned current/checkpoint generation without copying live layout metadata. - pub fn export_portable( + /// Resolve a pinned generation for portable export/preview helpers. + fn resolve_portable_generation( &self, - request: PortableExportRequest, - ) -> Result { + selection: &PortableSelection, + ) -> Result< + ( + graphforge_storage::ResolvedProjectGeneration, + &'static str, + Option, + ), + GfError, + > { let root = self.resolved_generation.container_root(); - let (generation, source, checkpoint) = match request.selection { - PortableSelection::Current => ( + match selection { + PortableSelection::Current => Ok(( graphforge_storage::resolve_project_generation(root)?, "current", None, - ), + )), PortableSelection::Checkpoint(name) => { let (_, generation) = graphforge_storage::open_checkpoint_generation_with_mode( root, - &name, + name, self.lifecycle_mode, )?; - (generation, "checkpoint", Some(name)) + Ok((generation, "checkpoint", Some(name.clone()))) } + } + } + + /// Preview one content-free portable-v2 component selection. + pub fn preview_portable_v2_selection( + &self, + request: &PortableV2SelectionPreviewRequest, + ) -> Result + { + let (generation, _, _) = self + .resolve_portable_generation(&request.selection) + .map_err(portable_resolve_err)?; + graphforge_storage::preview_portable_v2_selection( + &generation, + &request.request, + request.limits, + ) + } + + /// Preview one content-free portable-v2 graph-data subset. + pub fn preview_portable_v2_graph_subset( + &self, + request: &PortableV2SubsetPreviewRequest, + ) -> Result { + let (generation, _, _) = self + .resolve_portable_generation(&request.selection) + .map_err(portable_resolve_err)?; + graphforge_storage::preview_portable_v2_graph_subset( + &generation, + &request.request, + request.limits, + ) + } + + /// Export one pinned generation as an expanded or bundled portable-v2 package. + pub fn export_portable_v2( + &self, + request: &PortableV2ExportRequest, + cancelled: Option<&AtomicBool>, + progress: impl FnMut(graphforge_storage::PortableV2ExportProgress), + ) -> Result { + let (generation, source, checkpoint) = self + .resolve_portable_generation(&request.selection) + .map_err(portable_resolve_err)?; + let plan = if let Some(subset) = &request.subset { + let preview = graphforge_storage::preview_portable_v2_graph_subset( + &generation, + subset, + request.limits, + )?; + graphforge_storage::plan_graph_subset_portable_v2( + &generation, + &preview, + request.limits, + )? + } else { + let selection = graphforge_storage::preview_portable_v2_selection( + &generation, + &graphforge_storage::PortableV2SelectionRequest { + profile: request.profile.clone(), + strict: false, + }, + request.limits, + )?; + graphforge_storage::plan_selected_portable_v2(&generation, &selection, request.limits)? }; + let default_cancelled = AtomicBool::new(false); + let cancelled = cancelled.unwrap_or(&default_cancelled); + let receipt = graphforge_storage::export_complete_portable_v2( + &plan, + &request.output_path, + request.representation, + request.limits, + cancelled, + progress, + )?; + Ok(PortableV2ExportFacadeResult { + contract: "graphforge-portable-export/2", + source, + checkpoint, + generation_uuid: receipt.generation_uuid, + package_digest: format!("sha256:{}", hex(receipt.package_digest)), + transport_digest: format!("sha256:{}", hex(receipt.transport_digest)), + entry_count: receipt.entry_count, + payload_bytes: receipt.payload_bytes, + representation: match receipt.output { + graphforge_storage::PortableV2Output::Expanded => "expanded", + graphforge_storage::PortableV2Output::Bundle => "bundle", + }, + selection_fingerprint: receipt.selection_fingerprint, + output: request.output_path.clone(), + }) + } + + /// Export one pinned current/checkpoint generation without copying live layout metadata. + pub fn export_portable( + &self, + request: PortableExportRequest, + ) -> Result { + let (generation, source, checkpoint) = + self.resolve_portable_generation(&request.selection)?; let receipt = graphforge_storage::export_portable_project( &generation, &request.output, @@ -357,6 +530,13 @@ impl GraphForge { } } +fn portable_resolve_err(_error: GfError) -> graphforge_storage::PortableV2Error { + graphforge_storage::PortableV2Error::new( + graphforge_storage::PortableV2ErrorCode::Io, + "pinned project generation is not exportable", + ) +} + fn supported_capabilities() -> Vec { [ "epistemic", @@ -452,6 +632,92 @@ mod tests { GraphForge::new(target.to_str()).expect("imported CURRENT must reopen"); } + #[test] + fn public_v2_export_preview_and_verify_agree_on_package_digest() { + let root = tempfile::tempdir().unwrap(); + let source = root.path().join("source"); + std::fs::create_dir(&source).unwrap(); + let graph = GraphForge::new(source.to_str()).unwrap(); + let limits = graphforge_storage::PortableV2Limits::default(); + let preview = graph + .preview_portable_v2_selection(&PortableV2SelectionPreviewRequest { + selection: PortableSelection::Current, + request: graphforge_storage::PortableV2SelectionRequest { + profile: graphforge_storage::PortableV2SelectionProfile::Complete, + strict: false, + }, + limits, + }) + .unwrap(); + assert_eq!(preview.package_class, "complete"); + let expanded = root.path().join("expanded"); + let bundle = root.path().join("complete.gfpb"); + let expanded_export = graph + .export_portable_v2( + &PortableV2ExportRequest { + selection: PortableSelection::Current, + output_path: expanded.clone(), + representation: graphforge_storage::PortableV2Output::Expanded, + profile: graphforge_storage::PortableV2SelectionProfile::Complete, + subset: None, + limits, + }, + None, + |_| {}, + ) + .unwrap(); + let bundle_export = graph + .export_portable_v2( + &PortableV2ExportRequest { + selection: PortableSelection::Current, + output_path: bundle.clone(), + representation: graphforge_storage::PortableV2Output::Bundle, + profile: graphforge_storage::PortableV2SelectionProfile::Complete, + subset: None, + limits, + }, + None, + |_| {}, + ) + .unwrap(); + assert_eq!(expanded_export.package_digest, bundle_export.package_digest); + assert_eq!( + expanded_export.selection_fingerprint, + preview.selection_fingerprint + ); + let verified = verify_portable_v2( + &PortableVerifyRequest { + input: bundle, + mode: graphforge_storage::PortableV2Mode::Full, + limits, + }, + None, + ) + .unwrap(); + assert_eq!(verified.package_digest, bundle_export.package_digest); + let subset_error = graph + .preview_portable_v2_graph_subset(&PortableV2SubsetPreviewRequest { + selection: PortableSelection::Current, + request: graphforge_storage::PortableV2SubsetRequest { + selector: graphforge_storage::PortableV2GraphSelector::default(), + closure: graphforge_storage::PortableV2SubsetClosure::InducedEdges, + projection: graphforge_storage::PortableV2PropertyProjection::default(), + }, + limits, + }) + .unwrap_err(); + assert_eq!( + subset_error.code, + graphforge_storage::PortableV2ErrorCode::Incompatible + ); + assert!( + subset_error + .to_string() + .contains("pinned generation has no graph tree"), + "empty projects must fail closed before subset planning: {subset_error}" + ); + } + #[test] fn public_v2_import_facade_verifies_publishes_and_reopens() { let root = tempfile::tempdir().unwrap(); diff --git a/crates/graphforge-bindings-node/src/import_session.rs b/crates/graphforge-bindings-node/src/import_session.rs new file mode 100644 index 000000000..ffe659f06 --- /dev/null +++ b/crates/graphforge-bindings-node/src/import_session.rs @@ -0,0 +1,366 @@ +//! Thin Node bindings for durable staged graph-import sessions (#744 / #738). + +use std::path::PathBuf; +use std::sync::{Arc, Mutex, RwLock}; +use std::time::Duration; + +use arrow::ipc::reader::StreamReader; +use graphforge_api::{ + BulkInputKind, CancellationToken, GfError, GraphImportSession as ApiSession, ImportPhase, + ImportProgress, ImportSessionLimits, OperationId, +}; +use napi::bindgen_prelude::{AbortSignal, AsyncTask, BigInt, Buffer}; +use napi::{Env, Task}; +use napi_derive::napi; + +use crate::error::to_napi_err; +use crate::{Result, napi_validation}; + +fn phase_name(phase: ImportPhase) -> &'static str { + match phase { + ImportPhase::Open => "open", + ImportPhase::Validated => "validated", + ImportPhase::Committed => "committed", + ImportPhase::Aborted => "aborted", + ImportPhase::Quarantined => "quarantined", + } +} + +fn progress_output(progress: ImportProgress) -> ImportProgressOutput { + ImportProgressOutput { + rows_accepted: BigInt::from(progress.rows_accepted), + rows_rejected: BigInt::from(progress.rows_rejected), + bytes_accepted: BigInt::from(progress.bytes_accepted), + files_accepted: BigInt::from(progress.files_accepted), + files_pending: BigInt::from(progress.files_pending), + elapsed_millis: BigInt::from(progress.elapsed_millis), + peak_batch_rows: BigInt::from(progress.peak_batch_rows), + io_concurrency_limit: BigInt::from(progress.io_concurrency_limit), + } +} + +fn parse_kind(kind: &str) -> Result { + match kind { + "node" | "nodes" => Ok(BulkInputKind::Node), + "edge" | "edges" => Ok(BulkInputKind::Edge), + _ => Err(napi_validation("kind must be node or edge")), + } +} + +#[napi(object)] +pub struct ImportSessionLimitsInput { + pub batch_rows: Option, + pub max_source_bytes: Option, + pub max_files: Option, + pub max_rejected_rows: Option, + pub io_concurrency: Option, +} + +#[napi(object)] +pub struct ImportProgressOutput { + pub rows_accepted: BigInt, + pub rows_rejected: BigInt, + pub bytes_accepted: BigInt, + pub files_accepted: BigInt, + pub files_pending: BigInt, + pub elapsed_millis: BigInt, + pub peak_batch_rows: BigInt, + pub io_concurrency_limit: BigInt, +} + +#[napi(object)] +pub struct ImportSessionStatusOutput { + pub phase: String, + pub progress: ImportProgressOutput, +} + +fn parse_limits(input: Option) -> Result { + let defaults = ImportSessionLimits::default(); + let Some(input) = input else { + return Ok(defaults); + }; + Ok(ImportSessionLimits { + batch_rows: input + .batch_rows + .map_or(defaults.batch_rows, |value| value as usize), + max_source_bytes: match input.max_source_bytes { + Some(value) => crate::node_u64(Some(value), "maxSourceBytes")?, + None => defaults.max_source_bytes, + }, + max_files: match input.max_files { + Some(value) => crate::node_u64(Some(value), "maxFiles")?, + None => defaults.max_files, + }, + max_rejected_rows: match input.max_rejected_rows { + Some(value) => crate::node_u64(Some(value), "maxRejectedRows")?, + None => defaults.max_rejected_rows, + }, + io_concurrency: input + .io_concurrency + .map_or(defaults.io_concurrency, |value| value as usize), + }) +} + +/// Owned durable import-session handle. Contains no live rows. +#[napi(js_name = "GraphImportSession")] +pub struct GraphImportSession { + engine: Arc>, + closed: Arc, + inner: Arc>>, +} + +impl GraphImportSession { + fn with_mut( + &self, + f: impl FnOnce(&mut ApiSession) -> std::result::Result, + ) -> Result { + let mut guard = self + .inner + .lock() + .map_err(|_| to_napi_err(&GfError::Execution("import session lock poisoned".into())))?; + let session = guard.as_mut().ok_or_else(|| { + to_napi_err(&GfError::Lifecycle( + "import session handle is closed".into(), + )) + })?; + f(session).map_err(|error| to_napi_err(&error)) + } +} + +pub struct ValidateImportSessionTask { + engine: Arc>, + closed: Arc, + inner: Arc>>, + cancellation: CancellationToken, +} + +impl Task for ValidateImportSessionTask { + type Output = std::result::Result; + type JsValue = ImportProgressOutput; + + fn compute(&mut self) -> napi::Result { + if self.closed.load(std::sync::atomic::Ordering::Acquire) { + return Ok(Err(GfError::Lifecycle( + "operation on a closed GraphForge instance".into(), + ))); + } + let graph = self + .engine + .read() + .map_err(|_| napi::Error::from_reason("GraphForge lock poisoned"))?; + let mut guard = self + .inner + .lock() + .map_err(|_| napi::Error::from_reason("import session lock poisoned"))?; + let session = guard + .as_mut() + .ok_or_else(|| napi::Error::from_reason("import session handle is closed"))?; + Ok(session.validate_with_cancellation(&graph, Some(&self.cancellation))) + } + + fn resolve(&mut self, env: Env, output: Self::Output) -> napi::Result { + output + .map(progress_output) + .map_err(|error| crate::to_napi_deferred_err(env, &error)) + } +} + +pub struct CommitImportSessionTask { + engine: Arc>, + closed: Arc, + inner: Arc>>, + cancellation: CancellationToken, +} + +impl Task for CommitImportSessionTask { + type Output = std::result::Result; + type JsValue = String; + + fn compute(&mut self) -> napi::Result { + if self.closed.load(std::sync::atomic::Ordering::Acquire) { + return Ok(Err(GfError::Lifecycle( + "operation on a closed GraphForge instance".into(), + ))); + } + let graph = self + .engine + .read() + .map_err(|_| napi::Error::from_reason("GraphForge lock poisoned"))?; + let mut guard = self + .inner + .lock() + .map_err(|_| napi::Error::from_reason("import session lock poisoned"))?; + let session = guard + .as_mut() + .ok_or_else(|| napi::Error::from_reason("import session handle is closed"))?; + Ok(session + .commit(&graph, Some(&self.cancellation)) + .map(|uuid| uuid.to_string())) + } + + fn resolve(&mut self, env: Env, output: Self::Output) -> napi::Result { + output.map_err(|error| crate::to_napi_deferred_err(env, &error)) + } +} + +#[napi] +impl GraphImportSession { + /// Durable identifier used for resume. + #[napi(getter)] + pub fn session_uuid(&self) -> Result { + self.with_mut(|session| Ok(session.session_uuid().to_string())) + } + + /// Current durable phase and counters. + #[napi] + pub fn status(&self) -> Result { + let (phase, progress) = self.with_mut(|session| Ok(session.status()))?; + Ok(ImportSessionStatusOutput { + phase: phase_name(phase).to_owned(), + progress: progress_output(progress), + }) + } + + /// Append one Arrow IPC buffer without retaining live rows. + #[napi] + pub fn append_arrow(&self, kind: String, ipc: Buffer) -> Result<()> { + let kind = parse_kind(&kind)?; + let reader = + StreamReader::try_new(std::io::Cursor::new(ipc.to_vec()), None).map_err(|error| { + to_napi_err(&GfError::Validation(format!("invalid Arrow IPC: {error}"))) + })?; + let batches = reader + .collect::, _>>() + .map_err(|error| { + to_napi_err(&GfError::Validation(format!("invalid Arrow IPC: {error}"))) + })?; + self.with_mut(|session| session.append_arrow(kind, &batches)) + } + + /// Register a local Parquet source by copying it into durable ownership. + #[napi] + pub fn register_parquet(&self, kind: String, path: String) -> Result<()> { + let kind = parse_kind(&kind)?; + let path = PathBuf::from(path); + self.with_mut(|session| session.register_parquet(kind, &path)) + } + + /// Persist counters and source ordering without publishing graph state. + #[napi] + pub fn checkpoint(&self) -> Result { + let progress = self.with_mut(ApiSession::checkpoint)?; + Ok(progress_output(progress)) + } + + /// Validate and durably stage every source with optional cancellation. + #[napi] + pub fn validate(&self, signal: Option) -> AsyncTask { + let cancellation = CancellationToken::new(); + if let Some(signal) = signal { + let cancellation = cancellation.clone(); + signal.on_abort(move || cancellation.cancel()); + } + AsyncTask::new(ValidateImportSessionTask { + engine: Arc::clone(&self.engine), + closed: Arc::clone(&self.closed), + inner: Arc::clone(&self.inner), + cancellation, + }) + } + + /// Publish the fully staged graph as one generation. + #[napi] + pub fn commit(&self, signal: Option) -> AsyncTask { + let cancellation = CancellationToken::new(); + if let Some(signal) = signal { + let cancellation = cancellation.clone(); + signal.on_abort(move || cancellation.cancel()); + } + AsyncTask::new(CommitImportSessionTask { + engine: Arc::clone(&self.engine), + closed: Arc::clone(&self.closed), + inner: Arc::clone(&self.inner), + cancellation, + }) + } + + /// Abort without changing CURRENT. + #[napi] + pub fn abort(&self) -> Result { + let mut guard = self + .inner + .lock() + .map_err(|_| to_napi_err(&GfError::Execution("import session lock poisoned".into())))?; + let session = guard.take().ok_or_else(|| { + to_napi_err(&GfError::Lifecycle( + "import session handle is closed".into(), + )) + })?; + let progress = session.abort().map_err(|error| to_napi_err(&error))?; + Ok(progress_output(progress)) + } +} + +pub(crate) fn begin_import_session( + engine: Arc>, + closed: Arc, + operation_uuid: String, + limits: Option, +) -> Result { + if closed.load(std::sync::atomic::Ordering::Acquire) { + return Err(to_napi_err(&GfError::Lifecycle( + "operation on a closed GraphForge instance".into(), + ))); + } + let operation = crate::canonical_operation_id(&operation_uuid)?; + let limits = parse_limits(limits)?; + let graph = engine + .read() + .map_err(|_| to_napi_err(&GfError::Execution("GraphForge lock poisoned".into())))?; + let session = graph + .begin_import_session(operation, limits) + .map_err(|error| to_napi_err(&error))?; + Ok(GraphImportSession { + engine: Arc::clone(&engine), + closed, + inner: Arc::new(Mutex::new(Some(session))), + }) +} + +pub(crate) fn resume_import_session( + engine: Arc>, + closed: Arc, + session_uuid: String, +) -> Result { + if closed.load(std::sync::atomic::Ordering::Acquire) { + return Err(to_napi_err(&GfError::Lifecycle( + "operation on a closed GraphForge instance".into(), + ))); + } + let uuid = crate::canonical_operation_id(&session_uuid)?.0; + let graph = engine + .read() + .map_err(|_| to_napi_err(&GfError::Execution("GraphForge lock poisoned".into())))?; + let session = graph + .resume_import_session(uuid) + .map_err(|error| to_napi_err(&error))?; + Ok(GraphImportSession { + engine: Arc::clone(&engine), + closed, + inner: Arc::new(Mutex::new(Some(session))), + }) +} + +pub(crate) fn cleanup_stale_import_sessions( + graph: &graphforge_api::GraphForge, + max_age_secs: BigInt, +) -> Result { + let max_age_secs = crate::node_u64(Some(max_age_secs), "maxAgeSecs")?; + let cleaned = graph + .cleanup_stale_import_sessions(Duration::from_secs(max_age_secs)) + .map_err(|error| to_napi_err(&error))?; + Ok(BigInt::from(cleaned)) +} + +#[allow(dead_code)] +fn _keep_operation_id(_: OperationId) {} diff --git a/crates/graphforge-bindings-node/src/lib.rs b/crates/graphforge-bindings-node/src/lib.rs index a81b2af32..e330488ec 100644 --- a/crates/graphforge-bindings-node/src/lib.rs +++ b/crates/graphforge-bindings-node/src/lib.rs @@ -50,6 +50,8 @@ use napi_derive::napi; mod composite; mod error; +mod import_session; +mod portable; mod transaction; use composite::CompositeTransactionInput; use error::{NodeError, to_napi_err, type_error}; @@ -2260,7 +2262,22 @@ fn node_generation_identity( }) } -fn node_usize(value: Option, default: usize, name: &str) -> Result { +pub(crate) fn node_u64(value: Option, name: &str) -> Result { + let Some(value) = value else { + return Err(to_napi_err(&GfError::Validation(format!( + "{name} is required" + )))); + }; + let (negative, value, lossless) = value.get_u64(); + if negative || !lossless { + return Err(to_napi_err(&GfError::Validation(format!( + "{name} must be a lossless unsigned 64-bit integer" + )))); + } + Ok(value) +} + +pub(crate) fn node_usize(value: Option, default: usize, name: &str) -> Result { let Some(value) = value else { return Ok(default); }; @@ -3491,6 +3508,103 @@ impl GraphForge { })) } + /// Preview one content-free portable-v2 component selection. + #[napi] + pub fn preview_portable_v2_selection( + &self, + request: portable::PortableSelectionPreviewInput, + ) -> Result { + let graph = self.open_guard()?; + portable::preview_selection(&graph, request) + } + + /// Preview one content-free portable-v2 graph-data subset. + #[napi] + pub fn preview_portable_v2_graph_subset( + &self, + request: portable::PortableSubsetPreviewInput, + ) -> Result { + let graph = self.open_guard()?; + portable::preview_subset(&graph, request) + } + + /// Export one pinned generation as an expanded or bundled portable-v2 package. + #[napi] + pub fn export_portable_v2( + &self, + request: portable::PortableExportInput, + ) -> Result> { + self.ensure_open()?; + portable::build_export_task(Arc::clone(&self.inner), request) + } + + /// Verify portable-v2 content without opening or mutating a project. + #[napi] + pub fn verify_portable_v2( + request: portable::PortableVerifyInput, + ) -> Result> { + portable::build_verify_task(request) + } + + /// Verify and atomically import a complete portable-v2 package. + #[napi] + pub fn import_portable_v2( + request: portable::PortableImportInput, + ) -> Result> { + portable::build_import_task(request) + } + + /// Publish a verified portable-v2 package to an OCI Distribution registry. + #[napi] + pub fn publish_portable_v2_oci( + request: portable::PortableOciPublishInput, + ) -> Result> { + portable::build_publish_task(request) + } + + /// Pull and verify a portable-v2 package from an OCI Distribution registry. + #[napi] + pub fn pull_portable_v2_oci( + request: portable::PortableOciPullInput, + ) -> Result> { + portable::build_pull_task(request) + } + + /// Begin a durable staged import session. + #[napi] + pub fn begin_import_session( + &self, + operation_uuid: String, + limits: Option, + ) -> Result { + import_session::begin_import_session( + Arc::clone(&self.inner), + Arc::clone(&self.closed), + operation_uuid, + limits, + ) + } + + /// Resume one durable, non-terminal import session. + #[napi] + pub fn resume_import_session( + &self, + session_uuid: String, + ) -> Result { + import_session::resume_import_session( + Arc::clone(&self.inner), + Arc::clone(&self.closed), + session_uuid, + ) + } + + /// Abort and remove non-terminal sessions older than `maxAgeSecs`. + #[napi] + pub fn cleanup_stale_import_sessions(&self, max_age_secs: BigInt) -> Result { + let graph = self.open_guard()?; + import_session::cleanup_stale_import_sessions(&graph, max_age_secs) + } + /// Compare two checkpoint/current endpoints through the Rust diff engine. #[napi] pub fn diff_checkpoints( @@ -7631,7 +7745,7 @@ impl Task for AlgorithmRunEventsTask { } } -fn to_napi_deferred_err(env: Env, error: &GfError) -> napi::Error { +pub(crate) fn to_napi_deferred_err(env: Env, error: &GfError) -> napi::Error { let value = napi::JsError::from(to_napi_err(error)).into_unknown(env); napi::Error::from(value) } @@ -7687,17 +7801,44 @@ impl PlanHandle { g.explain(&self.cypher).map_err(|e| to_napi_err(&e)) } - /// Run the query and write the result to a Parquet file at `path`. + /// Run the query and write a streamed Parquet result with optional limits. #[napi] - #[must_use] - pub fn sink_parquet(&self, path: String) -> AsyncTask { - AsyncTask::new(SinkParquetTask { + pub fn sink_parquet( + &self, + path: String, + options: Option, + ) -> Result> { + let (options, cancellation) = portable::parse_sink_options(options)?; + Ok(AsyncTask::new(portable::SinkStreamTask { engine: Arc::clone(&self.engine), closed: Arc::clone(&self.closed), cypher: self.cypher.clone(), params: self.params.clone(), path, - }) + format: graphforge_api::ResultSinkFormat::Parquet, + options, + cancellation, + })) + } + + /// Run the query and write a streamed Arrow IPC result with optional limits. + #[napi] + pub fn sink_arrow_ipc( + &self, + path: String, + options: Option, + ) -> Result> { + let (options, cancellation) = portable::parse_sink_options(options)?; + Ok(AsyncTask::new(portable::SinkStreamTask { + engine: Arc::clone(&self.engine), + closed: Arc::clone(&self.closed), + cypher: self.cypher.clone(), + params: self.params.clone(), + path, + format: graphforge_api::ResultSinkFormat::ArrowIpc, + options, + cancellation, + })) } } @@ -7736,39 +7877,6 @@ impl Task for CollectIpcTask { } } -/// `AsyncTask` backing [`PlanHandle::sink_parquet`]. -pub struct SinkParquetTask { - engine: Arc>, - closed: Arc, - cypher: String, - params: HashMap, - path: String, -} - -impl Task for SinkParquetTask { - type Output = std::result::Result<(), GfError>; - type JsValue = (); - - fn compute(&mut self) -> napi::Result { - Ok((|| { - if self.closed.load(Ordering::Acquire) { - return Err(GfError::Lifecycle( - "operation on a closed GraphForge instance".into(), - )); - } - let graph = self - .engine - .read() - .map_err(|_| GfError::Execution("GraphForge lock poisoned".into()))?; - graph.execute_to_parquet_with_params(&self.cypher, &self.params, &self.path) - })()) - } - - fn resolve(&mut self, env: Env, output: Self::Output) -> napi::Result<()> { - output.map_err(|error| to_napi_deferred_err(env, &error)) - } -} - #[cfg(test)] mod tests { use std::sync::mpsc; diff --git a/crates/graphforge-bindings-node/src/portable.rs b/crates/graphforge-bindings-node/src/portable.rs new file mode 100644 index 000000000..e8c928356 --- /dev/null +++ b/crates/graphforge-bindings-node/src/portable.rs @@ -0,0 +1,872 @@ +//! Thin Node bindings for portable-v2 and streaming result sinks (#744). + +use std::path::PathBuf; +use std::sync::{Arc, RwLock}; + +use graphforge_api::{ + CancellationToken, GfError, PortableSelection, PortableV2Authenticity, PortableV2Compatibility, + PortableV2Error, PortableV2ErrorCode, PortableV2ExportRequest, PortableV2GraphSelector, + PortableV2ImportRequest, PortableV2Integrity, PortableV2Limits, PortableV2Mode, + PortableV2OciAuthenticityPolicy, PortableV2OciPublishFacadeRequest, + PortableV2OciPullFacadeRequest, PortableV2OciSignatureMaterial, PortableV2OciSignatureState, + PortableV2Output, PortableV2ParticipantId, PortableV2PropertyProjection, + PortableV2SelectionPreviewRequest, PortableV2SelectionProfile, PortableV2SelectionRequest, + PortableV2SubsetClosure, PortableV2SubsetPlan, PortableV2SubsetPreviewRequest, + PortableV2SubsetRequest, PortableVerifyRequest, ResultSinkFormat, ResultSinkOptions, + ResultSinkReceipt, +}; +use napi::bindgen_prelude::{AbortSignal, AsyncTask, BigInt, Buffer}; +use napi::{Env, Task}; +use napi_derive::napi; + +use crate::error::to_napi_err; +use crate::{Result, napi_validation}; + +pub(crate) fn to_portable_napi_err(error: &PortableV2Error) -> crate::NodeError { + let code = match error.code { + PortableV2ErrorCode::Cancelled => "Cancelled", + PortableV2ErrorCode::LimitExceeded => "LimitExceeded", + PortableV2ErrorCode::Io => "Io", + PortableV2ErrorCode::InvalidStructure => "InvalidStructure", + PortableV2ErrorCode::InvalidPath => "InvalidPath", + PortableV2ErrorCode::DuplicateEntry => "DuplicateEntry", + PortableV2ErrorCode::UnsupportedFuture => "UnsupportedFuture", + PortableV2ErrorCode::Incompatible => "Incompatible", + PortableV2ErrorCode::DigestMismatch => "DigestMismatch", + PortableV2ErrorCode::ConcurrentMutation => "ConcurrentMutation", + }; + napi::Error::new(code.to_owned(), error.to_string()) +} + +fn to_portable_deferred_err(env: Env, error: &PortableV2Error) -> napi::Error { + let value = napi::JsError::from(to_portable_napi_err(error)).into_unknown(env); + napi::Error::from(value) +} + +fn selection_from_checkpoint(checkpoint: Option) -> PortableSelection { + match checkpoint { + Some(name) => PortableSelection::Checkpoint(name), + None => PortableSelection::Current, + } +} + +fn parse_limits(input: Option) -> Result { + let defaults = PortableV2Limits::default(); + let Some(input) = input else { + return Ok(defaults); + }; + Ok(PortableV2Limits { + max_components: match input.max_components { + Some(value) => crate::node_u64(Some(value), "maxComponents")?, + None => defaults.max_components, + }, + max_entries: match input.max_entries { + Some(value) => crate::node_u64(Some(value), "maxEntries")?, + None => defaults.max_entries, + }, + max_entry_bytes: match input.max_entry_bytes { + Some(value) => crate::node_u64(Some(value), "maxEntryBytes")?, + None => defaults.max_entry_bytes, + }, + max_total_bytes: match input.max_total_bytes { + Some(value) => crate::node_u64(Some(value), "maxTotalBytes")?, + None => defaults.max_total_bytes, + }, + max_manifest_bytes: match input.max_manifest_bytes { + Some(value) => crate::node_u64(Some(value), "maxManifestBytes")?, + None => defaults.max_manifest_bytes, + }, + max_tag_manifest_bytes: match input.max_tag_manifest_bytes { + Some(value) => crate::node_u64(Some(value), "maxTagManifestBytes")?, + None => defaults.max_tag_manifest_bytes, + }, + max_path_bytes: match input.max_path_bytes { + Some(value) => crate::node_usize(Some(value), defaults.max_path_bytes, "maxPathBytes")?, + None => defaults.max_path_bytes, + }, + copy_buffer_bytes: match input.copy_buffer_bytes { + Some(value) => { + crate::node_usize(Some(value), defaults.copy_buffer_bytes, "copyBufferBytes")? + } + None => defaults.copy_buffer_bytes, + }, + }) +} + +fn parse_profile( + profile: Option, + identities: Option>, +) -> Result { + match profile.as_deref().unwrap_or("complete") { + "complete" => Ok(PortableV2SelectionProfile::Complete), + "ontology_only" => Ok(PortableV2SelectionProfile::OntologyOnly), + "data_components" => Ok(PortableV2SelectionProfile::DataComponents), + "artifacts" => Ok(PortableV2SelectionProfile::Artifacts), + "settings" => Ok(PortableV2SelectionProfile::Settings), + "custom" => { + let identities = identities.ok_or_else(|| { + to_napi_err(&GfError::Validation( + "custom profile requires identities".into(), + )) + })?; + Ok(PortableV2SelectionProfile::Custom( + identities + .into_iter() + .map(|identity| PortableV2ParticipantId { + capability_id: identity.capability_id, + record_family_id: identity.record_family_id, + }) + .collect(), + )) + } + _ => Err(napi_validation( + "profile must be complete, ontology_only, data_components, artifacts, settings, or custom", + )), + } +} + +fn parse_subset(input: Option) -> Result> { + let Some(input) = input else { + return Ok(None); + }; + let selector = input.selector.unwrap_or_default(); + let closure = match input.closure.as_deref().unwrap_or("induced_edges") { + "induced_edges" => PortableV2SubsetClosure::InducedEdges, + "referential" => PortableV2SubsetClosure::Referential, + _ => { + return Err(napi_validation( + "closure must be induced_edges or referential", + )); + } + }; + Ok(Some(PortableV2SubsetRequest { + selector: PortableV2GraphSelector { + node_uuids: selector.node_uuids.unwrap_or_default(), + edge_uuids: selector.edge_uuids.unwrap_or_default(), + }, + closure, + projection: PortableV2PropertyProjection { + exclude: input + .projection + .and_then(|projection| projection.exclude) + .unwrap_or_default(), + }, + })) +} + +fn parse_output(representation: Option<&str>) -> Result { + match representation.unwrap_or("bundle") { + "expanded" => Ok(PortableV2Output::Expanded), + "bundle" => Ok(PortableV2Output::Bundle), + _ => Err(napi_validation("representation must be expanded or bundle")), + } +} + +fn parse_mode(mode: Option<&str>) -> Result { + match mode.unwrap_or("full") { + "structure_only" => Ok(PortableV2Mode::StructureOnly), + "full" => Ok(PortableV2Mode::Full), + _ => Err(napi_validation("mode must be structure_only or full")), + } +} + +fn bind_signal(signal: Option) -> CancellationToken { + let cancellation = CancellationToken::new(); + if let Some(signal) = signal { + let cancellation = cancellation.clone(); + signal.on_abort(move || cancellation.cancel()); + } + cancellation +} + +fn selection_plan_json(plan: &graphforge_api::PortableV2SelectionPlan) -> serde_json::Value { + serde_json::json!({ + "sourceGenerationUuid": plan.source_generation_uuid, + "sourceManifestSha256": plan.source_manifest_sha256, + "packageClass": plan.package_class, + "included": plan.included, + "excluded": plan.excluded, + "redactions": plan.redactions, + "requiredCapabilities": plan.required_capabilities, + "estimatedPayloadBytes": plan.estimated_payload_bytes, + "selectionFingerprint": plan.selection_fingerprint, + }) +} + +fn subset_plan_json(plan: &PortableV2SubsetPlan) -> serde_json::Value { + serde_json::json!({ + "selection": selection_plan_json(&plan.selection), + "graphSubset": plan.graph_subset, + "selectedNodeCount": plan.selected_node_count, + "selectedEdgeCount": plan.selected_edge_count, + "endpointNodeCount": plan.endpoint_node_count, + "resultFingerprint": plan.result_fingerprint, + "subsetFingerprint": plan.subset_fingerprint, + }) +} + +fn export_result_output( + result: graphforge_api::PortableV2ExportFacadeResult, +) -> PortableExportOutput { + PortableExportOutput { + contract: result.contract.to_owned(), + source: result.source.to_owned(), + checkpoint: result.checkpoint, + generation_uuid: result.generation_uuid.to_string(), + package_digest: result.package_digest, + transport_digest: result.transport_digest, + entry_count: BigInt::from(u64::try_from(result.entry_count).unwrap_or(u64::MAX)), + payload_bytes: BigInt::from(result.payload_bytes), + representation: result.representation.to_owned(), + selection_fingerprint: result.selection_fingerprint, + output: result.output.display().to_string(), + } +} + +fn integrity_token(value: PortableV2Integrity) -> String { + match value { + PortableV2Integrity::NotChecked => "not_checked".into(), + PortableV2Integrity::Verified => "verified".into(), + PortableV2Integrity::Failed => "failed".into(), + } +} + +fn compatibility_token(value: PortableV2Compatibility) -> String { + match value { + PortableV2Compatibility::Supported => "supported".into(), + PortableV2Compatibility::UnsupportedFuture => "unsupported_future".into(), + PortableV2Compatibility::Failed => "failed".into(), + } +} + +fn authenticity_token(value: PortableV2Authenticity) -> String { + match value { + PortableV2Authenticity::NotEvaluated => "not_evaluated".into(), + PortableV2Authenticity::Unsigned => "unsigned".into(), + PortableV2Authenticity::Verified => "verified".into(), + PortableV2Authenticity::Failed => "failed".into(), + } +} + +fn signature_state_token(value: PortableV2OciSignatureState) -> String { + match value { + PortableV2OciSignatureState::Valid => "valid".into(), + PortableV2OciSignatureState::Invalid => "invalid".into(), + PortableV2OciSignatureState::Absent => "absent".into(), + PortableV2OciSignatureState::PolicyMismatched => "policy_mismatched".into(), + } +} + +fn verify_report_output(report: graphforge_api::PortableVerifyResult) -> PortableVerifyOutput { + PortableVerifyOutput { + contract: report.contract.to_owned(), + representation: match report.representation { + graphforge_api::PortableV2Representation::Expanded => "expanded".into(), + graphforge_api::PortableV2Representation::Bundle => "bundle".into(), + }, + package_digest: report.package_digest, + package_class: match report.package_class { + graphforge_api::PortableV2PackageClass::Complete => "complete".into(), + graphforge_api::PortableV2PackageClass::OntologyOnly => "ontology_only".into(), + graphforge_api::PortableV2PackageClass::ComponentSelective => { + "component_selective".into() + } + graphforge_api::PortableV2PackageClass::GraphDataSubset => "graph_data_subset".into(), + }, + component_count: BigInt::from(report.component_count), + entry_count: BigInt::from(report.entry_count), + payload_bytes: BigInt::from(report.payload_bytes), + integrity: integrity_token(report.integrity), + compatibility: compatibility_token(report.compatibility), + authenticity: authenticity_token(report.authenticity), + transport_digest: report.transport_digest, + } +} + +fn sink_receipt_output(receipt: ResultSinkReceipt) -> ResultSinkReceiptOutput { + ResultSinkReceiptOutput { + destination: receipt.destination.display().to_string(), + format: match receipt.format { + ResultSinkFormat::Parquet => "parquet".into(), + ResultSinkFormat::ArrowIpc => "arrow_ipc".into(), + }, + progress: ResultSinkProgressOutput { + phase: receipt.progress.phase.to_owned(), + rows: BigInt::from(receipt.progress.rows), + batches: BigInt::from(receipt.progress.batches), + bytes: BigInt::from(receipt.progress.bytes), + elapsed_ms: BigInt::from( + u64::try_from(receipt.progress.elapsed.as_millis()).unwrap_or(u64::MAX), + ), + complete: receipt.progress.complete, + }, + } +} + +#[napi(object)] +pub struct PortableV2LimitsInput { + pub max_components: Option, + pub max_entries: Option, + pub max_entry_bytes: Option, + pub max_total_bytes: Option, + pub max_manifest_bytes: Option, + pub max_tag_manifest_bytes: Option, + pub max_path_bytes: Option, + pub copy_buffer_bytes: Option, +} + +#[napi(object)] +pub struct PortableParticipantIdInput { + pub capability_id: String, + pub record_family_id: String, +} + +#[napi(object)] +#[derive(Default)] +pub struct PortableGraphSelectorInput { + pub node_uuids: Option>, + pub edge_uuids: Option>, +} + +#[napi(object)] +pub struct PortablePropertyProjectionInput { + pub exclude: Option>, +} + +#[napi(object)] +pub struct PortableSubsetInput { + pub selector: Option, + pub closure: Option, + pub projection: Option, +} + +#[napi(object)] +pub struct PortableSelectionPreviewInput { + pub checkpoint: Option, + pub profile: Option, + pub identities: Option>, + pub strict: Option, + pub limits: Option, +} + +#[napi(object)] +pub struct PortableSubsetPreviewInput { + pub checkpoint: Option, + pub subset: PortableSubsetInput, + pub limits: Option, +} + +#[napi(object, object_to_js = false)] +pub struct PortableExportInput { + pub output_path: String, + pub representation: Option, + pub profile: Option, + pub identities: Option>, + pub checkpoint: Option, + pub subset: Option, + pub limits: Option, + pub signal: Option, +} + +#[napi(object, object_to_js = false)] +pub struct PortableVerifyInput { + pub input: String, + pub mode: Option, + pub limits: Option, + pub signal: Option, +} + +#[napi(object, object_to_js = false)] +pub struct PortableImportInput { + pub project_root: String, + pub input: String, + pub operation_id: String, + pub limits: Option, + pub signal: Option, +} + +#[napi(object)] +pub struct PortableOciAuthenticityInput { + pub require_named_signer: Option, + pub verification_key: Option, +} + +#[napi(object)] +pub struct PortableOciSignatureInput { + pub signer: String, + pub key_id: String, + pub secret: Buffer, +} + +#[napi(object, object_to_js = false)] +pub struct PortableOciPublishInput { + pub package_path: String, + pub registry: String, + pub repository: String, + pub tag: Option, + pub limits: Option, + pub authenticity: Option, + pub signature: Option, + pub insecure_http: Option, + pub credential: Option, + pub signal: Option, +} + +#[napi(object, object_to_js = false)] +pub struct PortableOciPullInput { + pub registry: String, + pub repository: String, + pub reference: String, + pub destination: String, + pub expected_oci_digest: Option, + pub limits: Option, + pub authenticity: Option, + pub insecure_http: Option, + pub credential: Option, + pub signal: Option, +} + +#[napi(object)] +pub struct PortableExportOutput { + pub contract: String, + pub source: String, + pub checkpoint: Option, + pub generation_uuid: String, + pub package_digest: String, + pub transport_digest: String, + pub entry_count: BigInt, + pub payload_bytes: BigInt, + pub representation: String, + pub selection_fingerprint: String, + pub output: String, +} + +#[napi(object)] +pub struct PortableVerifyOutput { + pub contract: String, + pub representation: String, + pub package_digest: String, + pub package_class: String, + pub component_count: BigInt, + pub entry_count: BigInt, + pub payload_bytes: BigInt, + pub integrity: String, + pub compatibility: String, + pub authenticity: String, + pub transport_digest: Option, +} + +#[napi(object)] +pub struct PortableImportOutput { + pub package_digest: String, + pub transport_digest: Option, + pub generation_uuid: String, + pub idempotent_replay: bool, +} + +#[napi(object)] +pub struct PortableOciReferenceOutput { + pub registry: String, + pub repository: String, + pub oci_manifest_digest: String, + pub package_digest: String, + pub package_class: String, + pub tag: Option, + pub bytes_transferred: BigInt, + pub blob_count: BigInt, +} + +#[napi(object)] +pub struct PortableOciPullOutput { + pub reference: PortableOciReferenceOutput, + pub destination: String, + pub report: PortableVerifyOutput, + pub signature_state: String, +} + +#[napi(object, object_to_js = false)] +pub struct ResultSinkOptionsInput { + pub max_row_group_rows: Option, + pub max_batch_rows: Option, + pub signal: Option, +} + +#[napi(object)] +pub struct ResultSinkProgressOutput { + pub phase: String, + pub rows: BigInt, + pub batches: BigInt, + pub bytes: BigInt, + pub elapsed_ms: BigInt, + pub complete: bool, +} + +#[napi(object)] +pub struct ResultSinkReceiptOutput { + pub destination: String, + pub format: String, + pub progress: ResultSinkProgressOutput, +} + +pub(crate) fn preview_selection( + graph: &graphforge_api::GraphForge, + input: PortableSelectionPreviewInput, +) -> Result { + let request = PortableV2SelectionPreviewRequest { + selection: selection_from_checkpoint(input.checkpoint), + request: PortableV2SelectionRequest { + profile: parse_profile(input.profile, input.identities)?, + strict: input.strict.unwrap_or(false), + }, + limits: parse_limits(input.limits)?, + }; + let plan = graph + .preview_portable_v2_selection(&request) + .map_err(|error| to_portable_napi_err(&error))?; + Ok(selection_plan_json(&plan)) +} + +pub(crate) fn preview_subset( + graph: &graphforge_api::GraphForge, + input: PortableSubsetPreviewInput, +) -> Result { + let subset = parse_subset(Some(input.subset))? + .ok_or_else(|| to_napi_err(&GfError::Validation("subset is required".into())))?; + let request = PortableV2SubsetPreviewRequest { + selection: selection_from_checkpoint(input.checkpoint), + request: subset, + limits: parse_limits(input.limits)?, + }; + let plan = graph + .preview_portable_v2_graph_subset(&request) + .map_err(|error| to_portable_napi_err(&error))?; + Ok(subset_plan_json(&plan)) +} + +pub struct ExportPortableTask { + pub engine: Arc>, + pub request: PortableV2ExportRequest, + pub cancellation: CancellationToken, +} + +impl Task for ExportPortableTask { + type Output = + std::result::Result; + type JsValue = PortableExportOutput; + + fn compute(&mut self) -> napi::Result { + let graph = self + .engine + .read() + .map_err(|_| napi::Error::from_reason("GraphForge lock poisoned"))?; + Ok(graph.export_portable_v2(&self.request, Some(self.cancellation.flag()), |_| {})) + } + + fn resolve(&mut self, env: Env, output: Self::Output) -> napi::Result { + output + .map(export_result_output) + .map_err(|error| to_portable_deferred_err(env, &error)) + } +} + +pub struct VerifyPortableTask { + pub request: PortableVerifyRequest, + pub cancellation: CancellationToken, +} + +impl Task for VerifyPortableTask { + type Output = std::result::Result; + type JsValue = PortableVerifyOutput; + + fn compute(&mut self) -> napi::Result { + Ok(graphforge_api::verify_portable_v2( + &self.request, + Some(self.cancellation.flag()), + )) + } + + fn resolve(&mut self, env: Env, output: Self::Output) -> napi::Result { + output + .map(verify_report_output) + .map_err(|error| to_portable_deferred_err(env, &error)) + } +} + +pub struct ImportPortableTask { + pub project_root: PathBuf, + pub request: PortableV2ImportRequest, + pub cancellation: CancellationToken, +} + +impl Task for ImportPortableTask { + type Output = std::result::Result; + type JsValue = PortableImportOutput; + + fn compute(&mut self) -> napi::Result { + Ok(graphforge_api::GraphForge::import_portable_v2( + &self.project_root, + &self.request, + Some(self.cancellation.flag()), + )) + } + + fn resolve(&mut self, env: Env, output: Self::Output) -> napi::Result { + output + .map(|result| PortableImportOutput { + package_digest: result.package_digest, + transport_digest: result.transport_digest, + generation_uuid: result.generation_uuid.to_string(), + idempotent_replay: result.idempotent_replay, + }) + .map_err(|error| to_portable_deferred_err(env, &error)) + } +} + +fn authenticity_policy( + input: Option, +) -> PortableV2OciAuthenticityPolicy { + let Some(input) = input else { + return PortableV2OciAuthenticityPolicy::default(); + }; + PortableV2OciAuthenticityPolicy { + require_named_signer: input.require_named_signer, + verification_key: input.verification_key.map(|key| key.to_vec()), + } +} + +fn signature_material( + input: Option, +) -> Option { + input.map(|input| PortableV2OciSignatureMaterial { + signer: input.signer, + key_id: input.key_id, + secret: input.secret.to_vec(), + }) +} + +fn oci_reference_output( + reference: graphforge_api::PortableV2OciReference, +) -> PortableOciReferenceOutput { + PortableOciReferenceOutput { + registry: reference.registry, + repository: reference.repository, + oci_manifest_digest: reference.oci_manifest_digest, + package_digest: reference.package_digest, + package_class: match reference.package_class { + graphforge_api::PortableV2PackageClass::Complete => "complete".into(), + graphforge_api::PortableV2PackageClass::OntologyOnly => "ontology_only".into(), + graphforge_api::PortableV2PackageClass::ComponentSelective => { + "component_selective".into() + } + graphforge_api::PortableV2PackageClass::GraphDataSubset => "graph_data_subset".into(), + }, + tag: reference.tag, + bytes_transferred: BigInt::from(reference.bytes_transferred), + blob_count: BigInt::from(reference.blob_count), + } +} + +pub struct PublishOciTask { + pub request: PortableV2OciPublishFacadeRequest, + pub cancellation: CancellationToken, +} + +impl Task for PublishOciTask { + type Output = std::result::Result; + type JsValue = PortableOciReferenceOutput; + + fn compute(&mut self) -> napi::Result { + Ok(graphforge_api::publish_portable_v2_oci( + &self.request, + Some(self.cancellation.flag()), + )) + } + + fn resolve(&mut self, env: Env, output: Self::Output) -> napi::Result { + output + .map(oci_reference_output) + .map_err(|error| to_portable_deferred_err(env, &error)) + } +} + +pub struct PullOciTask { + pub request: PortableV2OciPullFacadeRequest, + pub cancellation: CancellationToken, +} + +impl Task for PullOciTask { + type Output = std::result::Result; + type JsValue = PortableOciPullOutput; + + fn compute(&mut self) -> napi::Result { + Ok(graphforge_api::pull_portable_v2_oci( + &self.request, + Some(self.cancellation.flag()), + )) + } + + fn resolve(&mut self, env: Env, output: Self::Output) -> napi::Result { + output + .map(|receipt| PortableOciPullOutput { + reference: oci_reference_output(receipt.reference), + destination: receipt.destination.display().to_string(), + report: verify_report_output(receipt.report), + signature_state: signature_state_token(receipt.signature_state), + }) + .map_err(|error| to_portable_deferred_err(env, &error)) + } +} + +pub fn build_export_task( + engine: Arc>, + input: PortableExportInput, +) -> Result> { + let cancellation = bind_signal(input.signal); + Ok(AsyncTask::new(ExportPortableTask { + engine, + request: PortableV2ExportRequest { + selection: selection_from_checkpoint(input.checkpoint), + output_path: PathBuf::from(input.output_path), + representation: parse_output(input.representation.as_deref())?, + profile: parse_profile(input.profile, input.identities)?, + subset: parse_subset(input.subset)?, + limits: parse_limits(input.limits)?, + }, + cancellation, + })) +} + +pub fn build_verify_task(input: PortableVerifyInput) -> Result> { + let cancellation = bind_signal(input.signal); + Ok(AsyncTask::new(VerifyPortableTask { + request: PortableVerifyRequest { + input: PathBuf::from(input.input), + mode: parse_mode(input.mode.as_deref())?, + limits: parse_limits(input.limits)?, + }, + cancellation, + })) +} + +pub fn build_import_task(input: PortableImportInput) -> Result> { + let cancellation = bind_signal(input.signal); + Ok(AsyncTask::new(ImportPortableTask { + project_root: PathBuf::from(input.project_root), + request: PortableV2ImportRequest { + input: PathBuf::from(input.input), + operation_id: crate::canonical_operation_id(&input.operation_id)?, + limits: parse_limits(input.limits)?, + }, + cancellation, + })) +} + +pub fn build_publish_task(input: PortableOciPublishInput) -> Result> { + let cancellation = bind_signal(input.signal); + Ok(AsyncTask::new(PublishOciTask { + request: PortableV2OciPublishFacadeRequest { + package_path: PathBuf::from(input.package_path), + registry: input.registry, + repository: input.repository, + tag: input.tag, + limits: parse_limits(input.limits)?, + authenticity: authenticity_policy(input.authenticity), + signature: signature_material(input.signature), + insecure_http: input.insecure_http.unwrap_or(false), + credential: input.credential, + }, + cancellation, + })) +} + +pub fn build_pull_task(input: PortableOciPullInput) -> Result> { + let cancellation = bind_signal(input.signal); + Ok(AsyncTask::new(PullOciTask { + request: PortableV2OciPullFacadeRequest { + registry: input.registry, + repository: input.repository, + reference: input.reference, + expected_oci_digest: input.expected_oci_digest, + destination: PathBuf::from(input.destination), + limits: parse_limits(input.limits)?, + authenticity: authenticity_policy(input.authenticity), + insecure_http: input.insecure_http.unwrap_or(false), + credential: input.credential, + }, + cancellation, + })) +} + +pub fn parse_sink_options( + input: Option, +) -> Result<(ResultSinkOptions, CancellationToken)> { + let defaults = ResultSinkOptions::default(); + let Some(input) = input else { + return Ok((defaults, CancellationToken::new())); + }; + let options = ResultSinkOptions { + max_row_group_rows: crate::node_usize( + input.max_row_group_rows, + defaults.max_row_group_rows, + "maxRowGroupRows", + )?, + max_batch_rows: crate::node_usize( + input.max_batch_rows, + defaults.max_batch_rows, + "maxBatchRows", + )?, + }; + Ok((options, bind_signal(input.signal))) +} + +pub struct SinkStreamTask { + pub engine: Arc>, + pub closed: Arc, + pub cypher: String, + pub params: std::collections::HashMap, + pub path: String, + pub format: ResultSinkFormat, + pub options: ResultSinkOptions, + pub cancellation: CancellationToken, +} + +impl Task for SinkStreamTask { + type Output = std::result::Result; + type JsValue = ResultSinkReceiptOutput; + + fn compute(&mut self) -> napi::Result { + Ok((|| { + if self.closed.load(std::sync::atomic::Ordering::Acquire) { + return Err(GfError::Lifecycle( + "operation on a closed GraphForge instance".into(), + )); + } + let graph = self + .engine + .read() + .map_err(|_| GfError::Execution("GraphForge lock poisoned".into()))?; + match self.format { + ResultSinkFormat::Parquet => graph.execute_to_parquet_stream_with_params( + &self.cypher, + &self.params, + &self.path, + &self.options, + Some(&self.cancellation), + ), + ResultSinkFormat::ArrowIpc => graph.execute_to_arrow_ipc_stream_with_params( + &self.cypher, + &self.params, + &self.path, + &self.options, + Some(&self.cancellation), + ), + } + })()) + } + + fn resolve(&mut self, env: Env, output: Self::Output) -> napi::Result { + output + .map(sink_receipt_output) + .map_err(|error| crate::to_napi_deferred_err(env, &error)) + } +} diff --git a/crates/graphforge-bindings-node/tests/async-errors.test.mjs b/crates/graphforge-bindings-node/tests/async-errors.test.mjs index 959530db4..dc9f898ac 100644 --- a/crates/graphforge-bindings-node/tests/async-errors.test.mjs +++ b/crates/graphforge-bindings-node/tests/async-errors.test.mjs @@ -1,7 +1,7 @@ // Structured native-addon Promise rejection acceptance (#2499). import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; +import { readdirSync, readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { tmpdir } from "node:os"; import { test } from "node:test"; @@ -22,11 +22,22 @@ async function rejectsWithCode(promise, code) { } test("every native task uses structured cooperative error transport", () => { - const source = readFileSync(join(here, "../src/lib.rs"), "utf8"); - const errors = readFileSync(join(here, "../src/error.rs"), "utf8"); - assert.equal(source.match(/impl Task for /g)?.length, 42); - assert.equal(source.match(/type Output = std::result::Result name.endsWith(".rs")) + .map((name) => readFileSync(join(srcDir, name), "utf8")) + .join("\n"); + const errors = readFileSync(join(srcDir, "error.rs"), "utf8"); + const taskCount = source.match(/impl Task for /g)?.length ?? 0; + assert.equal(taskCount, 49); + assert.equal( + source.match(/type Output =\s*(?:\n\s*)?std::result::Result;[\s\S]*?if self\.cancellation\.is_cancelled\(\)[\s\S]*?to_napi_deferred_err\(env,/, diff --git a/crates/graphforge-bindings-node/tests/import_session.test.mjs b/crates/graphforge-bindings-node/tests/import_session.test.mjs new file mode 100644 index 000000000..ba2aa8491 --- /dev/null +++ b/crates/graphforge-bindings-node/tests/import_session.test.mjs @@ -0,0 +1,42 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { randomUUID } from "node:crypto"; +import { + FixedSizeBinary, + Table, + Utf8, + tableToIPC, + vectorFromArray, +} from "apache-arrow"; +import { GraphForge } from "../index.js"; + +test("import session begin checkpoint resume abort lifecycle", () => { + const root = mkdtempSync(join(tmpdir(), "gf-import-")); + try { + const forge = new GraphForge(join(root, "project")); + const operation = randomUUID(); + const session = forge.beginImportSession(operation); + assert.equal(session.status().phase, "open"); + const nodeUuid = Buffer.from(randomUUID().replace(/-/g, ""), "hex"); + const table = new Table({ + node_uuid: vectorFromArray([nodeUuid], new FixedSizeBinary(16)), + label: vectorFromArray(["Person"], new Utf8()), + }); + session.appendArrow("node", Buffer.from(tableToIPC(table))); + const progress = session.checkpoint(); + assert.ok(progress.filesPending >= 1n); + assert.equal(typeof progress.bytesAccepted, "bigint"); + const sessionUuid = session.sessionUuid; + const resumed = forge.resumeImportSession(sessionUuid); + assert.equal(resumed.sessionUuid, sessionUuid); + const aborted = resumed.abort(); + assert.ok(aborted.filesAccepted >= 1n); + const cleaned = forge.cleanupStaleImportSessions(0n); + assert.equal(typeof cleaned, "bigint"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/crates/graphforge-bindings-node/tests/non-cypher-parity-policy.json b/crates/graphforge-bindings-node/tests/non-cypher-parity-policy.json index e7e6a7644..3ea19df05 100644 --- a/crates/graphforge-bindings-node/tests/non-cypher-parity-policy.json +++ b/crates/graphforge-bindings-node/tests/non-cypher-parity-policy.json @@ -1,8 +1,8 @@ { "contractVersion": 1, "rustManifest": "../../../tests/contracts/non-cypher-rust-surface.json", - "releaseSurfaceCount": 207, - "releaseSurfaceDigest": "01f642d893eeb78b9493363d6c4ac5eea6744e0a970b60a9e6a456fe9892a886", + "releaseSurfaceCount": 210, + "releaseSurfaceDigest": "9ca796af78a3ea51e4c0d404e9f74e3eb1cdd49a0b862dc15838cd9ada037877", "requiredEquivalent": [ "GraphForge.adopt_ontology", "GraphForge.clear_ontology", @@ -26,8 +26,9 @@ "streaming-errors-maintenance": "persistenceErrorsCancellation", "compatibility": "arrowIpc", "transaction-maintenance": "transactionMaintenance", - "resumable-import": "lifecycleConstruction", - "semantic-generation-diff": "semanticGenerationDiff" + "resumable-import": "importSession", + "semantic-generation-diff": "semanticGenerationDiff", + "portable-v2-facade": "portableV2" }, "classification": { "equivalent": [ @@ -47,12 +48,15 @@ "GraphForge.assess_confidence", "GraphForge.attach_evidence", "GraphForge.attach_resolved_run", + "GraphForge.begin_import_session", "GraphForge.begin_transaction", "GraphForge.bind_embedding_space_alias", "GraphForge.checkpoint", + "GraphForge.cleanup_stale_import_sessions", "GraphForge.clear", "GraphForge.clear_ontology", "GraphForge.cluster", + "GraphForge.committed_generation_identity", "GraphForge.compact_graph_delta", "GraphForge.confidence_assessment", "GraphForge.confidence_inputs", @@ -62,7 +66,6 @@ "GraphForge.create_hypothesis_group", "GraphForge.delete_checkpoint", "GraphForge.delete_embedding_space", - "GraphForge.committed_generation_identity", "GraphForge.diff_checkpoints", "GraphForge.diff_committed_generations", "GraphForge.embedding_refresh_project_policy", @@ -75,6 +78,7 @@ "GraphForge.execute_project_cleanup", "GraphForge.explain", "GraphForge.export_ontology", + "GraphForge.export_portable_v2", "GraphForge.find", "GraphForge.graph_delta_compaction_status", "GraphForge.graph_directedness", @@ -118,6 +122,8 @@ "GraphForge.prepare_rank_invocation", "GraphForge.prepare_similar_invocation", "GraphForge.preview_graph_delta_compaction", + "GraphForge.preview_portable_v2_graph_subset", + "GraphForge.preview_portable_v2_selection", "GraphForge.preview_project_cleanup", "GraphForge.profile_gsi", "GraphForge.project_capabilities", @@ -142,6 +148,7 @@ "GraphForge.remove_hypothesis_member", "GraphForge.resolve_belief_projection", "GraphForge.resolve_belief_subject", + "GraphForge.resume_import_session", "GraphForge.revert_to_checkpoint", "GraphForge.schema", "GraphForge.set_default_embedding_space", @@ -152,7 +159,14 @@ "GraphForge.suggest_ontology", "GraphForge.supersede_assertion", "GraphForge.validate_ontology", - "GraphForge.workspace_ontology" + "GraphForge.workspace_ontology", + "GraphImportSession.abort", + "GraphImportSession.append_arrow", + "GraphImportSession.checkpoint", + "GraphImportSession.commit", + "GraphImportSession.register_parquet", + "GraphImportSession.status", + "GraphImportSession.validate" ], "languageSpecific": { "GraphForge.execute_stream": { @@ -179,9 +193,9 @@ "GraphForge.execute_to_arrow_ipc_stream_with_params": { "nodeMembers": [ "GraphForge.plan", - "PlanHandle.collectIpc" + "PlanHandle.sinkArrowIpc" ], - "reason": "Node binds parameters on PlanHandle before Promise-delivered Arrow IPC." + "reason": "Node binds parameters on PlanHandle before the asynchronous Arrow IPC sink." }, "GraphForge.execute_to_parquet": { "nodeMembers": [ @@ -215,6 +229,12 @@ "GraphForge.algorithmDescriptorContracts" ], "reason": "Node projects the live Rust registry as a thin GraphForge method without opening knowledge or epistemic storage." + }, + "GraphImportSession.validate_with_cancellation": { + "nodeMembers": [ + "GraphImportSession.validate" + ], + "reason": "Node validate accepts an optional AbortSignal instead of a second method." } }, "nodeOnly": { @@ -225,14 +245,19 @@ "GraphForge.algorithmDescriptorContracts": "Thin projection of crate.algorithm_descriptor_contracts.", "GraphForge.path": "Node project-path getter corresponding to Rust introspection.", "GraphForge.plan": "Node asynchronous execution adapter classified above.", + "GraphForge.verifyPortableV2": "Static Node adapter for crate.verify_portable_v2.", + "GraphForge.importPortableV2": "Static Node adapter for GraphForge.import_portable_v2.", + "GraphForge.publishPortableV2Oci": "Static Node adapter for crate.publish_portable_v2_oci.", + "GraphForge.pullPortableV2Oci": "Static Node adapter for crate.pull_portable_v2_oci.", "PlanHandle.collectIpc": "Promise-delivered Arrow IPC adapter classified above.", "PlanHandle.explain": "Node-only plan introspection for the asynchronous execution adapter.", - "PlanHandle.sinkParquet": "Asynchronous Parquet sink adapter classified above." + "PlanHandle.sinkParquet": "Asynchronous Parquet sink adapter classified above.", + "PlanHandle.sinkArrowIpc": "Asynchronous Arrow IPC sink adapter classified above.", + "GraphImportSession.sessionUuid": "Node getter for the Rust session_uuid value object accessor." }, "notExposedDefaults": { "CheckpointView": "Node checkpoint views expose the bounded pinned read surface; unsupported Rust checkpoint operations require an explicit parity decision.", "GraphForge": "No shipped Node member exists for this Rust facade entry; adding one requires an explicit parity decision.", - "GraphImportSession": "Rust owns durable staged import sessions; binding projection is tracked separately.", "OpenRouterProviderSession": "Rust provider sessions are not exported; Node owns a configured native provider adapter instead.", "RepositoryContext": "Rust owns repository lifecycle behavior; the Node command wrapper parity is delivered by issue #226." } @@ -357,6 +382,9 @@ ], "cluster-vector.test.mjs": [ "k means cluster" + ], + "result_sink.test.mjs": [ + "result sinks stream parquet and arrow ipc with BigInt receipts" ] }, "transactionMaintenance": { @@ -365,6 +393,16 @@ "dropped wrapper handles roll back and never commit", "maintenance preview and execution reconcile candidate identities" ] + }, + "portableV2": { + "portable_v2.test.mjs": [ + "portable-v2 export verify and import preserve package digest" + ] + }, + "importSession": { + "import_session.test.mjs": [ + "import session begin checkpoint resume abort lifecycle" + ] } } } diff --git a/crates/graphforge-bindings-node/tests/non-cypher-release-parity.test.mjs b/crates/graphforge-bindings-node/tests/non-cypher-release-parity.test.mjs index 7bcfafefa..c92cd9d49 100644 --- a/crates/graphforge-bindings-node/tests/non-cypher-release-parity.test.mjs +++ b/crates/graphforge-bindings-node/tests/non-cypher-release-parity.test.mjs @@ -12,7 +12,12 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { test } from "node:test"; import { tableFromIPC } from "apache-arrow"; -import { CheckpointView, GraphForge, PlanHandle } from "../index.js"; +import { + CheckpointView, + GraphForge, + GraphImportSession, + PlanHandle, +} from "../index.js"; const here = dirname(fileURLToPath(import.meta.url)); const policy = JSON.parse( @@ -56,7 +61,12 @@ test("the Node classification is total, frozen, and backed by non-skipped native const languageSpecific = new Set( Object.keys(policy.classification.languageSpecific), ); - const receivers = { CheckpointView, GraphForge, PlanHandle }; + const receivers = { + CheckpointView, + GraphForge, + GraphImportSession, + PlanHandle, + }; for (const id of equivalent) { assert.ok(release.includes(id), `stale equivalent classification: ${id}`); assert.equal( @@ -136,6 +146,7 @@ test("the Node classification is total, frozen, and backed by non-skipped native for (const [receiver, constructor] of Object.entries({ CheckpointView, GraphForge, + GraphImportSession, PlanHandle, })) { const projected = new Set( @@ -155,6 +166,22 @@ test("the Node classification is total, frozen, and backed by non-skipped native `unclassified shipped Node member: ${receiver}.${method}`, ); } + for (const method of Object.getOwnPropertyNames(constructor)) { + if ( + method === "length" || + method === "name" || + method === "prototype" || + projected.has(method) + ) { + continue; + } + const descriptor = Object.getOwnPropertyDescriptor(constructor, method); + if (!descriptor || typeof descriptor.value !== "function") continue; + assert.ok( + nodeOnly.has(`${receiver}.${method}`), + `unclassified shipped Node static member: ${receiver}.${method}`, + ); + } } for (const [group, files] of Object.entries(policy.evidence)) { diff --git a/crates/graphforge-bindings-node/tests/portable_v2.test.mjs b/crates/graphforge-bindings-node/tests/portable_v2.test.mjs new file mode 100644 index 000000000..a11f2b3f1 --- /dev/null +++ b/crates/graphforge-bindings-node/tests/portable_v2.test.mjs @@ -0,0 +1,48 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { randomUUID } from "node:crypto"; +import { GraphForge } from "../index.js"; + +test("portable-v2 export verify and import preserve package digest", async () => { + const root = mkdtempSync(join(tmpdir(), "gf-portable-")); + try { + const source = join(root, "source"); + const forge = new GraphForge(source); + const preview = forge.previewPortableV2Selection({ profile: "complete" }); + assert.equal(preview.packageClass, "complete"); + const expanded = join(root, "expanded"); + const bundle = join(root, "complete.gfpb"); + const expandedExport = await forge.exportPortableV2({ + outputPath: expanded, + representation: "expanded", + profile: "complete", + }); + const bundleExport = await forge.exportPortableV2({ + outputPath: bundle, + representation: "bundle", + profile: "complete", + }); + assert.equal(expandedExport.packageDigest, bundleExport.packageDigest); + assert.equal( + expandedExport.selectionFingerprint, + preview.selectionFingerprint, + ); + assert.equal(typeof bundleExport.payloadBytes, "bigint"); + const verified = await GraphForge.verifyPortableV2({ + input: bundle, + mode: "full", + }); + assert.equal(verified.packageDigest, bundleExport.packageDigest); + const imported = await GraphForge.importPortableV2({ + projectRoot: join(root, "target"), + input: bundle, + operationId: randomUUID(), + }); + assert.equal(imported.packageDigest, bundleExport.packageDigest); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/crates/graphforge-bindings-node/tests/result_sink.test.mjs b/crates/graphforge-bindings-node/tests/result_sink.test.mjs new file mode 100644 index 000000000..cf04859ab --- /dev/null +++ b/crates/graphforge-bindings-node/tests/result_sink.test.mjs @@ -0,0 +1,42 @@ +import assert from "node:assert/strict"; +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { GraphForge } from "../index.js"; + +test("result sinks stream parquet and arrow ipc with BigInt receipts", async () => { + const forge = new GraphForge(); + for (const name of ["a", "b", "c"]) { + forge.execute(`CREATE (:Person {name: '${name}'})`); + } + const root = mkdtempSync(join(tmpdir(), "gf-sink-")); + try { + const plan = forge.plan( + "MATCH (p:Person) RETURN p.name AS name ORDER BY name", + ); + const parquet = join(root, "stream.parquet"); + const ipc = join(root, "stream.arrow"); + const parquetReceipt = await plan.sinkParquet(parquet, { + maxBatchRows: 64n, + maxRowGroupRows: 2n, + }); + const ipcReceipt = await plan.sinkArrowIpc(ipc, { + maxBatchRows: 64n, + maxRowGroupRows: 2n, + }); + assert.equal(parquetReceipt.progress.rows, 3n); + assert.equal(ipcReceipt.progress.rows, 3n); + assert.equal(typeof parquetReceipt.progress.bytes, "bigint"); + assert.equal(typeof parquetReceipt.progress.rows, "bigint"); + assert.equal(typeof parquetReceipt.progress.batches, "bigint"); + // Receipt counters stay BigInt end-to-end (lossless beyond Number.MAX_SAFE_INTEGER). + const oversized = 9_007_199_254_740_993n; // Number.MAX_SAFE_INTEGER + 2 + assert.notEqual(Number(oversized), Number(oversized + 1n)); + assert.ok(typeof parquetReceipt.progress.elapsedMs === "bigint"); + assert.ok(existsSync(parquet)); + assert.ok(existsSync(ipc)); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/crates/graphforge-bindings-py/python/graphforge/__init__.py b/crates/graphforge-bindings-py/python/graphforge/__init__.py index e8e4f7138..add384182 100644 --- a/crates/graphforge-bindings-py/python/graphforge/__init__.py +++ b/crates/graphforge-bindings-py/python/graphforge/__init__.py @@ -13,6 +13,7 @@ ExecutionError, GraphForge, GraphForgeError, + GraphImportSession, InvocationDescriptor, LifecycleError, NodeHandle, @@ -33,6 +34,7 @@ "ExecutionError", "GraphForge", "GraphForgeError", + "GraphImportSession", "InvocationDescriptor", "LifecycleError", "NodeHandle", diff --git a/crates/graphforge-bindings-py/python/graphforge/_graphforge_rs.pyi b/crates/graphforge-bindings-py/python/graphforge/_graphforge_rs.pyi index aea1de037..21a2ef716 100644 --- a/crates/graphforge-bindings-py/python/graphforge/_graphforge_rs.pyi +++ b/crates/graphforge-bindings-py/python/graphforge/_graphforge_rs.pyi @@ -6,6 +6,7 @@ Hand-written to match the PyO3 surface in `crates/graphforge-bindings-py/src/lib use `pyarrow` (a hard dependency). """ +from collections.abc import Callable from typing import Any, Literal, overload import uuid @@ -44,6 +45,17 @@ class CancellationToken: @property def is_cancelled(self) -> bool: ... +class GraphImportSession: + @property + def session_uuid(self) -> str: ... + def status(self) -> dict[str, Any]: ... + def append_arrow(self, kind: str, data: Any) -> None: ... + def register_parquet(self, kind: str, path: str) -> None: ... + def checkpoint(self) -> dict[str, Any]: ... + def validate(self, *, cancellation: CancellationToken | None = None) -> dict[str, Any]: ... + def commit(self, *, cancellation: CancellationToken | None = None) -> str: ... + def abort(self) -> dict[str, Any]: ... + class GraphTransaction: def status(self) -> dict[str, Any]: ... def stage_cypher(self, query: str, params: dict[str, Any] | None = None) -> None: ... @@ -243,6 +255,112 @@ class GraphForge: max_output_bytes: int = 268_435_456, cancellation: CancellationToken | None = None, ) -> dict[str, Any]: ... + def preview_portable_v2_selection( + self, + *, + checkpoint: str | None = None, + profile: str = "complete", + identities: list[dict[str, str]] | None = None, + strict: bool = False, + limits: dict[str, Any] | None = None, + ) -> dict[str, Any]: ... + def preview_portable_v2_graph_subset( + self, + *, + subset: dict[str, Any], + checkpoint: str | None = None, + limits: dict[str, Any] | None = None, + ) -> dict[str, Any]: ... + def export_portable_v2( + self, + *, + output_path: str, + representation: str = "bundle", + profile: str = "complete", + identities: list[dict[str, str]] | None = None, + checkpoint: str | None = None, + subset: dict[str, Any] | None = None, + limits: dict[str, Any] | None = None, + cancellation: CancellationToken | None = None, + progress: Callable[[dict[str, int]], object] | None = None, + ) -> dict[str, Any]: ... + @staticmethod + def verify_portable_v2( + input: str, + *, + mode: str = "full", + limits: dict[str, Any] | None = None, + cancellation: CancellationToken | None = None, + ) -> dict[str, Any]: ... + @staticmethod + def import_portable_v2( + project_root: str, + *, + input: str, + operation_id: str, + limits: dict[str, Any] | None = None, + cancellation: CancellationToken | None = None, + ) -> dict[str, Any]: ... + @staticmethod + def publish_portable_v2_oci( + *, + package_path: str, + registry: str, + repository: str, + tag: str | None = None, + limits: dict[str, Any] | None = None, + authenticity: dict[str, Any] | None = None, + signature: dict[str, Any] | None = None, + insecure_http: bool = False, + credential: str | None = None, + cancellation: CancellationToken | None = None, + ) -> dict[str, Any]: ... + @staticmethod + def pull_portable_v2_oci( + *, + registry: str, + repository: str, + reference: str, + destination: str, + expected_oci_digest: str | None = None, + limits: dict[str, Any] | None = None, + authenticity: dict[str, Any] | None = None, + insecure_http: bool = False, + credential: str | None = None, + cancellation: CancellationToken | None = None, + ) -> dict[str, Any]: ... + def begin_import_session( + self, + *, + operation_uuid: str, + batch_rows: int | None = None, + max_source_bytes: int | None = None, + max_files: int | None = None, + max_rejected_rows: int | None = None, + io_concurrency: int | None = None, + ) -> GraphImportSession: ... + def resume_import_session(self, session_uuid: str) -> GraphImportSession: ... + def cleanup_stale_import_sessions(self, *, max_age_secs: int) -> int: ... + def execute_to_parquet_stream( + self, + query: str, + path: str, + *, + params: dict[str, Any] | None = None, + max_row_group_rows: int = 65536, + max_batch_rows: int = 65536, + cancellation: CancellationToken | None = None, + ) -> dict[str, Any]: ... + def execute_to_arrow_ipc_stream( + self, + query: str, + path: str, + *, + params: dict[str, Any] | None = None, + max_row_group_rows: int = 65536, + max_batch_rows: int = 65536, + cancellation: CancellationToken | None = None, + ) -> dict[str, Any]: ... def diff_checkpoints( self, *, diff --git a/crates/graphforge-bindings-py/src/import_session.rs b/crates/graphforge-bindings-py/src/import_session.rs new file mode 100644 index 000000000..c003dd50b --- /dev/null +++ b/crates/graphforge-bindings-py/src/import_session.rs @@ -0,0 +1,331 @@ +//! Thin Python bindings for durable staged graph-import sessions (#744 / #738). + +use std::path::PathBuf; +use std::sync::Mutex; +use std::time::Duration; + +use graphforge_api::{ + BulkInputKind, GfError, GraphImportSession, ImportPhase, ImportProgress, ImportSessionLimits, +}; +use pyo3::prelude::*; +use pyo3::types::PyDict; +use uuid::Uuid; + +use crate::{ + GraphForge, PyCancellationToken, canonical_operation_id, py_bulk_input_to_batch, to_pyerr, +}; + +impl GraphForge { + /// Run durable import validation while releasing the GIL on `&self`. + pub(crate) fn run_import_validate( + &self, + py: Python<'_>, + session: &Mutex>, + cancellation: Option<&graphforge_api::CancellationToken>, + ) -> PyResult { + self.ensure_open()?; + // Lock only inside `detach` so a concurrent GIL-holding caller cannot + // wait on this mutex while the detached worker needs the GIL to return. + py.detach(|| { + let mut guard = session + .lock() + .map_err(|_| GfError::Execution("import session lock poisoned".into()))?; + let session = guard + .as_mut() + .ok_or_else(|| GfError::Lifecycle("import session handle is closed".into()))?; + session.validate_with_cancellation(&self.inner, cancellation) + }) + .map_err(|error| to_pyerr(py, &error)) + } + + /// Publish a fully staged import while releasing the GIL on `&self`. + pub(crate) fn run_import_commit( + &self, + py: Python<'_>, + session: &Mutex>, + cancellation: Option<&graphforge_api::CancellationToken>, + ) -> PyResult { + self.ensure_open()?; + py.detach(|| { + let mut guard = session + .lock() + .map_err(|_| GfError::Execution("import session lock poisoned".into()))?; + let session = guard + .as_mut() + .ok_or_else(|| GfError::Lifecycle("import session handle is closed".into()))?; + session + .commit(&self.inner, cancellation) + .map(|uuid| uuid.to_string()) + }) + .map_err(|error| to_pyerr(py, &error)) + } +} + +fn phase_name(phase: ImportPhase) -> &'static str { + match phase { + ImportPhase::Open => "open", + ImportPhase::Validated => "validated", + ImportPhase::Committed => "committed", + ImportPhase::Aborted => "aborted", + ImportPhase::Quarantined => "quarantined", + } +} + +fn progress_dict(py: Python<'_>, progress: &ImportProgress) -> PyResult> { + let out = PyDict::new(py); + out.set_item("rows_accepted", progress.rows_accepted)?; + out.set_item("rows_rejected", progress.rows_rejected)?; + out.set_item("bytes_accepted", progress.bytes_accepted)?; + out.set_item("files_accepted", progress.files_accepted)?; + out.set_item("files_pending", progress.files_pending)?; + out.set_item("elapsed_millis", progress.elapsed_millis)?; + out.set_item("peak_batch_rows", progress.peak_batch_rows)?; + out.set_item("io_concurrency_limit", progress.io_concurrency_limit)?; + Ok(out.into_any().unbind()) +} + +fn status_dict( + py: Python<'_>, + phase: ImportPhase, + progress: &ImportProgress, +) -> PyResult> { + let out = PyDict::new(py); + out.set_item("phase", phase_name(phase))?; + out.set_item("progress", progress_dict(py, progress)?)?; + Ok(out.into_any().unbind()) +} + +fn parse_kind(py: Python<'_>, kind: &str) -> PyResult { + match kind { + "node" | "nodes" => Ok(BulkInputKind::Node), + "edge" | "edges" => Ok(BulkInputKind::Edge), + _ => Err(to_pyerr( + py, + &GfError::Validation("kind must be node or edge".into()), + )), + } +} + +fn parse_limits( + batch_rows: Option, + max_source_bytes: Option, + max_files: Option, + max_rejected_rows: Option, + io_concurrency: Option, +) -> ImportSessionLimits { + let defaults = ImportSessionLimits::default(); + ImportSessionLimits { + batch_rows: batch_rows.unwrap_or(defaults.batch_rows), + max_source_bytes: max_source_bytes.unwrap_or(defaults.max_source_bytes), + max_files: max_files.unwrap_or(defaults.max_files), + max_rejected_rows: max_rejected_rows.unwrap_or(defaults.max_rejected_rows), + io_concurrency: io_concurrency.unwrap_or(defaults.io_concurrency), + } +} + +/// Owned durable import-session handle. Contains no live rows. +#[pyclass(name = "GraphImportSession", module = "graphforge")] +pub struct PyGraphImportSession { + parent: Py, + inner: Mutex>, +} + +impl PyGraphImportSession { + fn take_inner(&self, py: Python<'_>) -> PyResult { + self.inner + .lock() + .map_err(|_| { + to_pyerr( + py, + &GfError::Execution("import session lock poisoned".into()), + ) + })? + .take() + .ok_or_else(|| { + to_pyerr( + py, + &GfError::Lifecycle("import session handle is closed".into()), + ) + }) + } + + fn with_mut( + &self, + py: Python<'_>, + f: impl FnOnce(&mut GraphImportSession) -> Result, + ) -> PyResult { + let mut guard = self.inner.lock().map_err(|_| { + to_pyerr( + py, + &GfError::Execution("import session lock poisoned".into()), + ) + })?; + let session = guard.as_mut().ok_or_else(|| { + to_pyerr( + py, + &GfError::Lifecycle("import session handle is closed".into()), + ) + })?; + f(session).map_err(|error| to_pyerr(py, &error)) + } +} + +#[pymethods] +impl PyGraphImportSession { + /// Durable identifier used for resume. + #[getter] + fn session_uuid(&self, py: Python<'_>) -> PyResult { + self.with_mut(py, |session| Ok(session.session_uuid().to_string())) + } + + /// Current durable phase and counters. + fn status(&self, py: Python<'_>) -> PyResult> { + let (phase, progress) = self.with_mut(py, |session| Ok(session.status()))?; + status_dict(py, phase, &progress) + } + + /// Append one Arrow partition without retaining live rows. + fn append_arrow(&self, py: Python<'_>, kind: &str, data: &Bound<'_, PyAny>) -> PyResult<()> { + let kind = parse_kind(py, kind)?; + let batch = py_bulk_input_to_batch(py, data)?; + self.with_mut(py, |session| session.append_arrow(kind, &[batch])) + } + + /// Register a local Parquet source by copying it into durable ownership. + fn register_parquet(&self, py: Python<'_>, kind: &str, path: &str) -> PyResult<()> { + let kind = parse_kind(py, kind)?; + let path = PathBuf::from(path); + self.with_mut(py, |session| session.register_parquet(kind, &path)) + } + + /// Persist counters and source ordering without publishing graph state. + fn checkpoint(&self, py: Python<'_>) -> PyResult> { + let progress = self.with_mut(py, GraphImportSession::checkpoint)?; + progress_dict(py, &progress) + } + + /// Validate and durably stage every source with optional cancellation. + #[pyo3(signature = (*, cancellation=None))] + fn validate( + &self, + py: Python<'_>, + cancellation: Option<&PyCancellationToken>, + ) -> PyResult> { + let cancellation = cancellation.map(|token| token.inner.clone()); + let progress = self.parent.bind(py).borrow().run_import_validate( + py, + &self.inner, + cancellation.as_ref(), + )?; + progress_dict(py, &progress) + } + + /// Publish the fully staged graph as one generation. + #[pyo3(signature = (*, cancellation=None))] + fn commit( + &self, + py: Python<'_>, + cancellation: Option<&PyCancellationToken>, + ) -> PyResult { + let cancellation = cancellation.map(|token| token.inner.clone()); + self.parent + .bind(py) + .borrow() + .run_import_commit(py, &self.inner, cancellation.as_ref()) + } + + /// Abort without changing CURRENT. + fn abort(&self, py: Python<'_>) -> PyResult> { + let session = self.take_inner(py)?; + let progress = py + .detach(|| session.abort()) + .map_err(|error| to_pyerr(py, &error))?; + progress_dict(py, &progress) + } +} + +/// Begin a durable import pinned to the facade's current project generation. +#[allow(clippy::too_many_arguments)] +pub(crate) fn begin_import_session( + slf: &Bound<'_, GraphForge>, + py: Python<'_>, + operation_uuid: &str, + batch_rows: Option, + max_source_bytes: Option, + max_files: Option, + max_rejected_rows: Option, + io_concurrency: Option, +) -> PyResult> { + if slf.borrow().closed { + return Err(to_pyerr( + py, + &GfError::Lifecycle("operation on a closed GraphForge instance".into()), + )); + } + let operation = canonical_operation_id(operation_uuid).map_err(|error| to_pyerr(py, &error))?; + let limits = parse_limits( + batch_rows, + max_source_bytes, + max_files, + max_rejected_rows, + io_concurrency, + ); + let session = slf + .borrow() + .inner + .begin_import_session(operation, limits) + .map_err(|error| to_pyerr(py, &error))?; + Bound::new( + py, + PyGraphImportSession { + parent: slf.clone().unbind(), + inner: Mutex::new(Some(session)), + }, + ) + .map(Bound::unbind) +} + +/// Resume one durable, non-terminal session after process interruption. +pub(crate) fn resume_import_session( + forge: &Bound<'_, GraphForge>, + py: Python<'_>, + session_uuid: &str, +) -> PyResult> { + if forge.borrow().closed { + return Err(to_pyerr( + py, + &GfError::Lifecycle("operation on a closed GraphForge instance".into()), + )); + } + let uuid = Uuid::parse_str(session_uuid).map_err(|_| { + to_pyerr( + py, + &GfError::Validation("session_uuid must be a canonical UUID string".into()), + ) + })?; + let session = forge + .borrow() + .inner + .resume_import_session(uuid) + .map_err(|error| to_pyerr(py, &error))?; + Bound::new( + py, + PyGraphImportSession { + parent: forge.clone().unbind(), + inner: Mutex::new(Some(session)), + }, + ) + .map(Bound::unbind) +} + +/// Abort and remove durable staging for non-terminal sessions older than `max_age_secs`. +pub(crate) fn cleanup_stale_import_sessions( + forge: &GraphForge, + py: Python<'_>, + max_age_secs: u64, +) -> PyResult { + forge.ensure_open()?; + let max_age = Duration::from_secs(max_age_secs); + py.detach(|| forge.inner.cleanup_stale_import_sessions(max_age)) + .map_err(|error| to_pyerr(py, &error)) +} diff --git a/crates/graphforge-bindings-py/src/lib.rs b/crates/graphforge-bindings-py/src/lib.rs index ec78045c4..c435260e9 100644 --- a/crates/graphforge-bindings-py/src/lib.rs +++ b/crates/graphforge-bindings-py/src/lib.rs @@ -3,6 +3,8 @@ #![warn(unsafe_code)] mod composite; +mod import_session; +mod portable; mod transaction; use std::collections::{BTreeMap, HashMap}; @@ -445,7 +447,10 @@ fn py_to_json_value(value: &Bound<'_, PyAny>) -> PyResult { } } -fn json_value_to_python(py: Python<'_>, value: &serde_json::Value) -> PyResult> { +pub(crate) fn json_value_to_python( + py: Python<'_>, + value: &serde_json::Value, +) -> PyResult> { Ok(match value { serde_json::Value::Null => py.None(), serde_json::Value::Bool(value) => value.into_pyobject(py)?.to_owned().unbind().into_any(), @@ -530,7 +535,10 @@ fn pyarrow_table_to_batch(value: &Bound<'_, PyAny>) -> PyResult { /// Accepted forms: `pyarrow.Table`, Arrow-compatible DataFrame (`to_arrow` / /// pandas via `pyarrow.Table.from_pandas`), and `list[dict]` via /// `pyarrow.Table.from_pylist`. Ontology, identity, and publication stay in Rust. -fn py_bulk_input_to_batch(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn py_bulk_input_to_batch( + py: Python<'_>, + value: &Bound<'_, PyAny>, +) -> PyResult { if let Ok(batch) = pyarrow_table_to_batch(value) { return Ok(batch); } @@ -1572,7 +1580,9 @@ fn embedding_options_from_kwargs( } /// Build the `$param` map from a Python `dict` (or empty when `None`). -fn params_from_dict(params: Option<&Bound<'_, PyDict>>) -> PyResult> { +pub(crate) fn params_from_dict( + params: Option<&Bound<'_, PyDict>>, +) -> PyResult> { let mut out = HashMap::new(); if let Some(dict) = params { for (k, v) in dict.iter() { @@ -2548,7 +2558,7 @@ impl PyCancellationToken { impl GraphForge { /// Guard mirroring the v0.5 lifecycle contract: operations after `close()` /// raise `LifecycleError`. - fn ensure_open(&self) -> PyResult<()> { + pub(crate) fn ensure_open(&self) -> PyResult<()> { if self.closed { return Err(Python::attach(|py| { to_pyerr( @@ -2790,6 +2800,247 @@ impl GraphForge { Ok(out.into_any().unbind()) } + /// Preview one content-free portable-v2 component selection. + #[pyo3(signature = (*, checkpoint=None, profile="complete", identities=None, strict=false, limits=None))] + fn preview_portable_v2_selection( + &self, + py: Python<'_>, + checkpoint: Option, + profile: &str, + identities: Option<&Bound<'_, PyList>>, + strict: bool, + limits: Option<&Bound<'_, PyDict>>, + ) -> PyResult> { + portable::preview_portable_v2_selection( + self, py, checkpoint, profile, identities, strict, limits, + ) + } + + /// Preview one content-free portable-v2 graph-data subset. + #[pyo3(signature = (*, subset, checkpoint=None, limits=None))] + fn preview_portable_v2_graph_subset( + &self, + py: Python<'_>, + subset: &Bound<'_, PyDict>, + checkpoint: Option, + limits: Option<&Bound<'_, PyDict>>, + ) -> PyResult> { + portable::preview_portable_v2_graph_subset(self, py, checkpoint, subset, limits) + } + + /// Export one pinned generation as an expanded or bundled portable-v2 package. + #[pyo3(signature = (*, output_path, representation="bundle", profile="complete", identities=None, checkpoint=None, subset=None, limits=None, cancellation=None, progress=None))] + #[allow(clippy::too_many_arguments)] + fn export_portable_v2( + &self, + py: Python<'_>, + output_path: &str, + representation: &str, + profile: &str, + identities: Option<&Bound<'_, PyList>>, + checkpoint: Option, + subset: Option<&Bound<'_, PyDict>>, + limits: Option<&Bound<'_, PyDict>>, + cancellation: Option<&PyCancellationToken>, + progress: Option<&Bound<'_, PyAny>>, + ) -> PyResult> { + portable::export_portable_v2( + self, + py, + output_path, + representation, + profile, + identities, + checkpoint, + subset, + limits, + cancellation, + progress, + ) + } + + /// Verify portable-v2 content without opening or mutating a project. + #[staticmethod] + #[pyo3(signature = (input, *, mode="full", limits=None, cancellation=None))] + fn verify_portable_v2( + py: Python<'_>, + input: &str, + mode: &str, + limits: Option<&Bound<'_, PyDict>>, + cancellation: Option<&PyCancellationToken>, + ) -> PyResult> { + portable::verify_portable_v2(py, input, mode, limits, cancellation) + } + + /// Verify and atomically import a complete portable-v2 package. + #[staticmethod] + #[pyo3(signature = (project_root, *, input, operation_id, limits=None, cancellation=None))] + fn import_portable_v2( + py: Python<'_>, + project_root: &str, + input: &str, + operation_id: &str, + limits: Option<&Bound<'_, PyDict>>, + cancellation: Option<&PyCancellationToken>, + ) -> PyResult> { + portable::import_portable_v2(py, project_root, input, operation_id, limits, cancellation) + } + + /// Publish a verified portable-v2 package to an OCI Distribution registry. + #[staticmethod] + #[pyo3(signature = (*, package_path, registry, repository, tag=None, limits=None, authenticity=None, signature=None, insecure_http=false, credential=None, cancellation=None))] + #[allow(clippy::too_many_arguments)] + fn publish_portable_v2_oci( + py: Python<'_>, + package_path: &str, + registry: &str, + repository: &str, + tag: Option, + limits: Option<&Bound<'_, PyDict>>, + authenticity: Option<&Bound<'_, PyDict>>, + signature: Option<&Bound<'_, PyDict>>, + insecure_http: bool, + credential: Option, + cancellation: Option<&PyCancellationToken>, + ) -> PyResult> { + portable::publish_portable_v2_oci( + py, + package_path, + registry, + repository, + tag, + limits, + authenticity, + signature, + insecure_http, + credential, + cancellation, + ) + } + + /// Pull and verify a portable-v2 package from an OCI Distribution registry. + #[staticmethod] + #[pyo3(signature = (*, registry, repository, reference, destination, expected_oci_digest=None, limits=None, authenticity=None, insecure_http=false, credential=None, cancellation=None))] + #[allow(clippy::too_many_arguments)] + fn pull_portable_v2_oci( + py: Python<'_>, + registry: &str, + repository: &str, + reference: &str, + destination: &str, + expected_oci_digest: Option, + limits: Option<&Bound<'_, PyDict>>, + authenticity: Option<&Bound<'_, PyDict>>, + insecure_http: bool, + credential: Option, + cancellation: Option<&PyCancellationToken>, + ) -> PyResult> { + portable::pull_portable_v2_oci( + py, + registry, + repository, + reference, + destination, + expected_oci_digest, + limits, + authenticity, + insecure_http, + credential, + cancellation, + ) + } + + /// Begin a durable staged import session. + #[pyo3(signature = (*, operation_uuid, batch_rows=None, max_source_bytes=None, max_files=None, max_rejected_rows=None, io_concurrency=None))] + #[allow(clippy::too_many_arguments)] + fn begin_import_session( + slf: &Bound<'_, Self>, + py: Python<'_>, + operation_uuid: &str, + batch_rows: Option, + max_source_bytes: Option, + max_files: Option, + max_rejected_rows: Option, + io_concurrency: Option, + ) -> PyResult> { + import_session::begin_import_session( + slf, + py, + operation_uuid, + batch_rows, + max_source_bytes, + max_files, + max_rejected_rows, + io_concurrency, + ) + } + + /// Resume one durable, non-terminal import session. + #[pyo3(signature = (session_uuid,))] + fn resume_import_session( + slf: &Bound<'_, Self>, + py: Python<'_>, + session_uuid: &str, + ) -> PyResult> { + import_session::resume_import_session(slf, py, session_uuid) + } + + /// Abort and remove non-terminal sessions older than `max_age_secs`. + #[pyo3(signature = (*, max_age_secs))] + fn cleanup_stale_import_sessions(&self, py: Python<'_>, max_age_secs: u64) -> PyResult { + import_session::cleanup_stale_import_sessions(self, py, max_age_secs) + } + + /// Stream a query into an atomic Parquet file with explicit limits. + #[pyo3(signature = (query, path, *, params=None, max_row_group_rows=65536, max_batch_rows=65536, cancellation=None))] + #[allow(clippy::too_many_arguments)] + fn execute_to_parquet_stream( + &self, + py: Python<'_>, + query: &str, + path: &str, + params: Option<&Bound<'_, PyDict>>, + max_row_group_rows: usize, + max_batch_rows: usize, + cancellation: Option<&PyCancellationToken>, + ) -> PyResult> { + portable::execute_to_parquet_stream( + self, + py, + query, + path, + params, + max_row_group_rows, + max_batch_rows, + cancellation, + ) + } + + /// Stream a query into an atomic Arrow IPC stream file with explicit limits. + #[pyo3(signature = (query, path, *, params=None, max_row_group_rows=65536, max_batch_rows=65536, cancellation=None))] + #[allow(clippy::too_many_arguments)] + fn execute_to_arrow_ipc_stream( + &self, + py: Python<'_>, + query: &str, + path: &str, + params: Option<&Bound<'_, PyDict>>, + max_row_group_rows: usize, + max_batch_rows: usize, + cancellation: Option<&PyCancellationToken>, + ) -> PyResult> { + portable::execute_to_arrow_ipc_stream( + self, + py, + query, + path, + params, + max_row_group_rows, + max_batch_rows, + cancellation, + ) + } + /// Diff two checkpoint/current endpoints through the Rust-owned engine. #[pyo3(signature = (*, from_checkpoint=None, to_checkpoint=None, scope="summary", detail="summary", limit=100, after=None, cancellation=None))] #[allow(clippy::too_many_arguments)] @@ -6180,6 +6431,7 @@ fn _graphforge_rs(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(_cli_execute, m)?)?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/crates/graphforge-bindings-py/src/portable.rs b/crates/graphforge-bindings-py/src/portable.rs new file mode 100644 index 000000000..48bf43fa2 --- /dev/null +++ b/crates/graphforge-bindings-py/src/portable.rs @@ -0,0 +1,692 @@ +//! Thin Python bindings for portable-v2 preview/export/verify/import/OCI (#744). + +use std::path::PathBuf; + +use graphforge_api::{ + GfError, PortableSelection, PortableV2Error, PortableV2ErrorCode, PortableV2ExportRequest, + PortableV2GraphSelector, PortableV2ImportRequest, PortableV2Limits, PortableV2Mode, + PortableV2OciAuthenticityPolicy, PortableV2OciPublishFacadeRequest, + PortableV2OciPullFacadeRequest, PortableV2OciSignatureMaterial, PortableV2Output, + PortableV2ParticipantId, PortableV2PropertyProjection, PortableV2SelectionPreviewRequest, + PortableV2SelectionProfile, PortableV2SelectionRequest, PortableV2SubsetClosure, + PortableV2SubsetPlan, PortableV2SubsetPreviewRequest, PortableV2SubsetRequest, + PortableVerifyRequest, ResultSinkOptions, ResultSinkReceipt, +}; +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyList}; + +use crate::{ + GraphForge, PyCancellationToken, canonical_operation_id, json_value_to_python, + params_from_dict, to_pyerr, +}; + +/// Map a sanitized portable-v2 failure without credentials, headers, or host paths. +pub(crate) fn to_portable_pyerr(py: Python<'_>, error: &PortableV2Error) -> PyErr { + let message = error.to_string(); + let err = PyErr::new::(message); + let value = err.value(py); + let _ = value.setattr("code", portable_error_code(error.code)); + if let Some(entry) = &error.entry { + let _ = value.setattr("entry", entry.as_str()); + } + err +} + +fn portable_error_code(code: PortableV2ErrorCode) -> &'static str { + match code { + PortableV2ErrorCode::Cancelled => "Cancelled", + PortableV2ErrorCode::LimitExceeded => "LimitExceeded", + PortableV2ErrorCode::Io => "Io", + PortableV2ErrorCode::InvalidStructure => "InvalidStructure", + PortableV2ErrorCode::InvalidPath => "InvalidPath", + PortableV2ErrorCode::DuplicateEntry => "DuplicateEntry", + PortableV2ErrorCode::UnsupportedFuture => "UnsupportedFuture", + PortableV2ErrorCode::Incompatible => "Incompatible", + PortableV2ErrorCode::DigestMismatch => "DigestMismatch", + PortableV2ErrorCode::ConcurrentMutation => "ConcurrentMutation", + } +} + +fn selection_from_checkpoint(checkpoint: Option) -> PortableSelection { + match checkpoint { + Some(name) => PortableSelection::Checkpoint(name), + None => PortableSelection::Current, + } +} + +fn parse_limits(py: Python<'_>, value: Option<&Bound<'_, PyDict>>) -> PyResult { + let mut limits = PortableV2Limits::default(); + let Some(dict) = value else { + return Ok(limits); + }; + if let Some(item) = dict.get_item("max_components")? { + limits.max_components = item.extract()?; + } + if let Some(item) = dict.get_item("max_entries")? { + limits.max_entries = item.extract()?; + } + if let Some(item) = dict.get_item("max_entry_bytes")? { + limits.max_entry_bytes = item.extract()?; + } + if let Some(item) = dict.get_item("max_total_bytes")? { + limits.max_total_bytes = item.extract()?; + } + if let Some(item) = dict.get_item("max_manifest_bytes")? { + limits.max_manifest_bytes = item.extract()?; + } + if let Some(item) = dict.get_item("max_tag_manifest_bytes")? { + limits.max_tag_manifest_bytes = item.extract()?; + } + if let Some(item) = dict.get_item("max_path_bytes")? { + limits.max_path_bytes = item.extract()?; + } + if let Some(item) = dict.get_item("copy_buffer_bytes")? { + limits.copy_buffer_bytes = item.extract()?; + } + let _ = py; + Ok(limits) +} + +fn parse_profile( + py: Python<'_>, + profile: &str, + identities: Option<&Bound<'_, PyList>>, +) -> PyResult { + match profile { + "complete" => Ok(PortableV2SelectionProfile::Complete), + "ontology_only" => Ok(PortableV2SelectionProfile::OntologyOnly), + "data_components" => Ok(PortableV2SelectionProfile::DataComponents), + "artifacts" => Ok(PortableV2SelectionProfile::Artifacts), + "settings" => Ok(PortableV2SelectionProfile::Settings), + "custom" => { + let identities = identities.ok_or_else(|| { + to_pyerr( + py, + &GfError::Validation("custom profile requires identities".into()), + ) + })?; + let mut parsed = Vec::with_capacity(identities.len()); + for item in identities { + let dict = item.cast::().map_err(|_| { + to_pyerr( + py, + &GfError::Validation("identity entries must be dictionaries".into()), + ) + })?; + let capability_id = dict + .get_item("capability_id")? + .ok_or_else(|| { + to_pyerr( + py, + &GfError::Validation("identity requires capability_id".into()), + ) + })? + .extract::()?; + let record_family_id = dict + .get_item("record_family_id")? + .ok_or_else(|| { + to_pyerr( + py, + &GfError::Validation("identity requires record_family_id".into()), + ) + })? + .extract::()?; + parsed.push(PortableV2ParticipantId { + capability_id, + record_family_id, + }); + } + Ok(PortableV2SelectionProfile::Custom(parsed)) + } + _ => Err(to_pyerr( + py, + &GfError::Validation( + "profile must be complete, ontology_only, data_components, artifacts, settings, or custom" + .into(), + ), + )), + } +} + +fn parse_subset( + py: Python<'_>, + value: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let Some(dict) = value else { + return Ok(None); + }; + let selector_value = dict + .get_item("selector")? + .ok_or_else(|| to_pyerr(py, &GfError::Validation("subset requires selector".into())))?; + let selector = selector_value.cast::().map_err(|_| { + to_pyerr( + py, + &GfError::Validation("selector must be a dictionary".into()), + ) + })?; + let node_uuids = selector + .get_item("node_uuids")? + .map(|item| item.extract::>()) + .transpose()? + .unwrap_or_default(); + let edge_uuids = selector + .get_item("edge_uuids")? + .map(|item| item.extract::>()) + .transpose()? + .unwrap_or_default(); + let closure = match dict + .get_item("closure")? + .map(|item| item.extract::()) + .transpose()? + .as_deref() + .unwrap_or("induced_edges") + { + "induced_edges" => PortableV2SubsetClosure::InducedEdges, + "referential" => PortableV2SubsetClosure::Referential, + _ => { + return Err(to_pyerr( + py, + &GfError::Validation("closure must be induced_edges or referential".into()), + )); + } + }; + let exclude = dict + .get_item("projection")? + .and_then(|item| { + item.cast::().ok().and_then(|projection| { + projection + .get_item("exclude") + .ok() + .flatten() + .and_then(|exclude| exclude.extract::>().ok()) + }) + }) + .unwrap_or_default(); + Ok(Some(PortableV2SubsetRequest { + selector: PortableV2GraphSelector { + node_uuids, + edge_uuids, + }, + closure, + projection: PortableV2PropertyProjection { exclude }, + })) +} + +fn parse_output(py: Python<'_>, representation: &str) -> PyResult { + match representation { + "expanded" => Ok(PortableV2Output::Expanded), + "bundle" => Ok(PortableV2Output::Bundle), + _ => Err(to_pyerr( + py, + &GfError::Validation("representation must be expanded or bundle".into()), + )), + } +} + +fn parse_mode(py: Python<'_>, mode: &str) -> PyResult { + match mode { + "structure_only" => Ok(PortableV2Mode::StructureOnly), + "full" => Ok(PortableV2Mode::Full), + _ => Err(to_pyerr( + py, + &GfError::Validation("mode must be structure_only or full".into()), + )), + } +} + +fn selection_plan_json(plan: &graphforge_api::PortableV2SelectionPlan) -> serde_json::Value { + serde_json::json!({ + "source_generation_uuid": plan.source_generation_uuid, + "source_manifest_sha256": plan.source_manifest_sha256, + "package_class": plan.package_class, + "included": plan.included, + "excluded": plan.excluded, + "redactions": plan.redactions, + "required_capabilities": plan.required_capabilities, + "estimated_payload_bytes": plan.estimated_payload_bytes, + "selection_fingerprint": plan.selection_fingerprint, + }) +} + +fn subset_plan_json(plan: &PortableV2SubsetPlan) -> serde_json::Value { + serde_json::json!({ + "selection": selection_plan_json(&plan.selection), + "graph_subset": plan.graph_subset, + "selected_node_count": plan.selected_node_count, + "selected_edge_count": plan.selected_edge_count, + "endpoint_node_count": plan.endpoint_node_count, + "result_fingerprint": plan.result_fingerprint, + "subset_fingerprint": plan.subset_fingerprint, + }) +} + +fn export_result_json(result: &graphforge_api::PortableV2ExportFacadeResult) -> serde_json::Value { + serde_json::json!({ + "contract": result.contract, + "source": result.source, + "checkpoint": result.checkpoint, + "generation_uuid": result.generation_uuid.to_string(), + "package_digest": result.package_digest, + "transport_digest": result.transport_digest, + "entry_count": result.entry_count, + "payload_bytes": result.payload_bytes, + "representation": result.representation, + "selection_fingerprint": result.selection_fingerprint, + "output": result.output.display().to_string(), + }) +} + +fn verify_result_json( + py: Python<'_>, + report: &graphforge_api::PortableVerifyResult, +) -> PyResult { + serde_json::to_value(report) + .map_err(|error| to_pyerr(py, &GfError::Execution(error.to_string()))) +} + +fn import_result_json(result: &graphforge_api::PortableV2ImportResult) -> serde_json::Value { + serde_json::json!({ + "package_digest": result.package_digest, + "transport_digest": result.transport_digest, + "generation_uuid": result.generation_uuid.to_string(), + "idempotent_replay": result.idempotent_replay, + }) +} + +fn oci_reference_json( + py: Python<'_>, + reference: &graphforge_api::PortableV2OciReference, +) -> PyResult { + serde_json::to_value(reference) + .map_err(|error| to_pyerr(py, &GfError::Execution(error.to_string()))) +} + +fn oci_pull_json( + py: Python<'_>, + receipt: &graphforge_api::PortableV2OciPullReceipt, +) -> PyResult { + let reference = oci_reference_json(py, &receipt.reference)?; + let report = verify_result_json(py, &receipt.report)?; + Ok(serde_json::json!({ + "reference": reference, + "destination": receipt.destination.display().to_string(), + "report": report, + "signature_state": receipt.signature_state, + })) +} + +fn sink_receipt_dict(py: Python<'_>, receipt: &ResultSinkReceipt) -> PyResult> { + let out = PyDict::new(py); + out.set_item("destination", receipt.destination.display().to_string())?; + out.set_item( + "format", + match receipt.format { + graphforge_api::ResultSinkFormat::Parquet => "parquet", + graphforge_api::ResultSinkFormat::ArrowIpc => "arrow_ipc", + }, + )?; + let progress = PyDict::new(py); + progress.set_item("phase", receipt.progress.phase)?; + progress.set_item("rows", receipt.progress.rows)?; + progress.set_item("batches", receipt.progress.batches)?; + progress.set_item("bytes", receipt.progress.bytes)?; + progress.set_item( + "elapsed_ms", + u64::try_from(receipt.progress.elapsed.as_millis()).unwrap_or(u64::MAX), + )?; + progress.set_item("complete", receipt.progress.complete)?; + out.set_item("progress", progress)?; + Ok(out.into_any().unbind()) +} + +fn authenticity_policy( + py: Python<'_>, + value: Option<&Bound<'_, PyDict>>, +) -> PyResult { + let Some(dict) = value else { + return Ok(PortableV2OciAuthenticityPolicy::default()); + }; + let require_named_signer = dict + .get_item("require_named_signer")? + .map(|item| item.extract::()) + .transpose()?; + let verification_key = dict + .get_item("verification_key")? + .map(|item| item.extract::>()) + .transpose()?; + let _ = py; + Ok(PortableV2OciAuthenticityPolicy { + require_named_signer, + verification_key, + }) +} + +fn signature_material( + py: Python<'_>, + value: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let Some(dict) = value else { + return Ok(None); + }; + let signer = dict + .get_item("signer")? + .ok_or_else(|| to_pyerr(py, &GfError::Validation("signature requires signer".into())))? + .extract::()?; + let key_id = dict + .get_item("key_id")? + .ok_or_else(|| to_pyerr(py, &GfError::Validation("signature requires key_id".into())))? + .extract::()?; + let secret = dict + .get_item("secret")? + .ok_or_else(|| to_pyerr(py, &GfError::Validation("signature requires secret".into())))? + .extract::>()?; + Ok(Some(PortableV2OciSignatureMaterial { + signer, + key_id, + secret, + })) +} + +/// Preview one content-free portable-v2 component selection. +#[allow(clippy::too_many_arguments)] +pub(crate) fn preview_portable_v2_selection( + forge: &GraphForge, + py: Python<'_>, + checkpoint: Option, + profile: &str, + identities: Option<&Bound<'_, pyo3::types::PyList>>, + strict: bool, + limits: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + forge.ensure_open()?; + let request = PortableV2SelectionPreviewRequest { + selection: selection_from_checkpoint(checkpoint), + request: PortableV2SelectionRequest { + profile: parse_profile(py, profile, identities)?, + strict, + }, + limits: parse_limits(py, limits)?, + }; + let plan = py + .detach(|| forge.inner.preview_portable_v2_selection(&request)) + .map_err(|error| to_portable_pyerr(py, &error))?; + json_value_to_python(py, &selection_plan_json(&plan)) +} + +/// Preview one content-free portable-v2 graph-data subset. +pub(crate) fn preview_portable_v2_graph_subset( + forge: &GraphForge, + py: Python<'_>, + checkpoint: Option, + subset: &Bound<'_, PyDict>, + limits: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + forge.ensure_open()?; + let subset = parse_subset(py, Some(subset))?.ok_or_else(|| { + to_pyerr( + py, + &GfError::Validation("subset request is required".into()), + ) + })?; + let request = PortableV2SubsetPreviewRequest { + selection: selection_from_checkpoint(checkpoint), + request: subset, + limits: parse_limits(py, limits)?, + }; + let plan = py + .detach(|| forge.inner.preview_portable_v2_graph_subset(&request)) + .map_err(|error| to_portable_pyerr(py, &error))?; + json_value_to_python(py, &subset_plan_json(&plan)) +} + +/// Export one pinned generation as an expanded or bundled portable-v2 package. +#[allow(clippy::too_many_arguments)] +pub(crate) fn export_portable_v2( + forge: &GraphForge, + py: Python<'_>, + output_path: &str, + representation: &str, + profile: &str, + identities: Option<&Bound<'_, pyo3::types::PyList>>, + checkpoint: Option, + subset: Option<&Bound<'_, PyDict>>, + limits: Option<&Bound<'_, PyDict>>, + cancellation: Option<&PyCancellationToken>, + progress: Option<&Bound<'_, PyAny>>, +) -> PyResult> { + forge.ensure_open()?; + let request = PortableV2ExportRequest { + selection: selection_from_checkpoint(checkpoint), + output_path: PathBuf::from(output_path), + representation: parse_output(py, representation)?, + profile: parse_profile(py, profile, identities)?, + subset: parse_subset(py, subset)?, + limits: parse_limits(py, limits)?, + }; + let cancelled = cancellation.map(|token| token.inner.flag()); + let progress_cb = match progress { + Some(callback) => { + if !callback.is_callable() { + return Err(to_pyerr( + py, + &GfError::Validation("progress callback must be callable".into()), + )); + } + Some(callback.clone().unbind()) + } + None => None, + }; + let progress_error = std::sync::Mutex::new(None::); + let result = py + .detach(|| { + forge + .inner + .export_portable_v2(&request, cancelled, |event| { + if let Some(callback) = progress_cb.as_ref() { + let callback_result = Python::attach(|py| { + let payload = PyDict::new(py); + payload.set_item("entries_completed", event.entries_completed)?; + payload.set_item("bytes_completed", event.bytes_completed)?; + payload.set_item("entries_total", event.entries_total)?; + payload.set_item("bytes_total", event.bytes_total)?; + callback.bind(py).call1((payload,))?; + Ok::<(), PyErr>(()) + }); + if let Err(error) = callback_result { + let mut slot = progress_error + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if slot.is_none() { + *slot = Some(error); + } + } + } + }) + }) + .map_err(|error| to_portable_pyerr(py, &error))?; + if let Some(error) = progress_error + .into_inner() + .unwrap_or_else(std::sync::PoisonError::into_inner) + { + return Err(error); + } + json_value_to_python(py, &export_result_json(&result)) +} + +/// Verify portable-v2 content without opening a project. +pub(crate) fn verify_portable_v2( + py: Python<'_>, + input: &str, + mode: &str, + limits: Option<&Bound<'_, PyDict>>, + cancellation: Option<&PyCancellationToken>, +) -> PyResult> { + let request = PortableVerifyRequest { + input: PathBuf::from(input), + mode: parse_mode(py, mode)?, + limits: parse_limits(py, limits)?, + }; + let cancelled = cancellation.map(|token| token.inner.flag()); + let report = py + .detach(|| graphforge_api::verify_portable_v2(&request, cancelled)) + .map_err(|error| to_portable_pyerr(py, &error))?; + json_value_to_python(py, &verify_result_json(py, &report)?) +} + +/// Verify and atomically import a complete portable-v2 package. +pub(crate) fn import_portable_v2( + py: Python<'_>, + project_root: &str, + input: &str, + operation_id: &str, + limits: Option<&Bound<'_, PyDict>>, + cancellation: Option<&PyCancellationToken>, +) -> PyResult> { + let request = PortableV2ImportRequest { + input: PathBuf::from(input), + operation_id: canonical_operation_id(operation_id).map_err(|error| to_pyerr(py, &error))?, + limits: parse_limits(py, limits)?, + }; + let cancelled = cancellation.map(|token| token.inner.flag()); + let root = PathBuf::from(project_root); + let result = py + .detach(|| graphforge_api::GraphForge::import_portable_v2(&root, &request, cancelled)) + .map_err(|error| to_portable_pyerr(py, &error))?; + json_value_to_python(py, &import_result_json(&result)) +} + +/// Publish a verified portable-v2 package to an OCI registry. +#[allow(clippy::too_many_arguments)] +pub(crate) fn publish_portable_v2_oci( + py: Python<'_>, + package_path: &str, + registry: &str, + repository: &str, + tag: Option, + limits: Option<&Bound<'_, PyDict>>, + authenticity: Option<&Bound<'_, PyDict>>, + signature: Option<&Bound<'_, PyDict>>, + insecure_http: bool, + credential: Option, + cancellation: Option<&PyCancellationToken>, +) -> PyResult> { + let request = PortableV2OciPublishFacadeRequest { + package_path: PathBuf::from(package_path), + registry: registry.to_owned(), + repository: repository.to_owned(), + tag, + limits: parse_limits(py, limits)?, + authenticity: authenticity_policy(py, authenticity)?, + signature: signature_material(py, signature)?, + insecure_http, + credential, + }; + let cancelled = cancellation.map(|token| token.inner.flag()); + let reference = py + .detach(|| graphforge_api::publish_portable_v2_oci(&request, cancelled)) + .map_err(|error| to_portable_pyerr(py, &error))?; + json_value_to_python(py, &oci_reference_json(py, &reference)?) +} + +/// Pull and verify a portable-v2 package from an OCI registry. +#[allow(clippy::too_many_arguments)] +pub(crate) fn pull_portable_v2_oci( + py: Python<'_>, + registry: &str, + repository: &str, + reference: &str, + destination: &str, + expected_oci_digest: Option, + limits: Option<&Bound<'_, PyDict>>, + authenticity: Option<&Bound<'_, PyDict>>, + insecure_http: bool, + credential: Option, + cancellation: Option<&PyCancellationToken>, +) -> PyResult> { + let request = PortableV2OciPullFacadeRequest { + registry: registry.to_owned(), + repository: repository.to_owned(), + reference: reference.to_owned(), + expected_oci_digest, + destination: PathBuf::from(destination), + limits: parse_limits(py, limits)?, + authenticity: authenticity_policy(py, authenticity)?, + insecure_http, + credential, + }; + let cancelled = cancellation.map(|token| token.inner.flag()); + let receipt = py + .detach(|| graphforge_api::pull_portable_v2_oci(&request, cancelled)) + .map_err(|error| to_portable_pyerr(py, &error))?; + json_value_to_python(py, &oci_pull_json(py, &receipt)?) +} + +/// Stream a query into an atomic Parquet result with explicit limits. +#[allow(clippy::too_many_arguments)] +pub(crate) fn execute_to_parquet_stream( + forge: &GraphForge, + py: Python<'_>, + query: &str, + path: &str, + params: Option<&Bound<'_, PyDict>>, + max_row_group_rows: usize, + max_batch_rows: usize, + cancellation: Option<&PyCancellationToken>, +) -> PyResult> { + forge.ensure_open()?; + let params = params_from_dict(params)?; + let options = ResultSinkOptions { + max_row_group_rows, + max_batch_rows, + }; + let cancellation = cancellation.map(|token| token.inner.clone()); + let query = query.to_owned(); + let path = path.to_owned(); + let receipt = py + .detach(|| { + forge.inner.execute_to_parquet_stream_with_params( + &query, + ¶ms, + &path, + &options, + cancellation.as_ref(), + ) + }) + .map_err(|error| to_pyerr(py, &error))?; + sink_receipt_dict(py, &receipt) +} + +/// Stream a query into an atomic Arrow IPC stream file with explicit limits. +#[allow(clippy::too_many_arguments)] +pub(crate) fn execute_to_arrow_ipc_stream( + forge: &GraphForge, + py: Python<'_>, + query: &str, + path: &str, + params: Option<&Bound<'_, PyDict>>, + max_row_group_rows: usize, + max_batch_rows: usize, + cancellation: Option<&PyCancellationToken>, +) -> PyResult> { + forge.ensure_open()?; + let params = params_from_dict(params)?; + let options = ResultSinkOptions { + max_row_group_rows, + max_batch_rows, + }; + let cancellation = cancellation.map(|token| token.inner.clone()); + let query = query.to_owned(); + let path = path.to_owned(); + let receipt = py + .detach(|| { + forge.inner.execute_to_arrow_ipc_stream_with_params( + &query, + ¶ms, + &path, + &options, + cancellation.as_ref(), + ) + }) + .map_err(|error| to_pyerr(py, &error))?; + sink_receipt_dict(py, &receipt) +} diff --git a/crates/graphforge-bindings-py/tests/import_session_lifecycle.py b/crates/graphforge-bindings-py/tests/import_session_lifecycle.py new file mode 100644 index 000000000..06af91ebb --- /dev/null +++ b/crates/graphforge-bindings-py/tests/import_session_lifecycle.py @@ -0,0 +1,48 @@ +"""Import-session begin/checkpoint/resume/abort lifecycle through Python.""" + +from __future__ import annotations + +from pathlib import Path +import tempfile +import uuid + +import pyarrow as pa + + +def check_import_session_lifecycle() -> None: + import graphforge as gf + + with tempfile.TemporaryDirectory() as tmp: + project = Path(tmp) / "project" + project.mkdir() + forge = gf.GraphForge(str(project)) + operation = str(uuid.uuid4()) + session = forge.begin_import_session(operation_uuid=operation) + status = session.status() + assert status["phase"] == "open" + node_uuid = uuid.uuid4().bytes + table = pa.table( + { + "node_uuid": pa.array([node_uuid], type=pa.binary(16)), + "label": pa.array(["Person"]), + } + ) + session.append_arrow("node", table) + progress = session.checkpoint() + assert progress["files_pending"] >= 1 + session_uuid = session.session_uuid + del session + resumed = forge.resume_import_session(session_uuid) + assert resumed.session_uuid == session_uuid + aborted = resumed.abort() + assert aborted["files_accepted"] >= 1 + cleaned = forge.cleanup_stale_import_sessions(max_age_secs=0) + assert cleaned == 0 + + +def main() -> None: + check_import_session_lifecycle() + + +if __name__ == "__main__": + main() diff --git a/crates/graphforge-bindings-py/tests/non_cypher_release.py b/crates/graphforge-bindings-py/tests/non_cypher_release.py index 04334a3d6..9e0604481 100644 --- a/crates/graphforge-bindings-py/tests/non_cypher_release.py +++ b/crates/graphforge-bindings-py/tests/non_cypher_release.py @@ -23,8 +23,8 @@ RUST_MANIFEST = ROOT / "tests/contracts/non-cypher-rust-surface.json" RUST_GATE = ROOT / "scripts/ci/non-cypher-surface-gate.py" PYO3_SOURCE = ROOT / "crates/graphforge-bindings-py/src/lib.rs" -EXPECTED_RUST_DIGEST = "b63fefd89490c3a262ec6e869af5d97dddf7db51039b53981e68eb3c3aa2d4a7" -EXPECTED_RELEASE_DIGEST = "01f642d893eeb78b9493363d6c4ac5eea6744e0a970b60a9e6a456fe9892a886" +EXPECTED_RUST_DIGEST = "268d0832e1fa2bc823e1aa6a0f7a5129c29f4b0b23886a6fc2927616671a3b73" +EXPECTED_RELEASE_DIGEST = "9ca796af78a3ea51e4c0d404e9f74e3eb1cdd49a0b862dc15838cd9ada037877" PYTHON_ONLY_METHODS = frozenset( { @@ -118,6 +118,7 @@ }, "streaming-errors-maintenance": { "smoke.py": ["check_execute_stream", "check_lifecycle", "check_parse_error_span"], + "result_sink_stream.py": ["check_result_sink_stream"], }, "transaction-maintenance": { "transaction_parity.py": [ @@ -131,11 +132,14 @@ "non_cypher_release.py": ["check_native_artifact_and_no_fallback"], }, "resumable-import": { - "non_cypher_release.py": ["check_lifecycle_checkpoint_errors_and_reopen"], + "import_session_lifecycle.py": ["check_import_session_lifecycle"], }, "semantic-generation-diff": { "generation_diff.py": ["check_generation_diff"], }, + "portable-v2-facade": { + "portable_v2_parity.py": ["check_portable_v2_parity"], + }, } @@ -214,6 +218,7 @@ def _python_methods() -> set[str]: "PyResolvedBeliefProjection": "ResolvedBeliefProjection", "PyNodeHandle": "NodeHandle", "PyEdgeHandle": "EdgeHandle", + "PyGraphImportSession": "GraphImportSession", } found: set[str] = set() for rust_receiver, public_receiver in receiver_names.items(): @@ -243,7 +248,7 @@ def _classification_report() -> dict[str, object]: for group in manifest["method_evidence_groups"].values() for method_id in group["ids"] } - assert len(release_methods) == 207 + assert len(release_methods) == 210 assert _digest(release_methods) == EXPECTED_RELEASE_DIGEST assert set(EVIDENCE) == set(manifest["method_evidence_groups"]) @@ -283,7 +288,19 @@ def _classification_report() -> dict[str, object]: not_invoked = sorted(set(symbols) - {"main"} - invoked) assert not not_invoked, f"{group} evidence is not run by {filename}/main: {not_invoked}" - aliases = {"GraphForge.new": "GraphForge.__init__", "crate.version": "crate.version"} + aliases = { + "GraphForge.new": "GraphForge.__init__", + "crate.version": "crate.version", + "crate.verify_portable_v2": "GraphForge.verify_portable_v2", + "crate.publish_portable_v2_oci": "GraphForge.publish_portable_v2_oci", + "crate.pull_portable_v2_oci": "GraphForge.pull_portable_v2_oci", + "GraphForge.execute_to_parquet_stream_with_params": ( + "GraphForge.execute_to_parquet_stream" + ), + "GraphForge.execute_to_arrow_ipc_stream_with_params": ( + "GraphForge.execute_to_arrow_ipc_stream" + ), + } classifications: dict[str, dict[str, str]] = {} for rust_id in sorted(rust_methods): python_id = aliases.get(rust_id, rust_id) diff --git a/crates/graphforge-bindings-py/tests/portable_v2_parity.py b/crates/graphforge-bindings-py/tests/portable_v2_parity.py new file mode 100644 index 000000000..511aabe9b --- /dev/null +++ b/crates/graphforge-bindings-py/tests/portable_v2_parity.py @@ -0,0 +1,80 @@ +"""Portable-v2 export/verify/import identity parity through the Python binding.""" + +from __future__ import annotations + +from pathlib import Path +import tempfile +import uuid + + +def check_portable_v2_parity() -> None: + import graphforge as gf + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = root / "source" + source.mkdir() + forge = gf.GraphForge(str(source)) + preview = forge.preview_portable_v2_selection(profile="complete") + assert preview["package_class"] == "complete" + expanded = root / "expanded" + bundle = root / "complete.gfpb" + expanded_export = forge.export_portable_v2( + output_path=str(expanded), representation="expanded", profile="complete" + ) + bundle_export = forge.export_portable_v2( + output_path=str(bundle), representation="bundle", profile="complete" + ) + assert expanded_export["package_digest"] == bundle_export["package_digest"] + assert expanded_export["selection_fingerprint"] == preview["selection_fingerprint"] + + events: list[dict[str, object]] = [] + + def progress(event: dict[str, object]) -> None: + events.append(event) + raise RuntimeError("progress callback failed") + + try: + forge.export_portable_v2( + output_path=str(root / "callback-fail.gfpb"), + representation="bundle", + profile="complete", + progress=progress, + ) + raise AssertionError("expected progress callback failure to propagate") + except RuntimeError as error: + assert "progress callback failed" in str(error) + assert events, "progress callback must run at least once" + + try: + forge.export_portable_v2( + output_path=str(root / "not-callable.gfpb"), + representation="bundle", + profile="complete", + progress=object(), + ) + except gf.ValidationError as error: + assert "callable" in str(error).lower() + else: + raise AssertionError("expected non-callable progress to fail closed") + + verified = gf.GraphForge.verify_portable_v2(str(bundle), mode="full") + assert verified["package_digest"] == bundle_export["package_digest"] + target = root / "target" + imported = gf.GraphForge.import_portable_v2( + str(target), + input=str(bundle), + operation_id=str(uuid.uuid4()), + ) + assert imported["package_digest"] == bundle_export["package_digest"] + assert not imported["idempotent_replay"] + reopened = gf.GraphForge(str(target)) + assert reopened.path is not None + + +def main() -> None: + check_portable_v2_parity() + + +if __name__ == "__main__": + main() diff --git a/crates/graphforge-bindings-py/tests/result_sink_stream.py b/crates/graphforge-bindings-py/tests/result_sink_stream.py new file mode 100644 index 000000000..57cd21b14 --- /dev/null +++ b/crates/graphforge-bindings-py/tests/result_sink_stream.py @@ -0,0 +1,38 @@ +"""Streaming Parquet/Arrow IPC result sinks through the Python binding.""" + +from __future__ import annotations + +from pathlib import Path +import tempfile + + +def check_result_sink_stream() -> None: + import graphforge as gf + + forge = gf.GraphForge() + for name in ("a", "b", "c"): + forge.execute(f"CREATE (:Person {{name: '{name}'}})") + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + parquet = root / "stream.parquet" + ipc = root / "stream.arrow" + query = "MATCH (p:Person) RETURN p.name AS name ORDER BY name" + parquet_receipt = forge.execute_to_parquet_stream( + query, str(parquet), max_batch_rows=64, max_row_group_rows=2 + ) + ipc_receipt = forge.execute_to_arrow_ipc_stream( + query, str(ipc), max_batch_rows=64, max_row_group_rows=2 + ) + assert parquet_receipt["progress"]["rows"] == 3 + assert ipc_receipt["progress"]["rows"] == 3 + assert parquet_receipt["progress"]["complete"] is True + assert ipc_receipt["progress"]["complete"] is True + assert parquet.exists() and ipc.exists() + + +def main() -> None: + check_result_sink_stream() + + +if __name__ == "__main__": + main() diff --git a/crates/graphforge-cli/src/lib.rs b/crates/graphforge-cli/src/lib.rs index 6ca981987..35052e46e 100644 --- a/crates/graphforge-cli/src/lib.rs +++ b/crates/graphforge-cli/src/lib.rs @@ -20,6 +20,7 @@ use uuid::Uuid; include!(concat!(env!("OUT_DIR"), "/project_skills.rs")); mod maintenance_cli; +mod portable_cli; const MAX_SKILL_MANIFEST_BYTES: u64 = 256 * 1024; const MAX_SKILL_FILE_BYTES: u64 = 4 * 1024 * 1024; @@ -266,6 +267,18 @@ enum Command { Export(ExportArgs), /// Import a portable envelope into a new or empty project. Import(ImportArgs), + /// Portable-v2 preview, export, verify, import, and OCI promotion. + Portable { + #[command(subcommand)] + command: portable_cli::PortableCommand, + }, + /// Stream a Cypher result to Parquet or Arrow IPC without full materialization. + Query(portable_cli::QueryArgs), + /// Staged Arrow/Parquet graph-import sessions. + ImportSession { + #[command(subcommand)] + command: portable_cli::ImportSessionCommand, + }, /// Manage immutable named workspace checkpoints. Checkpoint { #[command(subcommand)] @@ -1054,6 +1067,36 @@ fn run(cli: Cli, output: &mut dyn Write) -> Result output, ); } + // Repository-independent portable commands must not call RepositoryContext::discover. + if let Command::Portable { command } = command { + match command { + portable_cli::PortableCommand::Verify(_) + | portable_cli::PortableCommand::PublishOci(_) + | portable_cli::PortableCommand::PullOci(_) => { + return portable_cli::run_portable_without_graph( + Path::new(""), + command, + cli.json, + output, + ) + .map(|()| 0); + } + portable_cli::PortableCommand::Import(_) => { + let path = resolve_project_path(cli.project, cli.project_dir)?; + return portable_cli::run_portable_without_graph(&path, command, cli.json, output) + .map(|()| 0); + } + command => { + let path = resolve_project_path(cli.project, cli.project_dir)?; + let path_text = path.to_str().ok_or_else(|| { + graphforge_api::GfError::Validation("--project must be valid UTF-8".into()) + })?; + let graph = GraphForge::new(Some(path_text))?; + return portable_cli::run_portable(&graph, &path, command, cli.json, output) + .map(|()| 0); + } + } + } let path = resolve_project_path(cli.project, cli.project_dir)?; let command = match command { Command::Import(args) => return run_import(args, &path, cli.json, output).map(|()| 0), @@ -1068,6 +1111,12 @@ fn run(cli: Cli, output: &mut dyn Write) -> Result let mut graph = GraphForge::new(Some(path_text))?; let command = match command { Command::Export(args) => return run_export(&graph, args, cli.json, output).map(|()| 0), + Command::Query(args) => { + return portable_cli::run_query(&graph, &args, cli.json, output).map(|()| 0); + } + Command::ImportSession { command } => { + return portable_cli::run_import_session(&graph, command, cli.json, output).map(|()| 0); + } Command::Recovery => { return maintenance_cli::run_recovery(&graph, cli.json, output).map(|()| 0); } diff --git a/crates/graphforge-cli/src/portable_cli.rs b/crates/graphforge-cli/src/portable_cli.rs new file mode 100644 index 000000000..a5976fc88 --- /dev/null +++ b/crates/graphforge-cli/src/portable_cli.rs @@ -0,0 +1,711 @@ +//! CLI surfaces for portable-v2, OCI promotion, streaming query sinks, and staged ingest (#744). + +use std::io::Write; +use std::path::PathBuf; +use std::time::Duration; + +use clap::{Args, Subcommand, ValueEnum}; +use graphforge_api::{ + BulkInputKind, GraphForge, ImportSessionLimits, OperationId, PortableSelection, + PortableV2Error, PortableV2ExportRequest, PortableV2ImportRequest, PortableV2Limits, + PortableV2Mode, PortableV2OciAuthenticityPolicy, PortableV2OciPublishFacadeRequest, + PortableV2OciPullFacadeRequest, PortableV2Output, PortableV2SelectionPreviewRequest, + PortableV2SelectionProfile, PortableV2SelectionRequest, PortableVerifyRequest, + ResultSinkOptions, publish_portable_v2_oci, pull_portable_v2_oci, verify_portable_v2, +}; +use uuid::Uuid; + +use crate::canonical_uuid; + +fn map_portable(error: &PortableV2Error) -> graphforge_api::GfError { + // Preserve the typed portable code in the message and map to the matching + // GfError fault domain so CLI exit codes distinguish IO/execution/validation. + let message = format!("{:?}: {error}", error.code); + match error.code { + graphforge_api::PortableV2ErrorCode::Io => graphforge_api::GfError::Storage(message), + graphforge_api::PortableV2ErrorCode::Cancelled + | graphforge_api::PortableV2ErrorCode::LimitExceeded + | graphforge_api::PortableV2ErrorCode::DigestMismatch + | graphforge_api::PortableV2ErrorCode::ConcurrentMutation => { + graphforge_api::GfError::Execution(message) + } + _ => graphforge_api::GfError::Validation(message), + } +} + +fn write_json( + value: &impl serde::Serialize, + output: &mut dyn Write, +) -> Result<(), graphforge_api::GfError> { + serde_json::to_writer(&mut *output, value) + .map_err(|error| graphforge_api::GfError::Execution(error.to_string()))?; + writeln!(output).map_err(|error| graphforge_api::GfError::Execution(error.to_string())) +} + +fn selection_flag( + current: bool, + checkpoint: Option, +) -> Result { + match (current, checkpoint) { + (true, None) => Ok(PortableSelection::Current), + (false, Some(name)) => Ok(PortableSelection::Checkpoint(name)), + _ => Err(graphforge_api::GfError::Validation( + "exactly one of --current or --checkpoint is required".into(), + )), + } +} + +#[derive(Subcommand)] +pub(crate) enum PortableCommand { + /// Preview a content-free portable-v2 component selection. + Preview(PortablePreviewArgs), + /// Export an expanded or bundled portable-v2 package. + Export(PortableV2ExportArgs), + /// Inspect or fully verify a portable-v2 package. + Verify(PortableVerifyArgs), + /// Import a complete portable-v2 package into a new/empty project. + Import(PortableV2ImportArgs), + /// Publish a verified package through an OCI Distribution registry. + PublishOci(PortablePublishOciArgs), + /// Pull and verify a digest-pinned package from an OCI registry. + PullOci(PortablePullOciArgs), +} + +#[derive(Clone, Copy, ValueEnum)] +enum PortableProfile { + Complete, + OntologyOnly, + DataComponents, + Artifacts, + Settings, +} + +impl From for PortableV2SelectionProfile { + fn from(value: PortableProfile) -> Self { + match value { + PortableProfile::Complete => Self::Complete, + PortableProfile::OntologyOnly => Self::OntologyOnly, + PortableProfile::DataComponents => Self::DataComponents, + PortableProfile::Artifacts => Self::Artifacts, + PortableProfile::Settings => Self::Settings, + } + } +} + +#[derive(Clone, Copy, ValueEnum)] +enum PortableFormat { + Expanded, + Bundle, +} + +impl From for PortableV2Output { + fn from(value: PortableFormat) -> Self { + match value { + PortableFormat::Expanded => Self::Expanded, + PortableFormat::Bundle => Self::Bundle, + } + } +} + +#[derive(Clone, Copy, ValueEnum)] +enum VerifyMode { + Inspect, + Full, +} + +impl From for PortableV2Mode { + fn from(value: VerifyMode) -> Self { + match value { + VerifyMode::Inspect => Self::StructureOnly, + VerifyMode::Full => Self::Full, + } + } +} + +#[derive(Args)] +pub(crate) struct PortablePreviewArgs { + #[arg( + long, + required_unless_present = "checkpoint", + conflicts_with = "checkpoint" + )] + current: bool, + #[arg(long, required_unless_present = "current", conflicts_with = "current")] + checkpoint: Option, + #[arg(long, value_enum, default_value_t = PortableProfile::Complete)] + profile: PortableProfile, + #[arg(long)] + strict: bool, +} + +#[derive(Args)] +pub(crate) struct PortableV2ExportArgs { + #[arg( + long, + required_unless_present = "checkpoint", + conflicts_with = "checkpoint" + )] + current: bool, + #[arg(long, required_unless_present = "current", conflicts_with = "current")] + checkpoint: Option, + #[arg(long)] + output: PathBuf, + #[arg(long, value_enum, default_value_t = PortableFormat::Bundle)] + format: PortableFormat, + #[arg(long, value_enum, default_value_t = PortableProfile::Complete)] + profile: PortableProfile, +} + +#[derive(Args)] +pub(crate) struct PortableVerifyArgs { + #[arg(long)] + input: PathBuf, + #[arg(long, value_enum, default_value_t = VerifyMode::Full)] + mode: VerifyMode, +} + +#[derive(Args)] +pub(crate) struct PortableV2ImportArgs { + #[arg(long)] + input: PathBuf, + #[arg(long)] + idempotency_key: String, +} + +#[derive(Args)] +pub(crate) struct PortablePublishOciArgs { + #[arg(long)] + package: PathBuf, + #[arg(long)] + registry: String, + #[arg(long)] + repository: String, + #[arg(long)] + tag: Option, + #[arg(long)] + insecure_http: bool, +} + +#[derive(Args)] +pub(crate) struct PortablePullOciArgs { + #[arg(long)] + registry: String, + #[arg(long)] + repository: String, + #[arg(long)] + reference: String, + #[arg(long)] + expected_digest: Option, + #[arg(long)] + destination: PathBuf, + #[arg(long)] + insecure_http: bool, +} + +fn oci_credential() -> Option { + std::env::var("GRAPHFORGE_OCI_CREDENTIAL").ok() +} + +#[allow( + clippy::too_many_lines, + reason = "CLI dispatch keeps preview/export in one portable command table" +)] +pub(crate) fn run_portable( + graph: &GraphForge, + project_root: &std::path::Path, + command: PortableCommand, + json: bool, + output: &mut dyn Write, +) -> Result<(), graphforge_api::GfError> { + match command { + PortableCommand::Preview(args) => { + let plan = graph + .preview_portable_v2_selection(&PortableV2SelectionPreviewRequest { + selection: selection_flag(args.current, args.checkpoint)?, + request: PortableV2SelectionRequest { + profile: args.profile.into(), + strict: args.strict, + }, + limits: PortableV2Limits::default(), + }) + .map_err(|error| map_portable(&error))?; + if json { + write_json(&plan, output)?; + } else { + writeln!( + output, + "selection class={} fingerprint={} estimated_bytes={}", + plan.package_class, plan.selection_fingerprint, plan.estimated_payload_bytes + ) + .map_err(|error| graphforge_api::GfError::Execution(error.to_string()))?; + } + } + PortableCommand::Export(args) => { + let result = graph + .export_portable_v2( + &PortableV2ExportRequest { + selection: selection_flag(args.current, args.checkpoint)?, + output_path: args.output, + representation: args.format.into(), + profile: args.profile.into(), + subset: None, + limits: PortableV2Limits::default(), + }, + None, + |progress| { + if !json { + let _ = writeln!( + output, + "export progress entries={}/{} bytes={}/{}", + progress.entries_completed, + progress.entries_total, + progress.bytes_completed, + progress.bytes_total + ); + } + }, + ) + .map_err(|error| map_portable(&error))?; + if json { + write_json(&result, output)?; + } else { + writeln!( + output, + "exported {} package_digest={} transport_digest={}", + result.representation, result.package_digest, result.transport_digest + ) + .map_err(|error| graphforge_api::GfError::Execution(error.to_string()))?; + } + } + command => { + return run_portable_without_graph(project_root, command, json, output); + } + } + Ok(()) +} + +/// Project-free portable operations that must not hold a live `GraphForge` lock. +#[allow( + clippy::too_many_lines, + reason = "CLI dispatch keeps verify/import/OCI in one project-free portable table" +)] +pub(crate) fn run_portable_without_graph( + project_root: &std::path::Path, + command: PortableCommand, + json: bool, + output: &mut dyn Write, +) -> Result<(), graphforge_api::GfError> { + match command { + PortableCommand::Preview(_) | PortableCommand::Export(_) => { + Err(graphforge_api::GfError::Validation( + "preview/export require an open project handle".into(), + )) + } + PortableCommand::Verify(args) => { + let report = verify_portable_v2( + &PortableVerifyRequest { + input: args.input, + mode: args.mode.into(), + limits: PortableV2Limits::default(), + }, + None, + ) + .map_err(|error| map_portable(&error))?; + if json { + write_json(&report, output)?; + } else { + writeln!( + output, + "verified package_digest={} integrity={:?} compatibility={:?}", + report.package_digest, report.integrity, report.compatibility + ) + .map_err(|error| graphforge_api::GfError::Execution(error.to_string()))?; + } + Ok(()) + } + PortableCommand::Import(args) => { + let result = GraphForge::import_portable_v2( + project_root, + &PortableV2ImportRequest { + input: args.input, + operation_id: OperationId(canonical_uuid(&args.idempotency_key)?), + limits: PortableV2Limits::default(), + }, + None, + ) + .map_err(|error| map_portable(&error))?; + if json { + write_json( + &serde_json::json!({ + "contract": "graphforge-portable-import/2", + "package_digest": result.package_digest, + "transport_digest": result.transport_digest, + "generation_uuid": result.generation_uuid, + "idempotent_replay": result.idempotent_replay, + }), + output, + )?; + } else { + writeln!( + output, + "imported generation {} package_digest={}", + result.generation_uuid, result.package_digest + ) + .map_err(|error| graphforge_api::GfError::Execution(error.to_string()))?; + } + Ok(()) + } + PortableCommand::PublishOci(args) => { + let reference = publish_portable_v2_oci( + &PortableV2OciPublishFacadeRequest { + package_path: args.package, + registry: args.registry, + repository: args.repository, + tag: args.tag, + limits: PortableV2Limits::default(), + authenticity: PortableV2OciAuthenticityPolicy::default(), + signature: None, + insecure_http: args.insecure_http, + credential: oci_credential(), + }, + None, + ) + .map_err(|error| map_portable(&error))?; + if json { + write_json( + &serde_json::json!({ + "contract": "graphforge-portable-oci-publish/2", + "registry": reference.registry, + "repository": reference.repository, + "oci_manifest_digest": reference.oci_manifest_digest, + "package_digest": reference.package_digest, + "tag": reference.tag, + }), + output, + )?; + } else { + writeln!( + output, + "published oci_manifest_digest={}", + reference.oci_manifest_digest + ) + .map_err(|error| graphforge_api::GfError::Execution(error.to_string()))?; + } + Ok(()) + } + PortableCommand::PullOci(args) => { + let receipt = pull_portable_v2_oci( + &PortableV2OciPullFacadeRequest { + registry: args.registry, + repository: args.repository, + reference: args.reference, + expected_oci_digest: args.expected_digest, + destination: args.destination, + limits: PortableV2Limits::default(), + authenticity: PortableV2OciAuthenticityPolicy::default(), + insecure_http: args.insecure_http, + credential: oci_credential(), + }, + None, + ) + .map_err(|error| map_portable(&error))?; + if json { + write_json( + &serde_json::json!({ + "contract": "graphforge-portable-oci-pull/2", + "oci_manifest_digest": receipt.reference.oci_manifest_digest, + "package_digest": receipt.reference.package_digest, + "destination": receipt.destination, + }), + output, + )?; + } else { + writeln!( + output, + "pulled package_digest={} oci_manifest_digest={}", + receipt.reference.package_digest, receipt.reference.oci_manifest_digest + ) + .map_err(|error| graphforge_api::GfError::Execution(error.to_string()))?; + } + Ok(()) + } + } +} + +#[derive(Clone, Copy, ValueEnum)] +pub(crate) enum QuerySinkFormat { + Parquet, + ArrowIpc, +} + +#[derive(Args)] +pub(crate) struct QueryArgs { + /// Cypher query to execute. + #[arg(long)] + cypher: String, + /// Streaming sink destination. + #[arg(long)] + output: PathBuf, + #[arg(long, value_enum, default_value_t = QuerySinkFormat::Parquet)] + format: QuerySinkFormat, + #[arg(long)] + max_batch_rows: Option, + #[arg(long)] + max_row_group_rows: Option, +} + +pub(crate) fn run_query( + graph: &GraphForge, + args: &QueryArgs, + json: bool, + output: &mut dyn Write, +) -> Result<(), graphforge_api::GfError> { + let options = ResultSinkOptions { + max_batch_rows: args.max_batch_rows.unwrap_or(65_536), + max_row_group_rows: args.max_row_group_rows.unwrap_or(65_536), + }; + let path = args.output.to_str().ok_or_else(|| { + graphforge_api::GfError::Validation("query --output must be valid UTF-8".into()) + })?; + let params = std::collections::HashMap::new(); + let receipt = match args.format { + QuerySinkFormat::Parquet => graph.execute_to_parquet_stream_with_params( + &args.cypher, + ¶ms, + path, + &options, + None, + )?, + QuerySinkFormat::ArrowIpc => graph.execute_to_arrow_ipc_stream_with_params( + &args.cypher, + ¶ms, + path, + &options, + None, + )?, + }; + if json { + write_json( + &serde_json::json!({ + "contract": "graphforge-result-sink/1", + "destination": receipt.destination, + "format": format!("{:?}", receipt.format), + "rows": receipt.progress.rows, + "batches": receipt.progress.batches, + "bytes": receipt.progress.bytes, + "complete": receipt.progress.complete, + }), + output, + )?; + } else { + writeln!( + output, + "wrote {} rows={} bytes={}", + receipt.destination.display(), + receipt.progress.rows, + receipt.progress.bytes + ) + .map_err(|error| graphforge_api::GfError::Execution(error.to_string()))?; + } + Ok(()) +} + +#[derive(Subcommand)] +pub(crate) enum ImportSessionCommand { + /// Open a new staged import session. + Begin(ImportSessionBeginArgs), + /// Resume an existing session by UUID. + Resume(ImportSessionResumeArgs), + /// Register a Parquet source path into the session. + RegisterParquet(ImportSessionRegisterArgs), + /// Checkpoint session progress. + Checkpoint(ImportSessionIdArgs), + /// Validate staged sources. + Validate(ImportSessionIdArgs), + /// Commit the session into the project. + Commit(ImportSessionIdArgs), + /// Abort the session. + Abort(ImportSessionIdArgs), + /// Cleanup stale sessions older than the given age. + Cleanup(ImportSessionCleanupArgs), +} + +#[derive(Args)] +pub(crate) struct ImportSessionBeginArgs { + #[arg(long)] + operation_uuid: String, +} + +#[derive(Args)] +pub(crate) struct ImportSessionResumeArgs { + #[arg(long)] + session_uuid: String, +} + +#[derive(Args)] +pub(crate) struct ImportSessionRegisterArgs { + #[arg(long)] + session_uuid: String, + #[arg(long)] + path: PathBuf, + #[arg(long, value_enum)] + kind: ImportSourceKindArg, +} + +#[derive(Args)] +pub(crate) struct ImportSessionIdArgs { + #[arg(long)] + session_uuid: String, +} + +#[derive(Args)] +pub(crate) struct ImportSessionCleanupArgs { + #[arg(long, default_value_t = 86_400)] + max_age_secs: u64, +} + +#[derive(Clone, Copy, ValueEnum)] +enum ImportSourceKindArg { + Nodes, + Edges, +} + +pub(crate) fn run_import_session( + graph: &GraphForge, + command: ImportSessionCommand, + json: bool, + output: &mut dyn Write, +) -> Result<(), graphforge_api::GfError> { + match command { + ImportSessionCommand::Begin(args) => { + let session = graph.begin_import_session( + OperationId(canonical_uuid(&args.operation_uuid)?), + ImportSessionLimits::default(), + )?; + write_session_receipt(session.session_uuid(), "begun", json, output) + } + ImportSessionCommand::Resume(args) => { + let session = graph.resume_import_session(canonical_uuid(&args.session_uuid)?)?; + write_session_receipt(session.session_uuid(), "resumed", json, output) + } + ImportSessionCommand::RegisterParquet(args) => { + let mut session = graph.resume_import_session(canonical_uuid(&args.session_uuid)?)?; + let kind = match args.kind { + ImportSourceKindArg::Nodes => BulkInputKind::Node, + ImportSourceKindArg::Edges => BulkInputKind::Edge, + }; + session.register_parquet(kind, &args.path)?; + write_session_receipt(session.session_uuid(), "registered", json, output) + } + ImportSessionCommand::Checkpoint(args) => { + let mut session = graph.resume_import_session(canonical_uuid(&args.session_uuid)?)?; + let progress = session.checkpoint()?; + write_progress( + session.session_uuid(), + "checkpointed", + &progress, + json, + output, + ) + } + ImportSessionCommand::Validate(args) => { + let mut session = graph.resume_import_session(canonical_uuid(&args.session_uuid)?)?; + let progress = session.validate(graph)?; + write_progress(session.session_uuid(), "validated", &progress, json, output) + } + ImportSessionCommand::Commit(args) => { + let mut session = graph.resume_import_session(canonical_uuid(&args.session_uuid)?)?; + let generation = session.commit(graph, None)?; + if json { + write_json( + &serde_json::json!({ + "contract": "graphforge-import-session/1", + "outcome": "committed", + "session_uuid": session.session_uuid(), + "generation_uuid": generation, + }), + output, + ) + } else { + writeln!( + output, + "committed session {} generation {generation}", + session.session_uuid() + ) + .map_err(|error| graphforge_api::GfError::Execution(error.to_string())) + } + } + ImportSessionCommand::Abort(args) => { + let session = graph.resume_import_session(canonical_uuid(&args.session_uuid)?)?; + let session_uuid = session.session_uuid(); + let progress = session.abort()?; + write_progress(session_uuid, "aborted", &progress, json, output) + } + ImportSessionCommand::Cleanup(args) => { + let removed = + graph.cleanup_stale_import_sessions(Duration::from_secs(args.max_age_secs))?; + if json { + write_json( + &serde_json::json!({ + "contract": "graphforge-import-session-cleanup/1", + "removed": removed, + }), + output, + ) + } else { + writeln!(output, "removed {removed} stale import sessions") + .map_err(|error| graphforge_api::GfError::Execution(error.to_string())) + } + } + } +} + +fn write_session_receipt( + session_uuid: Uuid, + outcome: &str, + json: bool, + output: &mut dyn Write, +) -> Result<(), graphforge_api::GfError> { + if json { + write_json( + &serde_json::json!({ + "contract": "graphforge-import-session/1", + "outcome": outcome, + "session_uuid": session_uuid, + }), + output, + ) + } else { + writeln!(output, "{outcome} session {session_uuid}") + .map_err(|error| graphforge_api::GfError::Execution(error.to_string())) + } +} + +fn write_progress( + session_uuid: Uuid, + outcome: &str, + progress: &graphforge_api::ImportProgress, + json: bool, + output: &mut dyn Write, +) -> Result<(), graphforge_api::GfError> { + if json { + write_json( + &serde_json::json!({ + "contract": "graphforge-import-session/1", + "outcome": outcome, + "session_uuid": session_uuid, + "rows_accepted": progress.rows_accepted, + "rows_rejected": progress.rows_rejected, + "bytes_accepted": progress.bytes_accepted, + }), + output, + ) + } else { + writeln!( + output, + "{outcome} session {session_uuid} rows_accepted={} bytes_accepted={}", + progress.rows_accepted, progress.bytes_accepted + ) + .map_err(|error| graphforge_api::GfError::Execution(error.to_string())) + } +} diff --git a/crates/graphforge-cli/tests/portable.rs b/crates/graphforge-cli/tests/portable.rs index 55f584f48..a491221ce 100644 --- a/crates/graphforge-cli/tests/portable.rs +++ b/crates/graphforge-cli/tests/portable.rs @@ -1,13 +1,19 @@ //! Same-binary integration coverage for portable project interchange. use std::fs; +use std::path::Path; use std::process::{Command, Output}; use serde_json::Value; use tempfile::TempDir; -fn gf(project: &std::path::Path, args: &[&str]) -> Output { - Command::new(env!("CARGO_BIN_EXE_gf")) +fn gf_bin() -> std::path::PathBuf { + // Bazel/cargo may provide a relative binary path; canonicalize before changing cwd. + fs::canonicalize(env!("CARGO_BIN_EXE_gf")).expect("resolve same-build gf binary") +} + +fn gf(project: &Path, args: &[&str]) -> Output { + Command::new(gf_bin()) .arg("--project") .arg(project) .args(args) @@ -15,8 +21,8 @@ fn gf(project: &std::path::Path, args: &[&str]) -> Output { .expect("run same-build gf binary") } -fn gf_repo(repository: &std::path::Path, args: &[&str]) -> Output { - Command::new(env!("CARGO_BIN_EXE_gf")) +fn gf_repo(repository: &Path, args: &[&str]) -> Output { + Command::new(gf_bin()) .arg("--project-dir") .arg(repository) .args(args) @@ -24,6 +30,17 @@ fn gf_repo(repository: &std::path::Path, args: &[&str]) -> Output { .expect("run same-build gf binary") } +fn gf_cwd(cwd: &Path, args: &[&str]) -> Output { + // Poison GF_REPOSITORY so accidental discovery fails deterministically. + let decoy = cwd.join("no-such-gf-repository"); + Command::new(gf_bin()) + .current_dir(cwd) + .env("GF_REPOSITORY", &decoy) + .args(args) + .output() + .expect("run same-build gf binary") +} + fn json(output: &Output) -> Value { assert!( output.status.success(), @@ -186,3 +203,113 @@ fn initialized_repository_can_import_into_its_pristine_state() { String::from_utf8_lossy(&imported.stderr) ); } + +#[test] +fn portable_verify_skips_repository_discovery() { + let outside = TempDir::new().expect("outside repository"); + let missing = outside.path().join("missing.gfpb"); + let output = gf_cwd( + outside.path(), + &[ + "--json", + "portable", + "verify", + "--mode", + "full", + "--input", + missing.to_str().unwrap(), + ], + ); + assert_ne!(output.status.code(), Some(0)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !stderr.to_ascii_lowercase().contains("repository"), + "verify must not require repository discovery: {stderr}" + ); + assert!( + !stderr.contains("GF_REPOSITORY"), + "verify must not fail as repository lookup: {stderr}" + ); +} + +#[test] +fn portable_v2_export_verify_and_import_round_trip() { + let root = TempDir::new().expect("temp root"); + let source_project = root.path().join("source"); + fs::create_dir(&source_project).expect("source project"); + let bundle = root.path().join("complete.gfpb"); + + let exported = json(&gf( + &source_project, + &[ + "--json", + "portable", + "export", + "--current", + "--format", + "bundle", + "--profile", + "complete", + "--output", + bundle.to_str().unwrap(), + ], + )); + assert_eq!(exported["contract"], "graphforge-portable-export/2"); + assert_eq!(exported["representation"], "bundle"); + let package_digest = exported["package_digest"].as_str().unwrap().to_owned(); + assert!(package_digest.starts_with("sha256:")); + + let verified = json(&gf( + &source_project, + &[ + "--json", + "portable", + "verify", + "--mode", + "full", + "--input", + bundle.to_str().unwrap(), + ], + )); + assert_eq!(verified["package_digest"], package_digest); + + let destination = root.path().join("destination"); + let imported = json(&gf( + &destination, + &[ + "--json", + "portable", + "import", + "--input", + bundle.to_str().unwrap(), + "--idempotency-key", + "00000000-0000-0000-0000-000000000744", + ], + )); + assert_eq!(imported["contract"], "graphforge-portable-import/2"); + assert_eq!(imported["package_digest"], package_digest); + assert_eq!(imported["idempotent_replay"], false); + + let listed = gf(&destination, &["checkpoint", "list"]); + assert!( + listed.status.success(), + "reopen failed: {}", + String::from_utf8_lossy(&listed.stderr) + ); + + // Verify is repository-independent: it must not require --project or a discovered repo. + let outside = TempDir::new().expect("outside repository"); + let verified_outside = json(&gf_cwd( + outside.path(), + &[ + "--json", + "portable", + "verify", + "--mode", + "full", + "--input", + bundle.to_str().unwrap(), + ], + )); + assert_eq!(verified_outside["package_digest"], package_digest); +} diff --git a/scripts/ci/test-non-cypher-surface-gate.py b/scripts/ci/test-non-cypher-surface-gate.py index 6154c570b..fbf05e25b 100644 --- a/scripts/ci/test-non-cypher-surface-gate.py +++ b/scripts/ci/test-non-cypher-surface-gate.py @@ -29,7 +29,7 @@ def validate(self, manifest: dict) -> list[str]: def test_checked_in_inventory_is_complete(self) -> None: self.assertEqual(GATE.validate(), []) - self.assertEqual(len(GATE.public_methods()), 321) + self.assertEqual(len(GATE.public_methods()), 324) self.assertEqual(len(GATE.algorithm_registry()), 94) def test_new_or_removed_public_method_fails_frozen_digest(self) -> None: diff --git a/tests/contracts/non-cypher-rust-surface.json b/tests/contracts/non-cypher-rust-surface.json index 57ab9edb1..6d3cb22dd 100644 --- a/tests/contracts/non-cypher-rust-surface.json +++ b/tests/contracts/non-cypher-rust-surface.json @@ -1,7 +1,7 @@ { "contract_version": 1, "scope": "Rust non-Cypher public release surface", - "public_method_digest": "b63fefd89490c3a262ec6e869af5d97dddf7db51039b53981e68eb3c3aa2d4a7", + "public_method_digest": "268d0832e1fa2bc823e1aa6a0f7a5129c29f4b0b23886a6fc2927616671a3b73", "method_policy": { "receiver_defaults": { "GraphForge": "release-tested", @@ -854,6 +854,19 @@ "symbol": "corrupt_and_incompatible_generations_are_typed_reload_required" } ] + }, + "portable-v2-facade": { + "ids": [ + "GraphForge.export_portable_v2", + "GraphForge.preview_portable_v2_graph_subset", + "GraphForge.preview_portable_v2_selection" + ], + "test_refs": [ + { + "path": "crates/graphforge-api/src/portable.rs", + "symbol": "public_v2_export_preview_and_verify_agree_on_package_digest" + } + ] } }, "algorithm_registry": {