From c8bd23d86a9358013734f0914e116c0c9dcfcbff Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 19 Aug 2026 00:42:55 +0530 Subject: [PATCH 1/6] =?UTF-8?q?Own=20the=20sync=20vocabulary:=20state,=20a?= =?UTF-8?q?udit,=20status=20(#18=20=C2=A7B1a/=C2=A7B2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The files under `core/src/sync/` accounted for a sync run in the engine's vocabulary: `SyncState`/`DailyBudget` were re-exports, the audit log was reachable only through engine wrappers, and the status types were the engine's. This lands the engine-neutral halves; the seven pipeline *calls* (`run_composio_connection` and friends) remain and are the next change's whole subject. What core now owns: - `sync::composio::providers::sync_state` -- `SyncState`, `DailyBudget` and the `SyncStateStore` KV seam, ported from the engine (the types are serde shapes over std/chrono; §B2's ask). The engine keeps its copy for its internal pipelines; both persist under one KV namespace, so two pin tests hold this copy to that contract: the namespace literal, and the full serialised shape. The shim's dead `extract_item_id` is not carried over -- its one apparent consumer uses `engine::backend::diff`'s function of the same name. - `sync::audit` -- `SyncAuditEntry` + append/read over `&Path`. The engine's rebuild pipeline appends to the same file with its own copy, so `audit_line_format_is_pinned` fixes the exact serialised line; drift between the two writers becomes a test failure, not corrupted history. The old engine wrappers swallowed append errors; call sites now warn explicitly instead. The engine-side wrappers are deleted, except best-effort `read_audit_log`, which OpenHuman reaches through the engine shim -- it is now backed by this module rather than the engine. - `sync::sync_status` -- `FreshnessLabel`/`MemorySyncStatus` owned with the engine's exact thresholds and serde shape. No core-side producer exists yet (OpenHuman still calls the engine's compute directly; its own allowlisted debt); these are the vocabulary that producer will fill. - The vault watcher imports `DocumentInput` through `ingest_pipeline` (core's designated ingest funnel) rather than naming the engine path itself. Engine references under `core/src/sync/`: 18 before, 11 after -- the seven pipeline calls, plus four doc comments that accurately describe where the pipelines live today. Rewording those before the orchestrator moves would make them lies. cargo test -p tinymemory-core: 811 passed (was 804), 0 failed cargo clippy -p tinymemory-core --all-targets: clean cargo fmt --all -- --check: clean --- core/src/engine/mod.rs | 11 +- core/src/engine/sync.rs | 69 +--- core/src/ingest_pipeline.rs | 5 + core/src/sources/sync.rs | 20 +- core/src/sync/audit.rs | 212 ++++++++++++ core/src/sync/composio/periodic.rs | 13 +- .../src/sync/composio/providers/sync_state.rs | 319 ++++++++++++++++-- core/src/sync/mod.rs | 1 + core/src/sync/sync_status/mod.rs | 84 ++++- core/src/sync/workspace/periodic.rs | 5 +- core/src/sync/workspace/watcher.rs | 2 +- 11 files changed, 619 insertions(+), 122 deletions(-) create mode 100644 core/src/sync/audit.rs diff --git a/core/src/engine/mod.rs b/core/src/engine/mod.rs index a6613e3..e30605b 100644 --- a/core/src/engine/mod.rs +++ b/core/src/engine/mod.rs @@ -65,10 +65,9 @@ pub use seal::{ }; pub use summariser::HostSummariser; pub use sync::{ - append_audit_entry, estimate_cost_usd, load_composio_sync_state, needs_rebuild, raw_coverage, - read_audit_log, rebuild_tree_from_raw, run_composio_connection, - run_composio_connection_with_budgets, run_github_sync, run_gmail_backfill, - run_slack_search_backfill, run_source_pipeline, sync_context, try_read_audit_log, - HostSyncAdapter, RawCoverage, RawFileRef, RealCostAccumulator, RebuildOutcome, - SourcePipelineFailure, SyncAuditEntry, HOST_SYNC_STATE_NAMESPACE, + estimate_cost_usd, load_composio_sync_state, needs_rebuild, raw_coverage, read_audit_log, + rebuild_tree_from_raw, run_composio_connection, run_composio_connection_with_budgets, + run_github_sync, run_gmail_backfill, run_slack_search_backfill, run_source_pipeline, + sync_context, HostSyncAdapter, RawCoverage, RawFileRef, RealCostAccumulator, RebuildOutcome, + SourcePipelineFailure, HOST_SYNC_STATE_NAMESPACE, }; diff --git a/core/src/engine/sync.rs b/core/src/engine/sync.rs index c72553c..5a82e65 100644 --- a/core/src/engine/sync.rs +++ b/core/src/engine/sync.rs @@ -24,9 +24,7 @@ use crate::Config; /// persisted sync cursor with no error anywhere. A duplicated literal is a /// drift hazard precisely when the thing it names is durable (#18 §B2). pub use tinycortex::memory::sync::state::STATE_NAMESPACE as HOST_SYNC_STATE_NAMESPACE; -pub use tinycortex::memory::sync::{ - RawCoverage, RawFileRef, RealCostAccumulator, RebuildOutcome, SyncAuditEntry, -}; +pub use tinycortex::memory::sync::{RawCoverage, RawFileRef, RealCostAccumulator, RebuildOutcome}; pub struct HostSyncAdapter { memory: MemoryClientRef, @@ -134,45 +132,13 @@ impl HostSyncAdapter { } } -/// Append one host sync audit record, logging failures without exposing source identifiers. -pub fn append_audit_entry(config: &Config, entry: &SyncAuditEntry) { - tracing::debug!( - source_kind = %entry.source_kind, - success = entry.success, - items_fetched = entry.items_fetched, - "[tinycortex:sync] audit append starting" - ); - let memory_config = super::memory_config_from(config, config.workspace_dir().clone()); - match tinycortex::memory::sync::append_audit_entry(&memory_config, entry) { - Ok(()) => tracing::debug!( - source_kind = %entry.source_kind, - success = entry.success, - "[tinycortex:sync] audit append completed" - ), - Err(error) => { - tracing::warn!(%error, source_kind = %entry.source_kind, "[tinycortex:sync] audit append failed"); - } - } -} - -/// Read persisted sync audit records while preserving storage failures for fail-closed callers. -pub fn try_read_audit_log(config: &Config) -> anyhow::Result> { - tracing::debug!("[tinycortex:sync] audit read starting"); - let memory_config = super::memory_config_from(config, config.workspace_dir().clone()); - let entries = tinycortex::memory::sync::read_audit_log(&memory_config).map_err(|error| { - tracing::warn!(%error, "[tinycortex:sync] audit read failed"); - error - })?; - tracing::debug!( - entries = entries.len(), - "[tinycortex:sync] audit read completed" - ); - Ok(entries) -} - /// Read persisted sync audit records for best-effort RPC and reporting surfaces. -pub fn read_audit_log(config: &Config) -> Vec { - try_read_audit_log(config).unwrap_or_default() +/// +/// Backed by `crate::sync::audit` — the host-owned log — not the engine; +/// this stays in the engine module only because OpenHuman reaches it through +/// the engine shim path. +pub fn read_audit_log(config: &Config) -> Vec { + crate::sync::audit::read_audit_log(config.workspace_dir()).unwrap_or_default() } /// Estimate sync inference cost using TinyCortex's canonical pricing model. @@ -798,10 +764,7 @@ fn stage_name(stage: SyncStage) -> &'static str { #[cfg(test)] mod tests { - use super::{ - build_pipeline, is_composio_toolkit_syncable, syncable_composio_toolkits, - try_read_audit_log, - }; + use super::{build_pipeline, is_composio_toolkit_syncable, syncable_composio_toolkits}; use crate::sources::MemorySourceEntry; use crate::sync::composio::{get_composio_sync_provider, init_default_composio_sync_providers}; @@ -917,22 +880,6 @@ mod tests { assert!(is_composio_toolkit_syncable(" slack ")); } - #[test] - fn fallible_audit_read_distinguishes_io_failure_from_empty_log() { - let workspace = tempfile::tempdir().expect("workspace"); - let audit_path = workspace.path().join("memory_tree/sync_audit.jsonl"); - std::fs::create_dir_all(&audit_path).expect("create directory at audit file path"); - - let mut config = tinymemory_api::host::test_support::TestHostConfig::default(); - config.workspace_dir = workspace.path().to_path_buf(); - - let error = try_read_audit_log(&config).expect_err("directory read must fail"); - assert!( - error.downcast_ref::().is_some(), - "expected the audit I/O error to remain distinguishable: {error:#}" - ); - } - /// Regression for #5473: a Composio connector sync must feed the memory tree, /// not just the `skill-` document store. The TinyCortex migration /// (#4794) dropped the tree-ingest half, so synced items stopped producing diff --git a/core/src/ingest_pipeline.rs b/core/src/ingest_pipeline.rs index 3d87d5c..c41e618 100644 --- a/core/src/ingest_pipeline.rs +++ b/core/src/ingest_pipeline.rs @@ -11,6 +11,11 @@ use crate::engine::backend::ingest::canonicalize::{ use crate::store::chunks::store::RawRef; use crate::Config; +// The input shapes this funnel accepts, re-exported so callers ingest through +// this module without naming the engine themselves (#18 §B1). The funnel is +// core's designated ingest seam; the engine reference belongs here, once. +pub use crate::engine::backend::ingest::canonicalize::document::DocumentInput as IngestDocumentInput; + pub use crate::engine::backend::ingest::IngestSummary as IngestResult; pub async fn ingest_chat( diff --git a/core/src/sources/sync.rs b/core/src/sources/sync.rs index 4ae194e..5f90167 100644 --- a/core/src/sources/sync.rs +++ b/core/src/sources/sync.rs @@ -142,9 +142,9 @@ pub async fn sync_source(source: MemorySourceEntry, config: Arc) -> Resu Some(&source.id), ); - use crate::engine::{append_audit_entry, SyncAuditEntry}; - append_audit_entry( - &*config, + use crate::sync::audit::{append_audit_entry, SyncAuditEntry}; + if let Err(error) = append_audit_entry( + config.workspace_dir(), &SyncAuditEntry { timestamp: chrono::Utc::now(), source_id: source.id.clone(), @@ -166,7 +166,9 @@ pub async fn sync_source(source: MemorySourceEntry, config: Arc) -> Resu success: true, error: None, }, - ); + ) { + tracing::warn!(%error, "[memory_sync:audit] append failed"); + } // Auto-rebuild: if raw files exist but the tree has // no summaries, build the tree now. @@ -185,9 +187,9 @@ pub async fn sync_source(source: MemorySourceEntry, config: Arc) -> Resu } Err(error) => { // Audit failed syncs too. - use crate::engine::{append_audit_entry, SyncAuditEntry}; - append_audit_entry( - &*config, + use crate::sync::audit::{append_audit_entry, SyncAuditEntry}; + if let Err(error) = append_audit_entry( + config.workspace_dir(), &SyncAuditEntry { timestamp: chrono::Utc::now(), source_id: source.id.clone(), @@ -209,7 +211,9 @@ pub async fn sync_source(source: MemorySourceEntry, config: Arc) -> Resu success: false, error: Some(error.clone()), }, - ); + ) { + tracing::warn!(%error, "[memory_sync:audit] append failed"); + } // Report internal failures to Sentry; known-expected // conditions (auth/network/rate-limit/missing config) are diff --git a/core/src/sync/audit.rs b/core/src/sync/audit.rs new file mode 100644 index 0000000..edf4aed --- /dev/null +++ b/core/src/sync/audit.rs @@ -0,0 +1,212 @@ +//! The sync audit log: one JSON line per sync run (#18 §B1/§B2). +//! +//! Owned here rather than re-exported from the engine so the files under +//! `core/src/sync/` can account for a run without naming an engine. The file +//! itself is shared infrastructure: +//! +//! - **Path**: `/memory_tree/sync_audit.jsonl` — fixed, because two +//! writers append to it. +//! - **The engine writes it too.** Its rebuild pipeline appends entries with +//! its own copy of this type. The `audit_line_format_is_pinned` test below +//! holds this copy to the exact serialised form so the two writers cannot +//! drift apart silently; if that test fails, the fix is a coordinated format +//! change on both sides, never a local edit. + +use std::io::Write; +use std::path::Path; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +const AUDIT_DIR: &str = "memory_tree"; +const AUDIT_FILENAME: &str = "sync_audit.jsonl"; + +/// One sync run, as the audit log records it. +/// +/// Field names are the on-disk format. See the module doc before changing +/// anything here. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SyncAuditEntry { + pub timestamp: DateTime, + pub source_id: String, + pub source_kind: String, + pub scope: String, + pub items_fetched: u32, + pub batches: u32, + pub input_tokens: u64, + pub output_tokens: u64, + pub estimated_cost_usd: f64, + #[serde(default)] + pub composio_actions_called: u32, + #[serde(default)] + pub composio_cost_usd: f64, + #[serde(default)] + pub actual_charged_usd: Option, + pub duration_ms: u64, + pub success: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl SyncAuditEntry { + /// The run's cost as the audit views it: the real charge when the + /// provider reported one, the estimate otherwise, plus Composio's own + /// action cost. + pub fn effective_cost_usd(&self) -> f64 { + self.actual_charged_usd.unwrap_or(self.estimated_cost_usd) + self.composio_cost_usd + } + + /// Alias for [`Self::effective_cost_usd`], kept because the periodic + /// scheduler's budget accounting already speaks this name. + pub fn combined_cost_usd(&self) -> f64 { + self.effective_cost_usd() + } +} + +/// Append one entry to the audit log under `workspace`. +/// +/// # Errors +/// +/// Returns an error when the directory cannot be created or the file cannot +/// be opened or written. +pub fn append_audit_entry(workspace: &Path, entry: &SyncAuditEntry) -> anyhow::Result<()> { + let directory = workspace.join(AUDIT_DIR); + std::fs::create_dir_all(&directory)?; + let mut file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(directory.join(AUDIT_FILENAME))?; + serde_json::to_writer(&mut file, entry)?; + writeln!(file)?; + tracing::debug!(source_id = %entry.source_id, success = entry.success, "[memory_sync:audit] entry appended"); + Ok(()) +} + +/// Read the audit log under `workspace`, newest first. +/// +/// A missing file is an empty log. Malformed lines are skipped with a +/// warning rather than failing the read: the log is append-only across +/// process crashes, so a torn final line must not hide the rest. +/// +/// # Errors +/// +/// Returns an error only when the file exists and cannot be read. +pub fn read_audit_log(workspace: &Path) -> anyhow::Result> { + let path = workspace.join(AUDIT_DIR).join(AUDIT_FILENAME); + let content = match std::fs::read_to_string(path) { + Ok(content) => content, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(error.into()), + }; + let mut entries: Vec<_> = content + .lines() + .filter(|line| !line.trim().is_empty()) + .filter_map(|line| match serde_json::from_str(line) { + Ok(entry) => Some(entry), + Err(error) => { + tracing::warn!(%error, "[memory_sync:audit] malformed audit line skipped"); + None + } + }) + .collect(); + entries.reverse(); + Ok(entries) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + fn entry() -> SyncAuditEntry { + SyncAuditEntry { + timestamp: DateTime::parse_from_rfc3339("2026-01-02T03:04:05Z") + .unwrap() + .with_timezone(&Utc), + source_id: "composio:gmail:conn-1".into(), + source_kind: "composio".into(), + scope: "user".into(), + items_fetched: 7, + batches: 2, + input_tokens: 100, + output_tokens: 40, + estimated_cost_usd: 0.5, + composio_actions_called: 3, + composio_cost_usd: 0.1, + actual_charged_usd: None, + duration_ms: 1234, + success: true, + error: None, + } + } + + /// The engine appends to the same file with its own copy of this type. + /// This pins the exact serialised line so the two writers cannot drift + /// apart silently — a failure here means a coordinated format change, + /// never a local edit. + #[test] + fn audit_line_format_is_pinned() { + let line = serde_json::to_string(&entry()).unwrap(); + assert_eq!( + line, + "{\"timestamp\":\"2026-01-02T03:04:05Z\",\ + \"source_id\":\"composio:gmail:conn-1\",\ + \"source_kind\":\"composio\",\ + \"scope\":\"user\",\ + \"items_fetched\":7,\ + \"batches\":2,\ + \"input_tokens\":100,\ + \"output_tokens\":40,\ + \"estimated_cost_usd\":0.5,\ + \"composio_actions_called\":3,\ + \"composio_cost_usd\":0.1,\ + \"actual_charged_usd\":null,\ + \"duration_ms\":1234,\ + \"success\":true}" + ); + } + + #[test] + fn append_then_read_round_trips_newest_first() { + let tmp = tempfile::tempdir().unwrap(); + let mut first = entry(); + first.source_id = "first".into(); + let mut second = entry(); + second.source_id = "second".into(); + append_audit_entry(tmp.path(), &first).unwrap(); + append_audit_entry(tmp.path(), &second).unwrap(); + + let entries = read_audit_log(tmp.path()).unwrap(); + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].source_id, "second"); + assert_eq!(entries[1].source_id, "first"); + } + + /// An unreadable log must surface as an error, never as an empty log — + /// budget accounting fails closed on it. + #[test] + fn io_failure_is_distinguishable_from_an_empty_log() { + let tmp = tempfile::tempdir().unwrap(); + // A directory where the file should be makes the read fail. + std::fs::create_dir_all(tmp.path().join(AUDIT_DIR).join(AUDIT_FILENAME)).unwrap(); + let error = read_audit_log(tmp.path()).expect_err("directory read must fail"); + assert!( + error.downcast_ref::().is_some(), + "expected the audit I/O error to remain distinguishable: {error:#}" + ); + } + + #[test] + fn missing_file_reads_as_empty_and_torn_lines_are_skipped() { + let tmp = tempfile::tempdir().unwrap(); + assert!(read_audit_log(tmp.path()).unwrap().is_empty()); + + append_audit_entry(tmp.path(), &entry()).unwrap(); + let path = tmp.path().join(AUDIT_DIR).join(AUDIT_FILENAME); + let mut content = std::fs::read_to_string(&path).unwrap(); + content.push_str("{\"torn\":"); + std::fs::write(&path, content).unwrap(); + + assert_eq!(read_audit_log(tmp.path()).unwrap().len(), 1); + } +} diff --git a/core/src/sync/composio/periodic.rs b/core/src/sync/composio/periodic.rs index 400e43b..0cbc9fa 100644 --- a/core/src/sync/composio/periodic.rs +++ b/core/src/sync/composio/periodic.rs @@ -58,7 +58,7 @@ use tinymemory_api::host::DEFAULT_MEMORY_SYNC_INTERVAL_SECS; use super::providers::{get_provider, ComposioUsage}; use crate::composio_host; -use crate::engine::{append_audit_entry, try_read_audit_log, SyncAuditEntry}; +use crate::sync::audit::{append_audit_entry, read_audit_log, SyncAuditEntry}; use chrono::{DateTime, Utc}; /// How often the scheduler wakes up to look for due syncs. Independent @@ -426,7 +426,8 @@ pub(crate) async fn run_one_tick() -> Result<(), String> { // "Sync every 24h" gap across app restarts. We index the persisted sync // audit log (wall-clock timestamps that survive restarts) and use it as the // due-check fallback whenever the in-memory monotonic record is absent. - let (audit_index, audit_available) = composio_audit_state(try_read_audit_log(&*config)); + let (audit_index, audit_available) = + composio_audit_state(read_audit_log(config.workspace_dir())); if !audit_available { tracing::warn!( "[memory_sync:periodic] audit unavailable; sources without in-memory cadence will be skipped" @@ -579,7 +580,9 @@ pub(crate) async fn run_one_tick() -> Result<(), String> { duration_ms, None, ); - append_audit_entry(&*config, &entry); + if let Err(error) = append_audit_entry(config.workspace_dir(), &entry) { + tracing::warn!(%error, "[memory_sync:audit] append failed"); + } record_sync_success(&conn.toolkit, &conn.id); fired += 1; } @@ -604,7 +607,9 @@ pub(crate) async fn run_one_tick() -> Result<(), String> { duration_ms, Some(e.to_string()), ); - append_audit_entry(&*config, &entry); + if let Err(error) = append_audit_entry(config.workspace_dir(), &entry) { + tracing::warn!(%error, "[memory_sync:audit] append failed"); + } // Intentionally do NOT update last_sync_at on failure // so the next tick retries immediately. } diff --git a/core/src/sync/composio/providers/sync_state.rs b/core/src/sync/composio/providers/sync_state.rs index d931f25..45459b2 100644 --- a/core/src/sync/composio/providers/sync_state.rs +++ b/core/src/sync/composio/providers/sync_state.rs @@ -1,41 +1,308 @@ -//! Compatibility exports for sync state now owned by tinycortex. +//! Cursor, dedup and daily-budget state for Composio sync (#18 §B2). +//! +//! Owned here, engine-neutral, persisted through the [`SyncStateStore`] KV +//! seam — any provider whose KV family can get/set a JSON value can carry +//! sync state. This was a re-export of the engine's copy; §B2 asks for the +//! state to be engine-neutral, and the type is nothing but serde shapes over +//! std/chrono, so owning it costs one copy. +//! +//! The engine keeps its own copy for its internal pipelines until §B1's +//! orchestrator move retires them. The two persist under the same KV +//! namespace with the same serde shape; `the_state_namespace_is_pinned` and +//! `state_line_format_is_pinned` below hold this copy to that contract. -pub use crate::engine::backend::sync::state::DEFAULT_DAILY_REQUEST_LIMIT; -pub use crate::engine::backend::sync::{DailyBudget, SyncState}; +use std::collections::{HashMap, HashSet}; -pub const KV_NAMESPACE: &str = crate::engine::HOST_SYNC_STATE_NAMESPACE; +use async_trait::async_trait; +use chrono::Utc; +use serde::{Deserialize, Serialize}; -pub fn extract_item_id(item: &serde_json::Value, paths: &[&str]) -> Option { - paths.iter().find_map(|path| { - let value = path - .split('.') - .try_fold(item, |current, segment| current.get(segment))?; - value - .as_str() - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_owned) - }) +/// The KV namespace every persisted sync cursor lives under. +/// +/// Durable: changing it strands every cursor. See the pin test. +pub const KV_NAMESPACE: &str = STATE_NAMESPACE; + +pub const DEFAULT_DAILY_REQUEST_LIMIT: u32 = 500; +pub const STATE_NAMESPACE: &str = "composio-sync-state"; + +#[async_trait] +pub trait SyncStateStore: Send + Sync { + async fn get(&self, namespace: &str, key: &str) -> anyhow::Result>; + async fn set( + &self, + namespace: &str, + key: &str, + value: &serde_json::Value, + ) -> anyhow::Result<()>; +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DailyBudget { + pub date: String, + pub requests_used: u32, + pub limit: u32, +} + +impl Default for DailyBudget { + fn default() -> Self { + Self { + date: today(), + requests_used: 0, + limit: DEFAULT_DAILY_REQUEST_LIMIT, + } + } +} + +impl DailyBudget { + pub fn remaining(&self) -> u32 { + if self.date != today() { + self.limit + } else { + self.limit.saturating_sub(self.requests_used) + } + } + + pub fn is_exhausted(&self) -> bool { + self.remaining() == 0 + } + + pub fn record_requests(&mut self, count: u32) { + let today = today(); + if self.date != today { + self.date = today; + self.requests_used = 0; + } + self.requests_used = self.requests_used.saturating_add(count); + } + + pub fn record_request(&mut self) { + self.record_requests(1); + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SyncState { + pub toolkit: String, + pub connection_id: String, + #[serde(default)] + pub cursor: Option, + #[serde(default)] + pub synced_ids: HashSet, + #[serde(default)] + pub item_versions: HashMap, + #[serde(default)] + pub daily_budget: DailyBudget, + #[serde(default)] + pub last_seen_id: Option, + #[serde(default)] + pub last_sync_at_ms: Option, + #[serde(skip)] + pub run_requests: u32, + #[serde(skip)] + pub run_provider_cost_usd: f64, +} + +impl SyncState { + pub fn new(toolkit: impl Into, connection_id: impl Into) -> Self { + Self { + toolkit: toolkit.into(), + connection_id: connection_id.into(), + cursor: None, + synced_ids: HashSet::new(), + item_versions: HashMap::new(), + daily_budget: DailyBudget::default(), + last_seen_id: None, + last_sync_at_ms: None, + run_requests: 0, + run_provider_cost_usd: 0.0, + } + } + + pub fn key(toolkit: &str, connection_id: &str) -> String { + format!("{toolkit}:{connection_id}") + } + + pub fn is_synced(&self, id: &str) -> bool { + self.synced_ids.contains(id) + } + + pub fn mark_synced(&mut self, id: impl Into) { + self.synced_ids.insert(id.into()); + } + + pub fn advance_cursor(&mut self, cursor: impl Into) { + self.cursor = Some(cursor.into()); + } + + pub fn set_last_seen_id(&mut self, id: impl Into) { + self.last_seen_id = Some(id.into()); + } + + pub fn set_last_sync_at_ms(&mut self, timestamp_ms: u64) { + self.last_sync_at_ms = Some(timestamp_ms); + } + + pub fn budget_exhausted(&self) -> bool { + self.daily_budget.is_exhausted() + } + + pub fn budget_remaining(&self) -> u32 { + self.daily_budget.remaining() + } + + pub fn record_requests(&mut self, count: u32) { + self.daily_budget.record_requests(count); + self.run_requests = self.run_requests.saturating_add(count); + } + + pub fn record_action(&mut self, attempts: u32, cost_usd: f64) { + self.record_requests(attempts.max(1)); + if cost_usd.is_finite() && cost_usd > 0.0 { + self.run_provider_cost_usd += cost_usd; + } + } + + pub async fn load( + store: &dyn SyncStateStore, + toolkit: &str, + connection_id: &str, + ) -> anyhow::Result { + let key = Self::key(toolkit, connection_id); + match store.get(STATE_NAMESPACE, &key).await? { + Some(value) => { + let mut state: Self = serde_json::from_value(value)?; + if state.daily_budget.date != today() { + state.daily_budget.date = today(); + state.daily_budget.requests_used = 0; + } + Ok(state) + } + None => Ok(Self::new(toolkit, connection_id)), + } + } + + pub async fn save(&self, store: &dyn SyncStateStore) -> anyhow::Result<()> { + let value = serde_json::to_value(self)?; + store + .set( + STATE_NAMESPACE, + &Self::key(&self.toolkit, &self.connection_id), + &value, + ) + .await + } +} + +fn today() -> String { + Utc::now().format("%Y-%m-%d").to_string() } #[cfg(test)] mod tests { - /// The namespace is durable, so changing it is a data migration. - /// - /// `KV_NAMESPACE` now re-exports the engine's constant, which makes host - /// and engine agree by construction — they previously agreed only because - /// two separate `const`s happened to hold the same literal. This pins the - /// *value* as well: every persisted Composio sync cursor lives under this - /// string, so a change upstream silently strands all of them. Failing here - /// turns that into a deliberate decision with a migration attached rather - /// than a quiet loss discovered when a sync re-runs from the beginning. + use std::collections::HashMap; + use std::sync::Mutex; + + use super::*; + + #[derive(Default)] + struct MemoryStateStore(Mutex>); + + #[async_trait] + impl SyncStateStore for MemoryStateStore { + async fn get( + &self, + namespace: &str, + key: &str, + ) -> anyhow::Result> { + Ok(self + .0 + .lock() + .unwrap() + .get(&format!("{namespace}:{key}")) + .cloned()) + } + + async fn set( + &self, + namespace: &str, + key: &str, + value: &serde_json::Value, + ) -> anyhow::Result<()> { + self.0 + .lock() + .unwrap() + .insert(format!("{namespace}:{key}"), value.clone()); + Ok(()) + } + } + + #[tokio::test] + async fn state_round_trips_cursor_dedup_and_budget() { + let store = MemoryStateStore::default(); + let mut state = SyncState::new("gmail", "conn-1"); + state.advance_cursor("cursor-2"); + state.mark_synced("message-1"); + state.record_requests(3); + state.save(&store).await.unwrap(); + + let loaded = SyncState::load(&store, "gmail", "conn-1").await.unwrap(); + assert_eq!(loaded.cursor.as_deref(), Some("cursor-2")); + assert!(loaded.is_synced("message-1")); + assert_eq!(loaded.daily_budget.requests_used, 3); + } + + /// The namespace is durable: every persisted Composio sync cursor lives + /// under this string, so a change strands all of them. The engine's copy + /// must agree; failing here means a coordinated migration, never a local + /// edit. #[test] fn the_state_namespace_is_pinned() { assert_eq!( - super::KV_NAMESPACE, - "composio-sync-state", + KV_NAMESPACE, "composio-sync-state", "the Composio sync-state KV namespace changed; every persisted \ cursor is stored under the old value and needs migrating" ); + assert_eq!(STATE_NAMESPACE, KV_NAMESPACE); + } + + /// The engine persists the same state with its own copy of this type. + /// Pins the serialised shape so the copies cannot drift silently. + #[test] + fn state_line_format_is_pinned() { + let mut state = SyncState::new("gmail", "conn-1"); + state.daily_budget.date = "2026-01-02".into(); + state.daily_budget.requests_used = 3; + state.advance_cursor("c2"); + state.mark_synced("m1"); + state.item_versions.insert("m1".into(), "v1".into()); + state.set_last_seen_id("m1"); + state.set_last_sync_at_ms(1_000); + let value = serde_json::to_value(&state).unwrap(); + assert_eq!( + value, + serde_json::json!({ + "toolkit": "gmail", + "connection_id": "conn-1", + "cursor": "c2", + "synced_ids": ["m1"], + "item_versions": {"m1": "v1"}, + "daily_budget": {"date": "2026-01-02", "requests_used": 3, "limit": 500}, + "last_seen_id": "m1", + "last_sync_at_ms": 1000 + }) + ); + } + + #[test] + fn stale_budget_reports_full_and_resets_on_record() { + let mut budget = DailyBudget { + date: "2000-01-01".into(), + requests_used: 499, + limit: 500, + }; + assert_eq!(budget.remaining(), 500); + budget.record_requests(1); + assert_eq!(budget.requests_used, 1); + assert_eq!(budget.remaining(), 499); } } diff --git a/core/src/sync/mod.rs b/core/src/sync/mod.rs index 33eb319..9d53ca0 100644 --- a/core/src/sync/mod.rs +++ b/core/src/sync/mod.rs @@ -26,6 +26,7 @@ //! own retry/backoff policy. The trait gives the orchestrator a //! single shape to call; everything else stays local. +pub mod audit; pub mod composio; pub mod mcp; pub mod sync_status; diff --git a/core/src/sync/sync_status/mod.rs b/core/src/sync/sync_status/mod.rs index 11c5b83..472d80f 100644 --- a/core/src/sync/sync_status/mod.rs +++ b/core/src/sync/sync_status/mod.rs @@ -1,16 +1,72 @@ -//! Memory sync status surface (#1136 — simplified rewrite). +//! Sync-status vocabulary (#18 §B1). //! -//! The earlier push-based design (phase events from each provider's -//! sync loop, persisted KV store, subscriber that mirrored events -//! into storage) was replaced because it drifted from reality — -//! "downloading 0/0" was a common lie while the chunks table told -//! the truth. The pull-based replacement is one SQL query against -//! `mem_tree_chunks` GROUPED BY `source_kind` on each RPC call. -//! -//! Public surface: -//! -//! * [`MemorySyncStatus`] / [`FreshnessLabel`] — what the RPC returns -//! * `openhuman.memory_sync_status_list` — handler in `rpc` -//! * Controller registration via `schemas::all_registered_controllers` +//! Owned here rather than re-exported from the engine. These are the shapes +//! the host's status RPC speaks; today the only *producer* is the engine's +//! SQLite-backed `list_sync_statuses`, which OpenHuman still calls directly +//! (its own containment debt, tracked in its `direct_engine_refs` allowlist). +//! When §B1's orchestrator move gives core a producer, it fills these types; +//! the serde shape matches the engine's copy field for field. + +use serde::{Deserialize, Serialize}; + +/// How fresh a provider's sync is, judged from its newest chunk. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FreshnessLabel { + /// Newest chunk is under 30 seconds old. + Active, + /// Newest chunk is under 5 minutes old. + Recent, + /// Anything older, or nothing synced yet. + Idle, +} + +impl FreshnessLabel { + /// Label for a provider whose newest chunk landed at + /// `last_chunk_at_ms`, judged at `now_ms`. + pub fn from_age_ms(last_chunk_at_ms: Option, now_ms: i64) -> Self { + match last_chunk_at_ms { + None => Self::Idle, + Some(timestamp) => match now_ms.saturating_sub(timestamp) { + age if age <= 30_000 => Self::Active, + age if age <= 5 * 60_000 => Self::Recent, + _ => Self::Idle, + }, + } + } +} + +/// One provider's sync progress, as the status RPC reports it. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct MemorySyncStatus { + pub provider: String, + pub chunks_synced: u64, + pub chunks_pending: u64, + pub batch_total: u64, + pub batch_processed: u64, + pub last_chunk_at_ms: Option, + pub freshness: FreshnessLabel, +} + +#[cfg(test)] +mod tests { + use super::*; -pub use crate::engine::backend::sync::{FreshnessLabel, MemorySyncStatus}; + #[test] + fn freshness_thresholds_match_the_engine() { + let now = 10_000_000; + assert_eq!(FreshnessLabel::from_age_ms(None, now), FreshnessLabel::Idle); + assert_eq!( + FreshnessLabel::from_age_ms(Some(now - 30_000), now), + FreshnessLabel::Active + ); + assert_eq!( + FreshnessLabel::from_age_ms(Some(now - 30_001), now), + FreshnessLabel::Recent + ); + assert_eq!( + FreshnessLabel::from_age_ms(Some(now - 300_001), now), + FreshnessLabel::Idle + ); + } +} diff --git a/core/src/sync/workspace/periodic.rs b/core/src/sync/workspace/periodic.rs index dbc3a0b..a02470a 100644 --- a/core/src/sync/workspace/periodic.rs +++ b/core/src/sync/workspace/periodic.rs @@ -33,10 +33,10 @@ use chrono::{DateTime, Utc}; use tokio::time::interval; use crate::config_loader as config_rpc; -use crate::engine::{try_read_audit_log, SyncAuditEntry}; use crate::scheduler_gate::resume_notify; use crate::sources::sync::sync_source; use crate::sources::types::{MemorySourceEntry, SourceKind}; +use crate::sync::audit::{read_audit_log, SyncAuditEntry}; use crate::sync::composio::periodic::{ connection_is_due, effective_interval_secs, periodic_pause_reason, }; @@ -181,7 +181,8 @@ pub(crate) async fn run_one_tick() -> Result<(), String> { return Ok(()); }; - let (audit_index, audit_available) = workspace_audit_state(try_read_audit_log(&*config)); + let (audit_index, audit_available) = + workspace_audit_state(read_audit_log(config.workspace_dir())); if !audit_available { tracing::warn!( "[memory_sync:workspace:periodic] audit unavailable; sources without in-memory cadence will be skipped" diff --git a/core/src/sync/workspace/watcher.rs b/core/src/sync/workspace/watcher.rs index c43e529..62b7f9c 100644 --- a/core/src/sync/workspace/watcher.rs +++ b/core/src/sync/workspace/watcher.rs @@ -56,7 +56,7 @@ use tokio::sync::mpsc; use crate::Config; use crate::config_loader as config_rpc; use crate::ingest_pipeline::ingest_document_with_scope; -use crate::engine::backend::ingest::canonicalize::document::DocumentInput; +use crate::ingest_pipeline::IngestDocumentInput as DocumentInput; use crate::sync::workspace::watcher::state::WatcherStateStore; use crate::scheduler_gate::current_policy; use crate::scheduler_gate::PauseReason; From 29f7c30c4c9063e01cc357c077b66788941465da Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 19 Aug 2026 02:11:34 +0530 Subject: [PATCH 2/6] =?UTF-8?q?Move=20the=20Composio=20sync=20pipelines=20?= =?UTF-8?q?off=20the=20engine=20(#18=20=C2=A7B1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seven remaining engine calls under `core/src/sync/` were the sync *execution*: pipelines living in the engine, reached through the `crate::engine` seam. This ports them -- orchestrator, dispatcher, HTTP client, connection lifecycle, and the twelve toolkit providers -- into `core/src/sync/pipelines/`, rewritten against what core already owns: the §B1a sync state, `tinymemory-sync`'s normalisers, and three sink traits an adapter implements over `MemoryClient`. `core/src/sync/` now names the engine zero times, code or comment. What the port taught, and the decisions inside it: - The pipelines were already store-neutral. They write through `SyncContext` sinks, and the engine's own adapter implemented those over core's `MemoryClient` all along -- the engine coupling was where the code *lived*, plus `MemoryConfig`. The ported `SyncContext` shrank to `{events, documents, state}`: the composio pipelines never used the summariser/local-documents/external-sources capabilities. - `PipelineConfig { composio, sync_depth_days, max_items }` replaces `MemoryConfig`. The pipelines read exactly three things; a pipeline that needs more must argue for the field. - The engine seam keeps `run_composio_connection`, `run_gmail_backfill` and `run_slack_search_backfill` as thin delegates onto the new runners, because OpenHuman reaches all three through the engine shim (including its `gmail_backfill_3d` binary). Zero downstream churn at the next pin bump; the seam's own pipeline plumbing is deleted, and its `build_pipeline` now refuses composio sources outright. - The #4957 unsupported-toolkit gate moved to `pipelines::host` with its tests: rejection still precedes credential resolution. - Gmail's canonical markdown moved to `tinymemory-sync` as `email_clean` + `email_markdown` -- pure text transforms, so they fit that crate's charter. The engine emits the same format from its copy; `thread_markdown_format_is_pinned` holds the two to one form, since the chunker splits on `---\nFrom:` boundaries. - `regex` returns to core's normal graph for Slack mention rewriting. #18 §D2 removed it as dead, which is no argument against a live consumer. - A local `ComposioMode` enum (Direct/Proxied) rather than the contract's `ComposioMode`, which is the host seam's *string-typed* setting -- same name, different concept; the mapping happens once, in `host::composio_config`. The tree-coupled source kinds (folder, repo, RSS, web page) still run through the engine seam by design: they summarise into the engine tree. Composio is what §B5's acceptance criterion names, and after this change every capability a Composio sync touches resolves through `MemoryClient` -- whatever driver the host bound. cargo test -p tinymemory-core: 843 passed, 0 failed cargo test -p tinymemory-sync: 124 passed (email modules + format pins) --- Cargo.lock | 103 ++++ core/Cargo.toml | 8 + core/src/engine/sync.rs | 273 ++++------ core/src/sync/audit.rs | 9 + core/src/sync/composio/mod.rs | 10 +- core/src/sync/composio/periodic.rs | 9 +- .../sync/composio/providers/clickup/mod.rs | 4 +- .../src/sync/composio/providers/github/mod.rs | 2 +- .../sync/composio/providers/gmail/provider.rs | 13 +- .../sync/composio/providers/gmail/tests.rs | 2 +- .../composio/providers/notion/provider.rs | 11 +- .../sync/composio/providers/slack/provider.rs | 22 +- core/src/sync/composio/providers/traits.rs | 2 +- core/src/sync/mod.rs | 1 + core/src/sync/pipelines/composio/client.rs | 304 +++++++++++ core/src/sync/pipelines/composio/connect.rs | 364 +++++++++++++ .../sync/pipelines/composio/connect_tests.rs | 305 +++++++++++ core/src/sync/pipelines/composio/gmail.rs | 380 +++++++++++++ .../sync/pipelines/composio/gmail_tests.rs | 101 ++++ core/src/sync/pipelines/composio/mod.rs | 22 + .../sync/pipelines/composio/orchestrator.rs | 505 ++++++++++++++++++ .../pipelines/composio/orchestrator_tests.rs | 236 ++++++++ core/src/sync/pipelines/composio/page_size.rs | 79 +++ .../pipelines/composio/page_size_tests.rs | 51 ++ .../pipelines/composio/providers/clickup.rs | 196 +++++++ .../pipelines/composio/providers/common.rs | 110 ++++ .../pipelines/composio/providers/github.rs | 178 ++++++ .../composio/providers/google_calendar.rs | 173 ++++++ .../composio/providers/google_docs.rs | 186 +++++++ .../composio/providers/google_drive.rs | 179 +++++++ .../composio/providers/google_sheets.rs | 186 +++++++ .../pipelines/composio/providers/linear.rs | 193 +++++++ .../sync/pipelines/composio/providers/mod.rs | 28 + .../pipelines/composio/providers/notion.rs | 193 +++++++ .../pipelines/composio/providers/outlook.rs | 205 +++++++ .../pipelines/composio/providers/slack.rs | 450 ++++++++++++++++ .../composio/providers/slack_parse.rs | 91 ++++ .../pipelines/composio/providers/todoist.rs | 226 ++++++++ core/src/sync/pipelines/dispatcher.rs | 123 +++++ core/src/sync/pipelines/dispatcher_tests.rs | 206 +++++++ core/src/sync/pipelines/host.rs | 379 +++++++++++++ core/src/sync/pipelines/mod.rs | 17 + core/src/sync/pipelines/traits.rs | 190 +++++++ .../composio_gmail_non_tinycortex_e2e.rs | 233 ++++++++ sync/Cargo.toml | 5 +- sync/src/email_clean.rs | 264 +++++++++ sync/src/email_clean_tests.rs | 143 +++++ sync/src/email_markdown.rs | 241 +++++++++ sync/src/lib.rs | 2 + 49 files changed, 7034 insertions(+), 179 deletions(-) create mode 100644 core/src/sync/pipelines/composio/client.rs create mode 100644 core/src/sync/pipelines/composio/connect.rs create mode 100644 core/src/sync/pipelines/composio/connect_tests.rs create mode 100644 core/src/sync/pipelines/composio/gmail.rs create mode 100644 core/src/sync/pipelines/composio/gmail_tests.rs create mode 100644 core/src/sync/pipelines/composio/mod.rs create mode 100644 core/src/sync/pipelines/composio/orchestrator.rs create mode 100644 core/src/sync/pipelines/composio/orchestrator_tests.rs create mode 100644 core/src/sync/pipelines/composio/page_size.rs create mode 100644 core/src/sync/pipelines/composio/page_size_tests.rs create mode 100644 core/src/sync/pipelines/composio/providers/clickup.rs create mode 100644 core/src/sync/pipelines/composio/providers/common.rs create mode 100644 core/src/sync/pipelines/composio/providers/github.rs create mode 100644 core/src/sync/pipelines/composio/providers/google_calendar.rs create mode 100644 core/src/sync/pipelines/composio/providers/google_docs.rs create mode 100644 core/src/sync/pipelines/composio/providers/google_drive.rs create mode 100644 core/src/sync/pipelines/composio/providers/google_sheets.rs create mode 100644 core/src/sync/pipelines/composio/providers/linear.rs create mode 100644 core/src/sync/pipelines/composio/providers/mod.rs create mode 100644 core/src/sync/pipelines/composio/providers/notion.rs create mode 100644 core/src/sync/pipelines/composio/providers/outlook.rs create mode 100644 core/src/sync/pipelines/composio/providers/slack.rs create mode 100644 core/src/sync/pipelines/composio/providers/slack_parse.rs create mode 100644 core/src/sync/pipelines/composio/providers/todoist.rs create mode 100644 core/src/sync/pipelines/dispatcher.rs create mode 100644 core/src/sync/pipelines/dispatcher_tests.rs create mode 100644 core/src/sync/pipelines/host.rs create mode 100644 core/src/sync/pipelines/mod.rs create mode 100644 core/src/sync/pipelines/traits.rs create mode 100644 core/tests/composio_gmail_non_tinycortex_e2e.rs create mode 100644 sync/src/email_clean.rs create mode 100644 sync/src/email_clean_tests.rs create mode 100644 sync/src/email_markdown.rs diff --git a/Cargo.lock b/Cargo.lock index 68faf84..8fb348a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,16 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "async-trait" version = "0.1.92" @@ -272,6 +282,24 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime", + "lazy_static", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" + [[package]] name = "digest" version = "0.10.7" @@ -379,6 +407,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.2.0" @@ -543,6 +577,25 @@ dependencies = [ "log", ] +[[package]] +name = "h2" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.16.1" @@ -567,6 +620,12 @@ dependencies = [ "hashbrown 0.16.1", ] +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "hex" version = "0.4.3" @@ -637,6 +696,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -785,6 +845,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.189" @@ -927,6 +993,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "objc2" version = "0.6.4" @@ -1850,6 +1926,7 @@ dependencies = [ "log", "parking_lot", "rand 0.8.7", + "regex", "reqwest", "rusqlite", "serde", @@ -1870,6 +1947,7 @@ dependencies = [ "url", "uuid", "walkdir", + "wiremock", ] [[package]] @@ -1916,6 +1994,7 @@ version = "0.1.0" dependencies = [ "chrono", "log", + "serde", "serde_json", "tracing", ] @@ -2012,6 +2091,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -2625,6 +2705,29 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +[[package]] +name = "wiremock" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" +dependencies = [ + "assert-json-diff", + "base64", + "deadpool", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "tokio", + "url", +] + [[package]] name = "wit-bindgen" version = "0.57.1" diff --git a/core/Cargo.toml b/core/Cargo.toml index 87f2e81..57c8dba 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -25,6 +25,11 @@ tinymemory-sync = { path = "../sync" } # crate's sync path drives the GitHub/RSS/web-page readers; a host that only # reads local folders can take the crate without them. tinymemory-sources = { path = "../sources", features = ["network"] } +# Slack mention tokens (`<@U…>`) in synced messages are rewritten to display +# names before ingestion. Returns to the normal graph for this one user — +# #18 §D2 removed it when nothing used it, which is no argument against a +# real consumer. +regex = "1.10" # The default embedded engine. `store/`, `tree/` and `sync/` drive it directly; # `tinycortex-api` is a direct dependency because `tinycortex::memory` aliases @@ -68,6 +73,9 @@ uuid = { version = "1", features = ["v4"] } walkdir = "2" [dev-dependencies] +# The ported Composio connect tests stand up a mock HTTP server (#18 §B1), +# exactly as they did in the engine. +wiremock = "0.6" # `TestHostConfig` — the concrete `MemoryHostConfig` the extracted test suites # build, since `Config` is a trait object and cannot be `Default`ed. tinymemory-api = { path = "../api", features = ["test-support"] } diff --git a/core/src/engine/sync.rs b/core/src/engine/sync.rs index 5a82e65..11ac63b 100644 --- a/core/src/engine/sync.rs +++ b/core/src/engine/sync.rs @@ -3,11 +3,9 @@ use async_trait::async_trait; use std::sync::Arc; use tinycortex::memory::sync::{ - ClickUpSyncPipeline, ComposioClient, ExternalSourceReader, GitHubSyncPipeline, - GithubRepoSyncPipeline, GmailSyncPipeline, LinearSyncPipeline, LocalDocument, - LocalDocumentSink, NotionSyncPipeline, SkillDocSink, SkillDocument, - SlackSearchBackfillPipeline, SlackSyncPipeline, SyncContext, SyncDispatcher, SyncEvent, - SyncEventSink, SyncOutcome, SyncPipeline, SyncStage, SyncStateStore, WorkspaceSourcePipeline, + ExternalSourceReader, GithubRepoSyncPipeline, LocalDocument, LocalDocumentSink, SkillDocSink, + SkillDocument, SyncContext, SyncDispatcher, SyncEvent, SyncEventSink, SyncOutcome, + SyncPipeline, SyncStage, SyncStateStore, WorkspaceSourcePipeline, }; use crate::sources::{MemorySourceEntry, SourceKind}; @@ -142,8 +140,10 @@ pub fn read_audit_log(config: &Config) -> Vec f64 { - tinycortex::memory::sync::estimate_cost_usd(input_tokens, output_tokens) + crate::sync::audit::estimate_cost_usd(input_tokens, output_tokens) } /// Measure coverage of a raw archive by its TinyCortex memory tree. @@ -309,6 +309,48 @@ pub async fn run_source_pipeline( source: &MemorySourceEntry, config: &Config, ) -> Result { + // Composio sources run on the engine-free pipelines (#18 §B1); this seam + // keeps only the tree-coupled kinds (folder/repo/rss/web — they summarise + // into the engine tree by design) and converts at the boundary for its + // OpenHuman-facing callers. + if source.kind == SourceKind::Composio { + let toolkit = source + .toolkit + .as_deref() + .map(str::trim) + .filter(|toolkit| !toolkit.is_empty()) + .ok_or_else(|| SourcePipelineFailure::without_usage("composio source missing toolkit"))? + .to_ascii_lowercase(); + let connection_id = source + .connection_id + .as_deref() + .map(str::trim) + .filter(|connection_id| !connection_id.is_empty()) + .ok_or_else(|| { + SourcePipelineFailure::without_usage("composio source missing connection_id") + })?; + let outcome = crate::sync::pipelines::host::run_composio_connection( + &toolkit, + connection_id, + config, + source.max_items, + source.sync_depth_days, + ) + .await + .map_err(|failure| SourcePipelineFailure { + message: failure.message, + actions_called: failure.actions_called, + provider_cost_usd: failure.provider_cost_usd, + })?; + return Ok(SyncOutcome { + records_ingested: outcome.records_ingested, + more_pending: outcome.more_pending, + actions_called: outcome.actions_called, + provider_cost_usd: outcome.provider_cost_usd, + note: outcome.note, + }); + } + let memory = crate::global::client_if_ready() .ok_or_else(|| SourcePipelineFailure::without_usage("memory client is not ready"))?; let mut memory_config = super::memory_config_from(config, config.workspace_dir().clone()); @@ -413,14 +455,22 @@ pub async fn run_composio_connection_with_budgets( run_source_pipeline(&source, config).await } +/// Load the persisted Composio sync state, in core's own vocabulary. +/// +/// Was typed with the engine's `SyncState`; the copies share one serde shape +/// and one KV namespace (pinned by tests in +/// `sync::composio::providers::sync_state`), so the retype changes no bytes. +/// Kept in the engine module only because OpenHuman reaches it through the +/// engine shim path. pub async fn load_composio_sync_state( toolkit: &str, connection_id: &str, -) -> anyhow::Result { +) -> anyhow::Result { let memory = crate::global::client_if_ready() .ok_or_else(|| anyhow::anyhow!("memory client is not ready"))?; - let adapter = HostSyncAdapter::new(memory); - tinycortex::memory::sync::SyncState::load(&adapter, toolkit, connection_id).await + let host = crate::sync::pipelines::host::PipelineHost::without_tree_ingest(memory); + crate::sync::composio::providers::sync_state::SyncState::load(&host, toolkit, connection_id) + .await } pub async fn run_slack_search_backfill( @@ -428,31 +478,30 @@ pub async fn run_slack_search_backfill( backfill_days: i64, config: &Config, ) -> Result { - let memory = crate::global::client_if_ready() - .ok_or_else(|| SourcePipelineFailure::without_usage("memory client is not ready"))?; - let mut memory_config = super::memory_config_from(config, config.workspace_dir().clone()); - let composio = composio_config(config).map_err(SourcePipelineFailure::without_usage)?; - memory_config.sync.composio = Some(composio.clone()); - let pipeline = std::sync::Arc::new(SlackSearchBackfillPipeline::new( - ComposioClient::new(composio), + // Delegates to the engine-free pipelines (#18 §B1); kept here because + // OpenHuman reaches this function through the engine shim path. + let outcome = crate::sync::pipelines::host::run_slack_search_backfill( connection_id, backfill_days, - )); - let pipeline_id = pipeline.id().to_owned(); - let mut dispatcher = SyncDispatcher::new(); - dispatcher - .register(pipeline) - .map_err(|error| SourcePipelineFailure::without_usage(error.to_string()))?; - dispatcher - .tick( - &pipeline_id, - &memory_config, - &source_sync_context(memory, config, false), - ) - .await - .map_err(|error| SourcePipelineFailure::without_usage(error.to_string())) + config, + ) + .await + .map_err(|failure| SourcePipelineFailure { + message: failure.message, + actions_called: failure.actions_called, + provider_cost_usd: failure.provider_cost_usd, + })?; + Ok(SyncOutcome { + records_ingested: outcome.records_ingested, + more_pending: outcome.more_pending, + actions_called: outcome.actions_called, + provider_cost_usd: outcome.provider_cost_usd, + note: outcome.note, + }) } +/// Delegates to the engine-free pipelines (#18 §B1); kept because OpenHuman's +/// backfill binary reaches it through the engine shim path. pub async fn run_gmail_backfill( connection_id: &str, query: &str, @@ -460,59 +509,32 @@ pub async fn run_gmail_backfill( page_size: usize, config: &Config, ) -> Result { - let memory = crate::global::client_if_ready() - .ok_or_else(|| SourcePipelineFailure::without_usage("memory client is not ready"))?; - let mut memory_config = super::memory_config_from(config, config.workspace_dir().clone()); - let composio = composio_config(config).map_err(SourcePipelineFailure::without_usage)?; - memory_config.sync.composio = Some(composio.clone()); - let pipeline = std::sync::Arc::new( - GmailSyncPipeline::new(ComposioClient::new(composio), connection_id) - .with_limits(max_pages, page_size) - .with_query(query), - ); - let pipeline_id = pipeline.id().to_owned(); - let mut dispatcher = SyncDispatcher::new(); - dispatcher - .register(pipeline) - .map_err(|error| SourcePipelineFailure::without_usage(error.to_string()))?; - dispatcher - .tick( - &pipeline_id, - &memory_config, - &source_sync_context(memory, config, false), - ) - .await - .map_err(|error| SourcePipelineFailure::without_usage(error.to_string())) -} - -/// Composio toolkit slugs that have a native memory-sync pipeline in -/// [`build_pipeline`] — the authoritative "can actually ingest into memory" set. -/// -/// This MUST stay in lockstep with the registered memory-sync providers -/// (`memory_sync::composio::all_composio_sync_providers`): the -/// `memory_sources.supported_toolkits` RPC advertises the provider registry, and -/// the `connection_created` auto-register gates on it, so any divergence -/// reintroduces the "connection reports ACTIVE but silently never ingests" -/// failure this guards against (#4957). The registry↔pipeline equality is pinned -/// by `composio_syncable_set_matches_provider_registry` in the tests below, and -/// the arms of the `match` in [`build_pipeline`] map 1:1 to these slugs. -pub fn syncable_composio_toolkits() -> &'static [&'static str] { - &["clickup", "github", "gmail", "linear", "notion", "slack"] -} - -/// Whether `toolkit` has a native memory-sync pipeline (case-insensitive). -/// Callers deciding *whether to offer/register* a Composio source should prefer -/// the provider registry (`get_composio_sync_provider`) so there is a single -/// advertised source of truth; this mirror exists for the sync layer itself. -pub fn is_composio_toolkit_syncable(toolkit: &str) -> bool { - let slug = toolkit.trim().to_ascii_lowercase(); - syncable_composio_toolkits().contains(&slug.as_str()) + let outcome = crate::sync::pipelines::host::run_gmail_backfill( + connection_id, + query, + max_pages, + page_size, + config, + ) + .await + .map_err(|failure| SourcePipelineFailure { + message: failure.message, + actions_called: failure.actions_called, + provider_cost_usd: failure.provider_cost_usd, + })?; + Ok(SyncOutcome { + records_ingested: outcome.records_ingested, + more_pending: outcome.more_pending, + actions_called: outcome.actions_called, + provider_cost_usd: outcome.provider_cost_usd, + note: outcome.note, + }) } fn build_pipeline( source: &MemorySourceEntry, - config: &Config, - memory_config: &mut tinycortex::memory::config::MemoryConfig, + _config: &Config, + _memory_config: &mut tinycortex::memory::config::MemoryConfig, ) -> Result, String> { if source.kind != SourceKind::Composio { let crate_source: tinycortex::memory::sources::MemorySourceEntry = serde_json::from_value( @@ -529,75 +551,13 @@ fn build_pipeline( .map_err(|error| error.to_string()); } - let toolkit = source - .toolkit - .as_deref() - .map(str::trim) - .filter(|toolkit| !toolkit.is_empty()) - .ok_or_else(|| "composio source missing toolkit".to_string())? - .to_ascii_lowercase(); - let connection_id = source - .connection_id - .as_deref() - .map(str::trim) - .filter(|connection_id| !connection_id.is_empty()) - .ok_or_else(|| "composio source missing connection_id".to_string())?; - // Fail closed *before* resolving credentials/client for any toolkit without a - // native pipeline. This keeps the unsupported-toolkit error identical to the - // match's fallback while making the syncable set a single, testable gate that - // stays pinned to the provider registry (#4957). - if !is_composio_toolkit_syncable(&toolkit) { - return Err(format!( - "tinycortex sync does not support toolkit '{toolkit}'" - )); - } - let composio = composio_config(config)?; - memory_config.sync.composio = Some(composio.clone()); - let client = ComposioClient::new(composio); - let pipeline: std::sync::Arc = match toolkit.as_str() { - "gmail" => std::sync::Arc::new(GmailSyncPipeline::new(client, connection_id)), - "github" => std::sync::Arc::new(GitHubSyncPipeline::new(client, connection_id)), - "notion" => std::sync::Arc::new(NotionSyncPipeline::new(client, connection_id)), - "linear" => std::sync::Arc::new(LinearSyncPipeline::new(client, connection_id)), - "clickup" => std::sync::Arc::new(ClickUpSyncPipeline::new(client, connection_id)), - "slack" => std::sync::Arc::new(SlackSyncPipeline::new(client, connection_id)), - _ => { - return Err(format!( - "tinycortex sync does not support toolkit '{toolkit}'" - )) - } - }; - Ok(pipeline) -} - -fn composio_config( - config: &Config, -) -> Result { - use tinycortex::memory::config::{ComposioMode, ComposioSyncConfig, SecretString}; - - if config.composio().mode.eq_ignore_ascii_case("direct") { - let api_key = crate::composio_host::api_key(config) - .or_else(|| config.composio().api_key.clone()) - .ok_or_else(|| "Composio direct API key is not configured".to_string())?; - Ok(ComposioSyncConfig { - mode: ComposioMode::Direct, - base_url: "https://backend.composio.dev/api/v3".into(), - api_key: Some(SecretString::new(api_key)), - bearer_token: None, - entity_id: Some(config.composio().entity_id.clone()), - }) - } else { - let bearer = config - .session_token()? - .ok_or_else(|| "OpenHuman backend bearer token is not configured".to_string())?; - Ok(ComposioSyncConfig { - mode: ComposioMode::Proxied, - base_url: config.effective_backend_api_url(), - api_key: None, - bearer_token: Some(SecretString::new(bearer)), - entity_id: Some(config.composio().entity_id.clone()), - }) - } + // Composio sources never reach this seam: `run_source_pipeline` routes + // them to `crate::sync::pipelines` (#18 §B1) before building. Only the + // tree-coupled kinds are built here. + Err(format!( + "engine seam does not build composio pipelines (kind {:?} unexpected here)", + source.kind + )) } #[async_trait] @@ -764,9 +724,10 @@ fn stage_name(stage: SyncStage) -> &'static str { #[cfg(test)] mod tests { - use super::{build_pipeline, is_composio_toolkit_syncable, syncable_composio_toolkits}; + use super::build_pipeline; use crate::sources::MemorySourceEntry; use crate::sync::composio::{get_composio_sync_provider, init_default_composio_sync_providers}; + use crate::sync::pipelines::host::{is_composio_toolkit_syncable, syncable_composio_toolkits}; /// The advertised set (`memory_sources.supported_toolkits`, sourced from the /// provider registry) and the syncable set (`build_pipeline`) must not @@ -838,7 +799,7 @@ mod tests { /// we must get the unsupported-toolkit error, proving the fail-closed /// ordering that stops an unsyncable toolkit from ever reaching a pipeline. #[test] - fn build_pipeline_rejects_unsupported_toolkit_before_resolving_config() { + fn build_pipeline_refuses_composio_sources() { // `googlecalendar` is a real Composio toolkit with no native pipeline — // exactly the prod case from #4957. let source: MemorySourceEntry = serde_json::from_value(serde_json::json!({ @@ -854,16 +815,16 @@ mod tests { let mut memory_config = tinycortex::memory::config::MemoryConfig::new("/tmp/openhuman-test-ws"); - // `build_pipeline` returns `Result, String>`; the - // Ok arm is not `Debug`, so match rather than `expect_err`. + // Composio never reaches this seam any more: `run_source_pipeline` + // routes it to the engine-free pipelines (#18 §B1). The seam's job is + // to say so, not to half-build one. let err = match build_pipeline(&source, &config, &mut memory_config) { - Ok(_) => panic!("unsupported toolkit must be rejected before config resolution"), + Ok(_) => panic!("the engine seam must refuse composio sources"), Err(e) => e, }; assert!( - err.contains("does not support toolkit 'googlecalendar'"), - "expected the unsupported-toolkit error (proving rejection precedes \ - config resolution), got: {err}" + err.contains("does not build composio pipelines"), + "expected the composio refusal, got: {err}" ); } diff --git a/core/src/sync/audit.rs b/core/src/sync/audit.rs index edf4aed..e54838f 100644 --- a/core/src/sync/audit.rs +++ b/core/src/sync/audit.rs @@ -63,6 +63,15 @@ impl SyncAuditEntry { } } +/// Estimated inference cost for a sync batch, in USD. +/// +/// The engine prices identically from its copy; both are estimates the audit +/// records alongside the real charge when one is reported. Owned here with +/// the audit log because this is where the number lands. +pub fn estimate_cost_usd(input_tokens: u64, output_tokens: u64) -> f64 { + input_tokens as f64 * 0.07 / 1_000_000.0 + output_tokens as f64 * 0.28 / 1_000_000.0 +} + /// Append one entry to the audit log under `workspace`. /// /// # Errors diff --git a/core/src/sync/composio/mod.rs b/core/src/sync/composio/mod.rs index b3df5eb..94219b4 100644 --- a/core/src/sync/composio/mod.rs +++ b/core/src/sync/composio/mod.rs @@ -161,8 +161,14 @@ pub async fn run_connection_sync( .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() .as_millis() as u64; - match crate::engine::run_composio_connection(&target.toolkit, &target.connection_id, &*config) - .await + match crate::sync::pipelines::host::run_composio_connection( + &target.toolkit, + &target.connection_id, + &*config, + None, + None, + ) + .await { Ok(outcome) => { let usage = ComposioUsage { diff --git a/core/src/sync/composio/periodic.rs b/core/src/sync/composio/periodic.rs index 0cbc9fa..cf64fd7 100644 --- a/core/src/sync/composio/periodic.rs +++ b/core/src/sync/composio/periodic.rs @@ -556,7 +556,14 @@ pub(crate) async fn run_one_tick() -> Result<(), String> { "[composio:periodic] firing sync" ); let sync_started = Instant::now(); - let result = crate::engine::run_source_pipeline(&source, &*config).await; + let result = crate::sync::pipelines::host::run_composio_connection( + &toolkit, + &conn.id, + &*config, + source.max_items, + source.sync_depth_days, + ) + .await; let duration_ms = sync_started.elapsed().as_millis() as u64; match result { diff --git a/core/src/sync/composio/providers/clickup/mod.rs b/core/src/sync/composio/providers/clickup/mod.rs index f38484a..7ca8115 100644 --- a/core/src/sync/composio/providers/clickup/mod.rs +++ b/core/src/sync/composio/providers/clickup/mod.rs @@ -6,8 +6,8 @@ //! re-learning a new shape: //! //! - `provider.rs` — `impl ComposioProvider for ClickUpProvider` -//! - `normalization` — payload-shape helpers, now reached through -//! `crate::engine::engine` (issue #18 §C1) +//! - `normalization` — payload-shape helpers, owned by `tinymemory-sync` +//! (issue #18 §B3) //! - `ingest.rs` — memory_tree document ingest (issue #2885) //! - `tools.rs` — `CLICKUP_CURATED` whitelist of Composio actions //! - `tests.rs` — unit tests for the helpers + trait metadata diff --git a/core/src/sync/composio/providers/github/mod.rs b/core/src/sync/composio/providers/github/mod.rs index 90633f3..89dbd5b 100644 --- a/core/src/sync/composio/providers/github/mod.rs +++ b/core/src/sync/composio/providers/github/mod.rs @@ -7,7 +7,7 @@ //! //! - `provider.rs` — `impl ComposioProvider for GitHubProvider` //! - `normalization` — payload-shape helpers, now reached through -//! `crate::engine::engine` (issue #18 §C1) +//! `tinymemory-sync` (issue #18 §B3) //! - `tools.rs` — `GITHUB_CURATED` whitelist of Composio actions //! - `tests.rs` — unit tests for the helpers + trait metadata //! diff --git a/core/src/sync/composio/providers/gmail/provider.rs b/core/src/sync/composio/providers/gmail/provider.rs index a46617e..701d56f 100644 --- a/core/src/sync/composio/providers/gmail/provider.rs +++ b/core/src/sync/composio/providers/gmail/provider.rs @@ -152,9 +152,14 @@ impl ComposioProvider for GmailProvider { let Some(connection_id) = ctx.connection_id.as_deref() else { return Err("[composio:gmail] trigger missing connection_id".to_string()); }; - if let Err(e) = - crate::engine::run_composio_connection("gmail", connection_id, ctx.config.as_ref()) - .await + if let Err(e) = crate::sync::pipelines::host::run_composio_connection( + "gmail", + connection_id, + ctx.config.as_ref(), + None, + None, + ) + .await { tracing::warn!( error = %e, @@ -168,5 +173,5 @@ impl ComposioProvider for GmailProvider { // Message fetching (the `GMAIL_FETCH_EMAILS` action, the search query, the // `max_items` cap math and the `sync_depth_days` `after:` floor) is owned -// by `crate::engine::backend::sync::GmailSyncPipeline`. What stays here is the +// by `crate::sync::pipelines::composio::GmailSyncPipeline`. What stays here is the // host-side provider surface: profile lookup and trigger dispatch. diff --git a/core/src/sync/composio/providers/gmail/tests.rs b/core/src/sync/composio/providers/gmail/tests.rs index 17a1825..4c6554b 100644 --- a/core/src/sync/composio/providers/gmail/tests.rs +++ b/core/src/sync/composio/providers/gmail/tests.rs @@ -1,7 +1,7 @@ //! Host-owned Gmail provider surface tests. //! //! Pagination, cursor, envelope parsing, and ingest behavior are owned and -//! tested by `crate::engine::backend::sync::GmailSyncPipeline`. +//! tested by `crate::sync::pipelines::composio::GmailSyncPipeline`. use super::GmailProvider; use crate::sync::composio::providers::ComposioProvider; diff --git a/core/src/sync/composio/providers/notion/provider.rs b/core/src/sync/composio/providers/notion/provider.rs index d02cc3e..af56b6d 100644 --- a/core/src/sync/composio/providers/notion/provider.rs +++ b/core/src/sync/composio/providers/notion/provider.rs @@ -278,9 +278,14 @@ impl ComposioProvider for NotionProvider { let Some(connection_id) = ctx.connection_id.as_deref() else { return Err("[composio:notion] trigger missing connection_id".to_string()); }; - if let Err(e) = - crate::engine::run_composio_connection("notion", connection_id, ctx.config.as_ref()) - .await + if let Err(e) = crate::sync::pipelines::host::run_composio_connection( + "notion", + connection_id, + ctx.config.as_ref(), + None, + None, + ) + .await { tracing::warn!( error = %e, diff --git a/core/src/sync/composio/providers/slack/provider.rs b/core/src/sync/composio/providers/slack/provider.rs index 3ed5270..3256d95 100644 --- a/core/src/sync/composio/providers/slack/provider.rs +++ b/core/src/sync/composio/providers/slack/provider.rs @@ -233,9 +233,14 @@ impl ComposioProvider for SlackProvider { let Some(connection_id) = ctx.connection_id.as_deref() else { return Err("[composio:slack] trigger missing connection_id".to_string()); }; - if let Err(e) = - crate::engine::run_composio_connection("slack", connection_id, ctx.config.as_ref()) - .await + if let Err(e) = crate::sync::pipelines::host::run_composio_connection( + "slack", + connection_id, + ctx.config.as_ref(), + None, + None, + ) + .await { tracing::warn!( error = %e, @@ -259,10 +264,13 @@ pub async fn run_backfill_via_search( .as_deref() .ok_or_else(|| "[composio:slack] search backfill missing connection_id".to_string())?; let started_at_ms = now_ms(); - let outcome = - crate::engine::run_slack_search_backfill(connection_id, backfill_days, ctx.config.as_ref()) - .await - .map_err(|error| error.to_string())?; + let outcome = crate::sync::pipelines::host::run_slack_search_backfill( + connection_id, + backfill_days, + ctx.config.as_ref(), + ) + .await + .map_err(|error| error.to_string())?; Ok(SyncOutcome { toolkit: "slack".into(), connection_id: Some(connection_id.into()), diff --git a/core/src/sync/composio/providers/traits.rs b/core/src/sync/composio/providers/traits.rs index b9a0e30..6f8b854 100644 --- a/core/src/sync/composio/providers/traits.rs +++ b/core/src/sync/composio/providers/traits.rs @@ -59,7 +59,7 @@ pub trait ComposioProvider: Send + Sync { ) })?; let started_at_ms = now_ms(); - let outcome = crate::engine::run_composio_connection_with_budgets( + let outcome = crate::sync::pipelines::host::run_composio_connection( self.toolkit_slug(), connection_id, ctx.config.as_ref(), diff --git a/core/src/sync/mod.rs b/core/src/sync/mod.rs index 9d53ca0..089b03c 100644 --- a/core/src/sync/mod.rs +++ b/core/src/sync/mod.rs @@ -29,5 +29,6 @@ pub mod audit; pub mod composio; pub mod mcp; +pub mod pipelines; pub mod sync_status; pub mod workspace; diff --git a/core/src/sync/pipelines/composio/client.rs b/core/src/sync/pipelines/composio/client.rs new file mode 100644 index 0000000..b7211f5 --- /dev/null +++ b/core/src/sync/pipelines/composio/client.rs @@ -0,0 +1,304 @@ +//! Minimal direct/proxied Composio action client. + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; + +use crate::sync::pipelines::traits::{ComposioMode, ComposioSyncConfig}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ExecuteResponse { + #[serde(default)] + pub data: serde_json::Value, + #[serde(default)] + pub successful: bool, + #[serde(default)] + pub error: Option, + #[serde(rename = "costUsd", default)] + pub cost_usd: f64, + #[serde(rename = "markdownFormatted", default)] + pub markdown_formatted: Option, + #[serde(skip, default = "one_attempt")] + pub attempts: u32, +} + +fn one_attempt() -> u32 { + 1 +} + +#[derive(Debug, thiserror::Error)] +#[error("{message}")] +pub struct ExecuteError { + pub attempts: u32, + message: String, +} + +#[derive(Clone)] +pub struct ComposioClient { + http: reqwest::Client, + config: ComposioSyncConfig, +} + +#[async_trait] +pub trait ActionExecutor: Send + Sync { + async fn execute( + &self, + action: &str, + arguments: serde_json::Value, + connection_id: Option<&str>, + ) -> anyhow::Result; +} + +#[async_trait] +impl ActionExecutor for ComposioClient { + async fn execute( + &self, + action: &str, + arguments: serde_json::Value, + connection_id: Option<&str>, + ) -> anyhow::Result { + ComposioClient::execute(self, action, arguments, connection_id).await + } +} + +impl ComposioClient { + pub fn new(config: ComposioSyncConfig) -> Self { + Self { + http: reqwest::Client::new(), + config, + } + } + + pub fn with_http_client(mut self, http: reqwest::Client) -> Self { + self.http = http; + self + } + + pub async fn execute( + &self, + action: &str, + arguments: serde_json::Value, + connection_id: Option<&str>, + ) -> anyhow::Result { + let action = action.trim(); + anyhow::ensure!(!action.is_empty(), "Composio action must not be empty"); + const MAX_ATTEMPTS: u32 = 3; + for attempt in 1..=MAX_ATTEMPTS { + let result = match self.config.mode { + ComposioMode::Direct => { + self.execute_direct(action, arguments.clone(), connection_id) + .await + } + ComposioMode::Proxied => self.execute_proxied(action, arguments.clone()).await, + }; + match result { + Ok(mut response) + if response.successful + || !retryable_provider_error(response.error.as_deref()) + || attempt == MAX_ATTEMPTS => + { + response.attempts = attempt; + return Ok(response); + } + Ok(_) => tracing::warn!( + action, + attempt, + "[sync:composio] retrying provider rate limit" + ), + Err(error) if retryable_transport_error(&error) && attempt < MAX_ATTEMPTS => { + tracing::warn!(action, attempt, %error, "[sync:composio] retrying transient transport failure"); + } + Err(error) => { + return Err(ExecuteError { + attempts: attempt, + message: error.to_string(), + } + .into()) + } + } + tokio::time::sleep(std::time::Duration::from_millis( + 250 * 2u64.pow(attempt - 1), + )) + .await; + } + unreachable!("retry loop always returns") + } + + async fn execute_direct( + &self, + action: &str, + arguments: serde_json::Value, + connection_id: Option<&str>, + ) -> anyhow::Result { + let key = self + .config + .api_key + .as_ref() + .filter(|key| !key.is_empty()) + .map(|key| key.expose().to_owned()) + .filter(|key| !key.trim().is_empty()) + .ok_or_else(|| anyhow::anyhow!("Composio direct API key is not configured"))?; + let url = format!( + "{}/tools/execute/{action}", + self.config.base_url.trim_end_matches('/') + ); + let mut body = serde_json::json!({ "arguments": arguments }); + if let Some(entity_id) = self + .config + .entity_id + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + { + body["user_id"] = serde_json::json!(entity_id); + } + if let Some(connection_id) = connection_id + .map(str::trim) + .filter(|value| !value.is_empty()) + { + body["connected_account_id"] = serde_json::json!(connection_id); + } + + let response = self + .http + .post(url) + .header("x-api-key", key) + .json(&body) + .send() + .await + .map_err(|error| anyhow::anyhow!("Composio direct request failed: {error}"))?; + let status = response.status(); + if !status.is_success() { + let _ = response.bytes().await; + anyhow::bail!("Composio direct request failed with HTTP {status}"); + } + let raw: serde_json::Value = decode_response(response, "direct").await?; + let successful = raw + .get("successful") + .and_then(serde_json::Value::as_bool) + .or_else(|| raw.get("success").and_then(serde_json::Value::as_bool)) + .unwrap_or(true); + let error = raw + .get("error") + .and_then(serde_json::Value::as_str) + .map(str::to_owned); + let data = raw.get("data").cloned().unwrap_or(raw); + Ok(ExecuteResponse { + data, + successful, + error, + cost_usd: 0.0, + markdown_formatted: None, + attempts: 1, + }) + } + + async fn execute_proxied( + &self, + action: &str, + arguments: serde_json::Value, + ) -> anyhow::Result { + let bearer = self + .config + .bearer_token + .as_ref() + .filter(|token| !token.is_empty()) + .ok_or_else(|| anyhow::anyhow!("Composio proxy bearer token is not configured"))?; + let url = format!( + "{}/agent-integrations/composio/execute", + self.config.base_url.trim_end_matches('/') + ); + let response = self + .http + .post(url) + .bearer_auth(bearer.expose()) + .json(&serde_json::json!({ "tool": action, "arguments": arguments })) + .send() + .await + .map_err(|error| anyhow::anyhow!("Composio proxy request failed: {error}"))?; + let status = response.status(); + if !status.is_success() { + let _ = response.bytes().await; + anyhow::bail!("Composio proxy request failed with HTTP {status}"); + } + let raw: serde_json::Value = response + .json() + .await + .map_err(|error| anyhow::anyhow!("Composio proxy response decode failed: {error}"))?; + decode_proxy_response(raw) + } +} + +fn decode_proxy_response(raw: serde_json::Value) -> anyhow::Result { + let payload = if raw.get("successful").is_some() { + raw + } else { + raw.get("data").cloned().unwrap_or(raw) + }; + serde_json::from_value(payload) + .map_err(|error| anyhow::anyhow!("Composio proxy response decode failed: {error}")) +} + +fn retryable_provider_error(error: Option<&str>) -> bool { + error.is_some_and(|error| { + let lower = error.to_ascii_lowercase(); + lower.contains("ratelimit") + || lower.contains("rate limit") + || lower.contains("too many requests") + }) +} + +fn retryable_transport_error(error: &anyhow::Error) -> bool { + let message = error.to_string(); + [ + "HTTP 429", + "HTTP 502", + "HTTP 503", + "HTTP 504", + "request failed", + ] + .iter() + .any(|needle| message.contains(needle)) +} + +async fn decode_response( + response: reqwest::Response, + mode: &str, +) -> anyhow::Result { + response + .json() + .await + .map_err(|error| anyhow::anyhow!("Composio {mode} response decode failed: {error}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn proxied_backend_envelope_decodes_provider_response() { + let response = decode_proxy_response(serde_json::json!({ + "success": true, + "data": { + "successful": true, + "data": {"messages": [{"messageId": "message-1"}]}, + "error": null + } + })) + .unwrap(); + + assert!(response.successful); + assert_eq!(response.data["messages"][0]["messageId"], "message-1"); + } + + #[test] + fn flat_proxy_response_remains_supported() { + let response = decode_proxy_response(serde_json::json!({ + "successful": true, + "data": {"items": [1]} + })) + .unwrap(); + + assert!(response.successful); + assert_eq!(response.data["items"], serde_json::json!([1])); + } +} diff --git a/core/src/sync/pipelines/composio/connect.rs b/core/src/sync/pipelines/composio/connect.rs new file mode 100644 index 0000000..1ea0998 --- /dev/null +++ b/core/src/sync/pipelines/composio/connect.rs @@ -0,0 +1,364 @@ +//! Composio v3 login/connect helpers. +//! +//! This module owns the reusable, host-agnostic pieces of the Composio +//! connection flow so a harness (or a server-side host) can drive an OAuth +//! login without re-deriving the wire contract: +//! +//! * a small per-integration **entity-id store** ([`EntityStore`]) that +//! remembers the `user_id` chosen for each toolkit across runs so re-runs +//! reuse the same Composio "entity" instead of orphaning connections; +//! * pure parsers for the connected-account **status** lifecycle; and +//! * thin async wrappers over the three v3 endpoints the connect flow needs. +//! +//! ## Verified v3 endpoints +//! +//! All confirmed against the Composio SDK source (the generated OpenAPI client +//! these SDKs wrap) — — and the v3 API +//! reference at : +//! +//! * `GET /api/v3/auth_configs?toolkit_slug={slug}` — list auth configs; +//! response `{ items: [ { id, toolkit: { slug } } ] }`. +//! (`ts/packages/core/src/models/AuthConfigs.ts`, `authConfigs.types.ts`.) +//! * `POST /api/v3/connected_accounts/link` — create a Composio Connect Link; +//! body `{ auth_config_id, user_id, callback_url? }`, response +//! `{ connected_account_id, redirect_url }`. +//! (`ConnectedAccounts.ts` `link()`, `connected_accounts.py` `link()`.) +//! * `GET /api/v3/connected_accounts/{nanoid}` — poll status; top-level +//! `status` in `INITIALIZING | INITIATED | ACTIVE | EXPIRED | FAILED | +//! REVOKED`. (`connected_accounts.py` `wait_for_connection`.) +//! +//! Authentication is the direct-mode `x-api-key` header, matching +//! [`super::client::ComposioClient`]. No secret is ever logged and error paths +//! discard raw response bodies (they can echo the key back verbatim). + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use uuid::Uuid; + +/// Connection statuses that will never recover on their own — polling should +/// stop and fail. Mirrors the Composio SDK's `terminalErrorStates` +/// (`FAILED`, `EXPIRED`, `REVOKED`); `INACTIVE` is intentionally excluded +/// because it can transition back to `ACTIVE`. +const TERMINAL_STATUSES: &[&str] = &["FAILED", "EXPIRED", "REVOKED", "DELETED"]; + +/// Generate a fresh Composio entity id (`user_id`) for a new connection. +/// +/// The `tinycortex-` prefix keeps harness-created entities recognisable in the +/// Composio dashboard while the UUID guarantees uniqueness. +pub fn generate_entity_id() -> String { + format!("tinycortex-{}", Uuid::new_v4()) +} + +/// True when a connected-account status string means the account is live and +/// usable for tool execution. Case-insensitive. +pub fn status_is_active(status: &str) -> bool { + status.trim().eq_ignore_ascii_case("ACTIVE") +} + +/// True when a status string is a terminal failure that polling must give up +/// on. Case-insensitive. +pub fn status_is_terminal(status: &str) -> bool { + let status = status.trim(); + TERMINAL_STATUSES + .iter() + .any(|terminal| status.eq_ignore_ascii_case(terminal)) +} + +/// Pull the connected-account `status` out of a get-by-id response. +/// +/// Composio has shipped the status both at the top level and nested under +/// `state.val` / `connectionData.val`; probe the known shapes. +pub fn extract_status(record: &Value) -> Option { + [ + record.get("status"), + record.pointer("/state/val/status"), + record.pointer("/connectionData/val/status"), + record.pointer("/connection_data/val/status"), + ] + .into_iter() + .flatten() + .find_map(Value::as_str) + .map(str::trim) + .filter(|status| !status.is_empty()) + .map(str::to_owned) +} + +/// Extract the OAuth redirect URL from a create-link response. The v3 `/link` +/// endpoint returns a flat `redirect_url`; older shapes nested it under +/// `connectionData.val.redirectUrl`, so probe both. +pub fn extract_redirect_url(record: &Value) -> Option { + [ + record.get("redirect_url"), + record.get("redirectUrl"), + record.pointer("/connectionData/val/redirectUrl"), + record.pointer("/connection_data/val/redirect_url"), + ] + .into_iter() + .flatten() + .find_map(Value::as_str) + .map(str::trim) + .filter(|url| !url.is_empty()) + .map(str::to_owned) +} + +/// Extract the connected-account id from a create-link response. The v3 +/// `/link` endpoint returns `connected_account_id`; legacy `initiate` returned +/// a top-level `id`. +pub fn extract_account_id(record: &Value) -> Option { + ["connected_account_id", "connectedAccountId", "id", "nanoid"] + .iter() + .find_map(|key| record.get(key).and_then(Value::as_str)) + .map(str::trim) + .filter(|id| !id.is_empty()) + .map(str::to_owned) +} + +/// Resolve an auth-config id for `toolkit` from a `GET /auth_configs` response. +/// +/// Prefers an item whose `toolkit.slug` matches (case-insensitively); falls +/// back to the first listed config when the toolkit was already used as a +/// server-side filter and the slug field is shaped differently. +pub fn resolve_auth_config_id(list: &Value, toolkit: &str) -> Option { + let items = list + .pointer("/items") + .and_then(Value::as_array) + .or_else(|| list.get("data").and_then(Value::as_array)) + .or_else(|| list.as_array())?; + + let matches_toolkit = |item: &Value| { + [ + item.pointer("/toolkit/slug"), + item.pointer("/toolkit/name"), + item.get("toolkit"), + ] + .into_iter() + .flatten() + .find_map(Value::as_str) + .map(|slug| slug.trim().eq_ignore_ascii_case(toolkit)) + .unwrap_or(false) + }; + let auth_config_id = |item: &Value| { + ["id", "nanoid"] + .iter() + .find_map(|key| item.get(key).and_then(Value::as_str)) + .map(str::trim) + .filter(|id| !id.is_empty()) + .map(str::to_owned) + }; + + items + .iter() + .find(|item| matches_toolkit(item)) + .and_then(auth_config_id) + .or_else(|| items.iter().find_map(auth_config_id)) +} + +/// A newly-created Composio Connect Link. +#[derive(Debug, Clone)] +pub struct ConnectionLink { + /// The pending connected-account id to poll for `ACTIVE`. + pub connected_account_id: String, + /// The OAuth URL the user must open to complete login, when the scheme is + /// redirect-based. `None` for schemes that activate without a browser step. + pub redirect_url: Option, +} + +/// Persistent, per-toolkit map of the entity id (`user_id`) chosen for each +/// integration. +/// +/// Stored as a small JSON object on disk (e.g. `.composio-harness.json`) so a +/// re-run reuses the same Composio entity instead of creating a fresh — and +/// therefore orphaned — connection every time. Load is best-effort: a missing +/// or corrupt file yields an empty store rather than an error. +#[derive(Debug, Clone)] +pub struct EntityStore { + path: PathBuf, + entries: BTreeMap, +} + +#[derive(Default, Serialize, Deserialize)] +struct EntityStoreFile { + /// toolkit slug -> entity id (`user_id`). + #[serde(default)] + entities: BTreeMap, +} + +impl EntityStore { + /// Load the store from `path`, tolerating a missing or unreadable file. + pub fn load(path: impl Into) -> Self { + let path = path.into(); + let entries = std::fs::read_to_string(&path) + .ok() + .and_then(|raw| serde_json::from_str::(&raw).ok()) + .map(|file| file.entities) + .unwrap_or_default(); + Self { path, entries } + } + + /// The backing file path. + pub fn path(&self) -> &Path { + &self.path + } + + /// The entity id recorded for `toolkit`, if any. + pub fn get(&self, toolkit: &str) -> Option<&str> { + self.entries.get(toolkit).map(String::as_str) + } + + /// Record `entity_id` for `toolkit` in memory (call [`Self::save`] to + /// persist). + pub fn set(&mut self, toolkit: &str, entity_id: impl Into) { + self.entries.insert(toolkit.to_owned(), entity_id.into()); + } + + /// Resolve the entity id to use when connecting `toolkit`, persisting the + /// choice so future runs are stable. + /// + /// Precedence: a value already stored for this toolkit wins; otherwise an + /// explicit `override_id` (e.g. `COMPOSIO_ENTITY_ID`) is adopted; otherwise + /// a fresh id is generated. The resolved id is written back and saved. + pub fn entity_id_for( + &mut self, + toolkit: &str, + override_id: Option<&str>, + ) -> std::io::Result { + if let Some(existing) = self.get(toolkit) { + return Ok(existing.to_owned()); + } + let chosen = override_id + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_owned) + .unwrap_or_else(generate_entity_id); + self.set(toolkit, chosen.clone()); + self.save()?; + Ok(chosen) + } + + /// Serialize the store to its backing file (pretty JSON). + pub fn save(&self) -> std::io::Result<()> { + let file = EntityStoreFile { + entities: self.entries.clone(), + }; + let json = serde_json::to_string_pretty(&file) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; + std::fs::write(&self.path, json) + } +} + +/// `GET /api/v3/auth_configs?toolkit_slug={toolkit}` — list auth configs, +/// optionally filtered to one toolkit. Returns the decoded JSON body so the +/// caller can resolve an id via [`resolve_auth_config_id`]. +pub async fn list_auth_configs( + http: &reqwest::Client, + base_url: &str, + api_key: &str, + toolkit: Option<&str>, +) -> anyhow::Result { + let mut request = http + .get(format!("{}/auth_configs", base_url.trim_end_matches('/'))) + .header("x-api-key", api_key); + if let Some(toolkit) = toolkit { + request = request.query(&[("toolkit_slug", toolkit)]); + } + let response = request + .send() + .await + .map_err(|error| anyhow::anyhow!("auth_configs request failed: {error}"))?; + let status = response.status(); + if !status.is_success() { + // Never echo the body; it can contain the key back verbatim. + let _ = response.bytes().await; + anyhow::bail!("auth_configs returned HTTP {status}"); + } + response + .json() + .await + .map_err(|error| anyhow::anyhow!("auth_configs decode failed: {error}")) +} + +/// `POST /api/v3/connected_accounts/link` — create a Composio Connect Link for +/// `auth_config_id` scoped to `user_id`. Returns the pending account id plus an +/// optional OAuth redirect URL. +pub async fn create_connection_link( + http: &reqwest::Client, + base_url: &str, + api_key: &str, + auth_config_id: &str, + user_id: &str, + callback_url: Option<&str>, +) -> anyhow::Result { + let mut body = serde_json::json!({ + "auth_config_id": auth_config_id, + "user_id": user_id, + }); + if let Some(callback_url) = callback_url + .map(str::trim) + .filter(|value| !value.is_empty()) + { + body["callback_url"] = serde_json::json!(callback_url); + } + let response = http + .post(format!( + "{}/connected_accounts/link", + base_url.trim_end_matches('/') + )) + .header("x-api-key", api_key) + .json(&body) + .send() + .await + .map_err(|error| anyhow::anyhow!("connected_accounts/link request failed: {error}"))?; + let status = response.status(); + if !status.is_success() { + let _ = response.bytes().await; + anyhow::bail!("connected_accounts/link returned HTTP {status}"); + } + let record: Value = response + .json() + .await + .map_err(|error| anyhow::anyhow!("connected_accounts/link decode failed: {error}"))?; + let connected_account_id = extract_account_id(&record).ok_or_else(|| { + anyhow::anyhow!("connected_accounts/link response missing a connected account id") + })?; + Ok(ConnectionLink { + connected_account_id, + redirect_url: extract_redirect_url(&record), + }) +} + +/// `GET /api/v3/connected_accounts/{account_id}` — fetch the current status of +/// a (possibly pending) connected account. Returns `None` if the response had +/// no recognisable status field. +pub async fn get_connection_status( + http: &reqwest::Client, + base_url: &str, + api_key: &str, + account_id: &str, +) -> anyhow::Result> { + let response = http + .get(format!( + "{}/connected_accounts/{account_id}", + base_url.trim_end_matches('/') + )) + .header("x-api-key", api_key) + .send() + .await + .map_err(|error| anyhow::anyhow!("connected_accounts/{{id}} request failed: {error}"))?; + let status = response.status(); + if !status.is_success() { + let _ = response.bytes().await; + anyhow::bail!("connected_accounts/{{id}} returned HTTP {status}"); + } + let record: Value = response + .json() + .await + .map_err(|error| anyhow::anyhow!("connected_accounts/{{id}} decode failed: {error}"))?; + Ok(extract_status(&record)) +} + +#[cfg(test)] +#[path = "connect_tests.rs"] +mod tests; diff --git a/core/src/sync/pipelines/composio/connect_tests.rs b/core/src/sync/pipelines/composio/connect_tests.rs new file mode 100644 index 0000000..7e893bc --- /dev/null +++ b/core/src/sync/pipelines/composio/connect_tests.rs @@ -0,0 +1,305 @@ +//! Unit tests for the pure Composio connect helpers: entity-id persistence, +//! status classification, and response-field extraction. No network I/O. + +use super::*; +use serde_json::json; +use wiremock::matchers::{body_partial_json, header, method, path as url_path, query_param}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +#[test] +fn generated_entity_id_is_prefixed_and_unique() { + let a = generate_entity_id(); + let b = generate_entity_id(); + assert!(a.starts_with("tinycortex-"), "unexpected id: {a}"); + assert_ne!(a, b, "two generated ids must differ"); +} + +#[test] +fn status_active_is_case_insensitive() { + assert!(status_is_active("ACTIVE")); + assert!(status_is_active("active")); + assert!(status_is_active(" Active ")); + assert!(!status_is_active("INITIATED")); + assert!(!status_is_active("FAILED")); +} + +#[test] +fn status_terminal_matches_failure_states_only() { + for terminal in ["FAILED", "expired", "Revoked", "DELETED"] { + assert!( + status_is_terminal(terminal), + "{terminal} should be terminal" + ); + } + for live in ["ACTIVE", "INITIATED", "INITIALIZING", "INACTIVE"] { + assert!(!status_is_terminal(live), "{live} should not be terminal"); + } +} + +#[test] +fn extract_status_probes_top_level_and_nested() { + assert_eq!( + extract_status(&json!({"status": "ACTIVE"})).as_deref(), + Some("ACTIVE") + ); + assert_eq!( + extract_status(&json!({"state": {"val": {"status": "INITIATED"}}})).as_deref(), + Some("INITIATED") + ); + assert_eq!( + extract_status(&json!({"connectionData": {"val": {"status": "EXPIRED"}}})).as_deref(), + Some("EXPIRED") + ); + assert_eq!(extract_status(&json!({"other": 1})), None); +} + +#[test] +fn extract_link_fields_from_v3_shape() { + let response = json!({ + "connected_account_id": "ca_abc123", + "redirect_url": "https://backend.composio.dev/oauth/start?token=xyz", + "link_token": "lt_123", + }); + assert_eq!(extract_account_id(&response).as_deref(), Some("ca_abc123")); + assert_eq!( + extract_redirect_url(&response).as_deref(), + Some("https://backend.composio.dev/oauth/start?token=xyz") + ); +} + +#[test] +fn extract_link_fields_tolerates_legacy_shape() { + let response = json!({ + "id": "ca_legacy", + "connectionData": {"val": {"status": "INITIATED", "redirectUrl": "https://x/y"}}, + }); + assert_eq!(extract_account_id(&response).as_deref(), Some("ca_legacy")); + assert_eq!( + extract_redirect_url(&response).as_deref(), + Some("https://x/y") + ); +} + +#[test] +fn resolve_auth_config_prefers_matching_toolkit() { + let list = json!({ + "items": [ + {"id": "ac_github", "toolkit": {"slug": "github"}}, + {"id": "ac_gmail", "toolkit": {"slug": "gmail"}}, + ] + }); + assert_eq!( + resolve_auth_config_id(&list, "gmail").as_deref(), + Some("ac_gmail") + ); + assert_eq!( + resolve_auth_config_id(&list, "github").as_deref(), + Some("ac_github") + ); +} + +#[test] +fn resolve_auth_config_falls_back_to_first_when_no_slug_match() { + // Server already filtered by toolkit_slug; items may not echo a slug shape + // we recognise, so fall back to the first listed config. + let list = json!({ "items": [ {"id": "ac_only"} ] }); + assert_eq!( + resolve_auth_config_id(&list, "gmail").as_deref(), + Some("ac_only") + ); + assert_eq!(resolve_auth_config_id(&json!({"items": []}), "gmail"), None); +} + +#[test] +fn entity_store_round_trips_and_reuses_ids() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(".composio-harness.json"); + + let mut store = EntityStore::load(&path); + assert_eq!(store.get("gmail"), None); + + // First resolve for gmail with an explicit override adopts + persists it. + let gmail_id = store.entity_id_for("gmail", Some("my-entity")).unwrap(); + assert_eq!(gmail_id, "my-entity"); + + // A second toolkit with no override generates a fresh id. + let github_id = store.entity_id_for("github", None).unwrap(); + assert!(github_id.starts_with("tinycortex-")); + assert_ne!(github_id, gmail_id); + + // Reload from disk: both ids survived and are stable on re-resolve. + let mut reloaded = EntityStore::load(&path); + assert_eq!(reloaded.get("gmail"), Some("my-entity")); + assert_eq!(reloaded.get("github"), Some(github_id.as_str())); + // Stored value wins even if a different override is passed on re-run. + assert_eq!( + reloaded.entity_id_for("gmail", Some("different")).unwrap(), + "my-entity" + ); +} + +#[test] +fn entity_store_load_tolerates_missing_and_corrupt_files() { + let dir = tempfile::tempdir().unwrap(); + let missing = dir.path().join("nope.json"); + assert_eq!(EntityStore::load(&missing).get("gmail"), None); + + let corrupt = dir.path().join("corrupt.json"); + std::fs::write(&corrupt, "{ not json ").unwrap(); + assert_eq!(EntityStore::load(&corrupt).get("gmail"), None); +} + +#[tokio::test] +async fn list_auth_configs_sends_filter_and_decodes_success() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(url_path("/api/v3/auth_configs")) + .and(query_param("toolkit_slug", "gmail")) + .and(header("x-api-key", "secret")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "items": [{"id": "ac_gmail", "toolkit": {"slug": "gmail"}}] + }))) + .mount(&server) + .await; + + let value = list_auth_configs( + &reqwest::Client::new(), + &format!("{}/api/v3/", server.uri()), + "secret", + Some("gmail"), + ) + .await + .unwrap(); + + assert_eq!( + resolve_auth_config_id(&value, "gmail").as_deref(), + Some("ac_gmail") + ); +} + +#[tokio::test] +async fn list_auth_configs_redacts_error_body_and_rejects_invalid_json() { + let failure = MockServer::start().await; + Mock::given(method("GET")) + .and(url_path("/auth_configs")) + .respond_with(ResponseTemplate::new(401).set_body_string("echoed-secret")) + .mount(&failure) + .await; + let error = list_auth_configs( + &reqwest::Client::new(), + &failure.uri(), + "echoed-secret", + None, + ) + .await + .unwrap_err() + .to_string(); + assert!(error.contains("HTTP 401")); + assert!(!error.contains("echoed-secret")); + + let malformed = MockServer::start().await; + Mock::given(method("GET")) + .and(url_path("/auth_configs")) + .respond_with(ResponseTemplate::new(200).set_body_string("not-json")) + .mount(&malformed) + .await; + assert!( + list_auth_configs(&reqwest::Client::new(), &malformed.uri(), "key", None) + .await + .unwrap_err() + .to_string() + .contains("decode failed") + ); +} + +#[tokio::test] +async fn create_connection_link_sends_trimmed_callback_and_extracts_fields() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(url_path("/connected_accounts/link")) + .and(header("x-api-key", "secret")) + .and(body_partial_json(json!({ + "auth_config_id": "ac_1", + "user_id": "user_1", + "callback_url": "https://callback" + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "connected_account_id": "ca_1", + "redirect_url": "https://oauth" + }))) + .mount(&server) + .await; + + let link = create_connection_link( + &reqwest::Client::new(), + &server.uri(), + "secret", + "ac_1", + "user_1", + Some(" https://callback "), + ) + .await + .unwrap(); + assert_eq!(link.connected_account_id, "ca_1"); + assert_eq!(link.redirect_url.as_deref(), Some("https://oauth")); +} + +#[tokio::test] +async fn create_connection_link_rejects_http_decode_and_missing_id_failures() { + for (status, body, expected) in [ + (500, "server error", "HTTP 500"), + (200, "not-json", "decode failed"), + (200, "{}", "missing a connected account id"), + ] { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(url_path("/connected_accounts/link")) + .respond_with(ResponseTemplate::new(status).set_body_string(body)) + .mount(&server) + .await; + let error = create_connection_link( + &reqwest::Client::new(), + &server.uri(), + "key", + "ac", + "user", + Some(" "), + ) + .await + .unwrap_err() + .to_string(); + assert!(error.contains(expected), "unexpected error: {error}"); + } +} + +#[tokio::test] +async fn get_connection_status_handles_success_and_safe_failures() { + let active = MockServer::start().await; + Mock::given(method("GET")) + .and(url_path("/connected_accounts/ca_1")) + .and(header("x-api-key", "key")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"status": "ACTIVE"}))) + .mount(&active) + .await; + assert_eq!( + get_connection_status(&reqwest::Client::new(), &active.uri(), "key", "ca_1") + .await + .unwrap() + .as_deref(), + Some("ACTIVE") + ); + + for (status, body, expected) in [(403, "key", "HTTP 403"), (200, "invalid", "decode failed")] { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(url_path("/connected_accounts/ca_2")) + .respond_with(ResponseTemplate::new(status).set_body_string(body)) + .mount(&server) + .await; + let error = get_connection_status(&reqwest::Client::new(), &server.uri(), "key", "ca_2") + .await + .unwrap_err() + .to_string(); + assert!(error.contains(expected)); + } +} diff --git a/core/src/sync/pipelines/composio/gmail.rs b/core/src/sync/pipelines/composio/gmail.rs new file mode 100644 index 0000000..de04774 --- /dev/null +++ b/core/src/sync/pipelines/composio/gmail.rs @@ -0,0 +1,380 @@ +//! Incremental Gmail synchronization through Composio. + +use std::sync::Arc; + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use serde_json::Value; + +use super::client::{ActionExecutor, ComposioClient}; +use super::orchestrator::{ + run_incremental_sync, IncrementalSource, PageFetch, SyncItem, SyncScope, +}; +use crate::sync::composio::providers::sync_state::SyncState; +use crate::sync::pipelines::traits::PipelineConfig; +use crate::sync::pipelines::traits::{ + SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, +}; +use tinymemory_sync::email_clean; +use tinymemory_sync::email_markdown::{self as email, EmailMessage, EmailThread}; + +const ACTION_FETCH_EMAILS: &str = "GMAIL_FETCH_EMAILS"; + +pub struct GmailSyncPipeline { + executor: Arc, + connection_id: String, + max_pages: usize, + page_size: usize, + query_override: Option, +} + +impl GmailSyncPipeline { + /// Sync through a plain Composio client. + pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { + Self::with_executor(Arc::new(client), connection_id) + } + + /// Sync through a caller-supplied executor. + /// + /// The seam exists for host-side response reshaping: the Gmail envelope + /// rewrite (verbose MIME payload → one slim record per message, body + /// pre-rendered into `markdown`) lives in the host, above this crate, so + /// wrapping the executor is the only way it can reach the fetched page + /// before [`document`](SyncPipeline) turns it into a stored document. + pub fn with_executor( + executor: Arc, + connection_id: impl Into, + ) -> Self { + Self { + executor, + connection_id: connection_id.into(), + max_pages: 10, + // Gmail fetches full message payloads (`include_payload: true`), so a + // large page overflows Composio's tool-response size cap with HTTP + // 413. 25 full messages/request stays comfortably under it; callers + // needing more throughput can raise it via `with_limits`. + page_size: 25, + query_override: None, + } + } + + pub fn with_limits(mut self, max_pages: usize, page_size: usize) -> Self { + self.max_pages = max_pages.max(1); + self.page_size = page_size.max(1); + self + } + + pub fn with_query(mut self, query: impl Into) -> Self { + self.query_override = Some(query.into()); + self + } +} + +#[async_trait] +impl SyncPipeline for GmailSyncPipeline { + fn id(&self) -> &str { + "composio:gmail" + } + + fn kind(&self) -> SyncPipelineKind { + SyncPipelineKind::Composio + } + + async fn init(&self, _config: &PipelineConfig, _context: &SyncContext) -> anyhow::Result<()> { + Ok(()) + } + + async fn tick( + &self, + _config: &PipelineConfig, + context: &SyncContext, + ) -> anyhow::Result { + run_incremental_sync( + self, + self.executor.as_ref(), + &self.connection_id, + _config, + context, + ) + .await + } +} + +#[async_trait] +impl IncrementalSource for GmailSyncPipeline { + fn toolkit(&self) -> &'static str { + "gmail" + } + + fn action(&self) -> &'static str { + ACTION_FETCH_EMAILS + } + + fn max_pages(&self) -> usize { + self.max_pages + } + fn stop_on_empty_pending(&self) -> bool { + true + } + + fn server_side_depth(&self) -> bool { + true + } + + /// Gmail pages are capped by `max_results`, and full message payloads make + /// a page's size depend on what is *in* the mail — a handful of large + /// attachments is enough for the provider to refuse 25 messages it accepted + /// yesterday. Naming the argument lets the orchestrator halve it and retry + /// rather than leaving the source stuck. + fn page_size_arg_key(&self) -> Option<&'static str> { + Some("max_results") + } + + fn arguments( + &self, + _scope: &SyncScope, + config: &PipelineConfig, + state: &SyncState, + page: Option<&str>, + ) -> Value { + let mut arguments = serde_json::json!({ + "max_results": self.page_size, + "include_payload": true, + }); + if let Some(token) = page { + arguments["page_token"] = serde_json::json!(token); + } + if let Some(query) = self.query_override.as_deref() { + arguments["query"] = Value::String(query.into()); + } else if let Some(cursor) = state.cursor.as_deref() { + arguments["query"] = serde_json::json!(format!( + "after:{}", + cursor_to_seconds(cursor).unwrap_or_default() + )); + } else if let Some(days) = config.sync_depth_days { + arguments["query"] = serde_json::json!(format!( + "after:{}", + (chrono::Utc::now() - chrono::Duration::days(days as i64)).timestamp() + )); + } + arguments + } + + fn extract_page(&self, data: &Value, _page: Option<&str>) -> PageFetch { + PageFetch { + items: extract_messages(data), + next: extract_page_token(data), + } + } + + fn dedup_key(&self, item: &Value) -> Option { + item_id(item) + } + + fn sort_cursor(&self, item: &Value) -> Option { + item_cursor(item) + } + + async fn document( + &self, + _scope: &SyncScope, + connection_id: &str, + item: SyncItem, + _executor: &dyn ActionExecutor, + _state: &mut SyncState, + ) -> anyhow::Result { + let id = item_id(&item.raw).unwrap_or_else(|| item.dedup_key.clone()); + Ok(SkillDocument { + namespace_skill_id: "gmail".into(), + connection_id: connection_id.into(), + document_id: format!("gmail:{id}"), + title: message_title(&item.raw), + content: canonical_markdown(&item.raw, &id), + toolkit: "gmail".into(), + metadata: serde_json::json!({ + "source": "composio-provider-incremental", + "taint": "external_sync", + "message_id": id, + }), + }) + } +} + +fn extract_messages(data: &Value) -> Vec { + [ + "/data/messages", + "/messages", + "/data/data/messages", + "/data/items", + "/items", + ] + .iter() + .find_map(|path| data.pointer(path).and_then(Value::as_array)) + .cloned() + .unwrap_or_default() +} + +fn extract_page_token(data: &Value) -> Option { + [ + "/data/nextPageToken", + "/nextPageToken", + "/data/data/nextPageToken", + ] + .iter() + .find_map(|path| data.pointer(path).and_then(Value::as_str)) + .map(str::trim) + .filter(|token| !token.is_empty()) + .map(str::to_owned) +} + +fn item_id(message: &Value) -> Option { + ["id", "messageId", "message_id"] + .iter() + .find_map(|key| message.get(key).and_then(Value::as_str)) + .map(str::trim) + .filter(|id| !id.is_empty()) + .map(str::to_owned) +} + +fn item_cursor(message: &Value) -> Option { + ["internalDate", "internal_date", "date"] + .iter() + .find_map(|key| message.get(key).and_then(Value::as_str)) + .map(str::trim) + .filter(|cursor| !cursor.is_empty()) + .map(str::to_owned) +} + +fn message_title(message: &Value) -> String { + ["subject", "title"] + .iter() + .find_map(|key| message.get(key).and_then(Value::as_str)) + .map(str::trim) + .filter(|title| !title.is_empty()) + .unwrap_or("Gmail message") + .to_owned() +} + +/// Render one Gmail message as canonical Markdown — the same shape the memory +/// tree ingests — rather than the provider's raw JSON. +/// +/// Storing `to_string_pretty(&item.raw)` puts a MIME tree, `Received:` headers +/// and base64 part bodies into the document: the literal words of the mail are +/// either absent or split mid-token by the chunker, so recall can never match +/// them. Routing through [`email::canonicalise`] reuses the canonicaliser the +/// tree already uses — headers as a small block, body through +/// `email_clean::clean_body` (reply chains and footer boilerplate stripped). +fn canonical_markdown(message: &Value, id: &str) -> String { + let thread = email_thread(message, id); + match email::thread_markdown(thread) { + Some(markdown) => markdown, + // The thread built here always holds exactly one message, so an empty + // thread (`None`) is unreachable in practice. Degrade to the bare body + // rather than dropping the message out of memory. + None => message_body(message), + } +} + +/// Adapt one provider message into the canonicaliser's input shape. A Gmail +/// sync item is a single message, so the thread wraps exactly one. +fn email_thread(message: &Value, id: &str) -> EmailThread { + let subject = message_title(message); + EmailThread { + provider: "gmail".into(), + thread_subject: subject.clone(), + messages: vec![EmailMessage { + from: message_sender(message), + to: message_recipients(message), + cc: Vec::new(), + subject, + sent_at: message_sent_at(message), + body: message_body(message), + source_ref: Some(format!("gmail:{id}")), + list_unsubscribe: None, + }], + } +} + +/// Body text for one message, best rendering first. +/// +/// `markdown` is what the Gmail response reshaper pins onto each message (HTML +/// stripped, URLs shortened, footers removed). `messageText` is the provider's +/// own plain-text rendering, used when the reshape did not run. `snippet` is a +/// last resort: truncated, but real prose — unlike the raw payload. +fn message_body(message: &Value) -> String { + ["markdown", "markdownFormatted", "messageText", "snippet"] + .iter() + .find_map(|key| nonempty_str(message, key)) + .unwrap_or_default() +} + +/// Sender header, rendered as `From:` and used by the canonicaliser as the +/// participant key. +fn message_sender(message: &Value) -> String { + ["from", "sender"] + .iter() + .find_map(|key| nonempty_str(message, key)) + .unwrap_or_else(|| "unknown".to_owned()) +} + +/// Recipients arrive as one comma-joined header string (some responses use an +/// array); split them so the canonicaliser can render a `To:` line. +fn message_recipients(message: &Value) -> Vec { + match message.get("to") { + Some(Value::String(header)) => header + .split(',') + .map(str::trim) + .filter(|address| !address.is_empty()) + .map(str::to_owned) + .collect(), + Some(Value::Array(values)) => values + .iter() + .filter_map(Value::as_str) + .map(str::trim) + .filter(|address| !address.is_empty()) + .map(str::to_owned) + .collect(), + _ => Vec::new(), + } +} + +/// Send time, preferring the canonicaliser's own `Value`-level date parser (it +/// already knows `date`, `internalDate`, and epoch-ms-as-string) and falling +/// back to the sync cursor. The epoch is the last resort because it is +/// *deterministic*: a message the provider dated with nothing must not rewrite +/// its own content — and so re-chunk and re-embed — on every sync. +fn message_sent_at(message: &Value) -> DateTime { + email_clean::parse_message_date(message) + .or_else(|| { + item_cursor(message) + .as_deref() + .and_then(cursor_to_seconds) + .and_then(|seconds| DateTime::from_timestamp(seconds, 0)) + }) + .unwrap_or_else(|| DateTime::from_timestamp(0, 0).expect("epoch is a valid timestamp")) +} + +/// Read `key` as a trimmed, non-empty string. Unlike a plain `get(..).as_str()` +/// chain over a candidate list, a present-but-blank field falls through to the +/// next candidate instead of ending the search. +fn nonempty_str(message: &Value, key: &str) -> Option { + message + .get(key) + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_owned) +} + +fn cursor_to_seconds(cursor: &str) -> Option { + if let Ok(milliseconds) = cursor.trim().parse::() { + return Some(milliseconds / 1000); + } + chrono::DateTime::parse_from_rfc3339(cursor) + .ok() + .map(|date| date.timestamp()) +} + +#[cfg(test)] +#[path = "gmail_tests.rs"] +mod tests; diff --git a/core/src/sync/pipelines/composio/gmail_tests.rs b/core/src/sync/pipelines/composio/gmail_tests.rs new file mode 100644 index 0000000..d4de768 --- /dev/null +++ b/core/src/sync/pipelines/composio/gmail_tests.rs @@ -0,0 +1,101 @@ +//! Tests for the Gmail message → canonical Markdown adapter. + +use serde_json::json; + +use super::{canonical_markdown, message_body, message_recipients, message_sent_at}; + +/// One message in the shape the Gmail response reshaper emits: a slim envelope +/// whose body is pre-rendered into `markdown`. +fn slim_message() -> serde_json::Value { + json!({ + "id": "18f0abc", + "threadId": "18f0abc", + "subject": "Boulder visit", + "from": "Advising ", + "to": "me@example.com, second@example.com", + "date": "2026-05-02T09:15:00Z", + "labels": ["INBOX"], + "markdown": "The University of Colorado orientation is on May 20.\n\nOn Fri, 1 May 2026, someone wrote:\n> please ignore this quoted reply", + }) +} + +#[test] +fn canonical_markdown_renders_headers_and_cleaned_body() { + let content = canonical_markdown(&slim_message(), "18f0abc"); + + // The literal words of the mail — the thing recall has to match — are + // present as prose, and the headers are readable rather than a MIME tree. + assert!( + content.contains("The University of Colorado orientation is on May 20."), + "body text must survive canonicalisation: {content}" + ); + assert!( + content.contains("From: Advising "), + "{content}" + ); + assert!(content.contains("Subject: Boulder visit"), "{content}"); + assert!( + content.contains("To: me@example.com, second@example.com"), + "{content}" + ); + + // Canonicalisation is what strips the quoted reply chain. + assert!( + !content.contains("please ignore this quoted reply"), + "reply chain must be stripped by clean_body: {content}" + ); + + // Nothing JSON-shaped is left: this is the regression the fix exists for. + assert!( + !content.contains("\"markdown\""), + "raw JSON must not be stored: {content}" + ); + assert!( + !content.contains("threadId"), + "envelope keys must not be stored: {content}" + ); +} + +#[test] +fn body_falls_back_to_message_text_when_the_reshape_did_not_run() { + // No `markdown` field — the provider's own plain text is used instead. + let raw = json!({ + "id": "18f0def", + "subject": "Direct", + "messageText": "Plain provider text about Colorado.", + }); + assert_eq!(message_body(&raw), "Plain provider text about Colorado."); + assert!(canonical_markdown(&raw, "18f0def").contains("Plain provider text about Colorado.")); +} + +#[test] +fn body_skips_a_present_but_blank_field() { + // A blank `markdown` must not shadow a usable `messageText`: the candidate + // list falls through on emptiness, not just on absence. + let raw = json!({ "markdown": " ", "messageText": "real body" }); + assert_eq!(message_body(&raw), "real body"); +} + +#[test] +fn recipients_split_from_either_a_header_string_or_an_array() { + let joined = json!({ "to": "a@x.com, b@y.com" }); + assert_eq!(message_recipients(&joined), vec!["a@x.com", "b@y.com"]); + + let array = json!({ "to": ["a@x.com", " b@y.com "] }); + assert_eq!(message_recipients(&array), vec!["a@x.com", "b@y.com"]); + + assert!(message_recipients(&json!({})).is_empty()); +} + +#[test] +fn sent_at_reads_epoch_millis_and_is_deterministic_when_undated() { + // Gmail's `internalDate` is epoch millis as a string. + let dated = json!({ "internalDate": "1777712100000" }); + assert_eq!(message_sent_at(&dated).timestamp(), 1_777_712_100); + + // Undated messages must resolve to the same value every sync, otherwise the + // rendered `Date:` header changes and the document re-chunks forever. + let undated = json!({ "subject": "no date anywhere" }); + assert_eq!(message_sent_at(&undated), message_sent_at(&undated)); + assert_eq!(message_sent_at(&undated).timestamp(), 0); +} diff --git a/core/src/sync/pipelines/composio/mod.rs b/core/src/sync/pipelines/composio/mod.rs new file mode 100644 index 0000000..6c8645b --- /dev/null +++ b/core/src/sync/pipelines/composio/mod.rs @@ -0,0 +1,22 @@ +//! Composio sync, engine-free: HTTP client, connection lifecycle, the +//! incremental-sync orchestrator, and one pipeline per toolkit. + +pub mod client; +pub mod connect; +pub mod gmail; +pub mod orchestrator; +pub(crate) mod page_size; +pub mod providers; + +pub use client::{ActionExecutor, ComposioClient, ExecuteError, ExecuteResponse}; +pub use connect::{ + create_connection_link, generate_entity_id, get_connection_status, list_auth_configs, + resolve_auth_config_id, status_is_active, status_is_terminal, ConnectionLink, EntityStore, +}; +pub use gmail::GmailSyncPipeline; +pub use orchestrator::{run_incremental_sync, IncrementalSource, PageFetch, SyncItem, SyncScope}; +pub use providers::{ + ClickUpSyncPipeline, GitHubSyncPipeline, GoogleCalendarSyncPipeline, GoogleDocsSyncPipeline, + GoogleDriveSyncPipeline, GoogleSheetsSyncPipeline, LinearSyncPipeline, NotionSyncPipeline, + OutlookSyncPipeline, SlackSearchBackfillPipeline, SlackSyncPipeline, TodoistSyncPipeline, +}; diff --git a/core/src/sync/pipelines/composio/orchestrator.rs b/core/src/sync/pipelines/composio/orchestrator.rs new file mode 100644 index 0000000..3a9340d --- /dev/null +++ b/core/src/sync/pipelines/composio/orchestrator.rs @@ -0,0 +1,505 @@ +//! Shared bounded incremental synchronization control flow. + +use async_trait::async_trait; +use serde_json::Value; + +use super::client::ActionExecutor; +use super::page_size::{apply_page_size, is_payload_too_large, shrink_page_size}; +use crate::sync::composio::providers::sync_state::SyncState; +use crate::sync::pipelines::traits::PipelineConfig; +use crate::sync::pipelines::traits::{ + SkillDocument, SyncContext, SyncEvent, SyncOutcome, SyncRunError, SyncStage, +}; + +#[derive(Debug)] +pub struct PageFetch { + pub items: Vec, + pub next: Option, +} + +#[derive(Debug)] +pub struct SyncItem { + pub dedup_key: String, + pub sort_cursor: Option, + pub raw: Value, +} + +#[derive(Clone, Debug, Default)] +pub struct SyncScope { + pub id: String, + pub label: String, + pub metadata: Value, +} + +impl SyncScope { + pub fn flat() -> Self { + Self::default() + } + + pub fn named(id: impl Into, label: impl Into) -> Self { + Self { + id: id.into(), + label: label.into(), + metadata: Value::Null, + } + } + + pub fn with_metadata(mut self, metadata: Value) -> Self { + self.metadata = metadata; + self + } +} + +#[async_trait] +pub trait IncrementalSource: Send + Sync { + fn toolkit(&self) -> &'static str; + fn action(&self) -> &'static str; + fn max_pages(&self) -> usize { + 10 + } + fn per_scope_cursors(&self) -> bool { + false + } + fn tolerate_scope_errors(&self) -> bool { + false + } + fn retain_dedup_keys(&self) -> bool { + true + } + fn stop_on_empty_pending(&self) -> bool { + false + } + fn server_side_depth(&self) -> bool { + false + } + fn depth_floor(&self, config: &PipelineConfig, state: &SyncState) -> Option { + if state.cursor.is_some() { + return None; + } + config.sync_depth_days.map(|days| { + (chrono::Utc::now() - chrono::Duration::days(days as i64)) + .format("%Y-%m-%dT%H:%M:%SZ") + .to_string() + }) + } + fn advance_scope_cursor(&self, _state: &mut SyncState, _scope: &SyncScope, _cursor: &str) {} + async fn scopes( + &self, + _executor: &dyn ActionExecutor, + _connection_id: &str, + _state: &mut SyncState, + ) -> anyhow::Result> { + Ok(vec![SyncScope::flat()]) + } + /// Name of the argument that caps how many items one page requests + /// (`max_results` for Gmail), when the action has one. + /// + /// Returning `Some` opts the source into the too-large-page retry: a page + /// the provider refuses purely for size is re-requested with the cap + /// halved, instead of failing the whole run. A source whose action has no + /// such knob returns `None` and keeps the previous behaviour. + fn page_size_arg_key(&self) -> Option<&'static str> { + None + } + fn arguments( + &self, + scope: &SyncScope, + config: &PipelineConfig, + state: &SyncState, + page: Option<&str>, + ) -> Value; + fn extract_page(&self, data: &Value, page: Option<&str>) -> PageFetch; + fn dedup_key(&self, item: &Value) -> Option; + fn sort_cursor(&self, item: &Value) -> Option; + async fn document( + &self, + scope: &SyncScope, + connection_id: &str, + item: SyncItem, + executor: &dyn ActionExecutor, + state: &mut SyncState, + ) -> anyhow::Result; +} + +pub async fn run_incremental_sync( + source: &dyn IncrementalSource, + executor: &dyn ActionExecutor, + connection_id: &str, + config: &PipelineConfig, + context: &SyncContext, +) -> anyhow::Result { + let toolkit = source.toolkit(); + emit(context, toolkit, connection_id, SyncStage::Fetching, None).await; + tracing::debug!(toolkit, connection_id, "[sync:orchestrator] sync starting"); + + let mut state = SyncState::load(context.state.as_ref(), toolkit, connection_id).await?; + if state.budget_exhausted() { + tracing::debug!( + toolkit, + connection_id, + "[sync:orchestrator] daily budget exhausted" + ); + return Ok(SyncOutcome { + note: Some("daily request budget exhausted".into()), + ..SyncOutcome::default() + }); + } + + let result = match source.scopes(executor, connection_id, &mut state).await { + Ok(scopes) => { + run_pages( + source, + executor, + connection_id, + config, + context, + &mut state, + &scopes, + ) + .await + } + Err(error) => Err(error), + }; + state.last_sync_at_ms = Some(now_ms()); + if let Err(error) = state.save(context.state.as_ref()).await { + emit( + context, + toolkit, + connection_id, + SyncStage::Failed, + Some("sync state persistence failed".into()), + ) + .await; + return Err(error); + } + + match result { + Ok(outcome) => { + emit( + context, + toolkit, + connection_id, + SyncStage::Stored, + Some(format!("{} records", outcome.records_ingested)), + ) + .await; + emit(context, toolkit, connection_id, SyncStage::Completed, None).await; + tracing::debug!( + toolkit, + connection_id, + records = outcome.records_ingested, + more_pending = outcome.more_pending, + "[sync:orchestrator] sync completed" + ); + Ok(outcome) + } + Err(error) => { + tracing::warn!(toolkit, connection_id, %error, "[sync:orchestrator] sync failed"); + emit( + context, + toolkit, + connection_id, + SyncStage::Failed, + Some(error.to_string()), + ) + .await; + Err(SyncRunError::new( + error.to_string(), + state.run_requests, + state.run_provider_cost_usd, + ) + .into()) + } + } +} + +async fn run_pages( + source: &dyn IncrementalSource, + executor: &dyn ActionExecutor, + connection_id: &str, + config: &PipelineConfig, + context: &SyncContext, + state: &mut SyncState, + scopes: &[SyncScope], +) -> anyhow::Result { + let mut newest_cursor = state.cursor.clone(); + let mut ingested = 0u32; + let mut more_pending = false; + let depth_floor = (!source.server_side_depth()) + .then(|| source.depth_floor(config, state)) + .flatten(); + + // Once a page proves too large for the provider, every later page of this + // run asks for the smaller size straight away rather than paying a rejected + // round-trip to rediscover the same limit. + let mut page_size_override: Option = None; + + 'scopes: for scope in scopes { + let mut page_token = None; + let mut scope_newest_cursor: Option = None; + let mut scope_failed = false; + 'pages: for page_index in 0..source.max_pages().max(1) { + if state.budget_exhausted() { + more_pending = true; + break 'scopes; + } + let mut arguments = source.arguments(scope, config, state, page_token.as_deref()); + apply_page_size( + &mut arguments, + source.page_size_arg_key(), + page_size_override, + ); + // The size a shrink most recently *tried*. Promoted to the run's + // sticky override only once the provider accepts a page at it — + // before that it names a size that may itself be refused. + let mut attempted_page_size: Option = None; + let response = loop { + let response = match executor + .execute(source.action(), arguments.clone(), Some(connection_id)) + .await + { + Ok(response) => response, + Err(error) if source.tolerate_scope_errors() => { + if let Some(execute_error) = + error.downcast_ref::() + { + state.record_requests(execute_error.attempts); + } + tracing::warn!(toolkit = source.toolkit(), connection_id, scope = %scope.label, %error, "[sync:orchestrator] scope fetch failed; continuing"); + scope_failed = true; + break 'pages; + } + Err(error) => { + if let Some(execute_error) = + error.downcast_ref::() + { + state.record_requests(execute_error.attempts); + } + return Err(error); + } + }; + // A completed provider round-trip is billable even when its + // envelope reports failure. Transport failures return before + // this point. + state.record_action(response.attempts, response.cost_usd); + if response.successful { + // Sticky for the rest of the run, and set HERE rather than + // at the shrink: with several halvings (25 → 12 → 6) an + // assignment per attempt leaves the last *rejected* size in + // the override on every step but the final one, and is + // correct at the end only because that step happens to be + // the accepted one. Recording the accepted size makes the + // intent independent of the retry order. + if let Some(accepted) = attempted_page_size { + page_size_override = Some(accepted); + } + break response; + } + // A page refused purely for its size is the one provider + // failure a *smaller request* can fix, so shrink and retry + // instead of failing the run. Without this a single oversized + // page stops the source dead until someone notices: on one live + // workspace Gmail sync sat broken for nine days that way. + if is_payload_too_large(response.error.as_deref()) { + if state.budget_exhausted() { + more_pending = true; + break 'scopes; + } + if let Some(reduced) = + shrink_page_size(&mut arguments, source.page_size_arg_key()) + { + attempted_page_size = Some(reduced); + tracing::warn!( + toolkit = source.toolkit(), + connection_id, + scope = %scope.label, + reduced_page_size = reduced, + "[sync:orchestrator] provider refused the page as too large; retrying with a smaller page" + ); + continue; + } + } + let error = anyhow::anyhow!( + "{} provider failure: {}", + source.toolkit(), + response + .error + .unwrap_or_else(|| "unknown provider error".into()) + ); + if source.tolerate_scope_errors() { + tracing::warn!(toolkit = source.toolkit(), connection_id, scope = %scope.label, %error, "[sync:orchestrator] provider rejected scope; continuing"); + scope_failed = true; + break 'pages; + } + return Err(error); + }; + + let fetched = source.extract_page(&response.data, page_token.as_deref()); + let mut reached_cursor_boundary = false; + let mut saw_unsynced_item = false; + for raw in fetched.items { + if config.max_items.is_some_and(|limit| ingested >= limit) { + more_pending = true; + break 'scopes; + } + if state.budget_exhausted() { + more_pending = true; + break 'scopes; + } + let Some(dedup_key) = source.dedup_key(&raw) else { + continue; + }; + if state.is_synced(&dedup_key) { + continue; + } + saw_unsynced_item = true; + let sort_cursor = source.sort_cursor(&raw); + if sort_cursor + .as_deref() + .zip(depth_floor.as_deref()) + .is_some_and(|(item_cursor, floor)| item_cursor < floor) + { + reached_cursor_boundary = true; + break; + } + if !source.per_scope_cursors() + && sort_cursor + .as_deref() + .zip(state.cursor.as_deref()) + .is_some_and(|(item_cursor, persisted_cursor)| { + item_cursor <= persisted_cursor + }) + { + tracing::debug!( + toolkit = source.toolkit(), + connection_id, + scope = %scope.label, + "[sync:orchestrator] reached persisted cursor boundary" + ); + reached_cursor_boundary = true; + break; + } + let document = match source + .document( + scope, + connection_id, + SyncItem { + dedup_key: dedup_key.clone(), + sort_cursor: sort_cursor.clone(), + raw, + }, + executor, + state, + ) + .await + { + Ok(document) => document, + Err(error) if source.tolerate_scope_errors() => { + tracing::warn!(toolkit = source.toolkit(), connection_id, scope = %scope.label, %error, "[sync:orchestrator] scope document conversion failed; continuing"); + scope_failed = true; + break; + } + Err(error) => return Err(error), + }; + if let Err(error) = context.documents.store(document).await { + if source.tolerate_scope_errors() { + tracing::warn!(toolkit = source.toolkit(), connection_id, scope = %scope.label, %error, "[sync:orchestrator] scope document store failed; continuing"); + scope_failed = true; + break; + } + return Err(error); + } + if source.retain_dedup_keys() { + state.mark_synced(dedup_key); + } + if let Some(cursor) = sort_cursor { + let target = if source.per_scope_cursors() { + &mut scope_newest_cursor + } else { + &mut newest_cursor + }; + if target + .as_deref() + .is_none_or(|current| cursor.as_str() > current) + { + *target = Some(cursor); + } + } + ingested = ingested.saturating_add(1); + if config.max_items.is_some_and(|limit| ingested >= limit) { + more_pending = true; + break 'scopes; + } + } + + page_token = fetched.next; + if source.stop_on_empty_pending() && !saw_unsynced_item { + tracing::debug!( + toolkit = source.toolkit(), + connection_id, + scope = %scope.label, + "[sync:orchestrator] stopping after all-deduplicated page" + ); + break; + } + if reached_cursor_boundary { + break; + } + if page_token.is_none() { + break; + } + if page_index + 1 == source.max_pages().max(1) { + more_pending = true; + } + } + if source.per_scope_cursors() && !scope_failed && !more_pending { + if let Some(cursor) = scope_newest_cursor.as_deref() { + source.advance_scope_cursor(state, scope, cursor); + state.save(context.state.as_ref()).await?; + } + } + } + + if !source.per_scope_cursors() && !more_pending { + if let Some(cursor) = newest_cursor { + state.advance_cursor(cursor); + } + } + Ok(SyncOutcome { + records_ingested: ingested, + more_pending, + actions_called: state.run_requests, + provider_cost_usd: state.run_provider_cost_usd, + note: None, + }) +} + +async fn emit( + context: &SyncContext, + toolkit: &str, + connection_id: &str, + stage: SyncStage, + message: Option, +) { + let _ = context + .events + .emit(SyncEvent { + source_id: format!("composio:{toolkit}:{connection_id}"), + toolkit: toolkit.into(), + connection_id: Some(connection_id.into()), + stage, + message, + }) + .await; +} + +fn now_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64 +} + +#[cfg(test)] +#[path = "orchestrator_tests.rs"] +mod tests; diff --git a/core/src/sync/pipelines/composio/orchestrator_tests.rs b/core/src/sync/pipelines/composio/orchestrator_tests.rs new file mode 100644 index 0000000..ba3f46a --- /dev/null +++ b/core/src/sync/pipelines/composio/orchestrator_tests.rs @@ -0,0 +1,236 @@ +//! Tests for the too-large-page retry. + +use std::sync::{Arc, Mutex}; + +use serde_json::json; + +use super::*; +use crate::sync::composio::providers::sync_state::SyncStateStore; +use crate::sync::pipelines::composio::client::ExecuteResponse; +use crate::sync::pipelines::traits::{SkillDocSink, SkillDocument, SyncEvent, SyncEventSink}; + +/// Executor that mimics a provider with a response-size ceiling: it refuses any +/// page asking for more than `accepts` items and records every size it was +/// asked for. +struct SizeLimitedExecutor { + accepts: u64, + requested: Mutex>, +} + +impl SizeLimitedExecutor { + fn new(accepts: u64) -> Self { + Self { + accepts, + requested: Mutex::new(Vec::new()), + } + } + + fn requested_sizes(&self) -> Vec { + self.requested.lock().unwrap().clone() + } +} + +#[async_trait] +impl ActionExecutor for SizeLimitedExecutor { + async fn execute( + &self, + _action: &str, + arguments: Value, + _connection_id: Option<&str>, + ) -> anyhow::Result { + let requested = arguments + .get("max_results") + .and_then(Value::as_u64) + .unwrap_or(0); + self.requested.lock().unwrap().push(requested); + + let mut response: ExecuteResponse = serde_json::from_value(json!({})).unwrap(); + if requested > self.accepts { + response.successful = false; + response.error = Some( + "413 {\"error\":{\"message\":\"The tool response payload is too large.\",\ + \"code\":1613,\"slug\":\"Upstream_PayloadTooLarge\"}}" + .to_string(), + ); + return Ok(response); + } + response.successful = true; + response.data = json!({ + "messages": [{ "id": format!("m{requested}"), "date": "1700000000000" }], + }); + Ok(response) + } +} + +/// Minimal paged source: one page, one item, page size declared as +/// `max_results` unless `declare_page_size` is off. +struct StubSource { + declare_page_size: bool, + initial_page_size: u64, +} + +#[async_trait] +impl IncrementalSource for StubSource { + fn toolkit(&self) -> &'static str { + "stub" + } + fn action(&self) -> &'static str { + "STUB_FETCH" + } + fn max_pages(&self) -> usize { + 1 + } + fn page_size_arg_key(&self) -> Option<&'static str> { + self.declare_page_size.then_some("max_results") + } + fn arguments( + &self, + _scope: &SyncScope, + _config: &PipelineConfig, + _state: &SyncState, + _page: Option<&str>, + ) -> Value { + json!({ "max_results": self.initial_page_size }) + } + fn extract_page(&self, data: &Value, _page: Option<&str>) -> PageFetch { + PageFetch { + items: data + .get("messages") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(), + next: None, + } + } + fn dedup_key(&self, item: &Value) -> Option { + item.get("id").and_then(Value::as_str).map(str::to_string) + } + fn sort_cursor(&self, item: &Value) -> Option { + item.get("date").and_then(Value::as_str).map(str::to_string) + } + async fn document( + &self, + _scope: &SyncScope, + connection_id: &str, + item: SyncItem, + _executor: &dyn ActionExecutor, + _state: &mut SyncState, + ) -> anyhow::Result { + Ok(SkillDocument { + namespace_skill_id: "stub".into(), + connection_id: connection_id.into(), + document_id: item.dedup_key, + title: "stub".into(), + content: "stub".into(), + toolkit: "stub".into(), + metadata: Value::Null, + }) + } +} + +#[derive(Default)] +struct NoopHost(Mutex>); + +#[async_trait] +impl SkillDocSink for NoopHost { + async fn store(&self, _: SkillDocument) -> anyhow::Result<()> { + Ok(()) + } + async fn delete(&self, _: &str, _: &str) -> anyhow::Result<()> { + Ok(()) + } +} + +#[async_trait] +impl SyncEventSink for NoopHost { + async fn emit(&self, _: SyncEvent) -> anyhow::Result<()> { + Ok(()) + } +} + +#[async_trait] +impl SyncStateStore for NoopHost { + async fn get(&self, namespace: &str, key: &str) -> anyhow::Result> { + Ok(self + .0 + .lock() + .unwrap() + .get(&format!("{namespace}:{key}")) + .cloned()) + } + async fn set(&self, namespace: &str, key: &str, value: &Value) -> anyhow::Result<()> { + self.0 + .lock() + .unwrap() + .insert(format!("{namespace}:{key}"), value.clone()); + Ok(()) + } +} + +fn context() -> SyncContext { + let host = Arc::new(NoopHost::default()); + SyncContext { + events: host.clone(), + documents: host.clone(), + state: host, + } +} + +#[tokio::test] +async fn an_oversized_page_is_halved_until_the_provider_accepts_it() { + // The provider takes 6 at a time; the source asks for 25. + let executor = SizeLimitedExecutor::new(6); + let source = StubSource { + declare_page_size: true, + initial_page_size: 25, + }; + + let outcome = run_incremental_sync( + &source, + &executor, + "conn-1", + &PipelineConfig::default(), + &context(), + ) + .await + .expect("a too-large page must not fail the run"); + + assert_eq!( + outcome.records_ingested, 1, + "the page is ingested after the retry" + ); + assert_eq!( + executor.requested_sizes(), + vec![25, 12, 6], + "each rejection halves the request until it fits" + ); +} + +#[tokio::test] +async fn a_source_without_a_page_size_argument_still_fails_fast() { + // No `page_size_arg_key` — there is nothing to shrink, so the old + // behaviour (surface the provider failure) must be preserved rather than + // looping on a request that can never change. + let executor = SizeLimitedExecutor::new(6); + let source = StubSource { + declare_page_size: false, + initial_page_size: 25, + }; + + let error = run_incremental_sync( + &source, + &executor, + "conn-1", + &PipelineConfig::default(), + &context(), + ) + .await + .expect_err("an unshrinkable too-large page is still a failure"); + + assert!(error.to_string().contains("provider failure"), "{error}"); + assert_eq!( + executor.requested_sizes(), + vec![25], + "no pointless retry of an identical request" + ); +} diff --git a/core/src/sync/pipelines/composio/page_size.rs b/core/src/sync/pipelines/composio/page_size.rs new file mode 100644 index 0000000..d970493 --- /dev/null +++ b/core/src/sync/pipelines/composio/page_size.rs @@ -0,0 +1,79 @@ +//! Page-size retry for a provider that refuses a page for its size. +//! +//! A page rejected purely because the response is too big is the one provider +//! failure a *smaller request* can fix. The orchestrator halves the page-size +//! argument and retries rather than failing the source, so a single oversized +//! page cannot stop a sync dead — on one live workspace a Gmail sync sat broken +//! for nine days that way. + +use serde_json::Value; + +/// Smallest page a shrink will ask for. One item is the point past which a +/// too-large response is about that single item, not the batch size. +pub(super) const MIN_PAGE_SIZE: u64 = 1; + +/// Whether the provider refused a page for its *size* rather than for anything +/// about the request's content — the only failure a smaller page can fix. +/// +/// Matched on the error text because that is all the envelope carries: Composio +/// reports it as HTTP 413 with a `Upstream_PayloadTooLarge` slug, and other +/// backends phrase it as "payload too large" / "response too large". +pub(super) fn is_payload_too_large(error: Option<&str>) -> bool { + error.is_some_and(|error| { + let lower = error.to_ascii_lowercase(); + lower.contains("payloadtoolarge") + || lower.contains("payload_too_large") + || mentions_status_413(&lower) + || (lower.contains("too large") + && (lower.contains("payload") || lower.contains("response"))) + }) +} + +/// Whether `text` names HTTP 413 as a status code. +/// +/// The digits have to stand alone. An unanchored `contains("413")` also matches +/// a message id, an amount, or a timestamp that merely contains those three +/// digits, and every such match costs a shrink-and-retry cycle before the real +/// error is finally surfaced — on a failure that a smaller page was never going +/// to fix. +fn mentions_status_413(lower: &str) -> bool { + lower.match_indices("413").any(|(at, _)| { + let before_is_digit = lower[..at] + .chars() + .next_back() + .is_some_and(|c| c.is_ascii_digit()); + let after_is_digit = lower[at + 3..] + .chars() + .next() + .is_some_and(|c| c.is_ascii_digit()); + !before_is_digit && !after_is_digit + }) +} + +/// Pin the page-size argument to `size`, if the source declared one. +pub(super) fn apply_page_size(arguments: &mut Value, key: Option<&str>, size: Option) { + if let (Some(key), Some(size)) = (key, size) { + if let Some(slot) = arguments.get_mut(key) { + *slot = Value::from(size); + } + } +} + +/// Halve the page-size argument in place, returning the new value. +/// +/// `None` means retrying is pointless — the source declares no page-size +/// argument, this request does not carry it, or it is already at the floor. +pub(super) fn shrink_page_size(arguments: &mut Value, key: Option<&str>) -> Option { + let key = key?; + let current = arguments.get(key)?.as_u64()?; + if current <= MIN_PAGE_SIZE { + return None; + } + let reduced = (current / 2).max(MIN_PAGE_SIZE); + arguments[key] = Value::from(reduced); + Some(reduced) +} + +#[cfg(test)] +#[path = "page_size_tests.rs"] +mod tests; diff --git a/core/src/sync/pipelines/composio/page_size_tests.rs b/core/src/sync/pipelines/composio/page_size_tests.rs new file mode 100644 index 0000000..6328c7b --- /dev/null +++ b/core/src/sync/pipelines/composio/page_size_tests.rs @@ -0,0 +1,51 @@ +//! Tests for the page-size retry helpers. + +use serde_json::json; + +use super::*; + +#[test] +fn payload_too_large_is_told_apart_from_other_provider_errors() { + assert!(is_payload_too_large(Some( + "413 {\"slug\":\"Upstream_PayloadTooLarge\"}" + ))); + assert!(is_payload_too_large(Some("Response too large for tool"))); + assert!(!is_payload_too_large(Some("rate limit exceeded"))); + assert!(!is_payload_too_large(Some("invalid grant"))); + assert!(!is_payload_too_large(None)); +} + +#[test] +fn shrinking_stops_at_the_floor() { + let mut arguments = json!({ "max_results": 3 }); + assert_eq!( + shrink_page_size(&mut arguments, Some("max_results")), + Some(1) + ); + assert_eq!(arguments["max_results"], json!(1)); + // At one item per page there is nothing left to halve. + assert_eq!(shrink_page_size(&mut arguments, Some("max_results")), None); + // A source that declares no key, or a request that lacks it, cannot shrink. + assert_eq!(shrink_page_size(&mut arguments, None), None); + assert_eq!( + shrink_page_size(&mut json!({ "other": 10 }), Some("max_results")), + None + ); +} + +/// The digits have to stand alone. Every false positive here costs a +/// shrink-and-retry cycle on a failure a smaller page was never going to fix, +/// and delays the real error reaching the caller. +#[test] +fn a_number_that_merely_contains_413_is_not_a_status_code() { + assert!(is_payload_too_large(Some("HTTP 413 Payload Too Large"))); + assert!(is_payload_too_large(Some("upstream returned 413."))); + assert!(is_payload_too_large(Some("(413)"))); + + assert!(!is_payload_too_large(Some("message id 4130 not found"))); + assert!(!is_payload_too_large(Some("amount 1413 exceeds the cap"))); + assert!(!is_payload_too_large(Some("thread 94137 is archived"))); + assert!(!is_payload_too_large(Some( + "at 1782891413 the token expired" + ))); +} diff --git a/core/src/sync/pipelines/composio/providers/clickup.rs b/core/src/sync/pipelines/composio/providers/clickup.rs new file mode 100644 index 0000000..d16176e --- /dev/null +++ b/core/src/sync/pipelines/composio/providers/clickup.rs @@ -0,0 +1,196 @@ +use async_trait::async_trait; +use serde_json::Value; + +use super::common::{checked_execute, document, first_array, pick_str}; +use crate::sync::composio::providers::sync_state::SyncState; +use crate::sync::pipelines::composio::{ + run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, + SyncScope, +}; +use crate::sync::pipelines::traits::PipelineConfig; +use crate::sync::pipelines::traits::{ + SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, +}; + +const ACTION_USER: &str = "CLICKUP_GET_AUTHORIZED_USER"; +const ACTION_WORKSPACES: &str = "CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES"; +const ACTION_TASKS: &str = "CLICKUP_GET_FILTERED_TEAM_TASKS"; + +pub struct ClickUpSyncPipeline { + client: ComposioClient, + connection_id: String, + max_pages: usize, + page_size: usize, +} + +impl ClickUpSyncPipeline { + pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { + Self { + client, + connection_id: connection_id.into(), + max_pages: 20, + page_size: 50, + } + } +} + +#[async_trait] +impl SyncPipeline for ClickUpSyncPipeline { + fn id(&self) -> &str { + "composio:clickup" + } + fn kind(&self) -> SyncPipelineKind { + SyncPipelineKind::Composio + } + async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { + Ok(()) + } + async fn tick( + &self, + config: &PipelineConfig, + context: &SyncContext, + ) -> anyhow::Result { + run_incremental_sync(self, &self.client, &self.connection_id, config, context).await + } +} + +#[async_trait] +impl IncrementalSource for ClickUpSyncPipeline { + fn toolkit(&self) -> &'static str { + "clickup" + } + fn action(&self) -> &'static str { + ACTION_TASKS + } + fn max_pages(&self) -> usize { + self.max_pages + } + fn depth_floor(&self, config: &PipelineConfig, state: &SyncState) -> Option { + if state.cursor.is_some() { + return None; + } + config.sync_depth_days.map(|days| { + (chrono::Utc::now() - chrono::Duration::days(days as i64)) + .timestamp_millis() + .to_string() + }) + } + async fn scopes( + &self, + executor: &dyn ActionExecutor, + connection_id: &str, + state: &mut SyncState, + ) -> anyhow::Result> { + let user_response = checked_execute( + executor, + ACTION_USER, + serde_json::json!({}), + connection_id, + state, + ) + .await?; + let user_id = ["/user/id", "/data/user/id", "/id", "/data/id"] + .iter() + .find_map(|path| user_response.data.pointer(path)) + .and_then(value_string) + .ok_or_else(|| anyhow::anyhow!("{ACTION_USER} returned no user id"))?; + if state.budget_exhausted() { + return Ok(Vec::new()); + } + let workspace_response = checked_execute( + executor, + ACTION_WORKSPACES, + serde_json::json!({}), + connection_id, + state, + ) + .await?; + let workspaces = first_array( + &workspace_response.data, + &["/teams", "/data/teams", "/workspaces", "/data/workspaces"], + ); + Ok(workspaces + .into_iter() + .filter_map(|workspace| pick_str(&workspace, &["id", "team_id", "workspace_id"])) + .map(|id| { + SyncScope::named(id.clone(), format!("workspace:{id}")) + .with_metadata(serde_json::json!({"user_id": user_id})) + }) + .collect()) + } + fn arguments( + &self, + scope: &SyncScope, + _: &PipelineConfig, + _: &SyncState, + page: Option<&str>, + ) -> Value { + serde_json::json!({"team_id": scope.id, "assignees": [scope.metadata.get("user_id").and_then(Value::as_str).unwrap_or_default()], "order_by": "updated", "reverse": true, "page": page.and_then(|value| value.parse::().ok()).unwrap_or(0), "page_size": self.page_size, "subtasks": true}) + } + fn extract_page(&self, data: &Value, page: Option<&str>) -> PageFetch { + let items = first_array( + data, + &[ + "/data/tasks", + "/tasks", + "/data/data/tasks", + "/data/results", + "/results", + "/data/items", + "/items", + ], + ); + let page_number = page + .and_then(|value| value.parse::().ok()) + .unwrap_or(0); + let next = (items.len() == self.page_size).then(|| (page_number + 1).to_string()); + PageFetch { items, next } + } + fn dedup_key(&self, item: &Value) -> Option { + let id = pick_str(item, &["id", "data.id", "task_id", "data.task_id"])?; + Some(match self.sort_cursor(item) { + Some(updated) => format!("{id}@{updated}"), + None => id, + }) + } + fn sort_cursor(&self, item: &Value) -> Option { + pick_str( + item, + &[ + "date_updated", + "data.date_updated", + "updated_at", + "data.updated_at", + "dateUpdated", + "data.dateUpdated", + ], + ) + } + async fn document( + &self, + scope: &SyncScope, + connection_id: &str, + item: SyncItem, + _: &dyn ActionExecutor, + _: &mut SyncState, + ) -> anyhow::Result { + let id = pick_str(&item.raw, &["id", "data.id", "task_id", "data.task_id"]) + .unwrap_or_else(|| item.dedup_key.clone()); + let title = pick_str(&item.raw, &["name", "data.name", "title", "data.title"]) + .unwrap_or_else(|| format!("ClickUp task {id}")); + let content = serde_json::to_string_pretty(&item.raw)?; + let mut result = document("clickup", connection_id, &id, title, content, item.raw); + result.metadata["workspace_id"] = Value::String(scope.id.clone()); + Ok(result) + } +} + +fn value_string(value: &Value) -> Option { + match value { + Value::String(value) => Some(value.clone()), + Value::Number(value) => Some(value.to_string()), + _ => None, + } + .map(|value| value.trim().to_owned()) + .filter(|value| !value.is_empty()) +} diff --git a/core/src/sync/pipelines/composio/providers/common.rs b/core/src/sync/pipelines/composio/providers/common.rs new file mode 100644 index 0000000..a76fa51 --- /dev/null +++ b/core/src/sync/pipelines/composio/providers/common.rs @@ -0,0 +1,110 @@ +use serde_json::Value; + +use crate::sync::pipelines::traits::SkillDocument; + +/// Walk a JSON document by dotted path and return the first non-empty scalar. +/// +/// # Not interchangeable with [`normalize::helpers::pick_str`] +/// +/// A second `pick_str` lives in [`normalize::helpers`], and the two differ. +/// This one resolves paths with [`Value::pointer`] (so a numeric segment +/// indexes into an array) and **coerces `Number` to its string form**; that +/// one walks with [`Value::get`] (objects only) and returns `None` for any +/// non-string leaf. Swapping one for the other changes what normalisers emit +/// for numeric fields. Keep them separate. +/// +/// [`normalize::helpers`]: tinymemory_sync::helpers +/// [`normalize::helpers::pick_str`]: tinymemory_sync::helpers::pick_str +pub fn pick_str(value: &Value, paths: &[&str]) -> Option { + paths.iter().find_map(|path| { + let pointer = format!("/{}", path.replace('.', "/")); + value + .pointer(&pointer) + .and_then(|value| match value { + Value::String(value) => Some(value.clone()), + Value::Number(value) => Some(value.to_string()), + _ => None, + }) + .map(|value| value.trim().to_owned()) + .filter(|value| !value.is_empty()) + }) +} + +pub fn first_array(data: &Value, pointers: &[&str]) -> Vec { + pointers + .iter() + .find_map(|pointer| data.pointer(pointer).and_then(Value::as_array)) + .cloned() + .unwrap_or_default() +} + +/// Reads a Google-style `nextPageToken` from the common Composio response +/// envelopes (single- and double-`data`-wrapped), trimming and dropping empty +/// tokens. Shared by the Google provider pipelines to avoid drift. +pub fn next_page_token(data: &Value) -> Option { + [ + "/data/nextPageToken", + "/nextPageToken", + "/data/data/nextPageToken", + ] + .iter() + .find_map(|path| data.pointer(path).and_then(Value::as_str)) + .map(str::trim) + .filter(|token| !token.is_empty()) + .map(str::to_owned) +} + +pub fn document( + toolkit: &str, + connection_id: &str, + id: &str, + title: String, + content: String, + raw: Value, +) -> SkillDocument { + SkillDocument { + namespace_skill_id: toolkit.into(), + connection_id: connection_id.into(), + document_id: format!("{toolkit}:{id}"), + title, + content, + toolkit: toolkit.into(), + metadata: serde_json::json!({ + "source": "composio-provider-incremental", + "taint": "external_sync", + "provider_id": id, + "raw": raw, + }), + } +} + +pub async fn checked_execute( + executor: &dyn super::super::client::ActionExecutor, + action: &str, + arguments: Value, + connection_id: &str, + state: &mut crate::sync::composio::providers::sync_state::SyncState, +) -> anyhow::Result { + let response = match executor + .execute(action, arguments, Some(connection_id)) + .await + { + Ok(response) => response, + Err(error) => { + if let Some(error) = error.downcast_ref::() { + state.record_requests(error.attempts); + } + return Err(error); + } + }; + state.record_action(response.attempts, response.cost_usd); + anyhow::ensure!( + response.successful, + "{action} provider failure: {}", + response + .error + .as_deref() + .unwrap_or("unknown provider error") + ); + Ok(response) +} diff --git a/core/src/sync/pipelines/composio/providers/github.rs b/core/src/sync/pipelines/composio/providers/github.rs new file mode 100644 index 0000000..032935a --- /dev/null +++ b/core/src/sync/pipelines/composio/providers/github.rs @@ -0,0 +1,178 @@ +use async_trait::async_trait; +use serde_json::Value; + +use super::common::{checked_execute, document, first_array, pick_str}; +use crate::sync::composio::providers::sync_state::SyncState; +use crate::sync::pipelines::composio::{ + run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, + SyncScope, +}; +use crate::sync::pipelines::traits::PipelineConfig; +use crate::sync::pipelines::traits::{ + SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, +}; + +const ACTION_USER: &str = "GITHUB_GET_THE_AUTHENTICATED_USER"; +const ACTION_SEARCH: &str = "GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS"; + +pub struct GitHubSyncPipeline { + client: ComposioClient, + connection_id: String, + max_pages: usize, + page_size: usize, +} + +impl GitHubSyncPipeline { + pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { + Self { + client, + connection_id: connection_id.into(), + max_pages: 20, + page_size: 50, + } + } +} + +#[async_trait] +impl SyncPipeline for GitHubSyncPipeline { + fn id(&self) -> &str { + "composio:github" + } + fn kind(&self) -> SyncPipelineKind { + SyncPipelineKind::Composio + } + async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { + Ok(()) + } + async fn tick( + &self, + config: &PipelineConfig, + context: &SyncContext, + ) -> anyhow::Result { + run_incremental_sync(self, &self.client, &self.connection_id, config, context).await + } +} + +#[async_trait] +impl IncrementalSource for GitHubSyncPipeline { + fn toolkit(&self) -> &'static str { + "github" + } + fn action(&self) -> &'static str { + ACTION_SEARCH + } + fn max_pages(&self) -> usize { + self.max_pages + } + fn server_side_depth(&self) -> bool { + true + } + async fn scopes( + &self, + executor: &dyn ActionExecutor, + connection_id: &str, + state: &mut SyncState, + ) -> anyhow::Result> { + let response = checked_execute( + executor, + ACTION_USER, + serde_json::json!({}), + connection_id, + state, + ) + .await?; + let login = pick_str(&response.data, &["login", "data.login"]) + .ok_or_else(|| anyhow::anyhow!("{ACTION_USER} returned no login"))?; + Ok(vec![SyncScope::named( + login.clone(), + format!("involves:{login}"), + )]) + } + fn arguments( + &self, + scope: &SyncScope, + config: &PipelineConfig, + state: &SyncState, + page: Option<&str>, + ) -> Value { + let mut query = format!("involves:{}", scope.id); + if let Some(cursor) = state.cursor.as_deref() { + query.push_str(&format!(" updated:>{cursor}")); + } else if let Some(days) = config.sync_depth_days { + let floor = chrono::Utc::now() - chrono::Duration::days(days as i64); + query.push_str(&format!(" updated:>{}", floor.format("%Y-%m-%dT%H:%M:%SZ"))); + } + serde_json::json!({ "q": query, "sort": "updated", "order": "desc", "per_page": self.page_size, "page": page.and_then(|value| value.parse::().ok()).unwrap_or(1) }) + } + fn extract_page(&self, data: &Value, page: Option<&str>) -> PageFetch { + let items = first_array( + data, + &[ + "/data/items", + "/items", + "/data/data/items", + "/data/results", + "/results", + ], + ); + let page_number = page + .and_then(|value| value.parse::().ok()) + .unwrap_or(1); + let next = (items.len() == self.page_size).then(|| (page_number + 1).to_string()); + PageFetch { items, next } + } + fn dedup_key(&self, item: &Value) -> Option { + let id = issue_id(item)?; + Some(match self.sort_cursor(item) { + Some(updated) => format!("{id}@{updated}"), + None => id, + }) + } + fn sort_cursor(&self, item: &Value) -> Option { + pick_str( + item, + &[ + "updated_at", + "data.updated_at", + "updatedAt", + "data.updatedAt", + ], + ) + } + async fn document( + &self, + _: &SyncScope, + connection_id: &str, + item: SyncItem, + _: &dyn ActionExecutor, + _: &mut SyncState, + ) -> anyhow::Result { + let id = issue_id(&item.raw).unwrap_or_else(|| item.dedup_key.clone()); + let title = pick_str(&item.raw, &["title", "data.title"]) + .unwrap_or_else(|| format!("GitHub issue {id}")); + let content = serde_json::to_string_pretty(&item.raw)?; + Ok(document( + "github", + connection_id, + &id, + title, + content, + item.raw, + )) + } +} + +fn issue_id(item: &Value) -> Option { + pick_str(item, &["id", "data.id"]).or_else(|| { + let url = pick_str(item, &["html_url", "data.html_url", "url", "data.url"])?; + let parts: Vec<_> = url.trim_end_matches('/').split('/').collect(); + (parts.len() >= 7).then(|| { + format!( + "{}/{}#{}", + parts[parts.len() - 4], + parts[parts.len() - 3], + parts[parts.len() - 1] + ) + }) + }) +} diff --git a/core/src/sync/pipelines/composio/providers/google_calendar.rs b/core/src/sync/pipelines/composio/providers/google_calendar.rs new file mode 100644 index 0000000..6e0a1f3 --- /dev/null +++ b/core/src/sync/pipelines/composio/providers/google_calendar.rs @@ -0,0 +1,173 @@ +use async_trait::async_trait; +use serde_json::Value; + +use super::common::{document, first_array, next_page_token, pick_str}; +use crate::sync::composio::providers::sync_state::SyncState; +use crate::sync::pipelines::composio::{ + run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, + SyncScope, +}; +use crate::sync::pipelines::traits::PipelineConfig; +use crate::sync::pipelines::traits::{ + SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, +}; + +const ACTION_EVENTS_LIST: &str = "GOOGLECALENDAR_EVENTS_LIST"; + +/// Incremental Google Calendar synchronization through Composio. +/// +/// Events are self-contained records (stable id + `updated` timestamp), so this +/// follows the document-shaped pattern (`LinearSyncPipeline`) rather than the +/// message-shaped one: a single list action, client-visible `updated` cursor, +/// content taken directly from the event payload with no secondary fetch. +pub struct GoogleCalendarSyncPipeline { + client: ComposioClient, + connection_id: String, + calendar_id: String, + max_pages: usize, + page_size: usize, +} + +impl GoogleCalendarSyncPipeline { + pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { + Self { + client, + connection_id: connection_id.into(), + calendar_id: "primary".into(), + max_pages: 10, + page_size: 50, + } + } + + pub fn with_limits(mut self, max_pages: usize, page_size: usize) -> Self { + self.max_pages = max_pages.max(1); + // Google Calendar caps `maxResults` at 2500; stay well under it. + self.page_size = page_size.clamp(1, 2500); + self + } +} + +#[async_trait] +impl SyncPipeline for GoogleCalendarSyncPipeline { + fn id(&self) -> &str { + "composio:googlecalendar" + } + fn kind(&self) -> SyncPipelineKind { + SyncPipelineKind::Composio + } + async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { + Ok(()) + } + async fn tick( + &self, + config: &PipelineConfig, + context: &SyncContext, + ) -> anyhow::Result { + run_incremental_sync(self, &self.client, &self.connection_id, config, context).await + } +} + +#[async_trait] +impl IncrementalSource for GoogleCalendarSyncPipeline { + fn toolkit(&self) -> &'static str { + "googlecalendar" + } + fn action(&self) -> &'static str { + ACTION_EVENTS_LIST + } + fn max_pages(&self) -> usize { + self.max_pages + } + // NB: `stop_on_empty_pending` is left at its default (false). The cursor only + // advances on a *complete* sync, so a run capped by `max_pages`/budget leaves + // it unadvanced; stopping early on an all-deduplicated first page would then + // permanently skip the still-unsynced tail. The persisted-cursor boundary + // already halts incremental runs at the right point. + fn server_side_depth(&self) -> bool { + true + } + fn arguments( + &self, + _: &SyncScope, + config: &PipelineConfig, + state: &SyncState, + page: Option<&str>, + ) -> Value { + // `single_events` expands recurring series into concrete instances so + // each carries a stable id; `order_by: "updated"` sorts ascending by + // modification time (oldest change first), matching the `updated` cursor. + let mut args = serde_json::json!({ + "calendar_id": self.calendar_id, + "max_results": self.page_size, + "single_events": true, + "order_by": "updated", + }); + if let Some(page) = page { + args["page_token"] = serde_json::json!(page); + } + // The cursor is a modification time (the item `updated` field), so it + // belongs on `updated_min` (last-modified lower bound) — NOT `time_min`, + // which filters by event *start* time and would drop recently-edited + // past events. `time_min` is only the start-time horizon for the first, + // cursorless backfill; once a cursor exists, `updated_min` fully bounds + // the incremental window. + if let Some(cursor) = state.cursor.as_deref() { + args["updated_min"] = serde_json::json!(cursor); + } else if let Some(days) = config.sync_depth_days { + args["time_min"] = serde_json::json!((chrono::Utc::now() + - chrono::Duration::days(days as i64)) + .to_rfc3339()); + } + args + } + fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { + PageFetch { + items: first_array( + data, + &[ + "/data/items", + "/items", + "/data/data/items", + "/data/events", + "/events", + ], + ), + next: next_page_token(data), + } + } + fn dedup_key(&self, item: &Value) -> Option { + let id = pick_str(item, &["id", "data.id", "iCalUID", "data.iCalUID"])?; + Some(match self.sort_cursor(item) { + Some(updated) => format!("{id}@{updated}"), + None => id, + }) + } + fn sort_cursor(&self, item: &Value) -> Option { + pick_str(item, &["updated", "data.updated"]) + } + async fn document( + &self, + _: &SyncScope, + connection_id: &str, + item: SyncItem, + _: &dyn ActionExecutor, + _: &mut SyncState, + ) -> anyhow::Result { + let id = pick_str(&item.raw, &["id", "data.id", "iCalUID", "data.iCalUID"]) + .unwrap_or_else(|| item.dedup_key.clone()); + let title = pick_str( + &item.raw, + &["summary", "data.summary", "title", "data.title"], + ) + .unwrap_or_else(|| format!("Calendar event {id}")); + let content = serde_json::to_string_pretty(&item.raw)?; + Ok(document( + "googlecalendar", + connection_id, + &id, + title, + content, + item.raw, + )) + } +} diff --git a/core/src/sync/pipelines/composio/providers/google_docs.rs b/core/src/sync/pipelines/composio/providers/google_docs.rs new file mode 100644 index 0000000..c2776fb --- /dev/null +++ b/core/src/sync/pipelines/composio/providers/google_docs.rs @@ -0,0 +1,186 @@ +use async_trait::async_trait; +use serde_json::Value; + +use super::common::{checked_execute, document, first_array, pick_str}; +use crate::sync::composio::providers::sync_state::SyncState; +use crate::sync::pipelines::composio::{ + run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, + SyncScope, +}; +use crate::sync::pipelines::traits::PipelineConfig; +use crate::sync::pipelines::traits::{ + SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, +}; + +const ACTION_SEARCH: &str = "GOOGLEDOCS_SEARCH_DOCUMENTS"; +const ACTION_PLAINTEXT: &str = "GOOGLEDOCS_GET_DOCUMENT_PLAINTEXT"; + +/// Incremental Google Docs synchronization through Composio. +/// +/// Two-step, document-shaped (like `NotionSyncPipeline`): `GOOGLEDOCS_SEARCH_DOCUMENTS` +/// enumerates accessible documents, then `GOOGLEDOCS_GET_DOCUMENT_PLAINTEXT` fetches the +/// body for each item inside [`IncrementalSource::document`]. +pub struct GoogleDocsSyncPipeline { + client: ComposioClient, + connection_id: String, + max_pages: usize, + page_size: usize, +} + +impl GoogleDocsSyncPipeline { + pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { + Self { + client, + connection_id: connection_id.into(), + // NOTE: SEARCH_DOCUMENTS' page-token arg name is not pinned by the + // curated catalog, so we do a single-page-per-tick fetch (no page + // token emitted) rather than guessing a pagination scheme. Capped at + // 1 page: since `arguments()` never advances the token, a >1 cap + // would re-fire the identical page-1 request and burn budget slots + // for silently-deduplicated items. + max_pages: 1, + page_size: 25, + } + } +} + +#[async_trait] +impl SyncPipeline for GoogleDocsSyncPipeline { + fn id(&self) -> &str { + "composio:googledocs" + } + fn kind(&self) -> SyncPipelineKind { + SyncPipelineKind::Composio + } + async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { + Ok(()) + } + async fn tick( + &self, + config: &PipelineConfig, + context: &SyncContext, + ) -> anyhow::Result { + run_incremental_sync(self, &self.client, &self.connection_id, config, context).await + } +} + +#[async_trait] +impl IncrementalSource for GoogleDocsSyncPipeline { + fn toolkit(&self) -> &'static str { + "googledocs" + } + fn action(&self) -> &'static str { + ACTION_SEARCH + } + fn max_pages(&self) -> usize { + self.max_pages + } + fn arguments( + &self, + _: &SyncScope, + _: &PipelineConfig, + _: &SyncState, + _page: Option<&str>, + ) -> Value { + // NOTE: an empty/broad `query` enumerates every accessible document; + // `max_results` bounds the batch. Both mirror the underlying Drive + // search parameters. No page token is emitted (see `max_pages`). + serde_json::json!({"query": "", "max_results": self.page_size}) + } + fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { + PageFetch { + items: first_array( + data, + &[ + "/data/documents", + "/documents", + "/data/files", + "/files", + "/data/results", + "/results", + "/data/items", + "/items", + ], + ), + // Bounded fetch: no page token consumed (see `max_pages`). The + // pointers are read defensively should Composio surface one. + next: [ + "/data/nextPageToken", + "/nextPageToken", + "/data/next_page_token", + "/next_page_token", + ] + .iter() + .find_map(|path| data.pointer(path).and_then(Value::as_str)) + .map(str::trim) + .filter(|token| !token.is_empty()) + .map(str::to_owned), + } + } + fn dedup_key(&self, item: &Value) -> Option { + let id = pick_str(item, &["id", "data.id", "documentId", "data.documentId"])?; + Some(match self.sort_cursor(item) { + Some(modified) => format!("{id}@{modified}"), + None => id, + }) + } + fn sort_cursor(&self, item: &Value) -> Option { + pick_str( + item, + &[ + "modifiedTime", + "data.modifiedTime", + "modified_time", + "updatedTime", + ], + ) + } + async fn document( + &self, + _: &SyncScope, + connection_id: &str, + item: SyncItem, + executor: &dyn ActionExecutor, + state: &mut SyncState, + ) -> anyhow::Result { + let id = pick_str( + &item.raw, + &["id", "data.id", "documentId", "data.documentId"], + ) + .unwrap_or_else(|| item.dedup_key.clone()); + let title = pick_str(&item.raw, &["title", "data.title", "name", "data.name"]) + .unwrap_or_else(|| format!("Google Doc {id}")); + // NOTE: GET_DOCUMENT_PLAINTEXT identifies the doc by an id argument; + // Composio commonly keys this as "id" (or "document_id"). We send "id". + let response = checked_execute( + executor, + ACTION_PLAINTEXT, + serde_json::json!({"id": id}), + connection_id, + state, + ) + .await?; + let content = [ + "/data/text", + "/text", + "/data/plaintext", + "/plaintext", + "/data/content", + "/content", + "/data/response_data/text", + ] + .iter() + .find_map(|path| response.data.pointer(path).and_then(Value::as_str)) + .filter(|value| !value.trim().is_empty()) + .map(str::to_owned) + .unwrap_or(serde_json::to_string_pretty(&item.raw)?); + Ok(document( + "googledocs", + connection_id, + &id, + title, + content, + item.raw, + )) + } +} diff --git a/core/src/sync/pipelines/composio/providers/google_drive.rs b/core/src/sync/pipelines/composio/providers/google_drive.rs new file mode 100644 index 0000000..d619aa2 --- /dev/null +++ b/core/src/sync/pipelines/composio/providers/google_drive.rs @@ -0,0 +1,179 @@ +use async_trait::async_trait; +use serde_json::Value; + +use super::common::{document, first_array, next_page_token, pick_str}; +use crate::sync::composio::providers::sync_state::SyncState; +use crate::sync::pipelines::composio::{ + run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, + SyncScope, +}; +use crate::sync::pipelines::traits::PipelineConfig; +use crate::sync::pipelines::traits::{ + SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, +}; + +// Composio deprecated `GOOGLEDRIVE_LIST_FILES` (2026-03-28) in favour of +// `GOOGLEDRIVE_FIND_FILE`, which is the current `files.list`-backed listing +// action (same paging/ordering/`q` filter surface). +const ACTION_FIND_FILE: &str = "GOOGLEDRIVE_FIND_FILE"; + +/// Incremental Google Drive synchronization through Composio. +/// +/// File-shaped: each Drive file is a record with a stable id and a +/// `modifiedTime`. This indexes file *metadata* only — it never downloads +/// binary bodies (which may be arbitrarily large and are not memory-shaped); +/// the document content is the file's structured metadata. +pub struct GoogleDriveSyncPipeline { + client: ComposioClient, + connection_id: String, + max_pages: usize, + page_size: usize, +} + +impl GoogleDriveSyncPipeline { + pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { + Self { + client, + connection_id: connection_id.into(), + max_pages: 10, + page_size: 50, + } + } + + pub fn with_limits(mut self, max_pages: usize, page_size: usize) -> Self { + self.max_pages = max_pages.max(1); + // Google Drive caps `pageSize` at 1000. + self.page_size = page_size.clamp(1, 1000); + self + } +} + +#[async_trait] +impl SyncPipeline for GoogleDriveSyncPipeline { + fn id(&self) -> &str { + "composio:googledrive" + } + fn kind(&self) -> SyncPipelineKind { + SyncPipelineKind::Composio + } + async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { + Ok(()) + } + async fn tick( + &self, + config: &PipelineConfig, + context: &SyncContext, + ) -> anyhow::Result { + run_incremental_sync(self, &self.client, &self.connection_id, config, context).await + } +} + +#[async_trait] +impl IncrementalSource for GoogleDriveSyncPipeline { + fn toolkit(&self) -> &'static str { + "googledrive" + } + fn action(&self) -> &'static str { + ACTION_FIND_FILE + } + fn max_pages(&self) -> usize { + self.max_pages + } + // NB: `stop_on_empty_pending` stays at its default (false) — see the note on + // the Calendar pipeline. The cursor only advances on a complete sync, so a + // capped run must not stop early on an all-deduplicated first page or it + // would permanently skip the unsynced tail. + fn server_side_depth(&self) -> bool { + true + } + fn arguments( + &self, + _: &SyncScope, + config: &PipelineConfig, + state: &SyncState, + page: Option<&str>, + ) -> Value { + let mut args = serde_json::json!({ + "page_size": self.page_size, + "order_by": "modifiedTime desc", + // Guarantee the fields the cursor/title/dedup depend on come back, + // regardless of the action's default projection. + "fields": "files(id,name,mimeType,modifiedTime),nextPageToken", + }); + if let Some(page) = page { + args["page_token"] = serde_json::json!(page); + } + // Depth window via a Drive `q` clause on modification time. Prefer the + // last-synced cursor, else the configured horizon. The cursor is + // validated as an RFC3339 timestamp before being interpolated into the + // query so a malformed persisted value can never inject into the `q` + // clause — on a bad value we simply omit the depth filter (full scan). + let floor = state + .cursor + .as_deref() + .filter(|cursor| chrono::DateTime::parse_from_rfc3339(cursor).is_ok()) + .map(str::to_owned) + .or_else(|| { + config.sync_depth_days.map(|days| { + (chrono::Utc::now() - chrono::Duration::days(days as i64)).to_rfc3339() + }) + }); + if let Some(floor) = floor { + // `GOOGLEDRIVE_FIND_FILE` names the Drive query parameter `q` (the + // native `files.list` name), not `query` — an unrecognised key would + // be ignored and defeat server-side depth bounding. + args["q"] = serde_json::json!(format!("modifiedTime > '{floor}'")); + } + args + } + fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { + PageFetch { + items: first_array( + data, + &[ + "/data/files", + "/files", + "/data/data/files", + "/data/items", + "/items", + ], + ), + next: next_page_token(data), + } + } + fn dedup_key(&self, item: &Value) -> Option { + let id = pick_str(item, &["id", "data.id", "fileId", "data.fileId"])?; + Some(match self.sort_cursor(item) { + Some(modified) => format!("{id}@{modified}"), + None => id, + }) + } + fn sort_cursor(&self, item: &Value) -> Option { + pick_str( + item, + &["modifiedTime", "data.modifiedTime", "modified_time"], + ) + } + async fn document( + &self, + _: &SyncScope, + connection_id: &str, + item: SyncItem, + _: &dyn ActionExecutor, + _: &mut SyncState, + ) -> anyhow::Result { + let id = pick_str(&item.raw, &["id", "data.id", "fileId", "data.fileId"]) + .unwrap_or_else(|| item.dedup_key.clone()); + let title = pick_str(&item.raw, &["name", "data.name", "title", "data.title"]) + .unwrap_or_else(|| format!("Drive file {id}")); + let content = serde_json::to_string_pretty(&item.raw)?; + Ok(document( + "googledrive", + connection_id, + &id, + title, + content, + item.raw, + )) + } +} diff --git a/core/src/sync/pipelines/composio/providers/google_sheets.rs b/core/src/sync/pipelines/composio/providers/google_sheets.rs new file mode 100644 index 0000000..fbc2665 --- /dev/null +++ b/core/src/sync/pipelines/composio/providers/google_sheets.rs @@ -0,0 +1,186 @@ +use async_trait::async_trait; +use serde_json::Value; + +use super::common::{checked_execute, document, first_array, pick_str}; +use crate::sync::composio::providers::sync_state::SyncState; +use crate::sync::pipelines::composio::{ + run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, + SyncScope, +}; +use crate::sync::pipelines::traits::PipelineConfig; +use crate::sync::pipelines::traits::{ + SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, +}; + +const ACTION_SEARCH: &str = "GOOGLESHEETS_SEARCH_SPREADSHEETS"; +const ACTION_INFO: &str = "GOOGLESHEETS_GET_SPREADSHEET_INFO"; + +/// Incremental Google Sheets synchronization through Composio. +/// +/// Two-step, document-shaped (like `NotionSyncPipeline`): `GOOGLESHEETS_SEARCH_SPREADSHEETS` +/// enumerates accessible spreadsheets, then `GOOGLESHEETS_GET_SPREADSHEET_INFO` fetches the +/// spreadsheet metadata for each item inside [`IncrementalSource::document`]. +pub struct GoogleSheetsSyncPipeline { + client: ComposioClient, + connection_id: String, + max_pages: usize, + page_size: usize, +} + +impl GoogleSheetsSyncPipeline { + pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { + Self { + client, + connection_id: connection_id.into(), + // NOTE: SEARCH_SPREADSHEETS' page-token arg name is not pinned by the + // curated catalog, so we do a single-page-per-tick fetch (no page + // token emitted) rather than guessing a pagination scheme. Capped at + // 1 page: since `arguments()` never advances the token, a >1 cap + // would re-fire the identical page-1 request and burn budget slots + // for silently-deduplicated items. + max_pages: 1, + page_size: 25, + } + } +} + +#[async_trait] +impl SyncPipeline for GoogleSheetsSyncPipeline { + fn id(&self) -> &str { + "composio:googlesheets" + } + fn kind(&self) -> SyncPipelineKind { + SyncPipelineKind::Composio + } + async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { + Ok(()) + } + async fn tick( + &self, + config: &PipelineConfig, + context: &SyncContext, + ) -> anyhow::Result { + run_incremental_sync(self, &self.client, &self.connection_id, config, context).await + } +} + +#[async_trait] +impl IncrementalSource for GoogleSheetsSyncPipeline { + fn toolkit(&self) -> &'static str { + "googlesheets" + } + fn action(&self) -> &'static str { + ACTION_SEARCH + } + fn max_pages(&self) -> usize { + self.max_pages + } + fn arguments( + &self, + _: &SyncScope, + _: &PipelineConfig, + _: &SyncState, + _page: Option<&str>, + ) -> Value { + // NOTE: an empty/broad `query` enumerates every accessible spreadsheet; + // `max_results` bounds the batch. Both mirror the underlying Drive + // search parameters. No page token is emitted (see `max_pages`). + serde_json::json!({"query": "", "max_results": self.page_size}) + } + fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { + PageFetch { + items: first_array( + data, + &[ + "/data/spreadsheets", + "/spreadsheets", + "/data/files", + "/files", + "/data/results", + "/results", + "/data/items", + "/items", + ], + ), + // Bounded fetch: no page token consumed (see `max_pages`). The + // pointers are read defensively should Composio surface one. + next: [ + "/data/nextPageToken", + "/nextPageToken", + "/data/next_page_token", + "/next_page_token", + ] + .iter() + .find_map(|path| data.pointer(path).and_then(Value::as_str)) + .map(str::trim) + .filter(|token| !token.is_empty()) + .map(str::to_owned), + } + } + fn dedup_key(&self, item: &Value) -> Option { + let id = pick_str( + item, + &["id", "data.id", "spreadsheetId", "data.spreadsheetId"], + )?; + Some(match self.sort_cursor(item) { + Some(modified) => format!("{id}@{modified}"), + None => id, + }) + } + fn sort_cursor(&self, item: &Value) -> Option { + pick_str( + item, + &["modifiedTime", "data.modifiedTime", "modified_time"], + ) + } + async fn document( + &self, + _: &SyncScope, + connection_id: &str, + item: SyncItem, + executor: &dyn ActionExecutor, + state: &mut SyncState, + ) -> anyhow::Result { + let id = pick_str( + &item.raw, + &["id", "data.id", "spreadsheetId", "data.spreadsheetId"], + ) + .unwrap_or_else(|| item.dedup_key.clone()); + let title = pick_str( + &item.raw, + &[ + "title", + "data.title", + "properties.title", + "data.properties.title", + "name", + ], + ) + .unwrap_or_else(|| format!("Google Sheet {id}")); + // NOTE: GET_SPREADSHEET_INFO identifies the spreadsheet by a + // "spreadsheet_id" argument (Google's canonical parameter name). + let response = checked_execute( + executor, + ACTION_INFO, + serde_json::json!({"spreadsheet_id": id}), + connection_id, + state, + ) + .await?; + // `response.data` is the already-unwrapped payload; the pointers catch + // any additional Composio wrapping, else we serialize the payload root. + let info = ["/data", "/data/data"] + .iter() + .find_map(|path| response.data.pointer(path)) + .unwrap_or(&response.data); + let content = serde_json::to_string_pretty(info)?; + Ok(document( + "googlesheets", + connection_id, + &id, + title, + content, + item.raw, + )) + } +} diff --git a/core/src/sync/pipelines/composio/providers/linear.rs b/core/src/sync/pipelines/composio/providers/linear.rs new file mode 100644 index 0000000..bfa08f7 --- /dev/null +++ b/core/src/sync/pipelines/composio/providers/linear.rs @@ -0,0 +1,193 @@ +use async_trait::async_trait; +use serde_json::Value; + +use super::common::{checked_execute, document, first_array, pick_str}; +use crate::sync::composio::providers::sync_state::SyncState; +use crate::sync::pipelines::composio::{ + run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, + SyncScope, +}; +use crate::sync::pipelines::traits::PipelineConfig; +use crate::sync::pipelines::traits::{ + SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, +}; + +const ACTION_USERS: &str = "LINEAR_LIST_LINEAR_USERS"; +const ACTION_ISSUES: &str = "LINEAR_LIST_LINEAR_ISSUES"; + +pub struct LinearSyncPipeline { + client: ComposioClient, + connection_id: String, + max_pages: usize, + page_size: usize, +} + +impl LinearSyncPipeline { + pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { + Self { + client, + connection_id: connection_id.into(), + max_pages: 20, + page_size: 50, + } + } +} + +#[async_trait] +impl SyncPipeline for LinearSyncPipeline { + fn id(&self) -> &str { + "composio:linear" + } + fn kind(&self) -> SyncPipelineKind { + SyncPipelineKind::Composio + } + async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { + Ok(()) + } + async fn tick( + &self, + config: &PipelineConfig, + context: &SyncContext, + ) -> anyhow::Result { + run_incremental_sync(self, &self.client, &self.connection_id, config, context).await + } +} + +#[async_trait] +impl IncrementalSource for LinearSyncPipeline { + fn toolkit(&self) -> &'static str { + "linear" + } + fn action(&self) -> &'static str { + ACTION_ISSUES + } + fn max_pages(&self) -> usize { + self.max_pages + } + async fn scopes( + &self, + executor: &dyn ActionExecutor, + connection_id: &str, + state: &mut SyncState, + ) -> anyhow::Result> { + let response = checked_execute( + executor, + ACTION_USERS, + serde_json::json!({"isMe": true}), + connection_id, + state, + ) + .await?; + let users = first_array( + &response.data, + &[ + "/data/nodes", + "/nodes", + "/data/data/nodes", + "/data/users/nodes", + ], + ); + let viewer = users.first().unwrap_or(&response.data); + let id = pick_str(viewer, &["id", "data.id"]) + .ok_or_else(|| anyhow::anyhow!("{ACTION_USERS} returned no viewer id"))?; + Ok(vec![SyncScope::named(id, "assignee:me")]) + } + fn arguments( + &self, + scope: &SyncScope, + _: &PipelineConfig, + _: &SyncState, + page: Option<&str>, + ) -> Value { + let mut args = serde_json::json!({"assigneeId": scope.id, "first": self.page_size, "orderBy": "updatedAt"}); + if let Some(page) = page { + args["after"] = serde_json::json!(page); + } + args + } + fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { + let items = first_array( + data, + &[ + "/data/nodes", + "/nodes", + "/data/data/nodes", + "/data/issues/nodes", + "/data/results", + "/results", + "/data/items", + "/items", + ], + ); + let page_info = [ + "/data/pageInfo", + "/pageInfo", + "/data/data/pageInfo", + "/data/issues/pageInfo", + ] + .iter() + .find_map(|path| data.pointer(path)); + let next = page_info + .filter(|info| { + info.get("hasNextPage") + .and_then(Value::as_bool) + .unwrap_or(false) + }) + .and_then(|info| info.get("endCursor").and_then(Value::as_str)) + .map(str::to_owned); + PageFetch { items, next } + } + fn dedup_key(&self, item: &Value) -> Option { + let id = pick_str(item, &["id", "data.id", "identifier", "data.identifier"])?; + Some(match self.sort_cursor(item) { + Some(updated) => format!("{id}@{updated}"), + None => id, + }) + } + fn sort_cursor(&self, item: &Value) -> Option { + pick_str( + item, + &[ + "updatedAt", + "data.updatedAt", + "updated_at", + "data.updated_at", + ], + ) + } + async fn document( + &self, + _: &SyncScope, + connection_id: &str, + item: SyncItem, + _: &dyn ActionExecutor, + _: &mut SyncState, + ) -> anyhow::Result { + let id = pick_str( + &item.raw, + &["id", "data.id", "identifier", "data.identifier"], + ) + .unwrap_or_else(|| item.dedup_key.clone()); + let title = pick_str( + &item.raw, + &[ + "title", + "data.title", + "name", + "data.name", + "identifier", + "data.identifier", + ], + ) + .unwrap_or_else(|| format!("Linear issue {id}")); + let content = serde_json::to_string_pretty(&item.raw)?; + Ok(document( + "linear", + connection_id, + &id, + title, + content, + item.raw, + )) + } +} diff --git a/core/src/sync/pipelines/composio/providers/mod.rs b/core/src/sync/pipelines/composio/providers/mod.rs new file mode 100644 index 0000000..dc86b2b --- /dev/null +++ b/core/src/sync/pipelines/composio/providers/mod.rs @@ -0,0 +1,28 @@ +//! One pipeline per Composio toolkit. Normalisation lives in +//! `tinymemory-sync`; these drive fetch, budget, and the write path. + +mod clickup; +mod common; +mod github; +mod google_calendar; +mod google_docs; +mod google_drive; +mod google_sheets; +mod linear; +mod notion; +mod outlook; +mod slack; +mod slack_parse; +mod todoist; + +pub use clickup::ClickUpSyncPipeline; +pub use github::GitHubSyncPipeline; +pub use google_calendar::GoogleCalendarSyncPipeline; +pub use google_docs::GoogleDocsSyncPipeline; +pub use google_drive::GoogleDriveSyncPipeline; +pub use google_sheets::GoogleSheetsSyncPipeline; +pub use linear::LinearSyncPipeline; +pub use notion::NotionSyncPipeline; +pub use outlook::OutlookSyncPipeline; +pub use slack::{SlackSearchBackfillPipeline, SlackSyncPipeline}; +pub use todoist::TodoistSyncPipeline; diff --git a/core/src/sync/pipelines/composio/providers/notion.rs b/core/src/sync/pipelines/composio/providers/notion.rs new file mode 100644 index 0000000..265f447 --- /dev/null +++ b/core/src/sync/pipelines/composio/providers/notion.rs @@ -0,0 +1,193 @@ +use async_trait::async_trait; +use serde_json::Value; + +use super::common::{checked_execute, document, first_array, pick_str}; +use crate::sync::composio::providers::sync_state::SyncState; +use crate::sync::pipelines::composio::{ + run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, + SyncScope, +}; +use crate::sync::pipelines::traits::PipelineConfig; +use crate::sync::pipelines::traits::{ + SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, +}; + +const ACTION_FETCH: &str = "NOTION_FETCH_DATA"; +const ACTION_MARKDOWN: &str = "NOTION_GET_PAGE_MARKDOWN"; + +pub struct NotionSyncPipeline { + client: ComposioClient, + connection_id: String, + max_pages: usize, + page_size: usize, +} + +impl NotionSyncPipeline { + pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { + Self { + client, + connection_id: connection_id.into(), + max_pages: 20, + page_size: 25, + } + } +} + +#[async_trait] +impl SyncPipeline for NotionSyncPipeline { + fn id(&self) -> &str { + "composio:notion" + } + fn kind(&self) -> SyncPipelineKind { + SyncPipelineKind::Composio + } + async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { + Ok(()) + } + async fn tick( + &self, + config: &PipelineConfig, + context: &SyncContext, + ) -> anyhow::Result { + run_incremental_sync(self, &self.client, &self.connection_id, config, context).await + } +} + +#[async_trait] +impl IncrementalSource for NotionSyncPipeline { + fn toolkit(&self) -> &'static str { + "notion" + } + fn action(&self) -> &'static str { + ACTION_FETCH + } + fn max_pages(&self) -> usize { + self.max_pages + } + fn arguments( + &self, + _: &SyncScope, + _: &PipelineConfig, + _: &SyncState, + page: Option<&str>, + ) -> Value { + let mut args = serde_json::json!({"page_size": self.page_size, "filter": {"value": "page", "property": "object"}, "sort": {"direction": "descending", "timestamp": "last_edited_time"}}); + if let Some(page) = page { + args["start_cursor"] = serde_json::json!(page); + } + args + } + fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { + PageFetch { + items: first_array( + data, + &[ + "/data/results", + "/results", + "/data/data/results", + "/data/items", + "/items", + ], + ), + next: [ + "/data/next_cursor", + "/next_cursor", + "/data/data/next_cursor", + ] + .iter() + .find_map(|path| data.pointer(path).and_then(Value::as_str)) + .map(str::to_owned), + } + } + fn dedup_key(&self, item: &Value) -> Option { + let id = pick_str(item, &["id", "data.id", "pageId", "data.pageId"])?; + Some(match self.sort_cursor(item) { + Some(edited) => format!("{id}@{edited}"), + None => id, + }) + } + fn sort_cursor(&self, item: &Value) -> Option { + pick_str( + item, + &[ + "last_edited_time", + "data.last_edited_time", + "lastEditedTime", + "data.lastEditedTime", + ], + ) + } + async fn document( + &self, + _: &SyncScope, + connection_id: &str, + item: SyncItem, + executor: &dyn ActionExecutor, + state: &mut SyncState, + ) -> anyhow::Result { + let id = pick_str(&item.raw, &["id", "data.id", "pageId", "data.pageId"]) + .unwrap_or_else(|| item.dedup_key.clone()); + let title = notion_title(&item.raw).unwrap_or_else(|| format!("Notion page {id}")); + let response = checked_execute( + executor, + ACTION_MARKDOWN, + serde_json::json!({"page_id": id}), + connection_id, + state, + ) + .await?; + let content = [ + "/markdown", + "/data/markdown", + "/data/response_data/markdown", + "/response_data/markdown", + "/data/content", + "/content", + "/text", + "/data/text", + ] + .iter() + .find_map(|path| response.data.pointer(path).and_then(Value::as_str)) + .filter(|value| !value.trim().is_empty()) + .map(str::to_owned) + .unwrap_or(serde_json::to_string_pretty(&item.raw)?); + Ok(document( + "notion", + connection_id, + &id, + title, + content, + item.raw, + )) + } +} + +fn notion_title(page: &Value) -> Option { + let properties = page + .get("properties") + .or_else(|| page.pointer("/data/properties")); + properties + .and_then(Value::as_object) + .and_then(|props| { + props.values().find_map(|property| { + (property.get("type").and_then(Value::as_str) == Some("title")) + .then(|| { + property + .get("title") + .and_then(Value::as_array) + .map(|parts| { + parts + .iter() + .filter_map(|part| { + part.get("plain_text").and_then(Value::as_str) + }) + .collect::>() + .join("") + }) + }) + .flatten() + .filter(|title| !title.is_empty()) + }) + }) + .or_else(|| pick_str(page, &["title", "data.title", "name", "data.name"])) +} diff --git a/core/src/sync/pipelines/composio/providers/outlook.rs b/core/src/sync/pipelines/composio/providers/outlook.rs new file mode 100644 index 0000000..bfe0780 --- /dev/null +++ b/core/src/sync/pipelines/composio/providers/outlook.rs @@ -0,0 +1,205 @@ +use async_trait::async_trait; +use serde_json::Value; + +use super::common::{document, first_array, pick_str}; +use crate::sync::composio::providers::sync_state::SyncState; +use crate::sync::pipelines::composio::{ + run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, + SyncScope, +}; +use crate::sync::pipelines::traits::PipelineConfig; +use crate::sync::pipelines::traits::{ + SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, +}; + +const ACTION_LIST_MESSAGES: &str = "OUTLOOK_LIST_MESSAGES"; + +/// Incremental Microsoft Outlook mail synchronization through Composio. +/// +/// Outlook messages carry a stable `id` and a `receivedDateTime` timestamp, so +/// this follows the message-shaped pattern (`GmailSyncPipeline`): a single list +/// action ordered newest-first, a client-visible `receivedDateTime` cursor, and +/// content taken directly from the message payload with no secondary fetch. +pub struct OutlookSyncPipeline { + client: ComposioClient, + connection_id: String, + max_pages: usize, + page_size: usize, +} + +impl OutlookSyncPipeline { + pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { + Self { + client, + connection_id: connection_id.into(), + max_pages: 10, + page_size: 25, + } + } + + pub fn with_limits(mut self, max_pages: usize, page_size: usize) -> Self { + self.max_pages = max_pages.max(1); + self.page_size = page_size.max(1); + self + } +} + +#[async_trait] +impl SyncPipeline for OutlookSyncPipeline { + fn id(&self) -> &str { + "composio:outlook" + } + fn kind(&self) -> SyncPipelineKind { + SyncPipelineKind::Composio + } + async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { + Ok(()) + } + async fn tick( + &self, + config: &PipelineConfig, + context: &SyncContext, + ) -> anyhow::Result { + run_incremental_sync(self, &self.client, &self.connection_id, config, context).await + } +} + +#[async_trait] +impl IncrementalSource for OutlookSyncPipeline { + fn toolkit(&self) -> &'static str { + "outlook" + } + fn action(&self) -> &'static str { + ACTION_LIST_MESSAGES + } + fn max_pages(&self) -> usize { + self.max_pages + } + fn stop_on_empty_pending(&self) -> bool { + true + } + fn server_side_depth(&self) -> bool { + true + } + fn arguments( + &self, + _: &SyncScope, + config: &PipelineConfig, + state: &SyncState, + page: Option<&str>, + ) -> Value { + // Microsoft Graph list-messages params passed through Composio: `top` + // bounds the page size, `orderby` sorts newest-first by receive time. + let mut args = serde_json::json!({ + "top": self.page_size, + "orderby": "receivedDateTime desc", + }); + if let Some(page) = page { + // Graph paginates via a `$skiptoken`; `extract_page` has already + // reduced the `@odata.nextLink` URL to the bare token. The exact + // Composio arg name for feeding it back is not fully certain — we + // send `skip_token` (the Graph-native name), so a mislabel here + // surfaces as a single-page fetch, not silent data loss. + args["skip_token"] = serde_json::json!(page); + } + // Depth window: prefer the last-synced cursor over the configured + // horizon (same precedence as the Gmail/Calendar pipelines). Graph + // filters server-side via `$filter` on `receivedDateTime`. + if let Some(cursor) = state.cursor.as_deref() { + args["filter"] = serde_json::json!(format!("receivedDateTime ge {cursor}")); + } else if let Some(days) = config.sync_depth_days { + let horizon = (chrono::Utc::now() - chrono::Duration::days(days as i64)).to_rfc3339(); + args["filter"] = serde_json::json!(format!("receivedDateTime ge {horizon}")); + } + args + } + fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { + PageFetch { + items: first_array( + data, + &[ + "/data/value", + "/value", + "/data/messages", + "/messages", + "/data/data/value", + "/data/items", + "/items", + ], + ), + next: [ + "/data/@odata.nextLink", + "/@odata.nextLink", + "/data/nextPageToken", + "/nextPageToken", + "/data/skip_token", + "/skip_token", + ] + .iter() + .find_map(|path| data.pointer(path).and_then(Value::as_str)) + .map(str::trim) + .filter(|token| !token.is_empty()) + .map(normalize_skip_token), + } + } + fn dedup_key(&self, item: &Value) -> Option { + let id = pick_str(item, &["id", "data.id", "messageId", "data.messageId"])?; + Some(match self.sort_cursor(item) { + Some(received) => format!("{id}@{received}"), + None => id, + }) + } + fn sort_cursor(&self, item: &Value) -> Option { + // Only `receivedDateTime` — the same field the `$filter` depth window + // keys on. A `lastModifiedDateTime` fallback would store a cursor in a + // different field than the filter compares, so on the next sync the + // `receivedDateTime ge ` window could skip valid messages. + pick_str( + item, + &[ + "receivedDateTime", + "data.receivedDateTime", + "received_date_time", + ], + ) + } + async fn document( + &self, + _: &SyncScope, + connection_id: &str, + item: SyncItem, + _: &dyn ActionExecutor, + _: &mut SyncState, + ) -> anyhow::Result { + let id = pick_str(&item.raw, &["id", "data.id", "messageId", "data.messageId"]) + .unwrap_or_else(|| item.dedup_key.clone()); + let title = pick_str(&item.raw, &["subject", "data.subject", "title"]) + .unwrap_or_else(|| format!("Outlook message {id}")); + let content = serde_json::to_string_pretty(&item.raw)?; + Ok(document( + "outlook", + connection_id, + &id, + title, + content, + item.raw, + )) + } +} + +/// Reduce a Graph paging token to the bare `$skiptoken` value. +/// +/// Graph returns `@odata.nextLink` as a full URL +/// (`https://graph.microsoft.com/v1.0/me/messages?$skiptoken=ABC...`). Feeding +/// that whole URL back as the paging arg would not resume pagination, so when +/// the token looks like a URL we extract just the `skiptoken` query value; +/// otherwise (Composio may already surface the bare token) we pass it through. +fn normalize_skip_token(token: &str) -> String { + let lower = token.to_ascii_lowercase(); + if let Some(pos) = lower.find("skiptoken=") { + let value = &token[pos + "skiptoken=".len()..]; + let end = value.find('&').unwrap_or(value.len()); + return value[..end].to_string(); + } + token.to_string() +} diff --git a/core/src/sync/pipelines/composio/providers/slack.rs b/core/src/sync/pipelines/composio/providers/slack.rs new file mode 100644 index 0000000..53bbe3c --- /dev/null +++ b/core/src/sync/pipelines/composio/providers/slack.rs @@ -0,0 +1,450 @@ +use std::collections::HashMap; + +use async_trait::async_trait; +use chrono::Utc; +use serde_json::Value; + +use super::common::{checked_execute, document, first_array, pick_str}; +use super::slack_parse::{ + decode_cursors, next_cursor, parse_ts, replace_mentions, search_matches, search_total_pages, +}; +use crate::sync::composio::providers::sync_state::SyncState; +use crate::sync::pipelines::composio::{ + run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, + SyncScope, +}; +use crate::sync::pipelines::traits::PipelineConfig; +use crate::sync::pipelines::traits::{ + SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, +}; + +const ACTION_CHANNELS: &str = "SLACK_LIST_CONVERSATIONS"; +const ACTION_HISTORY: &str = "SLACK_FETCH_CONVERSATION_HISTORY"; +const ACTION_SEARCH: &str = "SLACK_SEARCH_MESSAGES"; + +pub struct SlackSyncPipeline { + client: ComposioClient, + connection_id: String, + max_pages: usize, + page_size: usize, + backfill_days: i64, +} + +pub struct SlackSearchBackfillPipeline { + client: ComposioClient, + connection_id: String, + backfill_days: i64, + max_pages: u32, +} + +impl SlackSearchBackfillPipeline { + pub fn new( + client: ComposioClient, + connection_id: impl Into, + backfill_days: i64, + ) -> Self { + Self { + client, + connection_id: connection_id.into(), + backfill_days: backfill_days.max(1), + max_pages: 50, + } + } +} + +#[async_trait] +impl SyncPipeline for SlackSearchBackfillPipeline { + fn id(&self) -> &str { + "composio:slack:search-backfill" + } + + fn kind(&self) -> SyncPipelineKind { + SyncPipelineKind::Composio + } + + async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { + Ok(()) + } + + async fn tick(&self, _: &PipelineConfig, context: &SyncContext) -> anyhow::Result { + let mut state = + SyncState::load(context.state.as_ref(), "slack", &self.connection_id).await?; + if state.budget_exhausted() { + return Ok(SyncOutcome { + note: Some("slack search-backfill skipped: daily budget exhausted".into()), + ..SyncOutcome::default() + }); + } + + let directory = SlackSyncPipeline::new(self.client.clone(), self.connection_id.clone()); + let scopes = directory + .scopes(&self.client, &self.connection_id, &mut state) + .await?; + let channels: HashMap<_, _> = scopes + .into_iter() + .map(|scope| (scope.id.clone(), scope)) + .collect(); + let users = channels + .values() + .find_map(|scope| scope.metadata.get("users").and_then(Value::as_object)); + let after = (Utc::now() - chrono::Duration::days(self.backfill_days)) + .format("%Y-%m-%d") + .to_string(); + let mut page = 1u32; + let mut total_pages = 1u32; + let mut stored = 0u32; + + loop { + if state.budget_exhausted() || page > self.max_pages { + break; + } + let response = checked_execute( + &self.client, + ACTION_SEARCH, + serde_json::json!({ + "query": format!("after:{after}"), + "count": 100, + "sort": "timestamp", + "sort_dir": "asc", + "page": page, + }), + &self.connection_id, + &mut state, + ) + .await?; + if page == 1 { + total_pages = search_total_pages(&response.data).min(self.max_pages); + } + let matches = search_matches(&response.data); + let fetched = matches.len(); + for raw in matches { + let Some(ts) = pick_str(&raw, &["ts"]) else { + continue; + }; + if parse_ts(&ts).is_none() { + continue; + } + let Some(text) = pick_str(&raw, &["text"]).filter(|text| !text.trim().is_empty()) + else { + continue; + }; + let Some(channel_id) = pick_str(&raw, &["channel.id", "channel_id"]) else { + continue; + }; + let Some(scope) = channels.get(&channel_id) else { + tracing::warn!(channel_id, "[sync:slack-search] unknown channel skipped"); + continue; + }; + let author_id = + pick_str(&raw, &["user", "bot_id"]).unwrap_or_else(|| "unknown".into()); + let author = users + .and_then(|users| users.get(&author_id)) + .and_then(Value::as_str) + .unwrap_or(&author_id); + let text = replace_mentions(&text, users); + let mut doc = document( + "slack", + &self.connection_id, + &format!("{channel_id}:{ts}"), + format!("Slack {} from {author}", scope.label), + format!("[{ts}] {author}: {text}"), + raw, + ); + doc.metadata["channel_id"] = Value::String(channel_id); + doc.metadata["channel_label"] = Value::String(scope.label.clone()); + context.documents.store(doc).await?; + stored = stored.saturating_add(1); + } + if fetched == 0 || page >= total_pages { + break; + } + page = page.saturating_add(1); + } + + state.last_sync_at_ms = Some(Utc::now().timestamp_millis() as u64); + state.save(context.state.as_ref()).await?; + Ok(SyncOutcome { + records_ingested: stored, + more_pending: page < total_pages, + actions_called: state.run_requests, + provider_cost_usd: state.run_provider_cost_usd, + note: Some(format!( + "slack search-backfill: pages={page} records={stored}" + )), + }) + } +} + +impl SlackSyncPipeline { + pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { + Self { + client, + connection_id: connection_id.into(), + max_pages: 20, + page_size: 200, + backfill_days: 30, + } + } +} + +#[async_trait] +impl SyncPipeline for SlackSyncPipeline { + fn id(&self) -> &str { + "composio:slack" + } + fn kind(&self) -> SyncPipelineKind { + SyncPipelineKind::Composio + } + async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { + Ok(()) + } + async fn tick( + &self, + config: &PipelineConfig, + context: &SyncContext, + ) -> anyhow::Result { + run_incremental_sync(self, &self.client, &self.connection_id, config, context).await + } +} + +#[async_trait] +impl IncrementalSource for SlackSyncPipeline { + fn toolkit(&self) -> &'static str { + "slack" + } + fn action(&self) -> &'static str { + ACTION_HISTORY + } + fn max_pages(&self) -> usize { + self.max_pages + } + fn per_scope_cursors(&self) -> bool { + true + } + fn server_side_depth(&self) -> bool { + true + } + fn tolerate_scope_errors(&self) -> bool { + true + } + fn retain_dedup_keys(&self) -> bool { + true + } + + fn advance_scope_cursor(&self, state: &mut SyncState, scope: &SyncScope, cursor: &str) { + let mut cursors = decode_cursors(state.cursor.as_deref()); + cursors.insert(scope.id.clone(), cursor.into()); + state.cursor = serde_json::to_string(&cursors).ok(); + } + + async fn scopes( + &self, + executor: &dyn ActionExecutor, + connection_id: &str, + state: &mut SyncState, + ) -> anyhow::Result> { + let users = fetch_users(executor, connection_id, state).await; + let mut cursor: Option = None; + let mut channels = Vec::new(); + for _ in 0..20 { + if state.budget_exhausted() { + break; + } + let mut args = serde_json::json!({"limit": 200, "types": "public_channel,private_channel", "exclude_archived": true}); + if let Some(cursor) = cursor.as_deref() { + args["cursor"] = Value::String(cursor.into()); + } + let response = + checked_execute(executor, ACTION_CHANNELS, args, connection_id, state).await?; + channels.extend(first_array( + &response.data, + &["/data/channels", "/channels", "/data/data/channels"], + )); + cursor = next_cursor(&response.data); + if cursor.is_none() { + break; + } + } + Ok(channels + .into_iter() + .filter_map(|channel| { + let id = pick_str(&channel, &["id", "data.id"])?; + let name = pick_str(&channel, &["name", "data.name"]).unwrap_or_else(|| id.clone()); + let private = channel + .get("is_private") + .and_then(Value::as_bool) + .unwrap_or(false); + let label = if private { + format!("private:{name}") + } else { + format!("#{name}") + }; + Some( + SyncScope::named(id, label).with_metadata(serde_json::json!({ + "channel": channel, + "users": users, + })), + ) + }) + .collect()) + } + + fn arguments( + &self, + scope: &SyncScope, + config: &PipelineConfig, + state: &SyncState, + page: Option<&str>, + ) -> Value { + let cursors = decode_cursors(state.cursor.as_deref()); + let oldest = cursors.get(&scope.id).cloned().unwrap_or_else(|| { + format!( + "{}.000000", + (Utc::now() + - chrono::Duration::days( + config + .sync_depth_days + .map(i64::from) + .unwrap_or(self.backfill_days) + )) + .timestamp() + ) + }); + let mut args = serde_json::json!({"channel": scope.id, "oldest": oldest, "inclusive": false, "limit": self.page_size}); + if let Some(page) = page { + args["cursor"] = Value::String(page.into()); + } + args + } + + fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { + PageFetch { + items: first_array( + data, + &["/data/messages", "/messages", "/data/data/messages"], + ), + next: next_cursor(data), + } + } + + fn dedup_key(&self, item: &Value) -> Option { + let ts = pick_str(item, &["ts", "data.ts"])?; + parse_ts(&ts)?; + let text = pick_str(item, &["text", "data.text"])?; + (!text.trim().is_empty()).then_some(ts) + } + + fn sort_cursor(&self, item: &Value) -> Option { + pick_str(item, &["ts", "data.ts"]) + } + + async fn document( + &self, + scope: &SyncScope, + connection_id: &str, + item: SyncItem, + _: &dyn ActionExecutor, + _: &mut SyncState, + ) -> anyhow::Result { + let ts = pick_str(&item.raw, &["ts", "data.ts"]).unwrap_or(item.dedup_key); + let raw_text = pick_str(&item.raw, &["text", "data.text"]).unwrap_or_default(); + let author_id = pick_str( + &item.raw, + &["user", "data.user", "username", "data.username"], + ) + .unwrap_or_else(|| "unknown".into()); + let users = scope.metadata.get("users").and_then(Value::as_object); + let author = users + .and_then(|users| users.get(&author_id)) + .and_then(Value::as_str) + .unwrap_or(&author_id) + .to_owned(); + let text = replace_mentions(&raw_text, users); + let title = format!("Slack {} from {}", scope.label, author); + let content = format!("[{ts}] {author}: {text}"); + let mut result = document( + "slack", + connection_id, + &format!("{}:{ts}", scope.id), + title, + content, + item.raw, + ); + result.metadata["channel_id"] = Value::String(scope.id.clone()); + result.metadata["channel_label"] = Value::String(scope.label.clone()); + Ok(result) + } +} + +async fn fetch_users( + executor: &dyn ActionExecutor, + connection_id: &str, + state: &mut SyncState, +) -> HashMap { + let mut users = HashMap::new(); + let mut cursor: Option = None; + for page in 0..20 { + if state.budget_exhausted() { + break; + } + let mut arguments = serde_json::json!({"limit": 200}); + if let Some(cursor) = cursor.as_deref() { + arguments["cursor"] = Value::String(cursor.into()); + } + let response = match executor + .execute("SLACK_LIST_ALL_USERS", arguments, Some(connection_id)) + .await + { + Ok(response) => response, + Err(error) => { + if let Some(error) = error.downcast_ref::() { + state.record_requests(error.attempts); + } + tracing::warn!(page, %error, "[sync:slack] user directory fetch failed; using collected users"); + break; + } + }; + state.record_action(response.attempts, response.cost_usd); + if !response.successful { + tracing::warn!( + page, + error = response.error.as_deref().unwrap_or("provider failure"), + "[sync:slack] user directory rejected; using collected users" + ); + break; + } + let members = first_array( + &response.data, + &[ + "/data/members", + "/members", + "/data/users", + "/users", + "/data/data/members", + ], + ); + for member in members { + let Some(id) = pick_str(&member, &["id"]) else { + continue; + }; + if let Some(name) = [ + "profile.display_name", + "profile.real_name", + "real_name", + "name", + "profile.display_name_normalized", + "profile.real_name_normalized", + ] + .iter() + .find_map(|path| pick_str(&member, &[*path])) + { + users.insert(id, name); + } + } + cursor = next_cursor(&response.data); + if cursor.is_none() { + break; + } + } + users +} diff --git a/core/src/sync/pipelines/composio/providers/slack_parse.rs b/core/src/sync/pipelines/composio/providers/slack_parse.rs new file mode 100644 index 0000000..2419f60 --- /dev/null +++ b/core/src/sync/pipelines/composio/providers/slack_parse.rs @@ -0,0 +1,91 @@ +//! Slack response cursor, mention, and timestamp parsing. + +use std::collections::BTreeMap; +use std::sync::OnceLock; + +use serde_json::Value; + +use super::common::first_array; + +/// Return the cached matcher for Slack `<@USERID>` mentions. +pub(super) fn mention_regex() -> &'static regex::Regex { + static REGEX: OnceLock = OnceLock::new(); + REGEX.get_or_init(|| regex::Regex::new(r"<@(U[A-Z0-9]+)>").expect("Slack mention regex")) +} + +/// Replace Slack mention tokens with resolved display names, falling back to +/// the raw user id when the optional user map has no match. +pub(super) fn replace_mentions( + text: &str, + users: Option<&serde_json::Map>, +) -> String { + mention_regex() + .replace_all(text, |captures: ®ex::Captures<'_>| { + let id = &captures[1]; + let resolved = users + .and_then(|users| users.get(id)) + .and_then(Value::as_str) + .unwrap_or(id); + format!("@{resolved}") + }) + .into_owned() +} + +/// Read the first non-blank next cursor across supported response envelopes. +pub(super) fn next_cursor(data: &Value) -> Option { + [ + "/data/response_metadata/next_cursor", + "/response_metadata/next_cursor", + "/data/next_cursor", + "/next_cursor", + "/data/data/response_metadata/next_cursor", + ] + .iter() + .find_map(|path| data.pointer(path).and_then(Value::as_str)) + .map(str::trim) + .filter(|cursor| !cursor.is_empty()) + .map(str::to_owned) +} + +/// Extract Slack search matches across legacy and nested response envelopes. +pub(super) fn search_matches(data: &Value) -> Vec { + first_array( + data, + &[ + "/data/messages/matches", + "/messages/matches", + "/data/data/messages/matches", + "/messages", + ], + ) +} + +/// Extract the search page count, defaulting to one when paging is absent. +pub(super) fn search_total_pages(data: &Value) -> u32 { + [ + "/data/messages/paging/pages", + "/messages/paging/pages", + "/data/data/messages/paging/pages", + "/pages", + ] + .iter() + .find_map(|path| data.pointer(path).and_then(Value::as_u64)) + .unwrap_or(1) as u32 +} + +/// Decode persisted per-scope cursors, returning an empty map for absent or +/// malformed JSON so synchronization can restart safely. +pub(super) fn decode_cursors(raw: Option<&str>) -> BTreeMap { + raw.and_then(|raw| serde_json::from_str(raw).ok()) + .unwrap_or_default() +} + +/// Parse Slack's `seconds.fraction` timestamp into numeric components. +/// Missing fractions become zero; malformed numeric components return `None`. +pub(super) fn parse_ts(ts: &str) -> Option<(i64, u64)> { + let mut parts = ts.splitn(2, '.'); + Some(( + parts.next()?.parse().ok()?, + parts.next().unwrap_or("0").parse().ok()?, + )) +} diff --git a/core/src/sync/pipelines/composio/providers/todoist.rs b/core/src/sync/pipelines/composio/providers/todoist.rs new file mode 100644 index 0000000..cbbedc6 --- /dev/null +++ b/core/src/sync/pipelines/composio/providers/todoist.rs @@ -0,0 +1,226 @@ +use async_trait::async_trait; +use serde_json::Value; + +use super::common::{document, first_array, pick_str}; +use crate::sync::composio::providers::sync_state::SyncState; +use crate::sync::pipelines::composio::{ + run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, + SyncScope, +}; +use crate::sync::pipelines::traits::PipelineConfig; +use crate::sync::pipelines::traits::{ + SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, +}; + +const ACTION_GET_ALL_TASKS: &str = "TODOIST_GET_ALL_TASKS"; + +/// Incremental Todoist synchronization through Composio. +/// +/// Todoist tasks are self-contained records (stable id + `created_at` +/// timestamp), so this follows the document-shaped pattern +/// (`LinearSyncPipeline`) rather than the message-shaped one: a single list +/// action, content taken directly from the task payload with no secondary +/// fetch. Todoist's active-tasks endpoint returns a plain array and is not +/// paginated, so there is no server-side incremental filter, and a task carries +/// no modification timestamp. Incremental behavior is therefore driven by the +/// orchestrator's client-side dedup (`synced_ids`) keyed on a payload +/// fingerprint (see [`dedup_key`](Self::dedup_key)) — an unchanged task is +/// skipped, while any edit re-ingests. +pub struct TodoistSyncPipeline { + client: ComposioClient, + connection_id: String, + max_pages: usize, +} + +impl TodoistSyncPipeline { + pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { + Self { + client, + connection_id: connection_id.into(), + max_pages: 1, + } + } + + pub fn with_limits(mut self, max_pages: usize, _page_size: usize) -> Self { + self.max_pages = max_pages.max(1); + // Todoist active-tasks is unpaginated; the sibling `page_size` argument + // is accepted for signature parity but has no effect. + self + } +} + +#[async_trait] +impl SyncPipeline for TodoistSyncPipeline { + fn id(&self) -> &str { + "composio:todoist" + } + fn kind(&self) -> SyncPipelineKind { + SyncPipelineKind::Composio + } + async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { + Ok(()) + } + async fn tick( + &self, + config: &PipelineConfig, + context: &SyncContext, + ) -> anyhow::Result { + run_incremental_sync(self, &self.client, &self.connection_id, config, context).await + } +} + +#[async_trait] +impl IncrementalSource for TodoistSyncPipeline { + fn toolkit(&self) -> &'static str { + "todoist" + } + fn action(&self) -> &'static str { + ACTION_GET_ALL_TASKS + } + fn max_pages(&self) -> usize { + self.max_pages + } + fn stop_on_empty_pending(&self) -> bool { + true + } + fn server_side_depth(&self) -> bool { + false + } + fn arguments( + &self, + _: &SyncScope, + _: &PipelineConfig, + _: &SyncState, + _page: Option<&str>, + ) -> Value { + // Todoist "get all active tasks" needs no required arguments and ignores + // pagination; do not invent a page token. + serde_json::json!({}) + } + fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { + // Todoist's active-tasks response is sometimes the bare task array + // (already unwrapped from the Composio `data` envelope by the client) + // and sometimes wrapped under `tasks`/`items`. Handle the top-level + // array first, then the wrapped shapes. + let items = data.as_array().cloned().unwrap_or_else(|| { + first_array( + data, + &[ + "/data/tasks", + "/tasks", + "/data/items", + "/items", + "/data/data", + ], + ) + }); + PageFetch { + items, + // Todoist active tasks are returned as a single unpaginated array. + next: None, + } + } + fn dedup_key(&self, item: &Value) -> Option { + let id = pick_str(item, &["id", "data.id", "task_id", "data.task_id"])?; + // Todoist tasks have no modification timestamp, so `created_at` (which is + // immutable) would never change and edited tasks would never re-ingest. + // Key on a fingerprint of the task payload instead: any change to + // content/due/project yields a new key and re-ingests, while an + // unchanged task keeps its key and is deduped. + Some(format!("{id}@{}", payload_fingerprint(item))) + } + fn sort_cursor(&self, _item: &Value) -> Option { + // Todoist active tasks have no modification timestamp and the endpoint + // is unpaginated, so there is no meaningful sort cursor. Returning None + // is deliberate: the orchestrator's cursor-boundary short-circuit keys + // on `sort_cursor`, and using the immutable `created_at` would halt the + // scan (and skip re-ingest) for an edited task created before the + // persisted cursor. Freshness is handled entirely by `dedup_key`. + None + } + async fn document( + &self, + _: &SyncScope, + connection_id: &str, + item: SyncItem, + _: &dyn ActionExecutor, + _: &mut SyncState, + ) -> anyhow::Result { + let id = pick_str(&item.raw, &["id", "data.id", "task_id", "data.task_id"]) + .unwrap_or_else(|| item.dedup_key.clone()); + let title = pick_str( + &item.raw, + &["content", "data.content", "title", "data.title"], + ) + .unwrap_or_else(|| format!("Todoist task {id}")); + // A Todoist task's meaningful text is its `content` (title line) plus an + // optional `description`; store that as the document body so retrieval + // embeds the task text, not JSON syntax. Fall back to the raw payload + // only when the task carries no content field. + let content = match pick_str(&item.raw, &["content", "data.content"]) { + Some(text) => match pick_str(&item.raw, &["description", "data.description"]) { + Some(desc) if !desc.trim().is_empty() => format!("{text}\n\n{desc}"), + _ => text, + }, + None => serde_json::to_string_pretty(&item.raw)?, + }; + Ok(document( + "todoist", + connection_id, + &id, + title, + content, + item.raw, + )) + } +} + +/// Stable content fingerprint of a task payload, used as the freshness half of +/// the dedup key. Computed as FNV-1a over a canonical serialization (object keys +/// sorted recursively). The key is **persisted** in `SyncState`, so the hash +/// must be stable across Rust toolchains and independent of `serde_json` map +/// ordering — `DefaultHasher` guarantees neither, and an unstable value would +/// silently re-ingest every task on a toolchain bump. +fn payload_fingerprint(item: &Value) -> u64 { + let mut canonical = String::new(); + write_canonical(item, &mut canonical); + // FNV-1a 64-bit — a fixed, specified algorithm. + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for byte in canonical.as_bytes() { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + hash +} + +/// Serialize `value` with object keys sorted recursively so the byte stream is +/// canonical regardless of map insertion order. +fn write_canonical(value: &Value, out: &mut String) { + match value { + Value::Object(map) => { + out.push('{'); + let mut keys: Vec<&String> = map.keys().collect(); + keys.sort_unstable(); + for (index, key) in keys.iter().enumerate() { + if index > 0 { + out.push(','); + } + out.push_str(&serde_json::to_string(key).unwrap_or_default()); + out.push(':'); + write_canonical(&map[*key], out); + } + out.push('}'); + } + Value::Array(items) => { + out.push('['); + for (index, item) in items.iter().enumerate() { + if index > 0 { + out.push(','); + } + write_canonical(item, out); + } + out.push(']'); + } + other => out.push_str(&other.to_string()), + } +} diff --git a/core/src/sync/pipelines/dispatcher.rs b/core/src/sync/pipelines/dispatcher.rs new file mode 100644 index 0000000..4e40554 --- /dev/null +++ b/core/src/sync/pipelines/dispatcher.rs @@ -0,0 +1,123 @@ +//! Pipeline registry and fault-isolated synchronization dispatcher. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; + +use crate::sync::pipelines::traits::PipelineConfig; +use crate::sync::pipelines::traits::{SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SyncRunResult { + pub pipeline_id: String, + pub kind: SyncPipelineKind, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub outcome: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +#[derive(Default)] +pub struct SyncDispatcher { + pipelines: BTreeMap>, +} + +impl SyncDispatcher { + pub fn new() -> Self { + Self::default() + } + + pub fn register(&mut self, pipeline: Arc) -> anyhow::Result<()> { + let id = pipeline.id().trim(); + anyhow::ensure!(!id.is_empty(), "sync pipeline id must not be empty"); + anyhow::ensure!( + !self.pipelines.contains_key(id), + "sync pipeline already registered: {id}" + ); + tracing::debug!( + pipeline_id = id, + kind = pipeline.kind().as_str(), + "[memory_sync:dispatcher] registering pipeline" + ); + self.pipelines.insert(id.to_owned(), pipeline); + Ok(()) + } + + pub fn ids(&self) -> Vec<&str> { + self.pipelines.keys().map(String::as_str).collect() + } + + pub async fn init_all( + &self, + config: &PipelineConfig, + context: &SyncContext, + ) -> Vec { + let mut results = Vec::with_capacity(self.pipelines.len()); + for (id, pipeline) in &self.pipelines { + tracing::debug!( + pipeline_id = id, + "[memory_sync:dispatcher] initializing pipeline" + ); + let result = pipeline.init(config, context).await; + results.push(SyncRunResult { + pipeline_id: id.clone(), + kind: pipeline.kind(), + outcome: result.as_ref().ok().map(|_| SyncOutcome::default()), + error: result.err().map(|error| error.to_string()), + }); + } + results + } + + pub async fn tick( + &self, + pipeline_id: &str, + config: &PipelineConfig, + context: &SyncContext, + ) -> anyhow::Result { + let pipeline = self + .pipelines + .get(pipeline_id) + .ok_or_else(|| anyhow::anyhow!("unknown sync pipeline: {pipeline_id}"))?; + tracing::debug!( + pipeline_id, + "[memory_sync:dispatcher] pipeline tick starting" + ); + let outcome = pipeline.tick(config, context).await; + match &outcome { + Ok(outcome) => tracing::debug!( + pipeline_id, + records = outcome.records_ingested, + more_pending = outcome.more_pending, + "[memory_sync:dispatcher] pipeline tick completed" + ), + Err(error) => { + tracing::warn!(pipeline_id, %error, "[memory_sync:dispatcher] pipeline tick failed") + } + } + outcome + } + + pub async fn tick_all( + &self, + config: &PipelineConfig, + context: &SyncContext, + ) -> Vec { + let mut results = Vec::with_capacity(self.pipelines.len()); + for (id, pipeline) in &self.pipelines { + let result = pipeline.tick(config, context).await; + results.push(SyncRunResult { + pipeline_id: id.clone(), + kind: pipeline.kind(), + outcome: result.as_ref().ok().cloned(), + error: result.err().map(|error| error.to_string()), + }); + } + results + } +} + +#[cfg(test)] +#[path = "dispatcher_tests.rs"] +mod tests; diff --git a/core/src/sync/pipelines/dispatcher_tests.rs b/core/src/sync/pipelines/dispatcher_tests.rs new file mode 100644 index 0000000..a5b40d8 --- /dev/null +++ b/core/src/sync/pipelines/dispatcher_tests.rs @@ -0,0 +1,206 @@ +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; + +use super::*; +use crate::sync::composio::providers::sync_state::SyncStateStore; +use crate::sync::pipelines::traits::{SkillDocSink, SkillDocument, SyncEvent, SyncEventSink}; + +struct FakePipeline { + id: &'static str, + fail: bool, + init_fail: bool, +} + +#[async_trait] +impl SyncPipeline for FakePipeline { + fn id(&self) -> &str { + self.id + } + + fn kind(&self) -> SyncPipelineKind { + SyncPipelineKind::Workspace + } + + async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { + if self.init_fail { + anyhow::bail!("expected init failure") + } + Ok(()) + } + + async fn tick(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result { + if self.fail { + anyhow::bail!("expected failure") + } + Ok(SyncOutcome { + records_ingested: 3, + more_pending: false, + actions_called: 0, + provider_cost_usd: 0.0, + note: None, + }) + } +} + +#[derive(Default)] +struct NoopHost(Mutex>); + +#[async_trait] +impl SkillDocSink for NoopHost { + async fn store(&self, _: SkillDocument) -> anyhow::Result<()> { + Ok(()) + } + + async fn delete(&self, _: &str, _: &str) -> anyhow::Result<()> { + Ok(()) + } +} + +#[async_trait] +impl SyncEventSink for NoopHost { + async fn emit(&self, _: SyncEvent) -> anyhow::Result<()> { + Ok(()) + } +} + +#[async_trait] +impl SyncStateStore for NoopHost { + async fn get(&self, namespace: &str, key: &str) -> anyhow::Result> { + Ok(self + .0 + .lock() + .unwrap() + .get(&format!("{namespace}:{key}")) + .cloned()) + } + + async fn set( + &self, + namespace: &str, + key: &str, + value: &serde_json::Value, + ) -> anyhow::Result<()> { + self.0 + .lock() + .unwrap() + .insert(format!("{namespace}:{key}"), value.clone()); + Ok(()) + } +} + +fn context() -> SyncContext { + let host = Arc::new(NoopHost::default()); + SyncContext { + events: host.clone(), + documents: host.clone(), + state: host, + } +} + +#[tokio::test] +async fn tick_all_is_deterministic_and_isolates_failures() { + let mut dispatcher = SyncDispatcher::new(); + dispatcher + .register(Arc::new(FakePipeline { + id: "z-fail", + fail: true, + init_fail: false, + })) + .unwrap(); + dispatcher + .register(Arc::new(FakePipeline { + id: "a-ok", + fail: false, + init_fail: false, + })) + .unwrap(); + assert_eq!(dispatcher.ids(), vec!["a-ok", "z-fail"]); + assert!(dispatcher + .register(Arc::new(FakePipeline { + id: "a-ok", + fail: false, + init_fail: false, + })) + .is_err()); + let results = dispatcher + .tick_all(&PipelineConfig::default(), &context()) + .await; + assert_eq!(results.len(), 2); + assert_eq!(results[0].outcome.as_ref().unwrap().records_ingested, 3); + assert!(results[1] + .error + .as_deref() + .unwrap() + .contains("expected failure")); +} + +#[tokio::test] +async fn register_rejects_blank_ids_and_tick_reports_unknown_pipeline() { + let mut dispatcher = SyncDispatcher::new(); + assert!(dispatcher + .register(Arc::new(FakePipeline { + id: " ", + fail: false, + init_fail: false, + })) + .is_err()); + let error = dispatcher + .tick("missing", &PipelineConfig::default(), &context()) + .await + .unwrap_err(); + assert!(error.to_string().contains("unknown sync pipeline")); +} + +#[tokio::test] +async fn init_all_and_individual_tick_preserve_success_and_failure_details() { + let mut dispatcher = SyncDispatcher::new(); + dispatcher + .register(Arc::new(FakePipeline { + id: "a-init-fails", + fail: false, + init_fail: true, + })) + .unwrap(); + dispatcher + .register(Arc::new(FakePipeline { + id: "b-ok", + fail: false, + init_fail: false, + })) + .unwrap(); + dispatcher + .register(Arc::new(FakePipeline { + id: "c-tick-fails", + fail: true, + init_fail: false, + })) + .unwrap(); + let config = PipelineConfig::default(); + let context = context(); + + let initialized = dispatcher.init_all(&config, &context).await; + assert!(initialized[0] + .error + .as_deref() + .unwrap() + .contains("expected init failure")); + assert!(initialized[1].outcome.is_some()); + assert_eq!( + dispatcher + .tick("b-ok", &config, &context) + .await + .unwrap() + .records_ingested, + 3 + ); + assert!(dispatcher + .tick("c-tick-fails", &config, &context) + .await + .is_err()); + + let encoded = serde_json::to_value(&initialized[0]).unwrap(); + assert_eq!(encoded["pipeline_id"], "a-init-fails"); + assert!(encoded.get("outcome").is_none()); +} diff --git a/core/src/sync/pipelines/host.rs b/core/src/sync/pipelines/host.rs new file mode 100644 index 0000000..ec55b56 --- /dev/null +++ b/core/src/sync/pipelines/host.rs @@ -0,0 +1,379 @@ +//! The host side of the engine-free pipelines (#18 §B1): the sink adapter +//! over [`MemoryClient`], the Composio settings mapping, and the runners the +//! rest of `core/src/sync/` calls. +//! +//! This is the piece §B5's acceptance rests on: a pipeline sees three +//! capabilities — events, documents, state — and every one resolves through +//! [`MemoryClient`], so whatever driver the host bound serves the sync. + +use std::sync::Arc; + +use async_trait::async_trait; + +use crate::store::MemoryClientRef; +use crate::sync::composio::providers::sync_state::SyncStateStore; +use crate::sync::pipelines::composio::{ + ClickUpSyncPipeline, ComposioClient, GitHubSyncPipeline, GmailSyncPipeline, LinearSyncPipeline, + NotionSyncPipeline, SlackSearchBackfillPipeline, SlackSyncPipeline, +}; +use crate::sync::pipelines::dispatcher::SyncDispatcher; +use crate::sync::pipelines::traits::{ + ComposioMode, ComposioSyncConfig, PipelineConfig, SecretString, SkillDocSink, SkillDocument, + SyncContext, SyncEvent, SyncEventSink, SyncOutcome, SyncPipeline, SyncRunError, +}; +use crate::Config; + +/// A failed pipeline run, with whatever usage it burned before failing. +#[derive(Debug)] +pub struct PipelineFailure { + pub message: String, + pub actions_called: u32, + pub provider_cost_usd: f64, +} + +impl std::fmt::Display for PipelineFailure { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.message) + } +} + +impl std::error::Error for PipelineFailure {} + +impl PipelineFailure { + pub fn without_usage(message: impl Into) -> Self { + Self { + message: message.into(), + actions_called: 0, + provider_cost_usd: 0.0, + } + } +} + +/// Adapter giving the pipelines their three capabilities over the bound +/// memory client. The engine's `HostSyncAdapter` remains for the engine's own +/// pipelines; this one exists so a Composio sync never needs the engine. +pub struct PipelineHost { + memory: MemoryClientRef, + config: Option>, +} + +impl PipelineHost { + /// An adapter that also feeds the memory tree after each stored document + /// (parity with the engine adapter's #5473 behaviour). + pub fn new(memory: MemoryClientRef, config: Arc) -> Self { + Self { + memory, + config: Some(config), + } + } + + /// An adapter with no host config: documents are stored, tree ingest is + /// skipped. This is the shape a non-TinyCortex host uses. + pub fn without_tree_ingest(memory: MemoryClientRef) -> Self { + Self { + memory, + config: None, + } + } + + /// The pipeline context over this adapter. + pub fn context(self: &Arc) -> SyncContext { + SyncContext { + events: self.clone(), + documents: self.clone(), + state: self.clone(), + } + } +} + +#[async_trait] +impl SkillDocSink for PipelineHost { + async fn store(&self, document: SkillDocument) -> anyhow::Result<()> { + tracing::debug!( + toolkit = %document.toolkit, + connection_id = %document.connection_id, + document_id = %document.document_id, + "[memory_sync] storing synchronized document" + ); + self.memory + .store_skill_sync( + &document.namespace_skill_id, + &document.connection_id, + &document.title, + &document.content, + Some("tinycortex-sync".into()), + Some(document.metadata.clone()), + Some("medium".into()), + None, + None, + Some(document.document_id.clone()), + ) + .await + .map_err(anyhow::Error::msg)?; + + // #5473: additively reconnect the synced item to the memory tree — a + // best-effort secondary index; the skill store above is the source of + // truth and has committed. A failure here must NOT abort the sync (one + // poisonous item would stall the connection and re-buy the page on + // every retry). The config-less adapter skips tree ingest entirely. + if let Some(config) = self.config.as_deref() { + if let Err(error) = ingest_into_tree(config, &document).await { + tracing::warn!( + %error, + document_id = %document.document_id, + "[memory_sync] tree ingest failed; skill store remains authoritative" + ); + } + } + Ok(()) + } + + async fn delete(&self, namespace_skill_id: &str, document_id: &str) -> anyhow::Result<()> { + let namespace = format!("skill-{}", namespace_skill_id.trim()); + tracing::debug!( + namespace, + document_id, + "[memory_sync] deleting synchronized document" + ); + self.memory + .delete_document(&namespace, document_id) + .await + .map(|_| ()) + .map_err(anyhow::Error::msg) + } +} + +/// Mirror of the engine adapter's tree reconnect: route the stored document +/// through core's ingest funnel under the same source id scheme. +async fn ingest_into_tree(config: &Config, document: &SkillDocument) -> anyhow::Result<()> { + let source_id = format!( + "composio:{}:{}:{}", + document.toolkit, document.connection_id, document.document_id + ); + let doc = crate::ingest_pipeline::IngestDocumentInput { + provider: document.toolkit.clone(), + title: document.title.clone(), + body: document.content.clone(), + modified_at: chrono::Utc::now(), + source_ref: Some(source_id.clone()), + }; + let tags = vec!["composio_sync".to_string(), document.toolkit.clone()]; + crate::ingest_pipeline::ingest_document(config, &source_id, "", tags, doc) + .await + .map(|_| ()) + .map_err(|error| anyhow::anyhow!("{error}")) +} + +#[async_trait] +impl SyncEventSink for PipelineHost { + async fn emit(&self, event: SyncEvent) -> anyhow::Result<()> { + crate::events::publish(crate::events::MemoryEvent::SyncStageChanged { + trigger: "tinycortex".into(), + stage: super::traits::stage_name(event.stage).into(), + provider: Some(event.toolkit), + connection_id: event.connection_id, + detail: event.message, + source_id: Some(event.source_id), + }); + Ok(()) + } +} + +#[async_trait] +impl SyncStateStore for PipelineHost { + async fn get(&self, namespace: &str, key: &str) -> anyhow::Result> { + self.memory + .kv_get(Some(namespace), key) + .await + .map_err(anyhow::Error::msg) + } + + async fn set( + &self, + namespace: &str, + key: &str, + value: &serde_json::Value, + ) -> anyhow::Result<()> { + self.memory + .kv_set(Some(namespace), key, value) + .await + .map_err(anyhow::Error::msg) + } +} + +/// The Composio connection settings from the host's config — the same +/// resolution the engine seam performs, onto the local types. +pub fn composio_config(config: &Config) -> Result { + if config.composio().mode.eq_ignore_ascii_case("direct") { + let api_key = crate::composio_host::api_key(config) + .or_else(|| config.composio().api_key.clone()) + .ok_or_else(|| "Composio direct API key is not configured".to_string())?; + Ok(ComposioSyncConfig { + mode: ComposioMode::Direct, + base_url: "https://backend.composio.dev/api/v3".into(), + api_key: Some(SecretString::new(api_key)), + bearer_token: None, + entity_id: Some(config.composio().entity_id.clone()), + }) + } else { + let bearer = config + .session_token()? + .ok_or_else(|| "OpenHuman backend bearer token is not configured".to_string())?; + Ok(ComposioSyncConfig { + mode: ComposioMode::Proxied, + base_url: config.effective_backend_api_url(), + api_key: None, + bearer_token: Some(SecretString::new(bearer)), + entity_id: Some(config.composio().entity_id.clone()), + }) + } +} + +/// The toolkits with a native pipeline here. Kept identical to the engine +/// seam's list; `sync_status` advertising draws from the provider registry. +pub fn syncable_composio_toolkits() -> &'static [&'static str] { + &["clickup", "github", "gmail", "linear", "notion", "slack"] +} + +/// Whether `toolkit` has a native pipeline (case-insensitive). +pub fn is_composio_toolkit_syncable(toolkit: &str) -> bool { + let slug = toolkit.trim().to_ascii_lowercase(); + syncable_composio_toolkits().contains(&slug.as_str()) +} + +fn build_composio_pipeline( + toolkit: &str, + connection_id: &str, + composio: ComposioSyncConfig, +) -> Result, String> { + // Fail closed before resolving credentials for any toolkit without a + // native pipeline (#4957) — the gate stays a single testable list. + if !is_composio_toolkit_syncable(toolkit) { + return Err(format!("memory sync does not support toolkit '{toolkit}'")); + } + let client = ComposioClient::new(composio); + Ok(match toolkit { + "gmail" => Arc::new(GmailSyncPipeline::new(client, connection_id)), + "github" => Arc::new(GitHubSyncPipeline::new(client, connection_id)), + "notion" => Arc::new(NotionSyncPipeline::new(client, connection_id)), + "linear" => Arc::new(LinearSyncPipeline::new(client, connection_id)), + "clickup" => Arc::new(ClickUpSyncPipeline::new(client, connection_id)), + "slack" => Arc::new(SlackSyncPipeline::new(client, connection_id)), + _ => unreachable!("gated by is_composio_toolkit_syncable"), + }) +} + +/// Run one Composio connection through the engine-free pipelines. +pub async fn run_composio_connection( + toolkit: &str, + connection_id: &str, + config: &Config, + max_items: Option, + sync_depth_days: Option, +) -> Result { + let memory = crate::global::client_if_ready() + .ok_or_else(|| PipelineFailure::without_usage("memory client is not ready"))?; + let composio = composio_config(config).map_err(PipelineFailure::without_usage)?; + let pipeline = build_composio_pipeline(toolkit, connection_id, composio) + .map_err(PipelineFailure::without_usage)?; + let pipeline_config = PipelineConfig { + composio: None, // the client already holds the connection settings + sync_depth_days, + max_items, + }; + let host = Arc::new(PipelineHost::new(memory, config.to_arc())); + run_pipeline(pipeline, &pipeline_config, &host.context()).await +} + +/// Run a bounded Gmail backfill through the engine-free pipelines. +pub async fn run_gmail_backfill( + connection_id: &str, + query: &str, + max_pages: usize, + page_size: usize, + config: &Config, +) -> Result { + let memory = crate::global::client_if_ready() + .ok_or_else(|| PipelineFailure::without_usage("memory client is not ready"))?; + let composio = composio_config(config).map_err(PipelineFailure::without_usage)?; + let pipeline: Arc = Arc::new( + GmailSyncPipeline::new(ComposioClient::new(composio), connection_id) + .with_limits(max_pages, page_size) + .with_query(query), + ); + let host = Arc::new(PipelineHost::new(memory, config.to_arc())); + run_pipeline(pipeline, &PipelineConfig::default(), &host.context()).await +} + +/// Run the Slack search backfill through the engine-free pipelines. +pub async fn run_slack_search_backfill( + connection_id: &str, + backfill_days: i64, + config: &Config, +) -> Result { + let memory = crate::global::client_if_ready() + .ok_or_else(|| PipelineFailure::without_usage("memory client is not ready"))?; + let composio = composio_config(config).map_err(PipelineFailure::without_usage)?; + let client = ComposioClient::new(composio); + let pipeline: Arc = Arc::new(SlackSearchBackfillPipeline::new( + client, + connection_id, + backfill_days, + )); + let host = Arc::new(PipelineHost::new(memory, config.to_arc())); + run_pipeline(pipeline, &PipelineConfig::default(), &host.context()).await +} + +async fn run_pipeline( + pipeline: Arc, + config: &PipelineConfig, + context: &SyncContext, +) -> Result { + let pipeline_id = pipeline.id().to_owned(); + let mut dispatcher = SyncDispatcher::new(); + dispatcher + .register(pipeline) + .map_err(|error| PipelineFailure::without_usage(error.to_string()))?; + dispatcher + .tick(&pipeline_id, config, context) + .await + .map_err(|error| { + let usage = error.downcast_ref::(); + PipelineFailure { + message: error.to_string(), + actions_called: usage.map_or(0, |error| error.actions_called), + provider_cost_usd: usage.map_or(0.0, |error| error.provider_cost_usd), + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// #4957: an unsupported toolkit is rejected *before* credentials are + /// resolved — moved here with the gate itself from the engine seam. + #[test] + fn unsupported_toolkit_is_rejected_before_resolving_credentials() { + let err = + build_composio_pipeline("googlecalendar", "conn-1", ComposioSyncConfig::default()) + .err() + .expect("unsupported toolkit must be rejected"); + assert!( + err.contains("does not support toolkit 'googlecalendar'"), + "got: {err}" + ); + } + + #[test] + fn the_syncable_set_is_exactly_the_native_pipelines() { + for toolkit in syncable_composio_toolkits() { + assert!( + build_composio_pipeline(toolkit, "conn-1", ComposioSyncConfig::default()).is_ok(), + "advertised toolkit '{toolkit}' must build" + ); + } + assert!(!is_composio_toolkit_syncable("googlecalendar")); + assert!(is_composio_toolkit_syncable(" Gmail ")); + } +} diff --git a/core/src/sync/pipelines/mod.rs b/core/src/sync/pipelines/mod.rs new file mode 100644 index 0000000..dcd526f --- /dev/null +++ b/core/src/sync/pipelines/mod.rs @@ -0,0 +1,17 @@ +//! Engine-neutral sync pipelines (#18 §B1). +//! +//! The Composio orchestration that used to run inside the engine: fetch pages +//! within budget, normalise through `tinymemory-sync`, and write through the +//! [`traits::SyncContext`] sinks. A pipeline sees three capabilities — events, +//! documents, state — and whatever provider the host bound serves them, which +//! is the property §B5's acceptance test needs. +//! +//! The engine keeps its own copies for its internal pipelines (workspace +//! watcher, tree rebuild, repo summarisation — engine-tree features by +//! design). Sources of kind `Composio` route here; tree-coupled source kinds +//! still route through the engine seam. + +pub mod composio; +pub mod dispatcher; +pub mod host; +pub mod traits; diff --git a/core/src/sync/pipelines/traits.rs b/core/src/sync/pipelines/traits.rs new file mode 100644 index 0000000..62fe719 --- /dev/null +++ b/core/src/sync/pipelines/traits.rs @@ -0,0 +1,190 @@ +//! Host seams and pipeline contracts for live synchronization. + +use std::sync::Arc; + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SyncPipelineKind { + Composio, + Workspace, + Mcp, +} + +impl SyncPipelineKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Composio => "composio", + Self::Workspace => "workspace", + Self::Mcp => "mcp", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SyncStage { + Requested, + Fetching, + Stored, + Ingesting, + Completed, + Failed, +} + +/// Stable wire name for each stage, shared by every event adapter. +pub fn stage_name(stage: SyncStage) -> &'static str { + match stage { + SyncStage::Requested => "requested", + SyncStage::Fetching => "fetching", + SyncStage::Stored => "stored", + SyncStage::Ingesting => "ingesting", + SyncStage::Completed => "completed", + SyncStage::Failed => "failed", + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SyncEvent { + pub source_id: String, + pub toolkit: String, + pub connection_id: Option, + pub stage: SyncStage, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message: Option, +} + +#[async_trait] +pub trait SyncEventSink: Send + Sync { + async fn emit(&self, event: SyncEvent) -> anyhow::Result<()>; +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SkillDocument { + pub namespace_skill_id: String, + pub connection_id: String, + pub document_id: String, + pub title: String, + pub content: String, + pub toolkit: String, + #[serde(default)] + pub metadata: serde_json::Value, +} + +#[async_trait] +pub trait SkillDocSink: Send + Sync { + async fn store(&self, document: SkillDocument) -> anyhow::Result<()>; + async fn delete(&self, namespace_skill_id: &str, document_id: &str) -> anyhow::Result<()>; +} + +/// How the Composio client reaches the API: straight at it, or through the +/// backend proxy. The engine's enum, ported with the client — distinct from +/// `tinymemory_api::host::ComposioMode`, which is the *host seam's* +/// string-typed setting; the seam converts. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum ComposioMode { + /// Call api.composio.dev with the host's own key. + Direct, + /// Route through the backend proxy. + #[default] + Proxied, +} + +/// The Composio client's connection settings, owned here so the pipelines +/// take no engine config type (#18 §B1). The engine keeps its own copy for +/// its internal pipelines; the host constructs this one from its own config. +#[derive(Clone, Debug, Default)] +pub struct ComposioSyncConfig { + pub mode: ComposioMode, + pub base_url: String, + pub api_key: Option, + pub bearer_token: Option, + pub entity_id: Option, +} + +/// A string whose `Debug` never prints the value. +#[derive(Clone, Default, PartialEq, Eq)] +pub struct SecretString(String); + +impl SecretString { + pub fn new(value: impl Into) -> Self { + Self(value.into()) + } + + pub fn expose(&self) -> &str { + &self.0 + } + + pub fn is_empty(&self) -> bool { + self.0.trim().is_empty() + } +} + +impl std::fmt::Debug for SecretString { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("SecretString(redacted)") + } +} + +/// What a pipeline may read of the host's configuration: the Composio +/// connection settings and the sync-depth budget. Deliberately not the +/// host's whole config — a pipeline that needs more must argue for the +/// field here. +#[derive(Clone, Debug, Default)] +pub struct PipelineConfig { + pub composio: Option, + pub sync_depth_days: Option, + pub max_items: Option, +} + +/// Host capabilities required by sync pipelines. +#[derive(Clone)] +pub struct SyncContext { + pub events: Arc, + pub documents: Arc, + pub state: Arc, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct SyncOutcome { + pub records_ingested: u32, + pub more_pending: bool, + #[serde(default)] + pub actions_called: u32, + #[serde(default)] + pub provider_cost_usd: f64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub note: Option, +} + +#[derive(Debug, thiserror::Error)] +#[error("{message}")] +pub struct SyncRunError { + pub actions_called: u32, + pub provider_cost_usd: f64, + message: String, +} + +impl SyncRunError { + pub fn new(message: impl Into, actions_called: u32, provider_cost_usd: f64) -> Self { + Self { + actions_called, + provider_cost_usd, + message: message.into(), + } + } +} + +#[async_trait] +pub trait SyncPipeline: Send + Sync { + fn id(&self) -> &str; + fn kind(&self) -> SyncPipelineKind; + async fn init(&self, config: &PipelineConfig, context: &SyncContext) -> anyhow::Result<()>; + async fn tick( + &self, + config: &PipelineConfig, + context: &SyncContext, + ) -> anyhow::Result; +} diff --git a/core/tests/composio_gmail_non_tinycortex_e2e.rs b/core/tests/composio_gmail_non_tinycortex_e2e.rs new file mode 100644 index 0000000..4488a62 --- /dev/null +++ b/core/tests/composio_gmail_non_tinycortex_e2e.rs @@ -0,0 +1,233 @@ +//! Issue #18 §B5 / §E4 — the acceptance test for the sync section: +//! **Composio Gmail sync completes end to end against a driver that is not +//! TinyCortex.** +//! +//! The pieces under test, and what each proves: +//! +//! - A mock Composio (wiremock, loopback-only) serves two pages of +//! `GMAIL_FETCH_EMAILS` — pagination, cursor advance and dedup are real. +//! - The pipeline is `sync::pipelines::composio::GmailSyncPipeline`, run +//! through the real `SyncDispatcher` — the exact production path. +//! - The host is `PipelineHost::without_tree_ingest` over a `MemoryClient` +//! bound to the **namespace store** — the driver #42 (§A3) registers as its +//! own non-TinyCortex class. No engine is initialised, no tree exists, and +//! the pipeline code under `core/src/sync/` names no engine module. The +//! engine's `KvStore` appears only as a storage *library* inside the +//! namespace store's SQLite file — it is not the bound driver, and nothing +//! in the pipeline knows it is there. +//! +//! Offline by construction: the only socket is wiremock's 127.0.0.1 listener. + +use std::sync::Arc; + +use serde_json::json; +use wiremock::matchers::{body_partial_json, method, path}; +use wiremock::{Match, Mock, MockServer, Request, ResponseTemplate}; + +/// Matches the *first* fetch only: an execute body whose arguments carry no +/// `page_token`. `body_partial_json` cannot express absence, and without this +/// the page-1 mount also matches the page-2 request (which still contains +/// `max_results`), serving page 1 twice — dedup then eats the repeats and the +/// test fails honestly but confusingly. +struct NoPageToken; + +impl Match for NoPageToken { + fn matches(&self, request: &Request) -> bool { + serde_json::from_slice::(&request.body) + .map(|body| body["arguments"].get("page_token").is_none()) + .unwrap_or(false) + } +} + +use tinymemory_core::store::MemoryClient; + +/// The one piece of host wiring `MemoryClient` requires: an embedding host. +/// Noop — this test is about the sync path, and recall is not asserted. +#[derive(Debug)] +struct NoopEmbeddingHost; + +impl tinymemory_api::host::EmbeddingHost for NoopEmbeddingHost { + fn resolve_api_key(&self, _provider: &str) -> Option { + None + } + + fn ollama_base_url(&self) -> String { + "http://127.0.0.1:1".into() + } + + fn default_embedding_provider( + &self, + ) -> std::sync::Arc { + std::sync::Arc::new(tinymemory_api::host::NoopEmbedding) + } + + fn create_embedding_provider_with_credentials( + &self, + _provider: &str, + _model: &str, + _dims: usize, + _api_key: &str, + _custom_endpoint: Option<&str>, + ) -> Result, String> { + Ok(Box::new(tinymemory_api::host::NoopEmbedding)) + } + + fn model_supports_dimensions(&self, _model: &str) -> bool { + false + } + + fn cloud_embedding_provider( + &self, + _model: &str, + _dims: usize, + ) -> Result, String> { + Ok(Box::new(tinymemory_api::host::NoopEmbedding)) + } + + fn default_cloud_embedding_model(&self) -> &str { + "noop" + } + + fn default_cloud_embedding_dimensions(&self) -> usize { + 8 + } + + fn ollama_embedding_provider( + &self, + _base_url: &str, + _model: &str, + _dims: usize, + ) -> Result, String> { + Ok(Box::new(tinymemory_api::host::NoopEmbedding)) + } +} +use tinymemory_core::sync::composio::providers::sync_state::{SyncState, KV_NAMESPACE}; +use tinymemory_core::sync::pipelines::composio::ComposioClient; +use tinymemory_core::sync::pipelines::composio::GmailSyncPipeline; +use tinymemory_core::sync::pipelines::dispatcher::SyncDispatcher; +use tinymemory_core::sync::pipelines::host::PipelineHost; +use tinymemory_core::sync::pipelines::traits::{ + ComposioMode, ComposioSyncConfig, PipelineConfig, SecretString, SyncPipeline, +}; + +fn message(id: &str, subject: &str, body_md: &str) -> serde_json::Value { + json!({ + "id": id, + "subject": subject, + "from": "sender@example.com", + "markdown": body_md, + "messageTimestamp": "2026-01-02T03:04:05Z", + }) +} + +#[tokio::test(flavor = "multi_thread")] +async fn composio_gmail_sync_completes_against_the_namespace_driver() { + // ── The mock Composio ──────────────────────────────────────────────── + let server = MockServer::start().await; + + // Page 1: two messages and a cursor. Matched on the *absence* of a page + // token in the arguments, so retries stay deterministic. + Mock::given(method("POST")) + .and(path("/tools/execute/GMAIL_FETCH_EMAILS")) + .and(body_partial_json(json!({"arguments": {"max_results": 25}}))) + .and(NoPageToken) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "successful": true, + "data": { + "messages": [ + message("m1", "First", "hello one"), + message("m2", "Second", "hello two"), + ], + "nextPageToken": "page-2", + } + }))) + .mount(&server) + .await; + + // Page 2: one message, no cursor — the sync must stop here. + Mock::given(method("POST")) + .and(path("/tools/execute/GMAIL_FETCH_EMAILS")) + .and(body_partial_json( + json!({"arguments": {"page_token": "page-2"}}), + )) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "successful": true, + "data": { + "messages": [message("m3", "Third", "hello three")], + } + }))) + .mount(&server) + .await; + + // ── The non-TinyCortex driver ──────────────────────────────────────── + tinymemory_core::embedding_host::set_embedding_host(Arc::new(NoopEmbeddingHost)); + let workspace = tempfile::tempdir().expect("workspace"); + let memory = Arc::new( + MemoryClient::from_workspace_dir(workspace.path().to_path_buf()) + .expect("bind the namespace store"), + ); + + // ── The engine-free pipeline, on the production dispatcher ─────────── + let composio = ComposioSyncConfig { + mode: ComposioMode::Direct, + base_url: server.uri(), + api_key: Some(SecretString::new("test-key")), + bearer_token: None, + entity_id: Some("entity-1".into()), + }; + let pipeline = Arc::new(GmailSyncPipeline::new( + ComposioClient::new(composio), + "conn-1", + )); + let pipeline_id = pipeline.id().to_owned(); + + let host = Arc::new(PipelineHost::without_tree_ingest(memory.clone())); + let mut dispatcher = SyncDispatcher::new(); + dispatcher.register(pipeline).expect("register pipeline"); + let outcome = dispatcher + .tick(&pipeline_id, &PipelineConfig::default(), &host.context()) + .await + .expect("gmail sync must complete"); + + // ── End to end: the outcome ────────────────────────────────────────── + assert_eq!( + outcome.records_ingested, 3, + "all three messages ingest; outcome={outcome:?}" + ); + assert!(!outcome.more_pending, "page 2 carried no cursor"); + + // ── End to end: the documents landed in the bound store ────────────── + let docs = memory + .list_documents(Some("skill-gmail")) + .await + .expect("list synced documents"); + let listed = docs + .as_array() + .or_else(|| docs.get("documents").and_then(|d| d.as_array())) + .map(|a| a.len()) + .unwrap_or_default(); + assert_eq!(listed, 3, "three documents in skill-gmail: {docs}"); + + // ── End to end: the canonical markdown, not raw JSON ───────────────── + let doc = memory + .get_document("skill-gmail", "gmail:m1") + .await + .expect("read gmail:m1") + .expect("gmail:m1 stored"); + assert!( + doc.content.contains("From: sender@example.com") && doc.content.contains("hello one"), + "canonical markdown stored, got: {}", + doc.content + ); + + // ── End to end: cursor + dedup state persisted through the KV seam ─── + let state = SyncState::load(&*host, "gmail", "conn-1") + .await + .expect("load persisted state"); + assert!(state.is_synced("m1") && state.is_synced("m3"), "dedup ids"); + let raw = memory + .kv_get(Some(KV_NAMESPACE), "gmail:conn-1") + .await + .expect("kv read"); + assert!(raw.is_some(), "sync state persisted under {KV_NAMESPACE}"); +} diff --git a/sync/Cargo.toml b/sync/Cargo.toml index 223e667..8c9dd18 100644 --- a/sync/Cargo.toml +++ b/sync/Cargo.toml @@ -13,6 +13,8 @@ description = "Engine-neutral Composio payload normalisers for TinyMemory" # runtime. A dependency added here should have to argue for itself against that # sentence (issue #18 §B3). [dependencies] +# `email_markdown`'s thread shapes are (de)serialised at the pipeline seam. +serde = { version = "1", features = ["derive"] } serde_json = "1" # Two logging facades, neither an implementation, both carried over from the # engine layout this crate was extracted from: `gmail_post_process` traces @@ -31,7 +33,8 @@ log = "0.4" # alongside — but it means "pure `Value -> Value`" is true of every normaliser # here except that one. Better said out loud than discovered by someone whose # output moved when they changed TZ. -chrono = { version = "0.4", features = ["clock"] } +# `serde` joined for `email_markdown`'s timestamp (de)serialisers (#18 §B1). +chrono = { version = "0.4", features = ["clock", "serde"] } [lints.rust] unsafe_code = "forbid" diff --git a/sync/src/email_clean.rs b/sync/src/email_clean.rs new file mode 100644 index 0000000..c48f84a --- /dev/null +++ b/sync/src/email_clean.rs @@ -0,0 +1,264 @@ +//! Shared email rendering + cleaning helpers. +//! +//! Used by [`super::email`] when rendering canonical email markdown. The module +//! is intentionally pure-string-oriented plus a single `serde_json::Value` +//! helper (`parse_message_date`) for callers that work directly off slim +//! envelope JSON. Nothing here depends on the chunk-store types, which keeps the +//! helpers reusable. + +use chrono::{DateTime, NaiveDate, Utc}; +use serde_json::Value; + +/// Two-stage cleanup applied to each message body before it gets rendered into +/// a digest: +/// +/// 1. **Drop quoted reply chains** — once a message contains a +/// `On , wrote:` preamble, an `Original Message` / +/// `Forwarded message` separator, or a run of three+ consecutive +/// `>`-prefixed lines, everything from that point onward is the parent +/// message we already render directly above. +/// 2. **Drop footer noise** — `Unsubscribe`, `View in browser`, copyright +/// lines, legal disclaimers, and address blocks. We cut at the first line +/// containing a known footer trigger. +/// +/// The two passes run in order so a quoted-chain preamble below a +/// "view in browser" line still gets stripped on its own merits even if the +/// footer pass missed it. +pub fn clean_body(raw: &str) -> String { + let stage1 = drop_reply_chain(raw); + let stage2 = drop_footer_noise(&stage1); + collapse_blank_runs(stage2.trim()) +} + +/// Substrings that, when matched (case-insensitive) anywhere on a line, mark +/// the start of footer / boilerplate territory. Conservative list — every entry +/// should be unambiguous noise that wouldn't reasonably appear inside real +/// prose. +const FOOTER_TRIGGERS: &[&str] = &[ + "unsubscribe", + "view in browser", + "view this email in your browser", + "view it in your browser", + "update your email settings", + "manage your subscription", + "manage preferences", + "email preferences", + "you are receiving this email because", + "you received this email because", + "you're receiving this email because", + "to stop receiving", + "all rights reserved", + "© 20", + "(c) 20", + "copyright 20", + "powered by mailchimp", + "sent via sendgrid", + "this email and any files", + "confidentiality notice", + "if you are not the intended recipient", + "this communication may contain", +]; + +/// Strip quoted reply chains. See [`clean_body`] for details. +pub fn drop_reply_chain(s: &str) -> String { + let mut offset = 0usize; + let mut quoted_run_start: Option = None; + let mut quoted_run_len = 0u32; + + for line in s.split_inclusive('\n') { + let trimmed = line.trim(); + let lower = trimmed.to_ascii_lowercase(); + + // Explicit reply / forward markers. + let is_preamble = (lower.starts_with("on ") && lower.contains(" wrote:")) + || lower.contains("---------- forwarded message") + || lower.contains("----- original message") + || lower.contains("--------- original message") + || lower.contains("--- forwarded by"); + if is_preamble { + debug_assert!(s.is_char_boundary(offset)); + return s[..offset].trim_end().to_string(); + } + + // Three+ consecutive lines starting with `>` is a quoted reply chain in + // disguise (some clients de-quote on send). Treat the start of the run + // as the cut point. + if trimmed.starts_with('>') { + if quoted_run_start.is_none() { + quoted_run_start = Some(offset); + quoted_run_len = 1; + } else { + quoted_run_len += 1; + } + if quoted_run_len >= 3 { + let cut = quoted_run_start.unwrap_or(offset); + debug_assert!(s.is_char_boundary(cut)); + return s[..cut].trim_end().to_string(); + } + } else if !trimmed.is_empty() { + // Reset on a non-empty, non-quoted line. Blank lines don't break a + // quote run because senders often interleave them. + quoted_run_start = None; + quoted_run_len = 0; + } + + offset += line.len(); + } + s.to_string() +} + +/// Strip everything from the first line containing a footer trigger onward. +/// Uses the module's known footer-trigger list. +pub fn drop_footer_noise(s: &str) -> String { + let mut offset = 0usize; + for line in s.split_inclusive('\n') { + let lower = line.to_ascii_lowercase(); + if FOOTER_TRIGGERS.iter().any(|t| lower.contains(t)) { + debug_assert!(s.is_char_boundary(offset)); + return s[..offset].trim_end().to_string(); + } + offset += line.len(); + } + s.to_string() +} + +/// Collapse runs of 2+ blank lines into a single blank line. Trims trailing +/// newlines. +pub fn collapse_blank_runs(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut blank = 0u32; + for line in s.lines() { + if line.trim().is_empty() { + blank += 1; + if blank <= 1 { + out.push('\n'); + } + } else { + blank = 0; + out.push_str(line); + out.push('\n'); + } + } + while out.ends_with('\n') { + out.pop(); + } + out +} + +/// Truncate a body to at most `max_chars` characters, appending `…` when the +/// body is longer. Trims first so leading/trailing whitespace doesn't count +/// against the budget. +pub fn truncate_body(body: &str, max_chars: usize) -> String { + let trimmed = body.trim(); + if trimmed.chars().count() <= max_chars { + return trimmed.to_string(); + } + let mut out: String = trimmed.chars().take(max_chars).collect(); + out.push('…'); + out +} + +/// Escape only the few markdown chars that would visibly break the +/// header/inline contexts we use (#, |, *, _, `). Newlines collapse to spaces. +/// We leave most punctuation alone — the body is rendered as a blockquote +/// anyway. +pub fn md_escape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for ch in s.chars() { + match ch { + '\\' | '`' | '*' | '_' | '|' => { + out.push('\\'); + out.push(ch); + } + '\n' | '\r' => out.push(' '), + _ => out.push(ch), + } + } + out +} + +/// Pull the `` portion out of a `From` header, returning just the +/// bare email address. Falls back to `None` when no `<…>` brackets exist; in +/// that case the caller may use the raw From field. +pub fn extract_email(from: &str) -> Option { + let s = from.trim(); + if let (Some(start), Some(end)) = (s.rfind('<'), s.rfind('>')) { + if start < end { + debug_assert!(s.is_char_boundary(start + 1)); + debug_assert!(s.is_char_boundary(end)); + let inner = s[start + 1..end].trim(); + if inner.contains('@') { + return Some(inner.to_string()); + } + } + } + if s.contains('@') && !s.contains(' ') { + return Some(s.to_string()); + } + None +} + +/// If `s` starts with a 3-letter day-of-week prefix (`Mon, `, `Tue, `, …), +/// return the remainder; otherwise `None`. Used to feed a strict-rfc2822 reject +/// into a lenient retry. +fn strip_day_of_week_prefix(s: &str) -> Option<&str> { + const DAYS: &[&str] = &["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]; + let (prefix, rest) = s.split_once(", ")?; + if DAYS.iter().any(|d| d.eq_ignore_ascii_case(prefix)) { + Some(rest) + } else { + None + } +} + +/// Try a sequence of common date formats. The slim envelope sets `date` from +/// `messageTimestamp` (often ISO 8601 or epoch ms) when present, falling back +/// to the raw `Date:` header (RFC 2822). Operates on the raw `serde_json::Value` +/// so callers that work off the slim envelope JSON don't have to reshape it +/// first. +pub fn parse_message_date(m: &Value) -> Option> { + if let Some(dt) = m.get("date").and_then(parse_date_value) { + return Some(dt); + } + if let Some(dt) = m.get("internalDate").and_then(parse_date_value) { + return Some(dt); + } + m.get("data") + .and_then(|data| data.get("internalDate")) + .and_then(parse_date_value) +} + +fn parse_date_value(raw: &Value) -> Option> { + if let Some(s) = raw.as_str() { + let s = s.trim(); + if s.is_empty() { + return None; + } + // Epoch millis as a string? Gmail's `internalDate` uses this form. + if let Ok(ms) = s.parse::() { + return DateTime::from_timestamp_millis(ms); + } + if let Ok(dt) = DateTime::parse_from_rfc3339(s) { + return Some(dt.with_timezone(&Utc)); + } + if let Ok(dt) = DateTime::parse_from_rfc2822(s) { + return Some(dt.with_timezone(&Utc)); + } + // Lenient RFC 2822 fallback: strict `parse_from_rfc2822` rejects + // mismatched day-of-week. Strip a `, ` prefix and retry with + // the rfc2822 body format. + if let Some(rest) = strip_day_of_week_prefix(s) { + if let Ok(dt) = DateTime::parse_from_str(rest, "%d %b %Y %H:%M:%S %z") { + return Some(dt.with_timezone(&Utc)); + } + } + if let Ok(d) = NaiveDate::parse_from_str(s, "%Y-%m-%d") { + return d.and_hms_opt(0, 0, 0).map(|n| n.and_utc()); + } + } + raw.as_i64().and_then(DateTime::from_timestamp_millis) +} + +#[cfg(test)] +#[path = "email_clean_tests.rs"] +mod tests; diff --git a/sync/src/email_clean_tests.rs b/sync/src/email_clean_tests.rs new file mode 100644 index 0000000..e66b4b7 --- /dev/null +++ b/sync/src/email_clean_tests.rs @@ -0,0 +1,143 @@ +use super::*; +use serde_json::json; + +#[test] +fn drop_reply_chain_strips_on_x_wrote_preamble() { + let body = "Sounds good — let's do Tuesday.\n\nOn Mon, Apr 22, 2026 at 10:00 AM, Alice wrote:\n> Tuesday or Wednesday?\n> Let me know."; + let cleaned = drop_reply_chain(body); + assert_eq!(cleaned.trim(), "Sounds good — let's do Tuesday."); +} + +#[test] +fn drop_reply_chain_strips_forwarded_separator() { + let body = "FYI.\n\n---------- Forwarded message ---------\nFrom: bob\nSubject: hi"; + assert_eq!(drop_reply_chain(body).trim(), "FYI."); +} + +#[test] +fn drop_reply_chain_strips_consecutive_quoted_run() { + let body = "Thanks for the update.\n\n> earlier line 1\n> earlier line 2\n> earlier line 3\n> earlier line 4"; + assert_eq!(drop_reply_chain(body).trim(), "Thanks for the update."); +} + +#[test] +fn drop_reply_chain_keeps_short_quote() { + let body = "I think:\n> That sounds reasonable\n\nLet's proceed."; + let cleaned = drop_reply_chain(body); + assert!(cleaned.contains("Let's proceed")); + assert!(cleaned.contains("That sounds reasonable")); +} + +#[test] +fn drop_footer_noise_strips_unsubscribe_block() { + let body = + "Big news: GPT-5.5 is here.\n\nRead more at example.com\n\nUnsubscribe | © 2026 OpenAI"; + let cleaned = drop_footer_noise(body); + assert!(cleaned.contains("GPT-5.5")); + assert!(!cleaned.to_ascii_lowercase().contains("unsubscribe")); + assert!(!cleaned.contains("©")); +} + +#[test] +fn drop_footer_noise_strips_legal_disclaimer() { + let body = "Action item — review by Friday.\n\nThis email and any files transmitted with it are confidential and intended solely for the use of the individual to whom they are addressed."; + let cleaned = drop_footer_noise(body); + assert_eq!(cleaned.trim(), "Action item — review by Friday."); +} + +#[test] +fn clean_body_combines_passes() { + let body = + "Real content here.\n\nOn Mon, Apr 22, 2026, Alice wrote:\n> old stuff\n\nUnsubscribe"; + let cleaned = clean_body(body); + assert_eq!(cleaned, "Real content here."); +} + +#[test] +fn collapse_blank_runs_keeps_paragraph_breaks() { + let s = "a\n\n\n\nb\n\n\nc\n"; + assert_eq!(collapse_blank_runs(s), "a\n\nb\n\nc"); +} + +#[test] +fn truncate_body_adds_ellipsis() { + let s = "x".repeat(2000); + let t = truncate_body(&s, 1200); + assert!(t.ends_with('…')); + assert_eq!(t.chars().count(), 1201); +} + +#[test] +fn truncate_body_passthrough_when_short() { + let s = "hello"; + let t = truncate_body(s, 1200); + assert_eq!(t, "hello"); +} + +#[test] +fn md_escape_handles_special_chars() { + assert_eq!(md_escape("a*b_c"), "a\\*b\\_c"); + assert_eq!(md_escape("foo|bar"), "foo\\|bar"); + assert_eq!(md_escape("line1\nline2"), "line1 line2"); + assert_eq!(md_escape("plain text"), "plain text"); +} + +#[test] +fn extract_email_handles_both_forms() { + assert_eq!( + extract_email("Alice ").as_deref(), + Some("alice@example.com") + ); + assert_eq!( + extract_email("notify@github.com").as_deref(), + Some("notify@github.com") + ); + assert_eq!( + extract_email("\"Bot Name\" ").as_deref(), + Some("bot@x.io") + ); + assert!(extract_email("Alice").is_none()); +} + +#[test] +fn parse_message_date_handles_iso_and_rfc2822() { + let iso = json!({"date": "2026-04-21T10:00:00Z"}); + let rfc = json!({"date": "Mon, 21 Apr 2026 10:00:00 +0000"}); + let ms = json!({"date": 1745236800000_i64}); + let ms_str = json!({"date": "1745236800000"}); + let internal_ms_str = json!({"internalDate": "1745236800000"}); + let nested_internal_ms_str = json!({"data": {"internalDate": "1745236800000"}}); + let date_only = json!({"date": "2026-04-21"}); + assert!(parse_message_date(&iso).is_some()); + assert!(parse_message_date(&rfc).is_some()); + assert!(parse_message_date(&ms).is_some()); + assert!(parse_message_date(&ms_str).is_some()); + assert!(parse_message_date(&internal_ms_str).is_some()); + assert!(parse_message_date(&nested_internal_ms_str).is_some()); + assert!(parse_message_date(&date_only).is_some()); +} + +#[test] +fn parse_message_date_returns_none_when_missing_or_blank() { + assert!(parse_message_date(&json!({})).is_none()); + assert!(parse_message_date(&json!({"date": ""})).is_none()); + assert!(parse_message_date(&json!({"date": " "})).is_none()); +} + +#[test] +fn drop_reply_chain_handles_zwnj_in_body() { + let zwnj = "\u{200c}"; + let body = format!( + "سلام{}دوست عزیز، لطفاً بررسی کنید.\n\nOn Mon, Apr 22, 2026, Alice wrote:\n> old content", + zwnj + ); + + let cleaned = drop_reply_chain(&body); + + assert!(!cleaned.contains("old content")); + assert!( + cleaned.contains(zwnj), + "ZWNJ was incorrectly removed from real content" + ); + assert!(std::str::from_utf8(cleaned.as_bytes()).is_ok()); +} diff --git a/sync/src/email_markdown.rs b/sync/src/email_markdown.rs new file mode 100644 index 0000000..baff497 --- /dev/null +++ b/sync/src/email_markdown.rs @@ -0,0 +1,241 @@ +//! Email thread → markdown, shared shape with the engine's canonicaliser +//! (#18 §B1). +//! +//! The gmail pipeline stores one markdown document per message; the engine's +//! ingest path canonicalises full threads with the same header block and +//! body-cleaning rules. The exact output format is load-bearing twice over: +//! the chunker splits at `---\nFrom:` boundaries, and the engine writes the +//! same shape from its copy — `thread_markdown_format_is_pinned` holds the +//! two to one form. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Deserializer, Serialize}; + +use crate::email_clean; + +/// One message of a thread, in the canonicaliser's input shape. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct EmailMessage { + /// Sender, as the provider renders it. + pub from: String, + /// Direct recipients. + #[serde(default)] + pub to: Vec, + /// Carbon-copy recipients. + #[serde(default)] + pub cc: Vec, + /// Subject line. + pub subject: String, + #[serde( + default = "chrono_now", + serialize_with = "chrono::serde::ts_milliseconds::serialize", + deserialize_with = "deserialize_flexible_timestamp" + )] + /// When the message was sent. Accepts epoch-ms or RFC 3339 on the wire. + pub sent_at: DateTime, + /// Body text, best rendering the provider offers. + pub body: String, + /// Opaque pointer back to the raw source record. + #[serde(default)] + pub source_ref: Option, + /// `List-Unsubscribe` header, when present. + #[serde(default)] + pub list_unsubscribe: Option, +} + +/// A thread of messages from one provider. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct EmailThread { + /// Provider slug, e.g. `gmail`. + pub provider: String, + /// The thread's subject. + pub thread_subject: String, + /// Messages, any order; rendering sorts oldest-first. + pub messages: Vec, +} + +fn chrono_now() -> DateTime { + Utc::now() +} + +/// The thread as canonical markdown: one `---\nFrom:` block per message, +/// oldest first, bodies through [`email_clean::clean_body`]. `None` for an +/// empty thread. +pub fn thread_markdown(thread: EmailThread) -> Option { + if thread.messages.is_empty() { + return None; + } + let mut messages = thread.messages; + messages.sort_by_key(|m| m.sent_at); + + let mut md = String::new(); + // No leading `# Email thread — ...` header. Provider / subject info belongs + // in the MD front-matter. The chunker splits this output at `---\nFrom:` + // boundaries so each message becomes one chunk. + for msg in &messages { + md.push_str("---\n"); + md.push_str(&format!("From: {}\n", email_clean::md_escape(&msg.from))); + if !msg.to.is_empty() { + md.push_str(&format!( + "To: {}\n", + email_clean::md_escape(&msg.to.join(", ")) + )); + } + if !msg.cc.is_empty() { + md.push_str(&format!( + "Cc: {}\n", + email_clean::md_escape(&msg.cc.join(", ")) + )); + } + md.push_str(&format!( + "Subject: {}\n", + email_clean::md_escape(&msg.subject) + )); + md.push_str(&format!("Date: {}\n", msg.sent_at.to_rfc3339())); + + if let Some(unsub) = &msg.list_unsubscribe { + md.push_str(&format!( + "List-Unsubscribe: {}\n", + email_clean::md_escape(unsub) + )); + } + md.push('\n'); + let cleaned = email_clean::clean_body(msg.body.trim()); + if cleaned.is_empty() { + md.push('\n'); + } else { + let safe_body = cleaned + .lines() + .map(|line| { + if line.trim_end() == "---" { + format!("\\{line}") + } else { + line.to_string() + } + }) + .collect::>() + .join("\n"); + md.push_str(&safe_body); + } + md.push_str("\n\n"); + } + Some(md) +} + +/// Deserialise a `DateTime` from either: +/// - a JSON integer = epoch **milliseconds** (legacy callers — back-compat), +/// - a JSON string = RFC 3339 / ISO-8601 (e.g. `"2026-05-17T19:30:00Z"`), or +/// a decimal string containing epoch milliseconds. +/// +/// On an unparseable string a serde error is returned (no silent default). +/// Shared across chat, email, and document canonicalisers. +/// +fn deserialize_flexible_timestamp<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum RawTs { + Millis(i64), + Text(String), + Null, + } + + fn epoch_millis(ms: i64) -> Result, E> { + // Contemporary epoch seconds are ten digits while epoch milliseconds + // are thirteen. Reject the ambiguous near-epoch range so a seconds + // value cannot silently poison ordering and staleness calculations. + const MIN_PLAUSIBLE_EPOCH_MILLIS: u64 = 100_000_000_000; + if ms.unsigned_abs() < MIN_PLAUSIBLE_EPOCH_MILLIS { + return Err(E::custom(format!( + "epoch-ms value {ms} is too small; pass milliseconds, not seconds" + ))); + } + chrono::TimeZone::timestamp_millis_opt(&Utc, ms) + .single() + .ok_or_else(|| E::custom(format!("invalid epoch-ms: {ms}"))) + } + + let raw = RawTs::deserialize(deserializer)?; + match raw { + RawTs::Null => Ok(Utc::now()), + RawTs::Millis(ms) => epoch_millis(ms), + RawTs::Text(s) => { + if let Ok(dt) = DateTime::parse_from_rfc3339(&s) { + return Ok(dt.with_timezone(&Utc)); + } + if let Ok(ms) = s.parse::() { + return epoch_millis(ms); + } + Err(serde::de::Error::custom(format!( + "cannot parse '{s}' as RFC 3339 or epoch-ms" + ))) + } + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + /// The engine's canonicaliser emits this exact shape from its own copy of + /// this assembly, and the chunker splits on `---\nFrom:`. A failure here + /// is a coordinated format change, never a local edit. + #[test] + fn thread_markdown_format_is_pinned() { + let thread = EmailThread { + provider: "gmail".into(), + thread_subject: "Hello".into(), + messages: vec![EmailMessage { + from: "a@example.com".into(), + to: vec!["b@example.com".into()], + cc: Vec::new(), + subject: "Hello".into(), + sent_at: DateTime::parse_from_rfc3339("2026-01-02T03:04:05Z") + .unwrap() + .with_timezone(&Utc), + body: "Hi there".into(), + source_ref: Some("gmail:m1".into()), + list_unsubscribe: None, + }], + }; + assert_eq!( + thread_markdown(thread).unwrap(), + "---\nFrom: a@example.com\nTo: b@example.com\nSubject: Hello\nDate: 2026-01-02T03:04:05+00:00\n\nHi there\n\n" + ); + } + + #[test] + fn empty_thread_is_none_and_body_separators_are_escaped() { + assert!(thread_markdown(EmailThread { + provider: "gmail".into(), + thread_subject: String::new(), + messages: Vec::new(), + }) + .is_none()); + + let thread = EmailThread { + provider: "gmail".into(), + thread_subject: "s".into(), + messages: vec![EmailMessage { + from: "a".into(), + to: Vec::new(), + cc: Vec::new(), + subject: "s".into(), + sent_at: DateTime::parse_from_rfc3339("2026-01-02T03:04:05Z") + .unwrap() + .with_timezone(&Utc), + body: "x\n---\ny".into(), + source_ref: None, + list_unsubscribe: None, + }], + }; + let md = thread_markdown(thread).unwrap(); + assert!( + md.contains("\\---"), + "chunk separator must be escaped: {md}" + ); + } +} diff --git a/sync/src/lib.rs b/sync/src/lib.rs index 4eb3ba6..c00302e 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -34,5 +34,7 @@ pub mod notion; // The `_post_process` suffix is kept from the engine layout it came from, where // `slack.rs` and `github.rs` one directory up already held those names. Renaming // on the way out would have made this a rename *and* a move in one diff. +pub mod email_clean; +pub mod email_markdown; pub mod gmail_post_process; pub mod slack_post_process; From 172a3824b97b97886ec0b9c2cf1904eaed83b52e Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 19 Aug 2026 02:15:45 +0530 Subject: [PATCH 3/6] docs: give the moved module docs explicit link paths Docs CI (-D warnings) rejects two intra-doc links the moves carried along: email_clean's //! header pointed at its old engine sibling (super::email -> crate::email_markdown here), and host.rs's //! header used a bare [MemoryClient] that resolves in ///-position but not in module docs. The #44 commit recorded this exact asymmetry; it holds. RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --all-features: clean --- core/src/sync/pipelines/host.rs | 6 ++++-- sync/src/email_clean.rs | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/core/src/sync/pipelines/host.rs b/core/src/sync/pipelines/host.rs index ec55b56..e709164 100644 --- a/core/src/sync/pipelines/host.rs +++ b/core/src/sync/pipelines/host.rs @@ -1,10 +1,12 @@ //! The host side of the engine-free pipelines (#18 §B1): the sink adapter -//! over [`MemoryClient`], the Composio settings mapping, and the runners the +//! over [`MemoryClient`](crate::store::MemoryClient), the Composio settings +//! mapping, and the runners the //! rest of `core/src/sync/` calls. //! //! This is the piece §B5's acceptance rests on: a pipeline sees three //! capabilities — events, documents, state — and every one resolves through -//! [`MemoryClient`], so whatever driver the host bound serves the sync. +//! [`MemoryClient`](crate::store::MemoryClient), so whatever driver the host +//! bound serves the sync. use std::sync::Arc; diff --git a/sync/src/email_clean.rs b/sync/src/email_clean.rs index c48f84a..dbaa989 100644 --- a/sync/src/email_clean.rs +++ b/sync/src/email_clean.rs @@ -1,6 +1,6 @@ //! Shared email rendering + cleaning helpers. //! -//! Used by [`super::email`] when rendering canonical email markdown. The module +//! Used by [`crate::email_markdown`] when rendering canonical email markdown. The module //! is intentionally pure-string-oriented plus a single `serde_json::Value` //! helper (`parse_message_date`) for callers that work directly off slim //! envelope JSON. Nothing here depends on the chunk-store types, which keeps the From f0094eef9b3f8f79013ec5964b0e3c678d96d5ff Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 19 Aug 2026 10:41:43 +0530 Subject: [PATCH 4/6] =?UTF-8?q?ci:=20enforce=20engine=20containment=20(#18?= =?UTF-8?q?=20=C2=A7C1)=20as=20the=20criterion=20means=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #18's first acceptance criterion is spelled as a literal grep -- `grep -rl tinycortex core/src` -- and read literally it fails today with 59 files outside the engine module, every one a doc comment, string literal, or log tag like "[tinycortex:sync]". Prose will always match it; what the criterion means is that no file outside `core/src/engine/` reaches the engine through a *code path*. `scripts/ci/engine-containment.sh` tests that meaning: comment lines stripped, then `use tinycortex` or a `tinycortex::` path segment outside the engine module fails the build. On the current tree there are zero. Verified both ways before wiring it in: an injected `tinycortex::` call in `sync/audit.rs` fails it; an injected prose-only mention passes. Nothing enforced containment before this -- the criterion was met by inspection, and a later `use tinycortex` in core/src/store/ would have merged green. --- .github/workflows/ci.yml | 3 +++ scripts/ci/engine-containment.sh | 35 ++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100755 scripts/ci/engine-containment.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53d1d59..c306e53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,6 +86,9 @@ jobs: - name: Run the bundled example run: cargo run --example basic + - name: Assert engine containment (#18 §C1) + run: ./scripts/ci/engine-containment.sh + - name: Assert the contract crate stays free of heavy dependencies run: | forbidden="$(cargo tree -p tinymemory-api -e normal,build --prefix none \ diff --git a/scripts/ci/engine-containment.sh b/scripts/ci/engine-containment.sh new file mode 100755 index 0000000..6ead0e6 --- /dev/null +++ b/scripts/ci/engine-containment.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Issue #18 §C1 acceptance: nothing outside core's engine module names the +# tinycortex crate in code. +# +# The issue's literal check -- `grep -rl tinycortex core/src` -- counts prose: +# doc comments, string literals, and log tags like "[tinycortex:sync]" match it +# and always will. What the criterion *means* is that no file outside +# `core/src/engine/` reaches the engine through a code path. This script tests +# that: a `use tinycortex...` item or a `tinycortex::` path segment, in a +# non-comment position, outside the engine module. +set -euo pipefail + +cd "$(dirname "$0")/../.." + +# Strip comment lines (`//`, `///`, `//!`) before matching so prose cannot +# trip it; then require the crate name in path position. +offenders="$( + grep -rln --include='*.rs' 'tinycortex' core/src \ + | grep -v '^core/src/engine/' \ + | while read -r f; do + if sed -E 's://.*$::' "$f" \ + | grep -Eq '(^|[^A-Za-z0-9_])(use[[:space:]]+tinycortex\b|tinycortex::)'; then + echo "$f" + fi + done +)" + +if [ -n "$offenders" ]; then + echo "core/src files outside the engine module reach tinycortex in code:" >&2 + echo "$offenders" | sed 's/^/ /' >&2 + echo >&2 + echo "Route through core/src/engine/ (the seam) or the memory contract." >&2 + exit 1 +fi +echo "engine containment holds: no code path names tinycortex outside core/src/engine/" From 6555bfd0f0ffbdb25d049924ec0772418a53970d Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 19 Aug 2026 10:55:44 +0530 Subject: [PATCH 5/6] Match the toolkit on its normalised slug; forward registry caps Two CodeRabbit findings on #48, both taken: 1. `build_composio_pipeline` gated on `is_composio_toolkit_syncable`, which trims and lowercases, then matched on the RAW toolkit and put `unreachable!` in the fallthrough. `" Gmail "` passed the gate and panicked the sync task. Introduced in the port (the engine seam lowercased before this point); normalise once, match on the slug. Regression test feeds padded/mixed-case toolkits through the build. 2. `run_connection_sync` resolved the source's `max_items` / `sync_depth_days` from the registry, logged them as "caps from registry", then passed `None, None`. Pre-existing on main -- the old engine entry point had no budget parameters, so the caps had nowhere to go. The new runner takes them; forwarded. Manual/trigger syncs now honour the same caps the periodic loop already did. Closes the gap #49 was filed for. The other seven findings on this PR are defects in code the PR MOVED without changing (client timeouts, `successful` defaulting, the retry needle, slack backfill state-save-on-error, the two-write audit append, google_docs paging, token/cost caps). Each is real; each is filed as its own issue rather than fixed here, because a behaviour change hidden inside a relocation is exactly how regressions dodge review. cargo test -p tinymemory-core --lib: 844 passed --- core/src/sync/composio/mod.rs | 6 +++--- core/src/sync/pipelines/host.rs | 21 +++++++++++++++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/core/src/sync/composio/mod.rs b/core/src/sync/composio/mod.rs index 94219b4..6d5d907 100644 --- a/core/src/sync/composio/mod.rs +++ b/core/src/sync/composio/mod.rs @@ -156,7 +156,7 @@ pub async fn run_connection_sync( "[composio:sync] run_connection_sync: caps from registry" ); - let _ = (provider, src_max_items, src_sync_depth_days); + let _ = provider; let started_at_ms = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() @@ -165,8 +165,8 @@ pub async fn run_connection_sync( &target.toolkit, &target.connection_id, &*config, - None, - None, + src_max_items, + src_sync_depth_days, ) .await { diff --git a/core/src/sync/pipelines/host.rs b/core/src/sync/pipelines/host.rs index e709164..d8cd072 100644 --- a/core/src/sync/pipelines/host.rs +++ b/core/src/sync/pipelines/host.rs @@ -250,11 +250,16 @@ fn build_composio_pipeline( ) -> Result, String> { // Fail closed before resolving credentials for any toolkit without a // native pipeline (#4957) — the gate stays a single testable list. - if !is_composio_toolkit_syncable(toolkit) { + // + // Normalise once and match on the normalised slug: the gate accepts + // `" Gmail "` (trim + lowercase), so matching on the raw input would let a + // padded or mixed-case toolkit through the gate and into `unreachable!`. + let slug = toolkit.trim().to_ascii_lowercase(); + if !syncable_composio_toolkits().contains(&slug.as_str()) { return Err(format!("memory sync does not support toolkit '{toolkit}'")); } let client = ComposioClient::new(composio); - Ok(match toolkit { + Ok(match slug.as_str() { "gmail" => Arc::new(GmailSyncPipeline::new(client, connection_id)), "github" => Arc::new(GitHubSyncPipeline::new(client, connection_id)), "notion" => Arc::new(NotionSyncPipeline::new(client, connection_id)), @@ -378,4 +383,16 @@ mod tests { assert!(!is_composio_toolkit_syncable("googlecalendar")); assert!(is_composio_toolkit_syncable(" Gmail ")); } + + /// The gate normalises; the build must match on the same normalised + /// slug, or a padded/mixed-case toolkit passes the gate and panics. + #[test] + fn a_padded_or_mixed_case_toolkit_builds_rather_than_panicking() { + for toolkit in [" Gmail ", "GMAIL", "gmail\t", " Slack"] { + assert!( + build_composio_pipeline(toolkit, "conn-1", ComposioSyncConfig::default()).is_ok(), + "{toolkit:?} passes the gate and must build" + ); + } + } } From 7434e2bfcf8d79ed672ae17d973c316740a9c5bb Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 19 Aug 2026 11:04:20 +0530 Subject: [PATCH 6/6] Fix the seven pre-existing defects CodeRabbit found in the moved sync code These were filed as #52-#58 on the move-not-change rule; the decision is to fix them here rather than defer. Each is real and each predates the move -- the engine originals carry the same code -- so this commit is the one place in the PR that changes behaviour, and it says so. Composio client (#52, #53, #54): - Explicit connect (15s) and request (120s) timeouts via ClientBuilder. A hung connection stalled the sync task and held the state-mutation window open. Build failure panics rather than falling back to an untimed client -- unwrap_or_default() would drop the guarantee silently. - A payload that reports an error is not a success, whatever the `successful` flag says or omits. Extracted `decode_direct_response` so the rule is unit-tested; consumers gate document creation on the flag, and an error body must never be stored as content. - Retry classification is by status, not by the "request failed" substring both status-bail messages also matched. 400/401/403/404 no longer retry three times with backoff; transport failures are now reported as "... transport error: ..." and stay retryable. Slack search backfill (#55): run the body, then always save the state. `checked_execute` records billable requests before returning an error; propagating before the save lost the accounting and left the daily budget unadvanced. Same contract `run_incremental_sync` keeps. Audit append (#56): one buffer, one write_all. Two appenders share the file; a two-syscall append let their lines interleave and the reader then skipped both. The line format is unchanged (byte-pinned). Google Docs paging (#57): deterministic `order_by: modifiedTime desc` and the cursor (RFC 3339-validated, else omitted) as a `q` mod-time floor -- the same shape google_drive already used. The action returned the identical first batch every tick; documents past `max_results` were unreachable. Per-source spend caps (#58): `PipelineConfig` gains `max_tokens_per_sync` / `max_cost_per_sync_usd`; the orchestrator checks both beside `max_items` with the same stop-and-leave-pending contract, so a capped run resumes from its cursor. New `SourceCaps::from_source` + `run_composio_connection_with_caps`; the seam and the periodic loop pass the full source caps. cargo test -p tinymemory-core: 847 passed (+3 client tests, +1 padded- toolkit test from the previous commit), E2E acceptance test green cargo clippy -p tinymemory-core --all-targets: clean --- core/src/engine/sync.rs | 5 +- core/src/sync/audit.rs | 10 +- core/src/sync/composio/periodic.rs | 5 +- core/src/sync/pipelines/composio/client.rs | 139 ++++++++++++++---- .../sync/pipelines/composio/orchestrator.rs | 23 +++ .../composio/providers/google_docs.rs | 42 +++++- .../pipelines/composio/providers/slack.rs | 25 +++- core/src/sync/pipelines/host.rs | 48 +++++- core/src/sync/pipelines/traits.rs | 6 + 9 files changed, 258 insertions(+), 45 deletions(-) diff --git a/core/src/engine/sync.rs b/core/src/engine/sync.rs index 11ac63b..442521a 100644 --- a/core/src/engine/sync.rs +++ b/core/src/engine/sync.rs @@ -329,12 +329,11 @@ pub async fn run_source_pipeline( .ok_or_else(|| { SourcePipelineFailure::without_usage("composio source missing connection_id") })?; - let outcome = crate::sync::pipelines::host::run_composio_connection( + let outcome = crate::sync::pipelines::host::run_composio_connection_with_caps( &toolkit, connection_id, config, - source.max_items, - source.sync_depth_days, + crate::sync::pipelines::host::SourceCaps::from_source(source), ) .await .map_err(|failure| SourcePipelineFailure { diff --git a/core/src/sync/audit.rs b/core/src/sync/audit.rs index e54838f..a687698 100644 --- a/core/src/sync/audit.rs +++ b/core/src/sync/audit.rs @@ -85,8 +85,14 @@ pub fn append_audit_entry(workspace: &Path, entry: &SyncAuditEntry) -> anyhow::R .create(true) .append(true) .open(directory.join(AUDIT_FILENAME))?; - serde_json::to_writer(&mut file, entry)?; - writeln!(file)?; + // One buffer, one write. Two appenders share this file (the periodic loop + // and the manual source sync), and a two-syscall append lets their lines + // interleave; the reader would then skip both as malformed. A single + // `write_all` on an O_APPEND handle lands the whole line atomically for + // any plausible entry size. + let mut line = serde_json::to_vec(entry)?; + line.push(b'\n'); + file.write_all(&line)?; tracing::debug!(source_id = %entry.source_id, success = entry.success, "[memory_sync:audit] entry appended"); Ok(()) } diff --git a/core/src/sync/composio/periodic.rs b/core/src/sync/composio/periodic.rs index cf64fd7..f0c8c4e 100644 --- a/core/src/sync/composio/periodic.rs +++ b/core/src/sync/composio/periodic.rs @@ -556,12 +556,11 @@ pub(crate) async fn run_one_tick() -> Result<(), String> { "[composio:periodic] firing sync" ); let sync_started = Instant::now(); - let result = crate::sync::pipelines::host::run_composio_connection( + let result = crate::sync::pipelines::host::run_composio_connection_with_caps( &toolkit, &conn.id, &*config, - source.max_items, - source.sync_depth_days, + crate::sync::pipelines::host::SourceCaps::from_source(&source), ) .await; let duration_ms = sync_started.elapsed().as_millis() as u64; diff --git a/core/src/sync/pipelines/composio/client.rs b/core/src/sync/pipelines/composio/client.rs index b7211f5..ba1fcff 100644 --- a/core/src/sync/pipelines/composio/client.rs +++ b/core/src/sync/pipelines/composio/client.rs @@ -32,6 +32,12 @@ pub struct ExecuteError { message: String, } +/// Time to establish a TCP/TLS connection to Composio or the proxy. +const CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15); +/// Whole-request ceiling. Composio actions that page a large mailbox can run +/// long, so this is generous, but it is finite. +const REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(120); + #[derive(Clone)] pub struct ComposioClient { http: reqwest::Client, @@ -61,11 +67,23 @@ impl ActionExecutor for ComposioClient { } impl ComposioClient { + /// A client with explicit connect and request timeouts. + /// + /// `reqwest::Client::new()` has none: a hung Composio or proxy connection + /// would stall the sync task indefinitely and hold the sync-state + /// mutation window open with it. The builder is fallible only on TLS + /// backend initialisation, which cannot happen with the rustls feature this + /// crate compiles; if it ever did, an untimed fallback would silently drop + /// the guarantee, so it panics loudly instead of degrading. pub fn new(config: ComposioSyncConfig) -> Self { - Self { - http: reqwest::Client::new(), - config, - } + let http = reqwest::Client::builder() + .connect_timeout(CONNECT_TIMEOUT) + .timeout(REQUEST_TIMEOUT) + .build() + .unwrap_or_else(|error| { + panic!("Composio HTTP client failed to build (TLS backend unavailable): {error}") + }); + Self { http, config } } pub fn with_http_client(mut self, http: reqwest::Client) -> Self { @@ -165,31 +183,14 @@ impl ComposioClient { .json(&body) .send() .await - .map_err(|error| anyhow::anyhow!("Composio direct request failed: {error}"))?; + .map_err(|error| anyhow::anyhow!("Composio direct transport error: {error}"))?; let status = response.status(); if !status.is_success() { let _ = response.bytes().await; anyhow::bail!("Composio direct request failed with HTTP {status}"); } let raw: serde_json::Value = decode_response(response, "direct").await?; - let successful = raw - .get("successful") - .and_then(serde_json::Value::as_bool) - .or_else(|| raw.get("success").and_then(serde_json::Value::as_bool)) - .unwrap_or(true); - let error = raw - .get("error") - .and_then(serde_json::Value::as_str) - .map(str::to_owned); - let data = raw.get("data").cloned().unwrap_or(raw); - Ok(ExecuteResponse { - data, - successful, - error, - cost_usd: 0.0, - markdown_formatted: None, - attempts: 1, - }) + Ok(decode_direct_response(raw)) } async fn execute_proxied( @@ -214,7 +215,7 @@ impl ComposioClient { .json(&serde_json::json!({ "tool": action, "arguments": arguments })) .send() .await - .map_err(|error| anyhow::anyhow!("Composio proxy request failed: {error}"))?; + .map_err(|error| anyhow::anyhow!("Composio proxy transport error: {error}"))?; let status = response.status(); if !status.is_success() { let _ = response.bytes().await; @@ -228,6 +229,35 @@ impl ComposioClient { } } +/// Shape a direct-API payload into an [`ExecuteResponse`]. +/// +/// A payload that reports an `error` is not a success, whatever the +/// `successful` flag says or omits: consumers gate document creation on the +/// flag, and an error body must never be stored as content. +fn decode_direct_response(raw: serde_json::Value) -> ExecuteResponse { + let flagged = raw + .get("successful") + .and_then(serde_json::Value::as_bool) + .or_else(|| raw.get("success").and_then(serde_json::Value::as_bool)) + .unwrap_or(true); + let error = raw + .get("error") + .and_then(serde_json::Value::as_str) + .map(str::trim) + .filter(|error| !error.is_empty()) + .map(str::to_owned); + let successful = flagged && error.is_none(); + let data = raw.get("data").cloned().unwrap_or(raw); + ExecuteResponse { + data, + successful, + error, + cost_usd: 0.0, + markdown_formatted: None, + attempts: 1, + } +} + fn decode_proxy_response(raw: serde_json::Value) -> anyhow::Result { let payload = if raw.get("successful").is_some() { raw @@ -247,6 +277,14 @@ fn retryable_provider_error(error: Option<&str>) -> bool { }) } +/// Whether a failed execute is worth retrying with backoff. +/// +/// Retryable: rate limiting and upstream unavailability (429/502/503/504), +/// and transport failures (connect/read errors, timeouts) — reported by the +/// request paths as `"… transport error: …"`. NOT retryable: any other HTTP +/// status. 400/401/403/404 are permanent — an invalid API key must fail once, +/// not storm three times — and used to be caught by a `"request failed"` +/// needle that both status-bail messages also matched. fn retryable_transport_error(error: &anyhow::Error) -> bool { let message = error.to_string(); [ @@ -254,7 +292,7 @@ fn retryable_transport_error(error: &anyhow::Error) -> bool { "HTTP 502", "HTTP 503", "HTTP 504", - "request failed", + "transport error", ] .iter() .any(|needle| message.contains(needle)) @@ -274,6 +312,57 @@ async fn decode_response( mod tests { use super::*; + /// 4xx is permanent: an invalid key must fail once, not retry with + /// backoff. Only rate-limit/upstream statuses and transport failures + /// (connect/read errors, timeouts) are worth another attempt. + #[test] + fn retry_classification_is_by_status_not_by_substring() { + let retry = |m: &str| retryable_transport_error(&anyhow::anyhow!("{m}")); + assert!(retry( + "Composio direct request failed with HTTP 429 Too Many Requests" + )); + assert!(retry( + "Composio proxy request failed with HTTP 503 Service Unavailable" + )); + assert!(retry("Composio direct transport error: connection reset")); + assert!(!retry( + "Composio direct request failed with HTTP 401 Unauthorized" + )); + assert!(!retry( + "Composio proxy request failed with HTTP 404 Not Found" + )); + assert!(!retry( + "Composio direct request failed with HTTP 400 Bad Request" + )); + } + + /// An error payload is a failure even when the flag is absent or true. + #[test] + fn an_error_payload_is_never_a_success() { + let r = decode_direct_response(serde_json::json!({"error": "quota exceeded"})); + assert!(!r.successful, "missing flag + error must be a failure"); + assert_eq!(r.error.as_deref(), Some("quota exceeded")); + + let r = decode_direct_response(serde_json::json!({"successful": true, "error": " boom "})); + assert!(!r.successful, "flag=true + error must still be a failure"); + assert_eq!(r.error.as_deref(), Some("boom")); + + let r = decode_direct_response( + serde_json::json!({"successful": true, "error": " ", "data": {"x": 1}}), + ); + assert!(r.successful, "an empty error string is no error"); + assert!(r.error.is_none()); + assert_eq!(r.data["x"], 1); + } + + /// The client is built with finite timeouts; a build failure must not + /// silently degrade to an untimed client. + #[test] + fn client_builds_with_timeouts() { + let _ = ComposioClient::new(ComposioSyncConfig::default()); + assert!(CONNECT_TIMEOUT < REQUEST_TIMEOUT); + } + #[test] fn proxied_backend_envelope_decodes_provider_response() { let response = decode_proxy_response(serde_json::json!({ diff --git a/core/src/sync/pipelines/composio/orchestrator.rs b/core/src/sync/pipelines/composio/orchestrator.rs index 3a9340d..fd20137 100644 --- a/core/src/sync/pipelines/composio/orchestrator.rs +++ b/core/src/sync/pipelines/composio/orchestrator.rs @@ -224,6 +224,8 @@ async fn run_pages( ) -> anyhow::Result { let mut newest_cursor = state.cursor.clone(); let mut ingested = 0u32; + // Estimated tokens of stored content this run, for `max_tokens_per_sync`. + let mut tokens_ingested: u64 = 0; let mut more_pending = false; let depth_floor = (!source.server_side_depth()) .then(|| source.depth_floor(config, state)) @@ -346,6 +348,23 @@ async fn run_pages( more_pending = true; break 'scopes; } + // Per-source spend caps (#18): checked with the same "stop, + // leave the rest pending" contract as `max_items`, so a + // capped run resumes from its cursor next tick. + if config + .max_cost_per_sync_usd + .is_some_and(|cap| state.run_provider_cost_usd >= cap) + { + more_pending = true; + break 'scopes; + } + if config + .max_tokens_per_sync + .is_some_and(|cap| tokens_ingested >= cap) + { + more_pending = true; + break 'scopes; + } let Some(dedup_key) = source.dedup_key(&raw) else { continue; }; @@ -401,6 +420,10 @@ async fn run_pages( } Err(error) => return Err(error), }; + // Same rough estimate the tree's budgeting uses (~4 chars per + // token); a cap, not an invoice. + tokens_ingested = + tokens_ingested.saturating_add((document.content.len() / 4) as u64); if let Err(error) = context.documents.store(document).await { if source.tolerate_scope_errors() { tracing::warn!(toolkit = source.toolkit(), connection_id, scope = %scope.label, %error, "[sync:orchestrator] scope document store failed; continuing"); diff --git a/core/src/sync/pipelines/composio/providers/google_docs.rs b/core/src/sync/pipelines/composio/providers/google_docs.rs index c2776fb..e180974 100644 --- a/core/src/sync/pipelines/composio/providers/google_docs.rs +++ b/core/src/sync/pipelines/composio/providers/google_docs.rs @@ -78,14 +78,44 @@ impl IncrementalSource for GoogleDocsSyncPipeline { fn arguments( &self, _: &SyncScope, - _: &PipelineConfig, - _: &SyncState, + config: &PipelineConfig, + state: &SyncState, _page: Option<&str>, ) -> Value { - // NOTE: an empty/broad `query` enumerates every accessible document; - // `max_results` bounds the batch. Both mirror the underlying Drive - // search parameters. No page token is emitted (see `max_pages`). - serde_json::json!({"query": "", "max_results": self.page_size}) + // `GOOGLEDOCS_SEARCH_DOCUMENTS` fronts Drive's `files.list`, so it takes + // the same server-side controls Drive does. Order deterministically by + // modification time and bound the window with a `q` clause, so each + // tick fetches what changed since the cursor rather than the same + // first batch forever. Without this the action returned the identical + // page every tick and documents past `max_results` were unreachable. + let mut args = serde_json::json!({ + "query": "", + "max_results": self.page_size, + "order_by": "modifiedTime desc", + }); + // Prefer the last-synced cursor, else the configured horizon. The + // cursor is validated as RFC 3339 before it is interpolated into `q`, + // so a malformed persisted value can never inject into the query — on + // a bad value the depth filter is simply omitted (full scan). + let floor = state + .cursor + .as_deref() + .filter(|cursor| chrono::DateTime::parse_from_rfc3339(cursor).is_ok()) + .map(str::to_owned) + .or_else(|| { + config.sync_depth_days.map(|days| { + (chrono::Utc::now() - chrono::Duration::days(days as i64)).to_rfc3339() + }) + }); + if let Some(floor) = floor { + args["q"] = serde_json::json!(format!("modifiedTime > '{floor}'")); + } + args + } + fn server_side_depth(&self) -> bool { + // The `q` floor above bounds depth on the server, so the orchestrator + // must not additionally treat the cursor as a client-side stop. + true } fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { PageFetch { diff --git a/core/src/sync/pipelines/composio/providers/slack.rs b/core/src/sync/pipelines/composio/providers/slack.rs index 53bbe3c..3953d6f 100644 --- a/core/src/sync/pipelines/composio/providers/slack.rs +++ b/core/src/sync/pipelines/composio/providers/slack.rs @@ -76,9 +76,28 @@ impl SyncPipeline for SlackSearchBackfillPipeline { }); } + // Run the body, then save the state on BOTH paths. `checked_execute` + // records billable requests and provider cost into `state` before it + // returns an error; propagating that error before the save would lose + // the accounting and leave the daily budget unadvanced, so a backfill + // that fails repeatedly could keep calling the search action unbudgeted. + // Same contract `run_incremental_sync` keeps. + let result = self.run_backfill(&mut state, context).await; + state.last_sync_at_ms = Some(Utc::now().timestamp_millis() as u64); + state.save(context.state.as_ref()).await?; + result + } +} + +impl SlackSearchBackfillPipeline { + async fn run_backfill( + &self, + state: &mut SyncState, + context: &SyncContext, + ) -> anyhow::Result { let directory = SlackSyncPipeline::new(self.client.clone(), self.connection_id.clone()); let scopes = directory - .scopes(&self.client, &self.connection_id, &mut state) + .scopes(&self.client, &self.connection_id, state) .await?; let channels: HashMap<_, _> = scopes .into_iter() @@ -109,7 +128,7 @@ impl SyncPipeline for SlackSearchBackfillPipeline { "page": page, }), &self.connection_id, - &mut state, + state, ) .await?; if page == 1 { @@ -161,8 +180,6 @@ impl SyncPipeline for SlackSearchBackfillPipeline { page = page.saturating_add(1); } - state.last_sync_at_ms = Some(Utc::now().timestamp_millis() as u64); - state.save(context.state.as_ref()).await?; Ok(SyncOutcome { records_ingested: stored, more_pending: page < total_pages, diff --git a/core/src/sync/pipelines/host.rs b/core/src/sync/pipelines/host.rs index d8cd072..1b8a403 100644 --- a/core/src/sync/pipelines/host.rs +++ b/core/src/sync/pipelines/host.rs @@ -277,6 +277,48 @@ pub async fn run_composio_connection( config: &Config, max_items: Option, sync_depth_days: Option, +) -> Result { + run_composio_connection_with_caps( + toolkit, + connection_id, + config, + SourceCaps { + max_items, + sync_depth_days, + ..SourceCaps::default() + }, + ) + .await +} + +/// The per-source limits a run honours. All `None` = the source's defaults. +#[derive(Clone, Copy, Debug, Default)] +pub struct SourceCaps { + pub max_items: Option, + pub sync_depth_days: Option, + pub max_tokens_per_sync: Option, + pub max_cost_per_sync_usd: Option, +} + +impl SourceCaps { + /// The caps a registry entry carries. + pub fn from_source(source: &tinymemory_sources::MemorySourceEntry) -> Self { + Self { + max_items: source.max_items, + sync_depth_days: source.sync_depth_days, + max_tokens_per_sync: source.max_tokens_per_sync, + max_cost_per_sync_usd: source.max_cost_per_sync_usd, + } + } +} + +/// Run one Composio connection through the engine-free pipelines, honouring +/// every per-source cap. +pub async fn run_composio_connection_with_caps( + toolkit: &str, + connection_id: &str, + config: &Config, + caps: SourceCaps, ) -> Result { let memory = crate::global::client_if_ready() .ok_or_else(|| PipelineFailure::without_usage("memory client is not ready"))?; @@ -285,8 +327,10 @@ pub async fn run_composio_connection( .map_err(PipelineFailure::without_usage)?; let pipeline_config = PipelineConfig { composio: None, // the client already holds the connection settings - sync_depth_days, - max_items, + sync_depth_days: caps.sync_depth_days, + max_items: caps.max_items, + max_tokens_per_sync: caps.max_tokens_per_sync, + max_cost_per_sync_usd: caps.max_cost_per_sync_usd, }; let host = Arc::new(PipelineHost::new(memory, config.to_arc())); run_pipeline(pipeline, &pipeline_config, &host.context()).await diff --git a/core/src/sync/pipelines/traits.rs b/core/src/sync/pipelines/traits.rs index 62fe719..a3b76d3 100644 --- a/core/src/sync/pipelines/traits.rs +++ b/core/src/sync/pipelines/traits.rs @@ -137,6 +137,12 @@ pub struct PipelineConfig { pub composio: Option, pub sync_depth_days: Option, pub max_items: Option, + /// Stop the run once this many tokens (estimated from stored content) + /// have been ingested. `None` = unbounded. + pub max_tokens_per_sync: Option, + /// Stop the run once the provider has charged this much. `None` = + /// unbounded. + pub max_cost_per_sync_usd: Option, } /// Host capabilities required by sync pipelines.