From 4d2885481f0273b81dda774125d3eafa7e6660b6 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 2 Jun 2026 17:09:40 -0500 Subject: [PATCH 01/24] feat: add distributed project manifest primitives --- src/lib.rs | 6 + src/manifest.rs | 224 +++++++++++++++++++++++++++++++++++++ src/read_model/metadata.rs | 18 +-- 3 files changed, 239 insertions(+), 9 deletions(-) create mode 100644 src/manifest.rs diff --git a/src/lib.rs b/src/lib.rs index aca1294..7f44a50 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,6 +13,7 @@ mod commit_builder; pub mod emitter; mod hashmap_repo; pub mod lock; +pub mod manifest; pub mod microsvc; mod outbox; mod outbox_worker; @@ -131,6 +132,11 @@ pub use table::{ TableStoreError, TableWritePlan, DEFAULT_TABLE_VERSION_COLUMN, }; +pub use manifest::{ + DistributedManifestEnvelope, DistributedProjectManifest, MessageEndpointManifest, + ServiceManifest, TransportManifest, DISTRIBUTED_MANIFEST_SCHEMA_VERSION, +}; + // CommitBuilder: transactional batches of read models, outbox, and aggregates pub use commit_builder::{ CommitBuilder, CommitBuilderExt, ReadModelWritePlanCommitExt, StagedCommitBuilder, diff --git a/src/manifest.rs b/src/manifest.rs new file mode 100644 index 0000000..a8ae32d --- /dev/null +++ b/src/manifest.rs @@ -0,0 +1,224 @@ +use serde::{Deserialize, Serialize}; + +use crate::{ + generate_table_migration_artifacts, table_schema_statements, ReadModelError, + ReadModelMigrationArtifact, RelationalReadModel, TableSchema, TableSchemaRegistry, + TableSqlDialect, +}; + +pub const DISTRIBUTED_MANIFEST_SCHEMA_VERSION: u32 = 1; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DistributedManifestEnvelope { + pub schema_version: u32, + pub project: DistributedProjectManifest, +} + +impl DistributedManifestEnvelope { + pub fn new(project: DistributedProjectManifest) -> Self { + Self { + schema_version: DISTRIBUTED_MANIFEST_SCHEMA_VERSION, + project, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DistributedProjectManifest { + pub name: String, + pub tables: Vec, + pub services: Vec, +} + +impl DistributedProjectManifest { + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + tables: Vec::new(), + services: Vec::new(), + } + } + + pub fn read_model(mut self) -> Self + where + M: RelationalReadModel, + { + self.try_register_read_model::() + .expect("read model schema should be valid in distributed manifest"); + self + } + + pub fn try_read_model(mut self) -> Result + where + M: RelationalReadModel, + { + self.try_register_read_model::()?; + Ok(self) + } + + pub fn try_register_read_model(&mut self) -> Result<&mut Self, ReadModelError> + where + M: RelationalReadModel, + { + self.try_register_table_schema(M::schema()) + } + + pub fn table_schema(mut self, schema: TableSchema) -> Self { + self.try_register_table_schema(schema) + .expect("table schema should be valid in distributed manifest"); + self + } + + pub fn try_table_schema(mut self, schema: TableSchema) -> Result { + self.try_register_table_schema(schema)?; + Ok(self) + } + + pub fn try_register_table_schema( + &mut self, + schema: TableSchema, + ) -> Result<&mut Self, ReadModelError> { + let mut registry = self.table_registry()?; + registry.register_schema(schema.clone())?; + self.tables.push(schema); + Ok(self) + } + + pub fn service(mut self, service: ServiceManifest) -> Self { + self.services.push(service); + self + } + + pub fn table_registry(&self) -> Result { + let mut registry = TableSchemaRegistry::new(); + for schema in &self.tables { + registry.register_schema(schema.clone())?; + } + Ok(registry) + } + + pub fn sql_statements(&self, dialect: TableSqlDialect) -> Result, ReadModelError> { + table_schema_statements(&self.table_registry()?, dialect) + } + + pub fn sql_migration_artifacts( + &self, + dialect: TableSqlDialect, + ) -> Result, ReadModelError> { + generate_table_migration_artifacts(&self.table_registry()?, dialect) + } + + pub fn envelope(self) -> DistributedManifestEnvelope { + DistributedManifestEnvelope::new(self) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ServiceManifest { + pub name: String, + pub commands: Vec, + pub events: Vec, + pub transports: Vec, +} + +impl ServiceManifest { + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + commands: Vec::new(), + events: Vec::new(), + transports: Vec::new(), + } + } + + pub fn command(mut self, name: impl Into) -> Self { + self.commands.push(MessageEndpointManifest::new(name)); + self + } + + pub fn event(mut self, name: impl Into) -> Self { + self.events.push(MessageEndpointManifest::new(name)); + self + } + + pub fn transport(mut self, kind: impl Into) -> Self { + self.transports.push(TransportManifest::new(kind)); + self + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct MessageEndpointManifest { + pub name: String, +} + +impl MessageEndpointManifest { + pub fn new(name: impl Into) -> Self { + Self { name: name.into() } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TransportManifest { + pub kind: String, +} + +impl TransportManifest { + pub fn new(kind: impl Into) -> Self { + Self { kind: kind.into() } + } + + pub fn http() -> Self { + Self::new("http") + } +} + +#[cfg(test)] +mod tests { + use serde::{Deserialize, Serialize}; + + use super::*; + use crate::{outbox_message_schema, ReadModel}; + + #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, ReadModel)] + #[table("orders")] + struct OrderView { + #[id("order_id")] + order_id: String, + status: String, + } + + #[test] + fn manifest_collects_schema_service_metadata_and_renders_sql() { + let manifest = DistributedProjectManifest::new("checkout") + .read_model::() + .table_schema(outbox_message_schema()) + .service( + ServiceManifest::new("checkout-saga") + .command("checkout.start") + .event("seat.reserved") + .transport("http"), + ); + + let envelope = DistributedManifestEnvelope::new(manifest.clone()); + let json = serde_json::to_string(&envelope).expect("manifest should serialize"); + assert!(json.contains("\"schema_version\":1")); + assert!(json.contains("\"table_name\":\"orders\"")); + + let restored: DistributedManifestEnvelope = + serde_json::from_str(&json).expect("manifest should deserialize"); + assert_eq!(restored.project.name, "checkout"); + assert_eq!(restored.project.tables.len(), 2); + assert_eq!( + restored.project.services[0].commands[0].name, + "checkout.start" + ); + + let sql = manifest + .sql_statements(TableSqlDialect::Postgres) + .expect("manifest SQL should render") + .join("\n"); + assert!(sql.contains("CREATE TABLE IF NOT EXISTS \"orders\"")); + assert!(sql.contains("CREATE TABLE IF NOT EXISTS \"outbox_messages\"")); + } +} diff --git a/src/read_model/metadata.rs b/src/read_model/metadata.rs index b4ef724..1d2d6b1 100644 --- a/src/read_model/metadata.rs +++ b/src/read_model/metadata.rs @@ -1,13 +1,13 @@ use std::collections::{BTreeMap, BTreeSet}; -use serde::{de::DeserializeOwned, Serialize}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; use super::ReadModelError; pub const DEFAULT_READ_MODEL_VERSION_COLUMN: &str = "_sourced_version"; /// Logical storage type for a relational read-model column. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum ColumnType { Text, Boolean, @@ -21,7 +21,7 @@ pub enum ColumnType { } /// A foreign-key declaration from one read-model column to another table column. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct ForeignKey { pub table: String, pub column: String, @@ -37,7 +37,7 @@ impl ForeignKey { } /// Primary-key metadata for a relational read-model table. -#[derive(Clone, Debug, Default, PartialEq, Eq)] +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct PrimaryKey { pub columns: Vec, } @@ -86,7 +86,7 @@ impl RowKey { } /// Column metadata for a relational read-model table. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct ColumnDef { pub field_name: String, pub column_name: String, @@ -124,7 +124,7 @@ impl ColumnDef { } /// Index or unique-constraint metadata for a relational read-model table. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct IndexDef { pub name: Option, pub columns: Vec, @@ -142,7 +142,7 @@ impl IndexDef { } /// Relationship category for later workspace/write-plan lowering. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum RelationshipKind { HasMany, BelongsTo, @@ -150,7 +150,7 @@ pub enum RelationshipKind { } /// Relationship metadata for a relational read model. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct RelationshipDef { pub field_name: String, pub kind: RelationshipKind, @@ -160,7 +160,7 @@ pub struct RelationshipDef { } /// Schema metadata for one relational read-model table. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct ReadModelSchema { pub model_name: String, pub table_name: String, From 4e878631dc2255c6458034a3a755dda38e950ca3 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Wed, 3 Jun 2026 14:43:43 -0500 Subject: [PATCH 02/24] feat: return CommitReceipt from outbox commit OutboxCommit::commit now returns a CommitReceipt carrying the inserted outbox message id(s) instead of (), so an after-commit dispatcher can publish exactly the rows the transaction wrote. Source-compatible: ?-statement callers discard the receipt. Step 1 of [[tasks/durable-enqueue-outbox-dispatch-impl]] Implements [[specs/durable-enqueue-outbox-dispatch]] Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib.rs | 3 ++- src/outbox/commit.rs | 43 ++++++++++++++++++++++++++++++++++++++++--- src/outbox/mod.rs | 2 +- 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 7f44a50..7d40bb7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -67,7 +67,8 @@ pub use lock::{SqliteLock, SqliteLockManager}; // Outbox: commit concerns (aggregate + outbox in one commit) pub use outbox::{ outbox_message_insert_plan, outbox_message_key, outbox_message_row_values, - outbox_message_schema, OutboxCommit, OutboxMessage, OutboxMessageStatus, OUTBOX_MESSAGES_TABLE, + outbox_message_schema, CommitReceipt, OutboxCommit, OutboxMessage, OutboxMessageStatus, + OUTBOX_MESSAGES_TABLE, }; // Outbox Worker: drain and publish concerns diff --git a/src/outbox/commit.rs b/src/outbox/commit.rs index c5be7f9..539356e 100644 --- a/src/outbox/commit.rs +++ b/src/outbox/commit.rs @@ -4,6 +4,30 @@ use crate::repository::{ CommitBatch, RepositoryError, StreamIdentity, StreamWrite, TransactionalCommit, }; +/// Outcome of an outbox-bearing commit. +/// +/// Carries the ids of the outbox rows the transaction inserted so an +/// after-commit dispatcher knows exactly which rows to publish. This is the seam +/// the immediate-dispatch path hangs off (see +/// `specs/durable-enqueue-outbox-dispatch`). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct CommitReceipt { + /// Ids of the outbox messages inserted by this commit, in insertion order. + pub outbox_message_ids: Vec, +} + +impl CommitReceipt { + /// The outbox message ids inserted by this commit. + pub fn outbox_message_ids(&self) -> &[String] { + &self.outbox_message_ids + } + + /// Whether this commit inserted any outbox messages. + pub fn has_outbox_messages(&self) -> bool { + !self.outbox_message_ids.is_empty() + } +} + /// Helper returned by [`AggregateRepository::outbox`] to commit an aggregate /// and an outbox row in the same async transactional batch. /// @@ -20,13 +44,21 @@ where A: Aggregate + Send, { /// Commit the aggregate and outbox message together. - pub async fn commit(mut self, aggregate: &mut A) -> Result<(), RepositoryError> { + /// + /// Returns a [`CommitReceipt`] carrying the inserted outbox message id, so + /// an after-commit dispatcher can publish exactly the rows this transaction + /// wrote without re-scanning the outbox. + pub async fn commit(mut self, aggregate: &mut A) -> Result { self.message.set_source(aggregate); + let outbox_message_id = self.message.id().to_string(); let identity = StreamIdentity::new(A::aggregate_type(), aggregate.entity().id())?; let stream = StreamWrite::new(identity, aggregate.entity_mut()); let mut batch = CommitBatch::new(vec![stream]); batch.outbox_messages.push(self.message); - self.repo.repo().commit_batch(batch).await + self.repo.repo().commit_batch(batch).await?; + Ok(CommitReceipt { + outbox_message_ids: vec![outbox_message_id], + }) } } @@ -95,7 +127,12 @@ mod tests { let event = OutboxMessage::create("msg-1", "DummyTouched", b"{}".to_vec()).unwrap(); - repo.outbox(event).commit(&mut aggregate).await.unwrap(); + let receipt = repo.outbox(event).commit(&mut aggregate).await.unwrap(); + + // The receipt reports the inserted outbox row so an after-commit + // dispatcher knows what to publish. + assert!(receipt.has_outbox_messages()); + assert_eq!(receipt.outbox_message_ids(), ["msg-1".to_string()]); let pending = repo.repo().outbox_store().pending().unwrap(); assert_eq!(pending.len(), 1); diff --git a/src/outbox/mod.rs b/src/outbox/mod.rs index 4a02510..74143cb 100644 --- a/src/outbox/mod.rs +++ b/src/outbox/mod.rs @@ -49,4 +49,4 @@ pub use table::{ }; // Commit helpers -pub use commit::OutboxCommit; +pub use commit::{CommitReceipt, OutboxCommit}; From e620e6c693ecd5d9990c3ac9d2e4a6bedeebd283 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Wed, 3 Jun 2026 14:45:18 -0500 Subject: [PATCH 03/24] feat: add BusPublisher (Bus -> AsyncMessagePublisher) adapter Routes outbox-derived messages by MessageKind: commands to send_message (point-to-point), events to publish_message (fan-out). This is the missing adapter that lets the outbox dispatcher publish through any *Bus uniformly. Step 2 of [[tasks/durable-enqueue-outbox-dispatch-impl]] Implements [[specs/durable-enqueue-outbox-dispatch]] Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib.rs | 2 +- src/outbox_worker/bus_publisher.rs | 143 +++++++++++++++++++++++++++++ src/outbox_worker/mod.rs | 2 + 3 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 src/outbox_worker/bus_publisher.rs diff --git a/src/lib.rs b/src/lib.rs index 7d40bb7..cfb394d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -92,7 +92,7 @@ pub use outbox_worker::{ #[cfg(feature = "emitter")] pub use outbox_worker::LocalEmitterPublisher; pub use outbox_worker::{ - OutboxDispatchOutcome, OutboxDispatcher, OutboxSource, ReceivedOutboxMessage, + BusPublisher, OutboxDispatchOutcome, OutboxDispatcher, OutboxSource, ReceivedOutboxMessage, DEFAULT_OUTBOX_SOURCE_BATCH, DEFAULT_OUTBOX_SOURCE_LEASE, SOURCED_METADATA_PREFIX, }; diff --git a/src/outbox_worker/bus_publisher.rs b/src/outbox_worker/bus_publisher.rs new file mode 100644 index 0000000..0f62863 --- /dev/null +++ b/src/outbox_worker/bus_publisher.rs @@ -0,0 +1,143 @@ +//! `Bus` → `AsyncMessagePublisher` adapter. +//! +//! The outbox dispatcher publishes through a single [`AsyncMessagePublisher`], +//! but the command-vs-event topology split lives in the [`Bus`] as two methods +//! (`send_message` for point-to-point commands, `publish_message` for fan-out +//! events) backed by different publishers/topologies. This adapter bridges them +//! by routing on [`MessageKind`] — which the outbox → [`Message`] mapping +//! already sets (`Command` when a `destination` is present, else `Event`). +//! +//! This is what lets the outbox dispatcher publish through any `*Bus` uniformly, +//! for both the after-commit immediate path and the background poll loop. + +use std::sync::Arc; + +use crate::bus::{AsyncMessagePublisher, Bus, Message, MessageKind, TransportError}; + +/// Publishes outbox-derived [`Message`]s through a [`Bus`], routing by kind: +/// commands to `send_message` (point-to-point), events to `publish_message` +/// (fan-out). +pub struct BusPublisher { + bus: Arc, +} + +impl BusPublisher { + /// Wrap a shared bus as an [`AsyncMessagePublisher`]. + pub fn new(bus: Arc) -> Self { + Self { bus } + } + + /// The wrapped bus. + pub fn bus(&self) -> &Arc { + &self.bus + } +} + +impl Clone for BusPublisher { + fn clone(&self) -> Self { + Self { + bus: Arc::clone(&self.bus), + } + } +} + +impl AsyncMessagePublisher for BusPublisher { + async fn publish(&self, message: Message) -> Result<(), TransportError> { + match message.kind { + MessageKind::Command => self.bus.send_message(message).await, + MessageKind::Event => self.bus.publish_message(message).await, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::future::Future; + use std::sync::Mutex; + + fn block_on(future: F) -> F::Output { + use std::ptr; + use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; + const VTABLE: RawWakerVTable = RawWakerVTable::new( + |_| RawWaker::new(ptr::null(), &VTABLE), + |_| {}, + |_| {}, + |_| {}, + ); + let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &VTABLE)) }; + let mut cx = Context::from_waker(&waker); + let mut future = std::pin::pin!(future); + loop { + if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { + return output; + } + } + } + + #[derive(Debug, Clone, PartialEq, Eq)] + enum Call { + Send(String), + Publish(String), + } + + /// A bus that records which produce method was used for each message. + #[derive(Default)] + struct RecordingBus { + calls: Mutex>, + } + + impl RecordingBus { + fn calls(&self) -> Vec { + self.calls.lock().unwrap().clone() + } + } + + impl Bus for RecordingBus { + async fn send(&self, name: &str, payload: Vec) -> Result<(), TransportError> { + self.send_message(Message::new(name, MessageKind::Command, payload)) + .await + } + async fn publish(&self, name: &str, payload: Vec) -> Result<(), TransportError> { + self.publish_message(Message::new(name, MessageKind::Event, payload)) + .await + } + async fn send_message(&self, message: Message) -> Result<(), TransportError> { + self.calls.lock().unwrap().push(Call::Send(message.name)); + Ok(()) + } + async fn publish_message(&self, message: Message) -> Result<(), TransportError> { + self.calls.lock().unwrap().push(Call::Publish(message.name)); + Ok(()) + } + } + + #[test] + fn routes_command_to_send_and_event_to_publish() { + let bus = Arc::new(RecordingBus::default()); + let publisher = BusPublisher::new(bus.clone()); + + // A command-kind message (outbox row with a destination) → send_message. + block_on(publisher.publish(Message::new( + "ship.order", + MessageKind::Command, + b"{}".to_vec(), + ))) + .unwrap(); + // An event-kind message → publish_message. + block_on(publisher.publish(Message::new( + "order.shipped", + MessageKind::Event, + b"{}".to_vec(), + ))) + .unwrap(); + + assert_eq!( + bus.calls(), + vec![ + Call::Send("ship.order".to_string()), + Call::Publish("order.shipped".to_string()), + ] + ); + } +} diff --git a/src/outbox_worker/mod.rs b/src/outbox_worker/mod.rs index c051264..88b5c45 100644 --- a/src/outbox_worker/mod.rs +++ b/src/outbox_worker/mod.rs @@ -37,6 +37,7 @@ //! } //! ``` +mod bus_publisher; mod outbox_dispatch; mod outbox_source; mod publisher; @@ -59,6 +60,7 @@ pub use store::{ pub use worker::{DrainResult, OutboxWorker, ProcessOneResult}; // Outbox -> bus bridge (moved out of the bus module; depends up on bus traits). +pub use bus_publisher::BusPublisher; pub use outbox_dispatch::{OutboxDispatchOutcome, OutboxDispatcher, SOURCED_METADATA_PREFIX}; pub use outbox_source::{ OutboxSource, ReceivedOutboxMessage, DEFAULT_OUTBOX_SOURCE_BATCH, DEFAULT_OUTBOX_SOURCE_LEASE, From 8dc2170e9127898fcc60a49cbef09d6406377153 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Wed, 3 Jun 2026 15:05:03 -0500 Subject: [PATCH 04/24] feat: add HasOutboxStore capability for repo wrappers New trait abstracting 'produce a durable outbox store', resolving through the AggregateRepository -> QueuedRepository -> leaf repo wrapper chain. Lets the runtime build an OutboxDispatcher without naming the concrete repository type. Impls for HashMap (and feature-gated Sqlite/Postgres) leaves + the wrappers. Step 3 (store access) of [[tasks/durable-enqueue-outbox-dispatch-impl]] Implements [[specs/durable-enqueue-outbox-dispatch]] Co-Authored-By: Claude Opus 4.8 (1M context) --- src/microsvc/dependencies.rs | 92 ++++++++++++++++++++++++++++++++++++ src/microsvc/mod.rs | 2 +- 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/src/microsvc/dependencies.rs b/src/microsvc/dependencies.rs index 8ecdf68..453fc6c 100644 --- a/src/microsvc/dependencies.rs +++ b/src/microsvc/dependencies.rs @@ -1,6 +1,7 @@ //! Typed dependency wrappers for microsvc handlers. use crate::aggregate::AggregateRepository; +use crate::outbox_worker::AsyncOutboxStore; use crate::repository::{ReadModelWritePlanStore, RelationalReadModelQueryStore, Repository}; use crate::snapshot::SnapshotAggregateRepository; @@ -18,6 +19,77 @@ pub trait HasReadModelStore { fn read_model_store(&self) -> &Self::ReadModelStore; } +/// Dependency capability for repositories that expose a durable outbox store. +/// +/// A runtime uses this to build an `OutboxDispatcher` that drains committed +/// outbox rows to a transport, without naming the concrete repository type. The +/// capability resolves through the repository wrappers +/// (`AggregateRepository` -> `QueuedRepository` -> the leaf SQL/in-memory repo). +pub trait HasOutboxStore { + /// The concrete outbox store this repository produces. + type OutboxStore: AsyncOutboxStore; + + /// Produce a handle to the durable outbox store. + fn outbox_store(&self) -> Self::OutboxStore; +} + +// Leaf repositories own the store. Each calls its inherent `outbox_store()` — +// inherent methods take precedence over the trait method of the same name, so +// `self.outbox_store()` here does not recurse. +impl HasOutboxStore for crate::HashMapRepository { + type OutboxStore = crate::HashMapOutboxStore; + fn outbox_store(&self) -> Self::OutboxStore { + crate::HashMapRepository::outbox_store(self) + } +} + +#[cfg(feature = "sqlite")] +impl HasOutboxStore for crate::SqliteRepository { + type OutboxStore = crate::SqliteOutboxStore; + fn outbox_store(&self) -> Self::OutboxStore { + crate::SqliteRepository::outbox_store(self) + } +} + +#[cfg(feature = "postgres")] +impl HasOutboxStore for crate::PostgresRepository { + type OutboxStore = crate::PostgresOutboxStore; + fn outbox_store(&self) -> Self::OutboxStore { + crate::PostgresRepository::outbox_store(self) + } +} + +// Wrappers delegate inward. +impl HasOutboxStore for AggregateRepository +where + R: HasOutboxStore, +{ + type OutboxStore = R::OutboxStore; + fn outbox_store(&self) -> Self::OutboxStore { + self.repo().outbox_store() + } +} + +impl HasOutboxStore for crate::QueuedRepository +where + R: HasOutboxStore, +{ + type OutboxStore = R::OutboxStore; + fn outbox_store(&self) -> Self::OutboxStore { + self.inner().outbox_store() + } +} + +impl HasOutboxStore for SnapshotAggregateRepository +where + R: HasOutboxStore, +{ + type OutboxStore = as HasOutboxStore>::OutboxStore; + fn outbox_store(&self) -> Self::OutboxStore { + self.repo().outbox_store() + } +} + impl HasRepo for R where R: Repository, @@ -128,3 +200,23 @@ impl HasReadModelStore for RepoReadModelDependencies { &self.read_model_store } } + +#[cfg(test)] +mod tests { + use super::*; + + fn assert_has_outbox_store() {} + + #[test] + fn has_outbox_store_resolves_through_repo_wrappers() { + // The capability must resolve for the leaf repo and through the + // AggregateRepository -> QueuedRepository wrapper chain the canonical + // `repo.queued().aggregate::()` builder produces. `A` is unbounded, + // so the unit type stands in for any aggregate. + assert_has_outbox_store::(); + assert_has_outbox_store::>(); + assert_has_outbox_store::< + AggregateRepository, ()>, + >(); + } +} diff --git a/src/microsvc/mod.rs b/src/microsvc/mod.rs index ff2a01e..39c6947 100644 --- a/src/microsvc/mod.rs +++ b/src/microsvc/mod.rs @@ -63,7 +63,7 @@ mod session; pub use crate::bus::{Message, MessageKind, PayloadDecodeError, SubscriptionPlan}; pub use context::Context; pub use dependencies::{ - HasReadModelStore, HasRepo, ReadModelStoreDependencies, RepoDependencies, + HasOutboxStore, HasReadModelStore, HasRepo, ReadModelStoreDependencies, RepoDependencies, RepoReadModelDependencies, }; pub use error::HandlerError; From 023e5c814a25fa5158e4e30b258fb59e45795498 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Wed, 3 Jun 2026 15:10:22 -0500 Subject: [PATCH 05/24] feat: add Service::with_bus + Microservice runtime (produce side) Service::with_bus(bus) wraps the consumer Service into a Microservice carrying the transport config. Microservice::dispatcher() assembles an OutboxDispatcher over the service's own outbox store + a BusPublisher, so committed outbox rows drain to the bus routed by kind. Test proves commit -> dispatch -> published end to end over InMemoryBus. Consume side (run() auto listen/subscribe) and the in-transaction commit_outbox land next. Step 6 (runtime, produce side) of [[tasks/durable-enqueue-outbox-dispatch-impl]] Implements [[specs/durable-enqueue-outbox-dispatch]] Co-Authored-By: Claude Opus 4.8 (1M context) --- src/microsvc/mod.rs | 2 + src/microsvc/runtime.rs | 167 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 src/microsvc/runtime.rs diff --git a/src/microsvc/mod.rs b/src/microsvc/mod.rs index 39c6947..6ffc339 100644 --- a/src/microsvc/mod.rs +++ b/src/microsvc/mod.rs @@ -57,6 +57,7 @@ mod context; mod dependencies; mod error; mod message_router; +mod runtime; mod service; mod session; @@ -67,6 +68,7 @@ pub use dependencies::{ RepoReadModelDependencies, }; pub use error::HandlerError; +pub use runtime::{Microservice, DEFAULT_MAX_PUBLISH_ATTEMPTS, DEFAULT_PUBLISH_LEASE}; pub use service::{ CommandRequest, CommandResponse, DeliveryKind, HandlerBuilder, HandlerNames, HandlerSpec, Service, diff --git a/src/microsvc/runtime.rs b/src/microsvc/runtime.rs new file mode 100644 index 0000000..c5bad92 --- /dev/null +++ b/src/microsvc/runtime.rs @@ -0,0 +1,167 @@ +//! Runtime that ties a [`Service`] to a bus. +//! +//! `register_handlers!` builds a [`Service`] that is purely a consumer. Attaching +//! a bus with [`Service::with_bus`] turns it into a [`Microservice`] that carries +//! the transport config for both sides: it can drain committed outbox rows to the +//! bus (produce) and — once `run` lands — derive listen/subscribe from the +//! registered handlers (consume). +//! +//! The producing side is a thin assembly over [`OutboxDispatcher`] and +//! [`BusPublisher`]: [`Microservice::dispatcher`] hands back a dispatcher whose +//! store is the service's own outbox store and whose publisher routes through the +//! attached bus by [`MessageKind`](crate::bus::MessageKind). The same dispatcher +//! backs immediate after-commit dispatch and a background poll loop. + +use std::sync::Arc; +use std::time::Duration; + +use super::dependencies::{HasOutboxStore, HasRepo}; +use super::Service; +use crate::bus::Bus; +use crate::outbox_worker::{BusPublisher, OutboxDispatcher}; + +/// Default lease for an immediate after-commit outbox publish. +/// +/// Short by design: it only needs to cover commit → publish, so a crash before +/// the publish completes hands the row back to the polling worker quickly. +pub const DEFAULT_PUBLISH_LEASE: Duration = Duration::from_secs(5); + +/// Default publish-failure ceiling before an outbox row is permanently failed. +pub const DEFAULT_MAX_PUBLISH_ATTEMPTS: u32 = 5; + +/// A [`Service`] bound to a bus. +/// +/// Holds the service and bus behind `Arc`s so the produce side (the dispatcher) +/// and the consume side (listen/subscribe) can share them. +pub struct Microservice { + service: Arc>, + bus: Arc, + worker_id: String, + publish_lease: Duration, + max_attempts: u32, +} + +impl Microservice { + /// Bind a service to a bus with default dispatch settings. + pub fn new(service: Arc>, bus: Arc) -> Self { + Self { + service, + bus, + worker_id: format!("microsvc-immediate:{}", std::process::id()), + publish_lease: DEFAULT_PUBLISH_LEASE, + max_attempts: DEFAULT_MAX_PUBLISH_ATTEMPTS, + } + } + + /// The bound service. + pub fn service(&self) -> &Arc> { + &self.service + } + + /// The bound bus. + pub fn bus(&self) -> &Arc { + &self.bus + } + + /// Set the worker id used to scope outbox claims (default + /// `microsvc-immediate:`). + pub fn with_worker_id(mut self, worker_id: impl Into) -> Self { + self.worker_id = worker_id.into(); + self + } + + /// Set the lease taken when claiming an outbox row for publication. + pub fn with_publish_lease(mut self, lease: Duration) -> Self { + self.publish_lease = lease; + self + } + + /// Set the publish-failure ceiling before a row is permanently failed. + pub fn with_max_attempts(mut self, max_attempts: u32) -> Self { + self.max_attempts = max_attempts; + self + } +} + +impl Microservice +where + D: HasRepo + Send + Sync + 'static, + D::Repo: HasOutboxStore, + B: Bus, +{ + /// Build a dispatcher that drains committed outbox rows to the bus. + /// + /// The store is the service's own outbox store; the publisher routes each + /// message to the bus by kind (commands point-to-point, events fan-out). The + /// same dispatcher is used by immediate after-commit dispatch + /// (`dispatch_ids`) and a background poll loop (`dispatch_batch`). + pub fn dispatcher( + &self, + ) -> OutboxDispatcher<::OutboxStore, BusPublisher> { + OutboxDispatcher::new( + self.service.repo().outbox_store(), + BusPublisher::new(Arc::clone(&self.bus)), + self.worker_id.clone(), + self.publish_lease, + self.max_attempts, + ) + } +} + +impl Service { + /// Attach a bus, producing a [`Microservice`] that carries the transport + /// config for both producing (outbox dispatch) and consuming + /// (listen/subscribe). + pub fn with_bus(self, bus: B) -> Microservice { + Microservice::new(Arc::new(self), Arc::new(bus)) + } +} + +#[cfg(test)] +mod tests { + use crate::bus::InMemoryBus; + use crate::microsvc::Service; + use crate::{sourced, AggregateBuilder, Entity, HashMapRepository, OutboxMessage, Queueable}; + + #[derive(Default)] + struct Dummy { + entity: Entity, + } + + #[sourced(entity)] + impl Dummy { + #[event("touched")] + fn touch(&mut self) { + if self.entity.id().is_empty() { + self.entity.set_id("dummy-1"); + } + } + } + + #[tokio::test] + async fn dispatcher_drains_committed_outbox_row_to_the_bus() { + let service = + Service::with_repo(HashMapRepository::new().queued().aggregate::()); + + let microservice = service.with_bus(InMemoryBus::new()); + + // Commit an aggregate + outbox row through the bound service's repo. + let mut dummy = Dummy::default(); + dummy.touch().unwrap(); + let message = OutboxMessage::create("evt-1", "dummy.touched", b"{}".to_vec()).unwrap(); + let receipt = microservice + .service() + .repo() + .outbox(message) + .commit(&mut dummy) + .await + .unwrap(); + assert_eq!(receipt.outbox_message_ids(), ["evt-1".to_string()]); + + // The dispatcher (store + bus) drains the committed row to the bus. + let outcome = microservice.dispatcher().dispatch_batch(10).await.unwrap(); + assert_eq!(outcome.published, 1); + assert_eq!(outcome.released, 0); + assert_eq!(outcome.failed, 0); + } +} From 4bf0cd0600bdbe4d9247fa0dae9b63b23cebc397 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Wed, 3 Jun 2026 15:14:42 -0500 Subject: [PATCH 06/24] feat: add OutboxCommit::commit_claimed (claim-in-transaction) Claims the outbox row for publication in the same transaction that commits the aggregate: the row inserts already InFlight under the worker's lease (attempts = 1), so the after-commit publish needs no separate claim and cannot race the poller. Returns the claimed message clone so the caller can build the transport message and settle the claim. Test proves the row is in-flight, leased, and not poller-claimable. Step 4 (claim-in-transaction) of [[tasks/durable-enqueue-outbox-dispatch-impl]] Implements [[specs/durable-enqueue-outbox-dispatch]] Co-Authored-By: Claude Opus 4.8 (1M context) --- src/outbox/commit.rs | 67 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/src/outbox/commit.rs b/src/outbox/commit.rs index 539356e..f72d50a 100644 --- a/src/outbox/commit.rs +++ b/src/outbox/commit.rs @@ -1,3 +1,5 @@ +use std::time::{Duration, SystemTime}; + use crate::aggregate::{Aggregate, AggregateRepository}; use crate::outbox::OutboxMessage; use crate::repository::{ @@ -60,6 +62,42 @@ where outbox_message_ids: vec![outbox_message_id], }) } + + /// Commit the aggregate and outbox message together, **claiming the outbox + /// row for publication in the same transaction**. + /// + /// The row inserts already `InFlight` under `worker_id`'s lease + /// (`attempts = 1`), so the caller can publish it immediately after commit + /// without a separate claim and without racing the polling worker. While the + /// lease is held the poller skips the row; if the caller never publishes + /// (e.g. a crash), the lease expires and the worker reclaims it. + /// + /// Returns the receipt plus a clone of the claimed message so the caller can + /// build the transport message and settle the claim (`complete` on success, + /// `record_failure` on a publish error). The publish itself happens after + /// this returns — a broker call is never held open inside the transaction. + pub async fn commit_claimed( + mut self, + aggregate: &mut A, + worker_id: &str, + lease: Duration, + ) -> Result<(CommitReceipt, OutboxMessage), RepositoryError> { + self.message.set_source(aggregate); + self.message.claim_at(worker_id, lease, SystemTime::now())?; + let claimed = self.message.clone(); + let outbox_message_id = self.message.id().to_string(); + let identity = StreamIdentity::new(A::aggregate_type(), aggregate.entity().id())?; + let stream = StreamWrite::new(identity, aggregate.entity_mut()); + let mut batch = CommitBatch::new(vec![stream]); + batch.outbox_messages.push(self.message); + self.repo.repo().commit_batch(batch).await?; + Ok(( + CommitReceipt { + outbox_message_ids: vec![outbox_message_id], + }, + claimed, + )) + } } impl AggregateRepository { @@ -139,6 +177,35 @@ mod tests { assert_eq!(pending[0].id(), "msg-1"); } + #[tokio::test] + async fn commit_claimed_inserts_row_in_flight_under_lease() { + let repo = HashMapRepository::new().aggregate::(); + + let mut aggregate = Dummy::default(); + aggregate.touch().unwrap(); + + let event = OutboxMessage::create("msg-1", "DummyTouched", b"{}".to_vec()).unwrap(); + + let (receipt, claimed) = repo + .outbox(event) + .commit_claimed(&mut aggregate, "immediate:test", Duration::from_secs(5)) + .await + .unwrap(); + + // The committed row is claimed in the same transaction: in-flight, under + // this worker's lease, attempt 1. + assert_eq!(receipt.outbox_message_ids(), ["msg-1".to_string()]); + assert!(!claimed.is_pending()); + assert_eq!(claimed.attempts, 1); + + // A polling worker sees nothing claimable while the lease is held. + let pending = repo.repo().outbox_store().pending().unwrap(); + assert!( + pending.is_empty(), + "a row claimed in-transaction must not be claimable by the poller" + ); + } + #[tokio::test] async fn outbox_helper_failure_leaves_entities_uncommitted() { let repo = AggregateRepository::<_, Dummy>::new(FailingOutboxRepo::default()); From dcb82ca6a2b403e875c9c3dd9592788461474702 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Wed, 3 Jun 2026 15:19:57 -0500 Subject: [PATCH 07/24] =?UTF-8?q?feat:=20Context::commit=5Foutbox=20?= =?UTF-8?q?=E2=80=94=20publish-in-commit=20via=20attached=20bus?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires the durable-enqueue command path end to end: - DynPublisher: object-safe (boxed-future) form of AsyncMessagePublisher, so a publisher can sit behind Arc without making Service generic over it. - Service carries an optional ImmediatePublish (publisher + worker id + lease + attempts), set by with_bus; Context receives it. - Context::commit_outbox: with a bus attached, claims the outbox row in the commit transaction then publishes immediately through the bus, completing or releasing the claim; with no bus, commits pending for the poller. Best-effort publish never rolls back the committed aggregate. Test: dispatch -> commit_outbox -> row published immediately, none left pending. Steps 3+5 (DynPublisher + commit_outbox) of [[tasks/durable-enqueue-outbox-dispatch-impl]] Implements [[specs/durable-enqueue-outbox-dispatch]] Co-Authored-By: Claude Opus 4.8 (1M context) --- src/bus/mod.rs | 2 +- src/bus/publisher.rs | 29 ++++++++++++++++ src/microsvc/context.rs | 61 ++++++++++++++++++++++++++++++++- src/microsvc/runtime.rs | 76 ++++++++++++++++++++++++++++++++++++++--- src/microsvc/service.rs | 32 +++++++++++++++-- 5 files changed, 191 insertions(+), 9 deletions(-) diff --git a/src/bus/mod.rs b/src/bus/mod.rs index 48c66a3..414b6a7 100644 --- a/src/bus/mod.rs +++ b/src/bus/mod.rs @@ -139,7 +139,7 @@ pub use in_memory_bus::{InMemoryBus, InMemoryReceived}; pub use message::{Message, MessageKind, PayloadDecodeError, SubscriptionPlan}; #[cfg(feature = "postgres")] pub use postgres_bus::{LogReceived, PostgresBus, QueueReceived}; -pub use publisher::AsyncMessagePublisher; +pub use publisher::{AsyncMessagePublisher, DynPublisher}; pub use router::MessageRouter; pub use run_options::{ConsumerDeliveryMode, InboxHook, NoInbox, RunOptions}; pub use runner::run_source; diff --git a/src/bus/publisher.rs b/src/bus/publisher.rs index 19bc259..5011039 100644 --- a/src/bus/publisher.rs +++ b/src/bus/publisher.rs @@ -17,6 +17,7 @@ //! is acceptable under at-least-once, silent loss is not. use std::future::Future; +use std::pin::Pin; use super::{Message, TransportError}; @@ -54,6 +55,34 @@ pub trait AsyncMessagePublisher: Send + Sync { } } +/// Object-safe form of [`AsyncMessagePublisher`]. +/// +/// `AsyncMessagePublisher::publish` returns `impl Future` (RPITIT), which makes +/// the trait itself not object-safe — `dyn AsyncMessagePublisher` will not +/// compile. `DynPublisher` boxes the returned future so a publisher can be held +/// behind `Arc`, used where the concrete publisher type cannot +/// be a type parameter (for example on `microsvc::Service`, so attaching a bus +/// does not change the service's type). +/// +/// Blanket-implemented for every [`AsyncMessagePublisher`]; callers normally +/// produce one with `Arc::new(publisher) as Arc`. +pub trait DynPublisher: Send + Sync { + /// Publish a single message, returning a boxed future. + fn publish_dyn<'a>( + &'a self, + message: Message, + ) -> Pin> + Send + 'a>>; +} + +impl DynPublisher for P { + fn publish_dyn<'a>( + &'a self, + message: Message, + ) -> Pin> + Send + 'a>> { + Box::pin(self.publish(message)) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/microsvc/context.rs b/src/microsvc/context.rs index 6a7dbc2..adeb9e9 100644 --- a/src/microsvc/context.rs +++ b/src/microsvc/context.rs @@ -7,10 +7,15 @@ use serde::de::DeserializeOwned; use serde_json::Value; -use super::dependencies::{HasReadModelStore, HasRepo}; +use super::dependencies::{HasOutboxStore, HasReadModelStore, HasRepo}; use super::error::HandlerError; +use super::service::ImmediatePublish; use super::session::Session; +use crate::aggregate::{Aggregate, AggregateRepository}; use crate::bus::Message; +use crate::outbox::{CommitReceipt, OutboxMessage}; +use crate::outbox_worker::{AsyncOutboxStore, OutboxClaimRef}; +use crate::repository::TransactionalCommit; /// The context passed to every handler. /// @@ -38,6 +43,8 @@ pub struct Context<'a, D> { session: Session, /// Reference to the service dependencies. dependencies: &'a D, + /// After-commit publish config, present when a bus is attached. + immediate_publish: Option<&'a ImmediatePublish>, } impl<'a, D> Context<'a, D> { @@ -47,12 +54,14 @@ impl<'a, D> Context<'a, D> { input: Value, session: Session, dependencies: &'a D, + immediate_publish: Option<&'a ImmediatePublish>, ) -> Self { Self { message, input, session, dependencies, + immediate_publish, } } @@ -119,6 +128,56 @@ impl<'a, D> Context<'a, D> { self.dependencies.read_model_store() } + /// Commit an aggregate and an outbox message together, then publish the + /// message — the durable-enqueue command path. + /// + /// When the service was built with a bus (`Service::with_bus`), this claims + /// the outbox row in the commit transaction and publishes it immediately + /// after commit through the configured bus. The publish is best-effort: a + /// failure leaves the row claimed/retryable for the polling worker and does + /// **not** roll back the committed aggregate, so the command still succeeds. + /// + /// When no bus is configured, the row is committed `pending` and left for the + /// polling worker to publish. + pub async fn commit_outbox( + &self, + aggregate: &mut A, + message: OutboxMessage, + ) -> Result + where + D: HasRepo>, + R: TransactionalCommit + HasOutboxStore, + A: Aggregate + Send, + { + let repo = self.repo(); + + let Some(immediate) = self.immediate_publish else { + // No bus configured: durable enqueue only; the worker publishes. + return Ok(repo.outbox(message).commit(aggregate).await?); + }; + + // Claim the row in the commit transaction, then publish after commit. + // The lease hands the row to the poller if we crash before completing. + let (receipt, claimed) = repo + .outbox(message) + .commit_claimed(aggregate, &immediate.worker_id, immediate.lease) + .await?; + let claim = OutboxClaimRef::from_message(&claimed)?; + let transport = Message::from(&claimed); + let store = repo.outbox_store(); + match immediate.publisher.publish_dyn(transport).await { + Ok(()) => store.complete_async(&claim).await?, + Err(error) => { + // Best-effort: release/fail the claim for retry; the command + // still succeeded because the aggregate + row are committed. + store + .record_failure_async(&claim, &error.to_string(), immediate.max_attempts) + .await?; + } + } + Ok(receipt) + } + /// Check if the raw input contains a field. pub fn has_field(&self, field: &str) -> bool { self.input.get(field).is_some() diff --git a/src/microsvc/runtime.rs b/src/microsvc/runtime.rs index c5bad92..e19ae5b 100644 --- a/src/microsvc/runtime.rs +++ b/src/microsvc/runtime.rs @@ -16,8 +16,9 @@ use std::sync::Arc; use std::time::Duration; use super::dependencies::{HasOutboxStore, HasRepo}; +use super::service::ImmediatePublish; use super::Service; -use crate::bus::Bus; +use crate::bus::{Bus, DynPublisher}; use crate::outbox_worker::{BusPublisher, OutboxDispatcher}; /// Default lease for an immediate after-commit outbox publish. @@ -112,16 +113,40 @@ impl Service { /// Attach a bus, producing a [`Microservice`] that carries the transport /// config for both producing (outbox dispatch) and consuming /// (listen/subscribe). - pub fn with_bus(self, bus: B) -> Microservice { - Microservice::new(Arc::new(self), Arc::new(bus)) + /// + /// Attaching a bus also enables immediate after-commit publish: handlers + /// that call [`Context::commit_outbox`](crate::microsvc::Context::commit_outbox) + /// claim the outbox row in the commit transaction and publish it through this + /// bus. The immediate path uses [`DEFAULT_PUBLISH_LEASE`] and + /// [`DEFAULT_MAX_PUBLISH_ATTEMPTS`]; the `with_publish_lease` / + /// `with_max_attempts` setters configure the background poll loop. + pub fn with_bus(mut self, bus: B) -> Microservice + where + B: Bus + 'static, + { + let bus = Arc::new(bus); + let publisher: Arc = Arc::new(BusPublisher::new(Arc::clone(&bus))); + self.set_immediate_publish(ImmediatePublish { + publisher, + worker_id: format!("microsvc-immediate:{}", std::process::id()), + lease: DEFAULT_PUBLISH_LEASE, + max_attempts: DEFAULT_MAX_PUBLISH_ATTEMPTS, + }); + Microservice::new(Arc::new(self), bus) } } #[cfg(test)] mod tests { + use serde_json::{json, Value}; + use crate::bus::InMemoryBus; - use crate::microsvc::Service; - use crate::{sourced, AggregateBuilder, Entity, HashMapRepository, OutboxMessage, Queueable}; + use crate::microsvc::{Context, HandlerError, HasOutboxStore, Service, Session}; + use crate::outbox_worker::AsyncOutboxStore; + use crate::{ + sourced, AggregateBuilder, AggregateRepository, Entity, HashMapRepository, OutboxMessage, + OutboxMessageStatus, QueuedRepository, Queueable, + }; #[derive(Default)] struct Dummy { @@ -164,4 +189,45 @@ mod tests { assert_eq!(outcome.released, 0); assert_eq!(outcome.failed, 0); } + + type TouchRepo = AggregateRepository, Dummy>; + + // A named fn (not a closure) so the higher-ranked `Handler` bound resolves. + async fn touch_and_publish(ctx: &Context<'_, TouchRepo>) -> Result { + let mut dummy = Dummy::default(); + dummy.touch()?; + let message = OutboxMessage::create("evt-1", "dummy.touched", b"{}".to_vec())?; + ctx.commit_outbox(&mut dummy, message).await?; + Ok(json!({ "ok": true })) + } + + #[tokio::test] + async fn commit_outbox_publishes_immediately_when_bus_is_attached() { + let service = Service::with_repo(HashMapRepository::new().queued().aggregate::()) + .command("dummy.touch") + .handle(touch_and_publish); + let microservice = service.with_bus(InMemoryBus::new()); + + // Dispatching the command runs the handler, which calls `commit_outbox`: + // claim-in-transaction, then immediate publish through the attached bus. + microservice + .service() + .dispatch("dummy.touch", json!({}), Session::new()) + .await + .unwrap(); + + // The row was published immediately (claim-in-tx -> publish -> complete), + // so nothing is left pending for the poller. + let store = microservice.service().repo().outbox_store(); + let published = store + .messages_by_status_async(OutboxMessageStatus::Published) + .await + .unwrap(); + assert_eq!(published.len(), 1, "row should be published immediately"); + assert_eq!(published[0].id(), "evt-1"); + assert!( + store.pending_async().await.unwrap().is_empty(), + "no row should be left for the poller" + ); + } } diff --git a/src/microsvc/service.rs b/src/microsvc/service.rs index 4f4e917..4fbc7d0 100644 --- a/src/microsvc/service.rs +++ b/src/microsvc/service.rs @@ -23,6 +23,7 @@ use std::collections::HashMap; use std::future::Future; use std::pin::Pin; use std::sync::Arc; +use std::time::Duration; use serde_json::Value; @@ -30,7 +31,21 @@ use super::context::Context; use super::dependencies::{HasReadModelStore, HasRepo, RepoReadModelDependencies}; use super::error::HandlerError; use super::session::Session; -use crate::bus::{Message, MessageKind, SubscriptionPlan}; +use crate::bus::{DynPublisher, Message, MessageKind, SubscriptionPlan}; + +/// After-commit publish configuration, set when a bus is attached via +/// [`Service::with_bus`](crate::microsvc::Service::with_bus). +/// +/// When present, [`Context::commit_outbox`](crate::microsvc::Context::commit_outbox) +/// claims the outbox row in the commit transaction and publishes it through +/// `publisher`. When absent, `commit_outbox` writes the row `pending` and leaves +/// publication to the polling worker. +pub(crate) struct ImmediatePublish { + pub(crate) publisher: Arc, + pub(crate) worker_id: String, + pub(crate) lease: Duration, + pub(crate) max_attempts: u32, +} type GuardFn = dyn Fn(&Context) -> bool + Send + Sync; type HandlerFuture<'a> = Pin> + Send + 'a>>; @@ -177,6 +192,7 @@ pub struct Service { dependencies: D, handlers: HashMap<(MessageKind, String), RegisteredHandler>, handler_specs: Vec, + immediate_publish: Option, } impl Service { @@ -186,9 +202,15 @@ impl Service { dependencies, handlers: HashMap::new(), handler_specs: Vec::new(), + immediate_publish: None, } } + /// Attach the after-commit publish configuration (set by `with_bus`). + pub(crate) fn set_immediate_publish(&mut self, immediate_publish: ImmediatePublish) { + self.immediate_publish = Some(immediate_publish); + } + /// Create a service whose dependency type is an aggregate repository. pub fn with_repo(repo: D) -> Self where @@ -330,7 +352,13 @@ impl Service { (handler.guard.clone(), handler.handle.clone()) }; let name = message.name.clone(); - let ctx = Context::new(message, input, session, &self.dependencies); + let ctx = Context::new( + message, + input, + session, + &self.dependencies, + self.immediate_publish.as_ref(), + ); // Run guard (synchronous) if present. if let Some(guard) = &guard { From cabb902e57fd1cf90b87b5e5aa85dbde0b93181e Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Wed, 3 Jun 2026 15:25:02 -0500 Subject: [PATCH 08/24] feat: Microservice::run derives listen/subscribe from handlers run() reads the service's subscription_plan and drives the consumers concurrently on the caller's runtime: command handlers via competing listen, event handlers via fan-out subscribe. Uses an executor-agnostic poll-join (no spawn, no timer) so it works in core without pulling tokio. Returns on first error or when the consumers stop. Derive Clone for RunOptions/ConsumerDeliveryMode so one options value drives both consumers. Test: run() consumes a queued command and the handler's commit_outbox publishes immediately. Producing happy-path is commit_outbox (immediate); the backstop poll loop (needs a timer) is driven from dispatcher() by a runtime that provides one. Step 6 (runtime, consume side) of [[tasks/durable-enqueue-outbox-dispatch-impl]] Implements [[specs/durable-enqueue-outbox-dispatch]] Co-Authored-By: Claude Opus 4.8 (1M context) --- src/bus/run_options.rs | 2 + src/microsvc/runtime.rs | 94 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/src/bus/run_options.rs b/src/bus/run_options.rs index 9047eec..2a98ae4 100644 --- a/src/bus/run_options.rs +++ b/src/bus/run_options.rs @@ -32,6 +32,7 @@ pub trait InboxHook { /// /// Defaults to [`ConsumerDeliveryMode::Idempotent`]: the convention is /// idempotent handlers/projections, so a redelivered message is safe. +#[derive(Clone)] pub enum ConsumerDeliveryMode { /// Dispatch directly and acknowledge after handler success. Handlers are /// expected to be idempotent under redelivery. @@ -63,6 +64,7 @@ pub enum NoInbox {} /// /// Generic over the inbox hook type `I`, defaulting to [`NoInbox`] for the /// common idempotent case. +#[derive(Clone)] pub struct RunOptions { /// Whether consumer execution is plain idempotent dispatch or inbox-wrapped. pub delivery_mode: ConsumerDeliveryMode, diff --git a/src/microsvc/runtime.rs b/src/microsvc/runtime.rs index e19ae5b..c084e20 100644 --- a/src/microsvc/runtime.rs +++ b/src/microsvc/runtime.rs @@ -18,7 +18,7 @@ use std::time::Duration; use super::dependencies::{HasOutboxStore, HasRepo}; use super::service::ImmediatePublish; use super::Service; -use crate::bus::{Bus, DynPublisher}; +use crate::bus::{Bus, BusConsumer, DynPublisher, RunOptions, TransportError}; use crate::outbox_worker::{BusPublisher, OutboxDispatcher}; /// Default lease for an immediate after-commit outbox publish. @@ -109,6 +109,66 @@ where } } +impl Microservice +where + D: Send + Sync + 'static, + B: Bus + BusConsumer, +{ + /// Run the service against the attached bus. + /// + /// Derives the consumers from the registered handlers: command handlers are + /// consumed with competing (point-to-point) `listen`, event handlers with + /// fan-out `subscribe`. Both run concurrently on the caller's runtime; + /// `run` returns when the consumers stop (a pull source that drains, or the + /// first error). + /// + /// Producing is handled separately: the primary path is immediate publish via + /// [`Context::commit_outbox`](crate::microsvc::Context::commit_outbox); the + /// background poll loop (the crash backstop, which needs an async timer) is + /// driven from [`Self::dispatcher`] by a runtime that provides one. + pub async fn run(&self, options: RunOptions) -> Result<(), TransportError> { + use std::future::{poll_fn, Future}; + use std::pin::Pin; + use std::task::Poll; + + let plan = self.service.subscription_plan(); + let mut consumers: Vec< + Pin> + Send + '_>>, + > = Vec::new(); + if !plan.commands.is_empty() { + consumers.push(Box::pin( + self.bus.listen(Arc::clone(&self.service), options.clone()), + )); + } + if !plan.events.is_empty() { + consumers.push(Box::pin( + self.bus.subscribe(Arc::clone(&self.service), options), + )); + } + + // Drive every consumer concurrently on the caller's runtime — no spawn, + // no timer. Return on the first error; finish when all consumers stop. + poll_fn(move |cx| { + let mut index = 0; + while index < consumers.len() { + match consumers[index].as_mut().poll(cx) { + Poll::Ready(Ok(())) => { + consumers.remove(index); + } + Poll::Ready(Err(error)) => return Poll::Ready(Err(error)), + Poll::Pending => index += 1, + } + } + if consumers.is_empty() { + Poll::Ready(Ok(())) + } else { + Poll::Pending + } + }) + .await + } +} + impl Service { /// Attach a bus, producing a [`Microservice`] that carries the transport /// config for both producing (outbox dispatch) and consuming @@ -140,7 +200,7 @@ impl Service { mod tests { use serde_json::{json, Value}; - use crate::bus::InMemoryBus; + use crate::bus::{Bus, InMemoryBus, RunOptions}; use crate::microsvc::{Context, HandlerError, HasOutboxStore, Service, Session}; use crate::outbox_worker::AsyncOutboxStore; use crate::{ @@ -230,4 +290,34 @@ mod tests { "no row should be left for the poller" ); } + + #[tokio::test] + async fn run_consumes_registered_commands_from_the_bus() { + let service = Service::with_repo(HashMapRepository::new().queued().aggregate::()) + .command("dummy.touch") + .handle(touch_and_publish); + let microservice = service.with_bus(InMemoryBus::new()); + + // Enqueue a command on the bus, then run: `listen` is derived from the + // registered command, drains the message, and the handler runs + // (commit_outbox publishes immediately). `run` returns once the queue is + // empty (InMemoryBus source yields `None`). + microservice + .bus() + .send("dummy.touch", b"{}".to_vec()) + .await + .unwrap(); + microservice.run(RunOptions::idempotent()).await.unwrap(); + + let store = microservice.service().repo().outbox_store(); + let published = store + .messages_by_status_async(OutboxMessageStatus::Published) + .await + .unwrap(); + assert_eq!( + published.len(), + 1, + "run() should consume the command and publish its outbox row" + ); + } } From 8c2a5025ef29b7a86e4472d3727c0e90f4399a2e Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Wed, 3 Jun 2026 15:54:18 -0500 Subject: [PATCH 09/24] test: SQLite end-to-end durable-enqueue dispatch Exercises commit_outbox (claim-in-transaction + immediate publish) and run() against a real SQL backend (in-memory SQLite), not just HashMapRepository. Proves the HasOutboxStore impls and the SQL commit path persist the in-flight claim and complete it. Also fixes a must_use warning on the finished-consumer future in run(). [[tasks/durable-enqueue-outbox-dispatch-impl]] Implements [[specs/durable-enqueue-outbox-dispatch]] Co-Authored-By: Claude Opus 4.8 (1M context) --- src/microsvc/runtime.rs | 3 +- tests/durable_enqueue_sqlite/main.rs | 102 +++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 tests/durable_enqueue_sqlite/main.rs diff --git a/src/microsvc/runtime.rs b/src/microsvc/runtime.rs index c084e20..8085ed5 100644 --- a/src/microsvc/runtime.rs +++ b/src/microsvc/runtime.rs @@ -153,7 +153,8 @@ where while index < consumers.len() { match consumers[index].as_mut().poll(cx) { Poll::Ready(Ok(())) => { - consumers.remove(index); + // Drop the finished consumer future; nothing left to poll. + let _finished = consumers.remove(index); } Poll::Ready(Err(error)) => return Poll::Ready(Err(error)), Poll::Pending => index += 1, diff --git a/tests/durable_enqueue_sqlite/main.rs b/tests/durable_enqueue_sqlite/main.rs new file mode 100644 index 0000000..8e728d1 --- /dev/null +++ b/tests/durable_enqueue_sqlite/main.rs @@ -0,0 +1,102 @@ +//! End-to-end durable-enqueue dispatch over a real SQL backend (in-memory +//! SQLite). Exercises `commit_outbox` (claim-in-transaction + immediate publish) +//! and the `with_bus` runtime against a persistent repository, not just the +//! in-memory `HashMapRepository` covered by the unit tests. + +#![cfg(feature = "sqlite")] + +use serde_json::{json, Value}; + +use distributed::bus::{Bus, InMemoryBus, RunOptions}; +use distributed::microsvc::{Context, HandlerError, HasOutboxStore, Service, Session}; +use distributed::{ + sourced, AggregateBuilder, AggregateRepository, AsyncOutboxStore, Entity, OutboxMessage, + OutboxMessageStatus, QueuedRepository, Queueable, SqliteRepository, +}; + +#[derive(Default)] +struct Counter { + entity: Entity, + value: i64, +} + +#[sourced(entity, aggregate_type = "counter")] +impl Counter { + #[event("touched")] + fn touch(&mut self, id: String) { + self.entity.set_id(&id); + self.value += 1; + } +} + +type Repo = AggregateRepository, Counter>; + +// Named fn (not a closure) so the higher-ranked `Handler` bound resolves. +async fn handle_touch(ctx: &Context<'_, Repo>) -> Result { + let mut counter = Counter::default(); + counter.touch("c1".to_string())?; + let message = OutboxMessage::create("evt-c1", "counter.touched", b"{}".to_vec())?; + ctx.commit_outbox(&mut counter, message).await?; + Ok(json!({ "value": counter.value })) +} + +async fn service() -> Repo { + SqliteRepository::connect_and_migrate("sqlite::memory:") + .await + .expect("sqlite repository should migrate") + .queued() + .aggregate::() +} + +#[tokio::test] +async fn commit_outbox_publishes_immediately_over_sqlite() { + let microservice = Service::with_repo(service().await) + .command("counter.touch") + .handle(handle_touch) + .with_bus(InMemoryBus::new()); + + // The handler claims the outbox row in the SQL transaction, then publishes + // it immediately through the attached bus. + microservice + .service() + .dispatch("counter.touch", json!({}), Session::new()) + .await + .unwrap(); + + let store = microservice.service().repo().outbox_store(); + let published = store + .messages_by_status_async(OutboxMessageStatus::Published) + .await + .unwrap(); + assert_eq!(published.len(), 1, "row should be published immediately"); + assert_eq!(published[0].id(), "evt-c1"); + assert!( + store.pending_async().await.unwrap().is_empty(), + "nothing should be left for the poller" + ); +} + +#[tokio::test] +async fn run_consumes_command_and_publishes_over_sqlite() { + let microservice = Service::with_repo(service().await) + .command("counter.touch") + .handle(handle_touch) + .with_bus(InMemoryBus::new()); + + // Enqueue a command, then run: listen is derived from the registered command, + // drains it, and the handler publishes its outbox row through the bus. + microservice + .bus() + .send("counter.touch", b"{}".to_vec()) + .await + .unwrap(); + microservice.run(RunOptions::idempotent()).await.unwrap(); + + let store = microservice.service().repo().outbox_store(); + let published = store + .messages_by_status_async(OutboxMessageStatus::Published) + .await + .unwrap(); + assert_eq!(published.len(), 1); + assert_eq!(published[0].id(), "evt-c1"); +} From 85881cbd59d170dc003474e9e7161b138af28e49 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Wed, 3 Jun 2026 16:31:36 -0500 Subject: [PATCH 10/24] feat: commit_outbox works for all repo shapes incl snapshots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generalize the durable-enqueue command path with an OutboxCommitting trait that commits an aggregate + outbox row in one transaction, staging whatever the repo needs. Implemented for AggregateRepository (delegates to the existing OutboxCommit) and SnapshotAggregateRepository (stages the snapshot + outbox row together via CommitBatch — previously these could not compose). Context::commit_outbox now binds D::Repo: OutboxCommitting + HasOutboxStore instead of the concrete AggregateRepository, so snapshot-backed services get claim-in-transaction + immediate publish too. Test: snapshot-backed commit_outbox publishes immediately. [[tasks/durable-enqueue-outbox-dispatch-impl]] Builds on [[specs/transactional-commit-boundary]] Implements [[specs/durable-enqueue-outbox-dispatch]] Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib.rs | 4 +- src/microsvc/context.rs | 17 +++----- src/outbox/commit.rs | 58 +++++++++++++++++++++++++ src/outbox/mod.rs | 2 +- src/snapshot/repository.rs | 88 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 156 insertions(+), 13 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index cfb394d..d9f1007 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -67,8 +67,8 @@ pub use lock::{SqliteLock, SqliteLockManager}; // Outbox: commit concerns (aggregate + outbox in one commit) pub use outbox::{ outbox_message_insert_plan, outbox_message_key, outbox_message_row_values, - outbox_message_schema, CommitReceipt, OutboxCommit, OutboxMessage, OutboxMessageStatus, - OUTBOX_MESSAGES_TABLE, + outbox_message_schema, CommitReceipt, OutboxCommit, OutboxCommitting, OutboxMessage, + OutboxMessageStatus, OUTBOX_MESSAGES_TABLE, }; // Outbox Worker: drain and publish concerns diff --git a/src/microsvc/context.rs b/src/microsvc/context.rs index adeb9e9..4e89663 100644 --- a/src/microsvc/context.rs +++ b/src/microsvc/context.rs @@ -11,11 +11,9 @@ use super::dependencies::{HasOutboxStore, HasReadModelStore, HasRepo}; use super::error::HandlerError; use super::service::ImmediatePublish; use super::session::Session; -use crate::aggregate::{Aggregate, AggregateRepository}; use crate::bus::Message; -use crate::outbox::{CommitReceipt, OutboxMessage}; +use crate::outbox::{CommitReceipt, OutboxCommitting, OutboxMessage}; use crate::outbox_worker::{AsyncOutboxStore, OutboxClaimRef}; -use crate::repository::TransactionalCommit; /// The context passed to every handler. /// @@ -139,28 +137,27 @@ impl<'a, D> Context<'a, D> { /// /// When no bus is configured, the row is committed `pending` and left for the /// polling worker to publish. - pub async fn commit_outbox( + pub async fn commit_outbox( &self, aggregate: &mut A, message: OutboxMessage, ) -> Result where - D: HasRepo>, - R: TransactionalCommit + HasOutboxStore, - A: Aggregate + Send, + D: HasRepo, + D::Repo: OutboxCommitting + HasOutboxStore, + A: Send, { let repo = self.repo(); let Some(immediate) = self.immediate_publish else { // No bus configured: durable enqueue only; the worker publishes. - return Ok(repo.outbox(message).commit(aggregate).await?); + return Ok(repo.commit_outbox_pending(aggregate, message).await?); }; // Claim the row in the commit transaction, then publish after commit. // The lease hands the row to the poller if we crash before completing. let (receipt, claimed) = repo - .outbox(message) - .commit_claimed(aggregate, &immediate.worker_id, immediate.lease) + .commit_outbox_claimed(aggregate, message, &immediate.worker_id, immediate.lease) .await?; let claim = OutboxClaimRef::from_message(&claimed)?; let transport = Message::from(&claimed); diff --git a/src/outbox/commit.rs b/src/outbox/commit.rs index f72d50a..638eeca 100644 --- a/src/outbox/commit.rs +++ b/src/outbox/commit.rs @@ -110,6 +110,64 @@ impl AggregateRepository { } } +/// A repository that can commit an aggregate together with an outbox message in +/// one transaction, staging whatever else the repository requires (for example a +/// snapshot, for snapshot-backed repositories). +/// +/// This is the abstraction +/// [`Context::commit_outbox`](crate::microsvc::Context::commit_outbox) binds to, +/// so the durable-enqueue command path works for **any** repository shape — a +/// plain [`AggregateRepository`] or a `SnapshotAggregateRepository` — not just +/// one. The underlying `CommitBatch`/`TransactionalCommit` boundary already +/// applies streams, outbox rows, read models, and snapshots in one transaction; +/// this trait exposes that to the ergonomic command path. +pub trait OutboxCommitting { + /// Commit the aggregate and outbox row (left `pending`) in one transaction. + /// The polling worker publishes the row later. + fn commit_outbox_pending( + &self, + aggregate: &mut A, + message: OutboxMessage, + ) -> impl core::future::Future> + Send; + + /// Commit the aggregate and outbox row, claiming the row in the same + /// transaction for immediate publication. Returns a clone of the claimed + /// message so the caller can build the transport message and settle the claim. + fn commit_outbox_claimed( + &self, + aggregate: &mut A, + message: OutboxMessage, + worker_id: &str, + lease: Duration, + ) -> impl core::future::Future> + Send; +} + +impl OutboxCommitting for AggregateRepository +where + R: TransactionalCommit + Send + Sync, + A: Aggregate + Send + Sync, +{ + async fn commit_outbox_pending( + &self, + aggregate: &mut A, + message: OutboxMessage, + ) -> Result { + self.outbox(message).commit(aggregate).await + } + + async fn commit_outbox_claimed( + &self, + aggregate: &mut A, + message: OutboxMessage, + worker_id: &str, + lease: Duration, + ) -> Result<(CommitReceipt, OutboxMessage), RepositoryError> { + self.outbox(message) + .commit_claimed(aggregate, worker_id, lease) + .await + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/outbox/mod.rs b/src/outbox/mod.rs index 74143cb..1ad33e2 100644 --- a/src/outbox/mod.rs +++ b/src/outbox/mod.rs @@ -49,4 +49,4 @@ pub use table::{ }; // Commit helpers -pub use commit::{CommitReceipt, OutboxCommit}; +pub use commit::{CommitReceipt, OutboxCommit, OutboxCommitting}; diff --git a/src/snapshot/repository.rs b/src/snapshot/repository.rs index 2dcd782..7685704 100644 --- a/src/snapshot/repository.rs +++ b/src/snapshot/repository.rs @@ -1,5 +1,8 @@ +use std::time::{Duration, SystemTime}; + use crate::aggregate::{hydrate, AggregateRepository}; use crate::entity::{upcast_events, Entity}; +use crate::outbox::{CommitReceipt, OutboxCommitting, OutboxMessage}; use crate::repository::{ CommitBatch, GetStream, RepositoryError, SnapshotStore, SnapshotWrite, StreamIdentity, StreamWrite, TransactionalCommit, @@ -310,6 +313,91 @@ where } Ok(None) } + + /// Commit the aggregate, an outbox message, and (when due) a snapshot in one + /// transaction. With `claim` set, the outbox row is claimed in the same + /// transaction for immediate publication and a clone of the claimed message + /// is returned. This is what lets snapshot-backed repositories take part in + /// the durable-enqueue command path. + async fn commit_with_outbox( + &self, + aggregate: &mut A, + mut message: OutboxMessage, + claim: Option<(&str, Duration)>, + ) -> Result<(CommitReceipt, Option), RepositoryError> { + message.set_source(aggregate); + let claimed = match claim { + Some((worker_id, lease)) => { + message.claim_at(worker_id, lease, SystemTime::now())?; + Some(message.clone()) + } + None => None, + }; + let outbox_message_id = message.id().to_string(); + + let snapshot = self.snapshot_record(aggregate)?; + let snapshot_version = snapshot.as_ref().map(|record| record.version); + let identity = StreamIdentity::new(A::aggregate_type(), aggregate.entity().id())?; + let snapshots = snapshot + .into_iter() + .map(|record| SnapshotWrite::Save { + identity: identity.clone(), + record, + }) + .collect(); + + self.inner + .repo() + .commit_batch(CommitBatch { + streams: vec![StreamWrite::new(identity, aggregate.entity_mut())], + outbox_messages: vec![message], + read_model_plans: Vec::new(), + snapshots, + inbox_receipts: Vec::new(), + }) + .await?; + + if let Some(version) = snapshot_version { + aggregate.entity_mut().set_snapshot_version(version); + } + Ok(( + CommitReceipt { + outbox_message_ids: vec![outbox_message_id], + }, + claimed, + )) + } +} + +impl OutboxCommitting for SnapshotAggregateRepository +where + R: TransactionalCommit + Send + Sync, + A: Snapshottable + Send + Sync, +{ + async fn commit_outbox_pending( + &self, + aggregate: &mut A, + message: OutboxMessage, + ) -> Result { + let (receipt, _) = self.commit_with_outbox(aggregate, message, None).await?; + Ok(receipt) + } + + async fn commit_outbox_claimed( + &self, + aggregate: &mut A, + message: OutboxMessage, + worker_id: &str, + lease: Duration, + ) -> Result<(CommitReceipt, OutboxMessage), RepositoryError> { + let (receipt, claimed) = self + .commit_with_outbox(aggregate, message, Some((worker_id, lease))) + .await?; + Ok(( + receipt, + claimed.expect("claimed commit always returns the claimed message"), + )) + } } #[cfg(test)] From bef769186557e34580d0957616e5e78df04512ea Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 4 Jun 2026 20:29:03 -0500 Subject: [PATCH 11/24] refactor: snapshots are a transparent optimization (one repo type) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fold SnapshotAggregateRepository into AggregateRepository via an optional SnapshotPolicy whose Snapshottable/SnapshotStore requirements are captured as monomorphized fn-pointers at with_snapshots() time, keeping the generic get/commit methods unbounded. Now: - .with_snapshots(n) returns AggregateRepository (same type), so handler dependency types are identical with/without snapshots. - every method works either way; commit stages a snapshot (when due) in the same CommitBatch, get hydrates from a snapshot when present. The full repo surface (peek/abort/get_with/outbox/...) is available with snapshots on — previously the wrapper dropped most of it. - exactly ONE OutboxCommitting impl (on AggregateRepository); the snapshot- specific impl and the whole SnapshotAggregateRepository type are removed. with_snapshots now requires R: SnapshotStore (you can't cache snapshots in a store that can't hold them) — stricter and more correct than the old wrapper. Tests migrated to the unified type; assertions unchanged. Full suite + sqlite green. Implements [[specs/snapshots-as-transparent-optimization]] [[tasks/snapshots-transparent-optimization]] Co-Authored-By: Claude Opus 4.8 (1M context) --- src/aggregate/mod.rs | 1 + src/aggregate/repository.rs | 158 ++++++++++++++++-- src/lib.rs | 3 +- src/microsvc/dependencies.rs | 19 --- src/microsvc/runtime.rs | 61 ++++++- src/outbox/commit.rs | 12 ++ src/snapshot/mod.rs | 2 +- src/snapshot/repository.rs | 294 ++++++++------------------------- tests/snapshots/main.rs | 15 +- tests/sourced_snapshot/main.rs | 2 +- tests/upcasting/main.rs | 1 - 11 files changed, 295 insertions(+), 273 deletions(-) diff --git a/src/aggregate/mod.rs b/src/aggregate/mod.rs index 114e521..8ee68cf 100644 --- a/src/aggregate/mod.rs +++ b/src/aggregate/mod.rs @@ -2,4 +2,5 @@ mod aggregate; mod repository; pub use aggregate::{hydrate, Aggregate}; +pub(crate) use repository::SnapshotPolicy; pub use repository::{AggregateBuilder, AggregateRepository}; diff --git a/src/aggregate/repository.rs b/src/aggregate/repository.rs index f779d6a..445c1a4 100644 --- a/src/aggregate/repository.rs +++ b/src/aggregate/repository.rs @@ -1,10 +1,14 @@ +use std::future::Future; use std::marker::PhantomData; +use std::pin::Pin; use crate::entity::Entity; use crate::queued_repo::{GetAllWithOpts, GetWithOpts, ReadOpts, UnlockableRepository}; use crate::repository::{ - CommitBatch, GetStream, RepositoryError, StreamIdentity, StreamWrite, TransactionalCommit, + CommitBatch, GetStream, RepositoryError, SnapshotWrite, StreamIdentity, StreamWrite, + TransactionalCommit, }; +use crate::snapshot::SnapshotRecord; use super::{hydrate, Aggregate}; @@ -23,9 +27,55 @@ pub trait AggregateBuilder: Sized { impl AggregateBuilder for T {} +/// Snapshot behaviour for an [`AggregateRepository`], installed by +/// `with_snapshots`. +/// +/// A snapshot is a rebuildable cache over the event stream, so enabling it must +/// not change the repository's API. The `Snapshottable` / `SnapshotStore` +/// requirements are captured here as monomorphized function pointers at +/// `with_snapshots` time, which keeps the repository's generic get/commit methods +/// unbounded — they just consult `Option`. +pub(crate) struct SnapshotPolicy { + /// How many events between automatic snapshots. + frequency: u64, + /// Build a snapshot cache record for the aggregate when one is due. + record: fn(&A, u64) -> Result, RepositoryError>, + /// Load the cache record (if any) and hydrate the aggregate from it. + hydrate: HydrateFn, +} + +type HydrateFn = for<'a> fn( + &'a R, + &'a StreamIdentity, + Entity, +) -> Pin> + Send + 'a>>; + +impl SnapshotPolicy { + /// Construct a policy from its captured hooks. Called by `with_snapshots`, + /// which carries the `Snapshottable`/`SnapshotStore` bounds. + pub(crate) fn new( + frequency: u64, + record: fn(&A, u64) -> Result, RepositoryError>, + hydrate: HydrateFn, + ) -> Self { + Self { + frequency, + record, + hydrate, + } + } +} + /// Async repository wrapper for a specific aggregate type. +/// +/// Snapshots are an optional, transparent optimization: `with_snapshots(n)` +/// configures snapshot caching on this same type, and every method behaves +/// identically with or without it — on commit a snapshot is staged in the same +/// transaction when due, and on load the aggregate is hydrated from a snapshot +/// when one exists. pub struct AggregateRepository { repo: R, + snapshot: Option>, _marker: PhantomData, } @@ -33,6 +83,7 @@ impl AggregateRepository { pub fn new(repo: R) -> Self { Self { repo, + snapshot: None, _marker: PhantomData, } } @@ -44,6 +95,57 @@ impl AggregateRepository { pub fn repo_mut(&mut self) -> &mut R { &mut self.repo } + + /// Install the snapshot policy (used by `with_snapshots`). + pub(crate) fn set_snapshot_policy(&mut self, policy: SnapshotPolicy) { + self.snapshot = Some(policy); + } +} + +impl AggregateRepository +where + A: Aggregate + Send, +{ + /// Hydrate one entity into an aggregate, using the snapshot cache when a + /// policy is configured and a cache record is available, otherwise a full + /// replay. Same result either way. + async fn hydrate_entity( + &self, + identity: &StreamIdentity, + entity: Entity, + ) -> Result { + match &self.snapshot { + Some(policy) => (policy.hydrate)(&self.repo, identity, entity).await, + None => hydrate::(entity), + } + } + + /// Snapshot writes to stage alongside a commit of `aggregate`, plus the + /// covered version to record on the entity afterwards. Empty when no policy + /// is configured or a snapshot is not yet due. + fn snapshot_writes( + &self, + aggregate: &A, + ) -> Result<(Vec, Option), RepositoryError> { + let Some(policy) = &self.snapshot else { + return Ok((Vec::new(), None)); + }; + let Some(record) = (policy.record)(aggregate, policy.frequency)? else { + return Ok((Vec::new(), None)); + }; + let version = record.version; + let identity = stream_identity_for::(aggregate.entity().id())?; + Ok((vec![SnapshotWrite::Save { identity, record }], Some(version))) + } + + /// Snapshot writes for `aggregate`, exposed to the outbox/read-model commit + /// builders so they stage snapshots in the same transaction. + pub(crate) fn snapshot_writes_for( + &self, + aggregate: &A, + ) -> Result<(Vec, Option), RepositoryError> { + self.snapshot_writes(aggregate) + } } impl AggregateRepository @@ -57,7 +159,7 @@ where let Some(entity) = entity else { return Ok(None); }; - Ok(Some(hydrate::(entity)?)) + Ok(Some(self.hydrate_entity(&identity, entity).await?)) } /// Load existing aggregates for the provided ids. @@ -72,9 +174,21 @@ where .map(|id| stream_identity_for::(id)) .collect::, _>>()?; let entities = self.repo.get_streams(&identities).await?; + self.hydrate_entities(entities).await + } +} + +impl AggregateRepository +where + A: Aggregate + Send, +{ + /// Hydrate a batch of entities, deriving each identity from the entity id so + /// the snapshot cache can be consulted per aggregate. + async fn hydrate_entities(&self, entities: Vec) -> Result, RepositoryError> { let mut aggregates = Vec::with_capacity(entities.len()); for entity in entities { - aggregates.push(hydrate::(entity)?); + let identity = stream_identity_for::(entity.id())?; + aggregates.push(self.hydrate_entity(&identity, entity).await?); } Ok(aggregates) } @@ -86,18 +200,44 @@ where A: Aggregate + Send, { pub async fn commit(&self, aggregate: &mut A) -> Result<(), RepositoryError> { + let (snapshots, snapshot_version) = self.snapshot_writes(aggregate)?; let identity = stream_identity_for::(aggregate.entity().id())?; let stream = StreamWrite::new(identity, aggregate.entity_mut()); - self.repo.commit_batch(CommitBatch::new(vec![stream])).await + let mut batch = CommitBatch::new(vec![stream]); + batch.snapshots = snapshots; + self.repo.commit_batch(batch).await?; + if let Some(version) = snapshot_version { + aggregate.entity_mut().set_snapshot_version(version); + } + Ok(()) } pub async fn commit_all(&self, aggregates: &mut [&mut A]) -> Result<(), RepositoryError> { + // Compute snapshot writes (immutable borrows) before taking the mutable + // entity borrows for the streams. + let mut snapshots = Vec::new(); + let mut snapshot_versions = Vec::with_capacity(aggregates.len()); + for aggregate in aggregates.iter() { + let (mut writes, version) = self.snapshot_writes(aggregate)?; + snapshots.append(&mut writes); + snapshot_versions.push(version); + } + let mut streams = Vec::with_capacity(aggregates.len()); for aggregate in aggregates.iter_mut() { let identity = stream_identity_for::((*aggregate).entity().id())?; streams.push(StreamWrite::new(identity, (*aggregate).entity_mut())); } - self.repo.commit_batch(CommitBatch::new(streams)).await + let mut batch = CommitBatch::new(streams); + batch.snapshots = snapshots; + self.repo.commit_batch(batch).await?; + + for (aggregate, version) in aggregates.iter_mut().zip(snapshot_versions) { + if let Some(version) = version { + (*aggregate).entity_mut().set_snapshot_version(version); + } + } + Ok(()) } pub async fn commit_entities( @@ -124,7 +264,7 @@ where let Some(entity) = self.repo.get_stream_with(&identity, opts).await? else { return Ok(None); }; - Ok(Some(hydrate::(entity)?)) + Ok(Some(self.hydrate_entity(&identity, entity).await?)) } /// Non-locking read (alias for `get_with(ReadOpts::no_lock())`). @@ -149,11 +289,7 @@ where .map(|id| stream_identity_for::(id)) .collect::, _>>()?; let entities = self.repo.get_streams_with(&identities, opts).await?; - let mut aggregates = Vec::with_capacity(entities.len()); - for entity in entities { - aggregates.push(hydrate::(entity)?); - } - Ok(aggregates) + self.hydrate_entities(entities).await } /// Non-locking multi-read (alias for `get_all_with(ReadOpts::no_lock())`). diff --git a/src/lib.rs b/src/lib.rs index d9f1007..13574ad 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -145,8 +145,7 @@ pub use commit_builder::{ // Snapshot: state snapshot payloads and rebuildable cache records for hydration pub use snapshot::{ - hydrate_from_snapshot, InMemorySnapshotStore, SnapshotAggregateRepository, SnapshotRecord, - Snapshottable, + hydrate_from_snapshot, InMemorySnapshotStore, SnapshotRecord, Snapshottable, }; // Re-export the EventEmitter from the event_emitter_rs crate (requires "emitter" feature) diff --git a/src/microsvc/dependencies.rs b/src/microsvc/dependencies.rs index 453fc6c..a6f4d5e 100644 --- a/src/microsvc/dependencies.rs +++ b/src/microsvc/dependencies.rs @@ -3,7 +3,6 @@ use crate::aggregate::AggregateRepository; use crate::outbox_worker::AsyncOutboxStore; use crate::repository::{ReadModelWritePlanStore, RelationalReadModelQueryStore, Repository}; -use crate::snapshot::SnapshotAggregateRepository; /// Dependency capability for services that expose an aggregate repository. pub trait HasRepo { @@ -80,16 +79,6 @@ where } } -impl HasOutboxStore for SnapshotAggregateRepository -where - R: HasOutboxStore, -{ - type OutboxStore = as HasOutboxStore>::OutboxStore; - fn outbox_store(&self) -> Self::OutboxStore { - self.repo().outbox_store() - } -} - impl HasRepo for R where R: Repository, @@ -109,14 +98,6 @@ impl HasRepo for AggregateRepository { } } -impl HasRepo for SnapshotAggregateRepository { - type Repo = Self; - - fn repo(&self) -> &Self::Repo { - self - } -} - impl HasReadModelStore for S where S: ReadModelWritePlanStore + RelationalReadModelQueryStore, diff --git a/src/microsvc/runtime.rs b/src/microsvc/runtime.rs index 8085ed5..02a60d4 100644 --- a/src/microsvc/runtime.rs +++ b/src/microsvc/runtime.rs @@ -206,7 +206,7 @@ mod tests { use crate::outbox_worker::AsyncOutboxStore; use crate::{ sourced, AggregateBuilder, AggregateRepository, Entity, HashMapRepository, OutboxMessage, - OutboxMessageStatus, QueuedRepository, Queueable, + OutboxMessageStatus, QueuedRepository, Queueable, Snapshot, }; #[derive(Default)] @@ -321,4 +321,63 @@ mod tests { "run() should consume the command and publish its outbox row" ); } + + #[derive(Default, Snapshot)] + struct SnapCounter { + entity: Entity, + value: i64, + } + + #[sourced(entity, aggregate_type = "snap_counter")] + impl SnapCounter { + #[event("touched")] + fn touch(&mut self, id: String) { + self.entity.set_id(&id); + self.value += 1; + } + } + + // Snapshots are a transparent optimization: the repo type is unchanged. + type SnapRepo = AggregateRepository, SnapCounter>; + + async fn touch_snap(ctx: &Context<'_, SnapRepo>) -> Result { + let mut counter = SnapCounter::default(); + counter.touch("s1".to_string())?; + let message = OutboxMessage::create("evt-s1", "snap.touched", b"{}".to_vec())?; + ctx.commit_outbox(&mut counter, message).await?; + Ok(json!({})) + } + + #[tokio::test] + async fn commit_outbox_works_with_snapshot_backed_repo() { + // `commit_outbox` must work for a snapshot-backed repository too: the + // outbox row and the snapshot commit together in one transaction, then + // the row publishes immediately. + let repo = HashMapRepository::new() + .queued() + .aggregate::() + .with_snapshots(1); + let microservice = Service::with_repo(repo) + .command("snap.touch") + .handle(touch_snap) + .with_bus(InMemoryBus::new()); + + microservice + .service() + .dispatch("snap.touch", json!({}), Session::new()) + .await + .unwrap(); + + let store = microservice.service().repo().outbox_store(); + let published = store + .messages_by_status_async(OutboxMessageStatus::Published) + .await + .unwrap(); + assert_eq!( + published.len(), + 1, + "snapshot-backed commit_outbox should publish immediately" + ); + assert_eq!(published[0].id(), "evt-s1"); + } } diff --git a/src/outbox/commit.rs b/src/outbox/commit.rs index 638eeca..083d06f 100644 --- a/src/outbox/commit.rs +++ b/src/outbox/commit.rs @@ -53,11 +53,18 @@ where pub async fn commit(mut self, aggregate: &mut A) -> Result { self.message.set_source(aggregate); let outbox_message_id = self.message.id().to_string(); + // Stage a snapshot too when the repository has snapshots configured and + // one is due — same transaction as the events and the outbox row. + let (snapshots, snapshot_version) = self.repo.snapshot_writes_for(aggregate)?; let identity = StreamIdentity::new(A::aggregate_type(), aggregate.entity().id())?; let stream = StreamWrite::new(identity, aggregate.entity_mut()); let mut batch = CommitBatch::new(vec![stream]); batch.outbox_messages.push(self.message); + batch.snapshots = snapshots; self.repo.repo().commit_batch(batch).await?; + if let Some(version) = snapshot_version { + aggregate.entity_mut().set_snapshot_version(version); + } Ok(CommitReceipt { outbox_message_ids: vec![outbox_message_id], }) @@ -86,11 +93,16 @@ where self.message.claim_at(worker_id, lease, SystemTime::now())?; let claimed = self.message.clone(); let outbox_message_id = self.message.id().to_string(); + let (snapshots, snapshot_version) = self.repo.snapshot_writes_for(aggregate)?; let identity = StreamIdentity::new(A::aggregate_type(), aggregate.entity().id())?; let stream = StreamWrite::new(identity, aggregate.entity_mut()); let mut batch = CommitBatch::new(vec![stream]); batch.outbox_messages.push(self.message); + batch.snapshots = snapshots; self.repo.repo().commit_batch(batch).await?; + if let Some(version) = snapshot_version { + aggregate.entity_mut().set_snapshot_version(version); + } Ok(( CommitReceipt { outbox_message_ids: vec![outbox_message_id], diff --git a/src/snapshot/mod.rs b/src/snapshot/mod.rs index 8688d50..8394bf7 100644 --- a/src/snapshot/mod.rs +++ b/src/snapshot/mod.rs @@ -4,6 +4,6 @@ mod snapshottable; mod store; pub use in_memory::InMemorySnapshotStore; -pub use repository::{hydrate_from_snapshot, SnapshotAggregateRepository}; +pub use repository::hydrate_from_snapshot; pub use snapshottable::Snapshottable; pub use store::SnapshotRecord; diff --git a/src/snapshot/repository.rs b/src/snapshot/repository.rs index 7685704..eee1055 100644 --- a/src/snapshot/repository.rs +++ b/src/snapshot/repository.rs @@ -1,12 +1,9 @@ -use std::time::{Duration, SystemTime}; +use std::future::Future; +use std::pin::Pin; -use crate::aggregate::{hydrate, AggregateRepository}; +use crate::aggregate::{hydrate, AggregateRepository, SnapshotPolicy}; use crate::entity::{upcast_events, Entity}; -use crate::outbox::{CommitReceipt, OutboxCommitting, OutboxMessage}; -use crate::repository::{ - CommitBatch, GetStream, RepositoryError, SnapshotStore, SnapshotWrite, StreamIdentity, - StreamWrite, TransactionalCommit, -}; +use crate::repository::{RepositoryError, SnapshotStore, StreamIdentity}; use super::snapshottable::Snapshottable; use super::store::SnapshotRecord; @@ -172,237 +169,64 @@ fn hydrate_with_optional_snapshot( .map_err(snapshot_hydration_error_to_repository_error) } -/// Async repository wrapper that treats aggregate snapshots as rebuildable -/// hydration cache records. -pub struct SnapshotAggregateRepository { - inner: AggregateRepository, - frequency: u64, -} - -impl SnapshotAggregateRepository { - pub fn new(inner: AggregateRepository, frequency: u64) -> Self { - Self { inner, frequency } - } - - pub fn repo(&self) -> &AggregateRepository { - &self.inner - } -} - -impl AggregateRepository { - /// Wrap this async repository with snapshot cache support at the given event - /// frequency. - pub fn with_snapshots(self, frequency: u64) -> SnapshotAggregateRepository { - SnapshotAggregateRepository::new(self, frequency) - } -} - -impl SnapshotAggregateRepository +impl AggregateRepository where - R: GetStream + SnapshotStore, + R: SnapshotStore + Sync, A: Snapshottable + Send, { - pub async fn get(&self, id: &str) -> Result, RepositoryError> { - let identity = StreamIdentity::new(A::aggregate_type(), id)?; - let entity = self.inner.repo().get_stream(&identity).await?; - let Some(entity) = entity else { - return Ok(None); - }; - let snapshot = self.inner.repo().get_snapshot(&identity).await?; - Ok(Some(hydrate_with_optional_snapshot::(entity, snapshot)?)) - } - - pub async fn get_all(&self, ids: &[&str]) -> Result, RepositoryError> { - let identities = ids - .iter() - .map(|id| StreamIdentity::new(A::aggregate_type(), *id)) - .collect::, _>>()?; - let entities = self.inner.repo().get_streams(&identities).await?; - let mut aggregates = Vec::with_capacity(entities.len()); - for entity in entities { - let identity = StreamIdentity::new(A::aggregate_type(), entity.id())?; - let snapshot = self.inner.repo().get_snapshot(&identity).await?; - aggregates.push(hydrate_with_optional_snapshot::(entity, snapshot)?); - } - Ok(aggregates) + /// Enable snapshot caching at the given event frequency. + /// + /// Snapshots are a transparent optimization: this configures snapshot + /// behaviour on the **same** repository type and returns it. On commit a + /// snapshot is staged (when due) in the same transaction; on load the + /// aggregate is hydrated from a snapshot when one exists. Every other method + /// behaves identically with or without snapshots. + pub fn with_snapshots(mut self, frequency: u64) -> Self { + self.set_snapshot_policy(SnapshotPolicy::new( + frequency, + snapshot_record_if_due::, + hydrate_from_store::, + )); + self } } -impl SnapshotAggregateRepository -where - R: TransactionalCommit, - A: Snapshottable + Send, -{ - pub async fn commit(&self, aggregate: &mut A) -> Result<(), RepositoryError> { - let snapshot = self.snapshot_record(aggregate)?; - let snapshot_version = snapshot.as_ref().map(|record| record.version); - let identity = StreamIdentity::new(A::aggregate_type(), aggregate.entity().id())?; - let snapshots = snapshot - .into_iter() - .map(|record| SnapshotWrite::Save { - identity: identity.clone(), - record, - }) - .collect(); - - self.inner - .repo() - .commit_batch(CommitBatch { - streams: vec![StreamWrite::new(identity, aggregate.entity_mut())], - outbox_messages: Vec::new(), - read_model_plans: Vec::new(), - snapshots, - inbox_receipts: Vec::new(), - }) - .await?; - - if let Some(version) = snapshot_version { - aggregate.entity_mut().set_snapshot_version(version); - } - Ok(()) - } - - pub async fn commit_all(&self, aggregates: &mut [&mut A]) -> Result<(), RepositoryError> { - let mut snapshot_versions = Vec::with_capacity(aggregates.len()); - let mut snapshots = Vec::new(); - for aggregate in aggregates.iter() { - let snapshot = self.snapshot_record(*aggregate)?; - snapshot_versions.push(snapshot.as_ref().map(|record| record.version)); - if let Some(record) = snapshot { - snapshots.push(SnapshotWrite::Save { - identity: StreamIdentity::new( - A::aggregate_type(), - record.aggregate_id.as_str(), - )?, - record, - }); - } - } - - let mut streams = Vec::with_capacity(aggregates.len()); - for aggregate in aggregates.iter_mut() { - let identity = StreamIdentity::new(A::aggregate_type(), (*aggregate).entity().id())?; - streams.push(StreamWrite::new(identity, (*aggregate).entity_mut())); - } - - self.inner - .repo() - .commit_batch(CommitBatch { - streams, - outbox_messages: Vec::new(), - read_model_plans: Vec::new(), - snapshots, - inbox_receipts: Vec::new(), - }) - .await?; - - for (aggregate, snapshot_version) in aggregates.iter_mut().zip(snapshot_versions) { - if let Some(version) = snapshot_version { - aggregate.entity_mut().set_snapshot_version(version); - } - } - Ok(()) - } - - fn snapshot_record(&self, aggregate: &A) -> Result, RepositoryError> { - let version = aggregate.entity().version(); - let snap_version = aggregate.entity().snapshot_version(); - - if snapshot_due(version, snap_version, self.frequency) { - return snapshot_record_for(aggregate).map(Some); - } +/// Build a snapshot cache record for `aggregate` when one is due at `frequency`. +/// Captured as the `record` hook of a `SnapshotPolicy`. +fn snapshot_record_if_due( + aggregate: &A, + frequency: u64, +) -> Result, RepositoryError> { + let version = aggregate.entity().version(); + let snap_version = aggregate.entity().snapshot_version(); + if snapshot_due(version, snap_version, frequency) { + snapshot_record_for(aggregate).map(Some) + } else { Ok(None) } - - /// Commit the aggregate, an outbox message, and (when due) a snapshot in one - /// transaction. With `claim` set, the outbox row is claimed in the same - /// transaction for immediate publication and a clone of the claimed message - /// is returned. This is what lets snapshot-backed repositories take part in - /// the durable-enqueue command path. - async fn commit_with_outbox( - &self, - aggregate: &mut A, - mut message: OutboxMessage, - claim: Option<(&str, Duration)>, - ) -> Result<(CommitReceipt, Option), RepositoryError> { - message.set_source(aggregate); - let claimed = match claim { - Some((worker_id, lease)) => { - message.claim_at(worker_id, lease, SystemTime::now())?; - Some(message.clone()) - } - None => None, - }; - let outbox_message_id = message.id().to_string(); - - let snapshot = self.snapshot_record(aggregate)?; - let snapshot_version = snapshot.as_ref().map(|record| record.version); - let identity = StreamIdentity::new(A::aggregate_type(), aggregate.entity().id())?; - let snapshots = snapshot - .into_iter() - .map(|record| SnapshotWrite::Save { - identity: identity.clone(), - record, - }) - .collect(); - - self.inner - .repo() - .commit_batch(CommitBatch { - streams: vec![StreamWrite::new(identity, aggregate.entity_mut())], - outbox_messages: vec![message], - read_model_plans: Vec::new(), - snapshots, - inbox_receipts: Vec::new(), - }) - .await?; - - if let Some(version) = snapshot_version { - aggregate.entity_mut().set_snapshot_version(version); - } - Ok(( - CommitReceipt { - outbox_message_ids: vec![outbox_message_id], - }, - claimed, - )) - } } -impl OutboxCommitting for SnapshotAggregateRepository +/// Load the snapshot cache record (if any) and hydrate `entity` from it. +/// Captured as the `hydrate` hook of a `SnapshotPolicy`. +fn hydrate_from_store<'a, R, A>( + repo: &'a R, + identity: &'a StreamIdentity, + entity: Entity, +) -> Pin> + Send + 'a>> where - R: TransactionalCommit + Send + Sync, - A: Snapshottable + Send + Sync, + R: SnapshotStore + Sync, + A: Snapshottable + Send, { - async fn commit_outbox_pending( - &self, - aggregate: &mut A, - message: OutboxMessage, - ) -> Result { - let (receipt, _) = self.commit_with_outbox(aggregate, message, None).await?; - Ok(receipt) - } - - async fn commit_outbox_claimed( - &self, - aggregate: &mut A, - message: OutboxMessage, - worker_id: &str, - lease: Duration, - ) -> Result<(CommitReceipt, OutboxMessage), RepositoryError> { - let (receipt, claimed) = self - .commit_with_outbox(aggregate, message, Some((worker_id, lease))) - .await?; - Ok(( - receipt, - claimed.expect("claimed commit always returns the claimed message"), - )) - } + Box::pin(async move { + let snapshot = repo.get_snapshot(identity).await?; + hydrate_with_optional_snapshot::(entity, snapshot) + }) } #[cfg(test)] mod tests { use super::*; + use crate::repository::{CommitBatch, TransactionalCommit}; use crate::{sourced, Aggregate, EventRecord}; #[derive(Default)] @@ -456,11 +280,34 @@ mod tests { } } + // Snapshots can only be enabled on a repo that can store them; this stub + // satisfies the bound (the commit-failure test never loads a snapshot). + impl SnapshotStore for FailingSnapshotRepo { + async fn get_snapshot( + &self, + _identity: &StreamIdentity, + ) -> Result, RepositoryError> { + Ok(None) + } + async fn save_snapshot( + &self, + _identity: &StreamIdentity, + _record: SnapshotRecord, + ) -> Result<(), RepositoryError> { + Ok(()) + } + async fn delete_snapshot( + &self, + _identity: &StreamIdentity, + ) -> Result { + Ok(false) + } + } + #[tokio::test] async fn snapshot_batch_failure_leaves_aggregate_uncommitted() { let repo = FailingSnapshotRepo::default(); - let aggregate_repo = AggregateRepository::new(repo); - let snapshot_repo = SnapshotAggregateRepository::new(aggregate_repo, 1); + let snapshot_repo = AggregateRepository::new(repo).with_snapshots(1); let mut aggregate = TestAggregate::default(); aggregate.touch().unwrap(); @@ -469,7 +316,6 @@ mod tests { assert_eq!(err, RepositoryError::Model("snapshot write failed".into())); assert!(snapshot_repo - .repo() .repo() .saw_snapshot .load(std::sync::atomic::Ordering::SeqCst)); diff --git a/tests/snapshots/main.rs b/tests/snapshots/main.rs index df5d145..a51573c 100644 --- a/tests/snapshots/main.rs +++ b/tests/snapshots/main.rs @@ -61,7 +61,6 @@ async fn snapshot_created_at_frequency_threshold() { // Version 1 — below threshold of 2, no snapshot yet assert!(repo - .repo() .repo() .get_snapshot(&identity) .await @@ -74,7 +73,7 @@ async fn snapshot_created_at_frequency_threshold() { repo.commit(&mut todo).await.unwrap(); // Version 2 >= 0 + 2 — snapshot should now exist - let snap = repo.repo().repo().get_snapshot(&identity).await.unwrap(); + let snap = repo.repo().get_snapshot(&identity).await.unwrap(); assert!(snap.is_some()); let snap = snap.unwrap(); assert_eq!(snap.version, 2); @@ -108,7 +107,6 @@ async fn no_snapshot_before_threshold() { // Only 1 event, threshold is 5 assert!(repo - .repo() .repo() .get_snapshot(&identity) .await @@ -132,7 +130,6 @@ async fn load_from_snapshot_produces_correct_state() { // Snapshot at version 2 assert!(repo - .repo() .repo() .get_snapshot(&identity) .await @@ -168,7 +165,6 @@ async fn snapshot_plus_newer_events() { // Snapshot exists at version 2, completed = true let snap = repo - .repo() .repo() .get_snapshot(&identity) .await @@ -265,7 +261,6 @@ async fn no_snapshot_falls_back_to_full_replay() { // Snapshot exists assert!(repo - .repo() .repo() .get_snapshot(&identity) .await @@ -273,9 +268,8 @@ async fn no_snapshot_falls_back_to_full_replay() { .is_some()); // Delete the snapshot - repo.repo().repo().delete_snapshot(&identity).await.unwrap(); + repo.repo().delete_snapshot(&identity).await.unwrap(); assert!(repo - .repo() .repo() .get_snapshot(&identity) .await @@ -305,7 +299,6 @@ async fn snapshot_version_advances_on_second_snapshot() { // First snapshot at version 1 let snap = repo - .repo() .repo() .get_snapshot(&identity) .await @@ -320,7 +313,6 @@ async fn snapshot_version_advances_on_second_snapshot() { // Second snapshot at version 2 let snap = repo - .repo() .repo() .get_snapshot(&identity) .await @@ -354,7 +346,6 @@ async fn with_queued_repo() { // Snapshot should exist through the queued + snapshot chain let snap = repo - .repo() .repo() .inner() .get_snapshot(&identity) @@ -426,7 +417,6 @@ async fn commit_all_with_snapshots() { // Both should have snapshots at version 2 let snap1 = repo - .repo() .repo() .get_snapshot(&identity1) .await @@ -434,7 +424,6 @@ async fn commit_all_with_snapshots() { .unwrap(); assert_eq!(snap1.version, 2); let snap2 = repo - .repo() .repo() .get_snapshot(&identity2) .await diff --git a/tests/sourced_snapshot/main.rs b/tests/sourced_snapshot/main.rs index 22d1c02..670c51e 100644 --- a/tests/sourced_snapshot/main.rs +++ b/tests/sourced_snapshot/main.rs @@ -176,7 +176,7 @@ async fn sourced_attr_snapshot_roundtrip_via_repo() { // At version 2, should have a snapshot let identity = StreamIdentity::new(Counter::aggregate_type(), "c1").unwrap(); - let snap_record = repo.repo().repo().get_snapshot(&identity).await.unwrap(); + let snap_record = repo.repo().get_snapshot(&identity).await.unwrap(); assert!(snap_record.is_some()); let loaded = repo.get("c1").await.unwrap().unwrap(); diff --git a/tests/upcasting/main.rs b/tests/upcasting/main.rs index d4cc3ff..71ee7aa 100644 --- a/tests/upcasting/main.rs +++ b/tests/upcasting/main.rs @@ -352,7 +352,6 @@ async fn snapshot_plus_upcasting_post_snapshot_events() { // Snapshot should now exist at version 1 let snapshot_identity = StreamIdentity::new(TodoV2::aggregate_type(), "t1").unwrap(); assert!(repo - .repo() .repo() .get_snapshot(&snapshot_identity) .await From 8d6eda41392e1a895569d3681d51b2c2c8f1d4b0 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 4 Jun 2026 21:01:34 -0500 Subject: [PATCH 12/24] =?UTF-8?q?refactor:=20repo.outbox(msg).commit(agg)?= =?UTF-8?q?=20publishes=20=E2=80=94=20no=20wrapper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the existing API do the new functionality instead of adding a method. Attaching a bus (Service::with_bus) installs an outbox publisher on the repository; OutboxCommit::commit then claims the row in the commit transaction and publishes it immediately via that bus, settling the claim (complete, or release for the worker on failure). No bus configured -> commit stays pending for the worker, exactly as before. Removed: ctx.commit_outbox, the OutboxCommitting trait, OutboxCommit::commit_claimed, Service's ImmediatePublish + the Context publisher plumbing, and the now-unused DynPublisher (the snapshot unification already collapsed the two repo types, so the polymorphism trait was dead weight). Added: OutboxPublishHook (object-safe) + OutboxPublisherConfig on the repo, BusOutboxPublishHook (store + BusPublisher), ConfigurableOutboxPublisher. Tests migrated to repo.outbox(msg).commit(agg); full default + sqlite suites green. Implements [[specs/durable-enqueue-outbox-dispatch]] Co-Authored-By: Claude Opus 4.8 (1M context) --- src/aggregate/repository.rs | 15 ++ src/bus/mod.rs | 2 +- src/bus/publisher.rs | 29 ---- src/lib.rs | 9 +- src/microsvc/context.rs | 58 +------- src/microsvc/dependencies.rs | 18 +++ src/microsvc/mod.rs | 4 +- src/microsvc/runtime.rs | 89 ++++++----- src/microsvc/service.rs | 34 +---- src/outbox/commit.rs | 212 ++++++++++----------------- src/outbox/mod.rs | 2 +- src/outbox_worker/mod.rs | 2 + src/outbox_worker/publish_hook.rs | 61 ++++++++ tests/durable_enqueue_sqlite/main.rs | 11 +- 14 files changed, 245 insertions(+), 301 deletions(-) create mode 100644 src/outbox_worker/publish_hook.rs diff --git a/src/aggregate/repository.rs b/src/aggregate/repository.rs index 445c1a4..b771d9d 100644 --- a/src/aggregate/repository.rs +++ b/src/aggregate/repository.rs @@ -3,6 +3,7 @@ use std::marker::PhantomData; use std::pin::Pin; use crate::entity::Entity; +use crate::outbox::OutboxPublisherConfig; use crate::queued_repo::{GetAllWithOpts, GetWithOpts, ReadOpts, UnlockableRepository}; use crate::repository::{ CommitBatch, GetStream, RepositoryError, SnapshotWrite, StreamIdentity, StreamWrite, @@ -76,6 +77,7 @@ impl SnapshotPolicy { pub struct AggregateRepository { repo: R, snapshot: Option>, + outbox_publisher: Option, _marker: PhantomData, } @@ -84,6 +86,7 @@ impl AggregateRepository { Self { repo, snapshot: None, + outbox_publisher: None, _marker: PhantomData, } } @@ -100,6 +103,18 @@ impl AggregateRepository { pub(crate) fn set_snapshot_policy(&mut self, policy: SnapshotPolicy) { self.snapshot = Some(policy); } + + /// Install the outbox publisher so commits publish immediately (used by + /// `Service::with_bus`). + pub(crate) fn set_outbox_publisher(&mut self, config: OutboxPublisherConfig) { + self.outbox_publisher = Some(config); + } + + /// The configured outbox publisher, if any. Consulted by + /// `OutboxCommit::commit`. + pub(crate) fn outbox_publisher(&self) -> Option<&OutboxPublisherConfig> { + self.outbox_publisher.as_ref() + } } impl AggregateRepository diff --git a/src/bus/mod.rs b/src/bus/mod.rs index 414b6a7..48c66a3 100644 --- a/src/bus/mod.rs +++ b/src/bus/mod.rs @@ -139,7 +139,7 @@ pub use in_memory_bus::{InMemoryBus, InMemoryReceived}; pub use message::{Message, MessageKind, PayloadDecodeError, SubscriptionPlan}; #[cfg(feature = "postgres")] pub use postgres_bus::{LogReceived, PostgresBus, QueueReceived}; -pub use publisher::{AsyncMessagePublisher, DynPublisher}; +pub use publisher::AsyncMessagePublisher; pub use router::MessageRouter; pub use run_options::{ConsumerDeliveryMode, InboxHook, NoInbox, RunOptions}; pub use runner::run_source; diff --git a/src/bus/publisher.rs b/src/bus/publisher.rs index 5011039..19bc259 100644 --- a/src/bus/publisher.rs +++ b/src/bus/publisher.rs @@ -17,7 +17,6 @@ //! is acceptable under at-least-once, silent loss is not. use std::future::Future; -use std::pin::Pin; use super::{Message, TransportError}; @@ -55,34 +54,6 @@ pub trait AsyncMessagePublisher: Send + Sync { } } -/// Object-safe form of [`AsyncMessagePublisher`]. -/// -/// `AsyncMessagePublisher::publish` returns `impl Future` (RPITIT), which makes -/// the trait itself not object-safe — `dyn AsyncMessagePublisher` will not -/// compile. `DynPublisher` boxes the returned future so a publisher can be held -/// behind `Arc`, used where the concrete publisher type cannot -/// be a type parameter (for example on `microsvc::Service`, so attaching a bus -/// does not change the service's type). -/// -/// Blanket-implemented for every [`AsyncMessagePublisher`]; callers normally -/// produce one with `Arc::new(publisher) as Arc`. -pub trait DynPublisher: Send + Sync { - /// Publish a single message, returning a boxed future. - fn publish_dyn<'a>( - &'a self, - message: Message, - ) -> Pin> + Send + 'a>>; -} - -impl DynPublisher for P { - fn publish_dyn<'a>( - &'a self, - message: Message, - ) -> Pin> + Send + 'a>> { - Box::pin(self.publish(message)) - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/lib.rs b/src/lib.rs index 13574ad..f48617e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -67,8 +67,8 @@ pub use lock::{SqliteLock, SqliteLockManager}; // Outbox: commit concerns (aggregate + outbox in one commit) pub use outbox::{ outbox_message_insert_plan, outbox_message_key, outbox_message_row_values, - outbox_message_schema, CommitReceipt, OutboxCommit, OutboxCommitting, OutboxMessage, - OutboxMessageStatus, OUTBOX_MESSAGES_TABLE, + outbox_message_schema, CommitReceipt, OutboxCommit, OutboxMessage, OutboxMessageStatus, + OutboxPublishHook, OutboxPublisherConfig, OUTBOX_MESSAGES_TABLE, }; // Outbox Worker: drain and publish concerns @@ -92,8 +92,9 @@ pub use outbox_worker::{ #[cfg(feature = "emitter")] pub use outbox_worker::LocalEmitterPublisher; pub use outbox_worker::{ - BusPublisher, OutboxDispatchOutcome, OutboxDispatcher, OutboxSource, ReceivedOutboxMessage, - DEFAULT_OUTBOX_SOURCE_BATCH, DEFAULT_OUTBOX_SOURCE_LEASE, SOURCED_METADATA_PREFIX, + BusOutboxPublishHook, BusPublisher, OutboxDispatchOutcome, OutboxDispatcher, OutboxSource, + ReceivedOutboxMessage, DEFAULT_OUTBOX_SOURCE_BATCH, DEFAULT_OUTBOX_SOURCE_LEASE, + SOURCED_METADATA_PREFIX, }; pub use queued_repo::{ diff --git a/src/microsvc/context.rs b/src/microsvc/context.rs index 4e89663..6a7dbc2 100644 --- a/src/microsvc/context.rs +++ b/src/microsvc/context.rs @@ -7,13 +7,10 @@ use serde::de::DeserializeOwned; use serde_json::Value; -use super::dependencies::{HasOutboxStore, HasReadModelStore, HasRepo}; +use super::dependencies::{HasReadModelStore, HasRepo}; use super::error::HandlerError; -use super::service::ImmediatePublish; use super::session::Session; use crate::bus::Message; -use crate::outbox::{CommitReceipt, OutboxCommitting, OutboxMessage}; -use crate::outbox_worker::{AsyncOutboxStore, OutboxClaimRef}; /// The context passed to every handler. /// @@ -41,8 +38,6 @@ pub struct Context<'a, D> { session: Session, /// Reference to the service dependencies. dependencies: &'a D, - /// After-commit publish config, present when a bus is attached. - immediate_publish: Option<&'a ImmediatePublish>, } impl<'a, D> Context<'a, D> { @@ -52,14 +47,12 @@ impl<'a, D> Context<'a, D> { input: Value, session: Session, dependencies: &'a D, - immediate_publish: Option<&'a ImmediatePublish>, ) -> Self { Self { message, input, session, dependencies, - immediate_publish, } } @@ -126,55 +119,6 @@ impl<'a, D> Context<'a, D> { self.dependencies.read_model_store() } - /// Commit an aggregate and an outbox message together, then publish the - /// message — the durable-enqueue command path. - /// - /// When the service was built with a bus (`Service::with_bus`), this claims - /// the outbox row in the commit transaction and publishes it immediately - /// after commit through the configured bus. The publish is best-effort: a - /// failure leaves the row claimed/retryable for the polling worker and does - /// **not** roll back the committed aggregate, so the command still succeeds. - /// - /// When no bus is configured, the row is committed `pending` and left for the - /// polling worker to publish. - pub async fn commit_outbox( - &self, - aggregate: &mut A, - message: OutboxMessage, - ) -> Result - where - D: HasRepo, - D::Repo: OutboxCommitting + HasOutboxStore, - A: Send, - { - let repo = self.repo(); - - let Some(immediate) = self.immediate_publish else { - // No bus configured: durable enqueue only; the worker publishes. - return Ok(repo.commit_outbox_pending(aggregate, message).await?); - }; - - // Claim the row in the commit transaction, then publish after commit. - // The lease hands the row to the poller if we crash before completing. - let (receipt, claimed) = repo - .commit_outbox_claimed(aggregate, message, &immediate.worker_id, immediate.lease) - .await?; - let claim = OutboxClaimRef::from_message(&claimed)?; - let transport = Message::from(&claimed); - let store = repo.outbox_store(); - match immediate.publisher.publish_dyn(transport).await { - Ok(()) => store.complete_async(&claim).await?, - Err(error) => { - // Best-effort: release/fail the claim for retry; the command - // still succeeded because the aggregate + row are committed. - store - .record_failure_async(&claim, &error.to_string(), immediate.max_attempts) - .await?; - } - } - Ok(receipt) - } - /// Check if the raw input contains a field. pub fn has_field(&self, field: &str) -> bool { self.input.get(field).is_some() diff --git a/src/microsvc/dependencies.rs b/src/microsvc/dependencies.rs index a6f4d5e..7b44283 100644 --- a/src/microsvc/dependencies.rs +++ b/src/microsvc/dependencies.rs @@ -1,6 +1,7 @@ //! Typed dependency wrappers for microsvc handlers. use crate::aggregate::AggregateRepository; +use crate::outbox::OutboxPublisherConfig; use crate::outbox_worker::AsyncOutboxStore; use crate::repository::{ReadModelWritePlanStore, RelationalReadModelQueryStore, Repository}; @@ -18,6 +19,23 @@ pub trait HasReadModelStore { fn read_model_store(&self) -> &Self::ReadModelStore; } +/// Dependency capability for repositories whose outbox commits can publish +/// immediately. +/// +/// `Service::with_bus` installs an [`OutboxPublisherConfig`] through this so that +/// `repo.outbox(msg).commit(agg)` publishes the row right after commit. Without +/// it, commits leave the row `pending` for the polling worker. +pub trait ConfigurableOutboxPublisher { + /// Install the outbox publisher. + fn configure_outbox_publisher(&mut self, config: OutboxPublisherConfig); +} + +impl ConfigurableOutboxPublisher for AggregateRepository { + fn configure_outbox_publisher(&mut self, config: OutboxPublisherConfig) { + self.set_outbox_publisher(config); + } +} + /// Dependency capability for repositories that expose a durable outbox store. /// /// A runtime uses this to build an `OutboxDispatcher` that drains committed diff --git a/src/microsvc/mod.rs b/src/microsvc/mod.rs index 6ffc339..bbc1142 100644 --- a/src/microsvc/mod.rs +++ b/src/microsvc/mod.rs @@ -64,8 +64,8 @@ mod session; pub use crate::bus::{Message, MessageKind, PayloadDecodeError, SubscriptionPlan}; pub use context::Context; pub use dependencies::{ - HasOutboxStore, HasReadModelStore, HasRepo, ReadModelStoreDependencies, RepoDependencies, - RepoReadModelDependencies, + ConfigurableOutboxPublisher, HasOutboxStore, HasReadModelStore, HasRepo, + ReadModelStoreDependencies, RepoDependencies, RepoReadModelDependencies, }; pub use error::HandlerError; pub use runtime::{Microservice, DEFAULT_MAX_PUBLISH_ATTEMPTS, DEFAULT_PUBLISH_LEASE}; diff --git a/src/microsvc/runtime.rs b/src/microsvc/runtime.rs index 02a60d4..92994a0 100644 --- a/src/microsvc/runtime.rs +++ b/src/microsvc/runtime.rs @@ -15,11 +15,11 @@ use std::sync::Arc; use std::time::Duration; -use super::dependencies::{HasOutboxStore, HasRepo}; -use super::service::ImmediatePublish; +use super::dependencies::{ConfigurableOutboxPublisher, HasOutboxStore, HasRepo}; use super::Service; -use crate::bus::{Bus, BusConsumer, DynPublisher, RunOptions, TransportError}; -use crate::outbox_worker::{BusPublisher, OutboxDispatcher}; +use crate::bus::{Bus, BusConsumer, RunOptions, TransportError}; +use crate::outbox::OutboxPublisherConfig; +use crate::outbox_worker::{BusOutboxPublishHook, BusPublisher, OutboxDispatcher}; /// Default lease for an immediate after-commit outbox publish. /// @@ -122,10 +122,10 @@ where /// `run` returns when the consumers stop (a pull source that drains, or the /// first error). /// - /// Producing is handled separately: the primary path is immediate publish via - /// [`Context::commit_outbox`](crate::microsvc::Context::commit_outbox); the - /// background poll loop (the crash backstop, which needs an async timer) is - /// driven from [`Self::dispatcher`] by a runtime that provides one. + /// Producing is handled separately: the primary path is immediate publish on + /// `repo.outbox(msg).commit(agg)` (enabled by `with_bus`); the background poll + /// loop (the crash backstop, which needs an async timer) is driven from + /// [`Self::dispatcher`] by a runtime that provides one. pub async fn run(&self, options: RunOptions) -> Result<(), TransportError> { use std::future::{poll_fn, Future}; use std::pin::Pin; @@ -170,29 +170,37 @@ where } } -impl Service { +impl Service +where + D: Send + Sync + 'static + HasOutboxStore + ConfigurableOutboxPublisher, +{ /// Attach a bus, producing a [`Microservice`] that carries the transport - /// config for both producing (outbox dispatch) and consuming - /// (listen/subscribe). + /// config for both producing and consuming. /// - /// Attaching a bus also enables immediate after-commit publish: handlers - /// that call [`Context::commit_outbox`](crate::microsvc::Context::commit_outbox) - /// claim the outbox row in the commit transaction and publish it through this - /// bus. The immediate path uses [`DEFAULT_PUBLISH_LEASE`] and - /// [`DEFAULT_MAX_PUBLISH_ATTEMPTS`]; the `with_publish_lease` / - /// `with_max_attempts` setters configure the background poll loop. + /// Attaching a bus installs an outbox publisher on the repository, so + /// `repo.outbox(msg).commit(agg)` (and `ctx.repo().outbox(...).commit(...)`) + /// claims the row in the commit transaction and publishes it immediately + /// through this bus — no separate call. The immediate path uses + /// [`DEFAULT_PUBLISH_LEASE`] and [`DEFAULT_MAX_PUBLISH_ATTEMPTS`]; the polling + /// worker remains the crash/retry backstop. pub fn with_bus(mut self, bus: B) -> Microservice where B: Bus + 'static, { let bus = Arc::new(bus); - let publisher: Arc = Arc::new(BusPublisher::new(Arc::clone(&bus))); - self.set_immediate_publish(ImmediatePublish { - publisher, - worker_id: format!("microsvc-immediate:{}", std::process::id()), - lease: DEFAULT_PUBLISH_LEASE, - max_attempts: DEFAULT_MAX_PUBLISH_ATTEMPTS, - }); + // Build the publish hook over the service's own outbox store + this bus, + // and install it on the repository so commits publish immediately. + let hook = BusOutboxPublishHook::new( + self.dependencies().outbox_store(), + BusPublisher::new(Arc::clone(&bus)), + DEFAULT_MAX_PUBLISH_ATTEMPTS, + ); + let config = OutboxPublisherConfig::new( + Arc::new(hook), + format!("microsvc-immediate:{}", std::process::id()), + DEFAULT_PUBLISH_LEASE, + ); + self.dependencies_mut().configure_outbox_publisher(config); Microservice::new(Arc::new(self), bus) } } @@ -225,13 +233,11 @@ mod tests { } #[tokio::test] - async fn dispatcher_drains_committed_outbox_row_to_the_bus() { - let service = - Service::with_repo(HashMapRepository::new().queued().aggregate::()); - - let microservice = service.with_bus(InMemoryBus::new()); + async fn commit_publishes_immediately_leaving_nothing_for_the_dispatcher() { + let microservice = Service::with_repo(HashMapRepository::new().queued().aggregate::()) + .with_bus(InMemoryBus::new()); - // Commit an aggregate + outbox row through the bound service's repo. + // The plain commit API publishes immediately because a bus is attached. let mut dummy = Dummy::default(); dummy.touch().unwrap(); let message = OutboxMessage::create("evt-1", "dummy.touched", b"{}".to_vec()).unwrap(); @@ -244,9 +250,11 @@ mod tests { .unwrap(); assert_eq!(receipt.outbox_message_ids(), ["evt-1".to_string()]); - // The dispatcher (store + bus) drains the committed row to the bus. + // The row was already published at commit time, so the backstop + // dispatcher (poll loop) finds nothing to drain. let outcome = microservice.dispatcher().dispatch_batch(10).await.unwrap(); - assert_eq!(outcome.published, 1); + assert_eq!(outcome.claimed, 0, "row was already published at commit"); + assert_eq!(outcome.published, 0); assert_eq!(outcome.released, 0); assert_eq!(outcome.failed, 0); } @@ -258,18 +266,19 @@ mod tests { let mut dummy = Dummy::default(); dummy.touch()?; let message = OutboxMessage::create("evt-1", "dummy.touched", b"{}".to_vec())?; - ctx.commit_outbox(&mut dummy, message).await?; + // The good old API — commit publishes immediately because a bus is attached. + ctx.repo().outbox(message).commit(&mut dummy).await?; Ok(json!({ "ok": true })) } #[tokio::test] - async fn commit_outbox_publishes_immediately_when_bus_is_attached() { + async fn commit_publishes_immediately_when_bus_is_attached() { let service = Service::with_repo(HashMapRepository::new().queued().aggregate::()) .command("dummy.touch") .handle(touch_and_publish); let microservice = service.with_bus(InMemoryBus::new()); - // Dispatching the command runs the handler, which calls `commit_outbox`: + // Dispatching the command runs the handler, which calls `outbox().commit()`: // claim-in-transaction, then immediate publish through the attached bus. microservice .service() @@ -301,7 +310,7 @@ mod tests { // Enqueue a command on the bus, then run: `listen` is derived from the // registered command, drains the message, and the handler runs - // (commit_outbox publishes immediately). `run` returns once the queue is + // (commit publishes immediately). `run` returns once the queue is // empty (InMemoryBus source yields `None`). microservice .bus() @@ -344,13 +353,13 @@ mod tests { let mut counter = SnapCounter::default(); counter.touch("s1".to_string())?; let message = OutboxMessage::create("evt-s1", "snap.touched", b"{}".to_vec())?; - ctx.commit_outbox(&mut counter, message).await?; + ctx.repo().outbox(message).commit(&mut counter).await?; Ok(json!({})) } #[tokio::test] - async fn commit_outbox_works_with_snapshot_backed_repo() { - // `commit_outbox` must work for a snapshot-backed repository too: the + async fn outbox_commit_publishes_with_snapshot_backed_repo() { + // `outbox().commit()` must work for a snapshot-backed repository too: the // outbox row and the snapshot commit together in one transaction, then // the row publishes immediately. let repo = HashMapRepository::new() @@ -376,7 +385,7 @@ mod tests { assert_eq!( published.len(), 1, - "snapshot-backed commit_outbox should publish immediately" + "snapshot-backed outbox commit should publish immediately" ); assert_eq!(published[0].id(), "evt-s1"); } diff --git a/src/microsvc/service.rs b/src/microsvc/service.rs index 4fbc7d0..3858b08 100644 --- a/src/microsvc/service.rs +++ b/src/microsvc/service.rs @@ -23,7 +23,6 @@ use std::collections::HashMap; use std::future::Future; use std::pin::Pin; use std::sync::Arc; -use std::time::Duration; use serde_json::Value; @@ -31,21 +30,7 @@ use super::context::Context; use super::dependencies::{HasReadModelStore, HasRepo, RepoReadModelDependencies}; use super::error::HandlerError; use super::session::Session; -use crate::bus::{DynPublisher, Message, MessageKind, SubscriptionPlan}; - -/// After-commit publish configuration, set when a bus is attached via -/// [`Service::with_bus`](crate::microsvc::Service::with_bus). -/// -/// When present, [`Context::commit_outbox`](crate::microsvc::Context::commit_outbox) -/// claims the outbox row in the commit transaction and publishes it through -/// `publisher`. When absent, `commit_outbox` writes the row `pending` and leaves -/// publication to the polling worker. -pub(crate) struct ImmediatePublish { - pub(crate) publisher: Arc, - pub(crate) worker_id: String, - pub(crate) lease: Duration, - pub(crate) max_attempts: u32, -} +use crate::bus::{Message, MessageKind, SubscriptionPlan}; type GuardFn = dyn Fn(&Context) -> bool + Send + Sync; type HandlerFuture<'a> = Pin> + Send + 'a>>; @@ -192,7 +177,6 @@ pub struct Service { dependencies: D, handlers: HashMap<(MessageKind, String), RegisteredHandler>, handler_specs: Vec, - immediate_publish: Option, } impl Service { @@ -202,13 +186,13 @@ impl Service { dependencies, handlers: HashMap::new(), handler_specs: Vec::new(), - immediate_publish: None, } } - /// Attach the after-commit publish configuration (set by `with_bus`). - pub(crate) fn set_immediate_publish(&mut self, immediate_publish: ImmediatePublish) { - self.immediate_publish = Some(immediate_publish); + /// Mutable access to the dependencies, used by `with_bus` to install the + /// outbox publisher on the repository before the service is shared. + pub(crate) fn dependencies_mut(&mut self) -> &mut D { + &mut self.dependencies } /// Create a service whose dependency type is an aggregate repository. @@ -352,13 +336,7 @@ impl Service { (handler.guard.clone(), handler.handle.clone()) }; let name = message.name.clone(); - let ctx = Context::new( - message, - input, - session, - &self.dependencies, - self.immediate_publish.as_ref(), - ); + let ctx = Context::new(message, input, session, &self.dependencies); // Run guard (synchronous) if present. if let Some(guard) = &guard { diff --git a/src/outbox/commit.rs b/src/outbox/commit.rs index 083d06f..c494ea9 100644 --- a/src/outbox/commit.rs +++ b/src/outbox/commit.rs @@ -1,3 +1,6 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; use std::time::{Duration, SystemTime}; use crate::aggregate::{Aggregate, AggregateRepository}; @@ -6,6 +9,48 @@ use crate::repository::{ CommitBatch, RepositoryError, StreamIdentity, StreamWrite, TransactionalCommit, }; +/// Publishes an already-committed, claimed outbox row and settles its claim. +/// +/// Implemented by the outbox → bus bridge and installed on an +/// [`AggregateRepository`] (by `Service::with_bus`) so that +/// `repo.outbox(msg).commit(agg)` publishes immediately — no separate call. The +/// hook owns the publisher and the outbox store; it is given the claimed message +/// the commit just wrote, publishes it, and completes the claim (or releases it +/// for the polling worker on failure). It is object-safe so the repository can +/// hold it without naming the transport/store types. +pub trait OutboxPublishHook: Send + Sync { + /// Publish a committed, claimed outbox row and settle its claim. Publish + /// failures are absorbed (the row stays retryable for the worker); only a + /// store error surfaces. + fn publish_claimed<'a>( + &'a self, + claimed: OutboxMessage, + ) -> Pin> + Send + 'a>>; +} + +/// Outbox publisher installed on a repository so commits publish immediately. +pub struct OutboxPublisherConfig { + pub(crate) hook: Arc, + pub(crate) worker_id: String, + pub(crate) lease: Duration, +} + +impl OutboxPublisherConfig { + /// Build the config from a publish hook, the worker id used to scope the + /// in-transaction claim, and the publish lease. + pub fn new( + hook: Arc, + worker_id: impl Into, + lease: Duration, + ) -> Self { + Self { + hook, + worker_id: worker_id.into(), + lease, + } + } +} + /// Outcome of an outbox-bearing commit. /// /// Carries the ids of the outbox rows the transaction inserted so an @@ -45,16 +90,36 @@ where R: TransactionalCommit, A: Aggregate + Send, { - /// Commit the aggregate and outbox message together. + /// Commit the aggregate and outbox message together, and — when the + /// repository has a bus configured (via `Service::with_bus`) — publish the + /// row immediately. /// - /// Returns a [`CommitReceipt`] carrying the inserted outbox message id, so - /// an after-commit dispatcher can publish exactly the rows this transaction - /// wrote without re-scanning the outbox. + /// With a bus configured, the row is **claimed in this same transaction** + /// (born `InFlight` under a short lease) and published right after commit, so + /// publication needs no separate claim and cannot race the polling worker; a + /// crash before publish hands the row back to the worker at lease expiry, and + /// a publish failure leaves it retryable. Without a bus, the row is committed + /// `pending` for the polling worker to publish. A snapshot is also staged when + /// the repository has snapshots configured and one is due — all in the one + /// transaction. + /// + /// Returns a [`CommitReceipt`] carrying the inserted outbox message id. pub async fn commit(mut self, aggregate: &mut A) -> Result { self.message.set_source(aggregate); let outbox_message_id = self.message.id().to_string(); - // Stage a snapshot too when the repository has snapshots configured and - // one is due — same transaction as the events and the outbox row. + + // When a bus is configured, claim the row in this transaction so it can + // be published immediately after commit; otherwise leave it `pending`. + let publisher = self.repo.outbox_publisher(); + let claimed = match publisher { + Some(config) => { + self.message + .claim_at(&config.worker_id, config.lease, SystemTime::now())?; + Some(self.message.clone()) + } + None => None, + }; + let (snapshots, snapshot_version) = self.repo.snapshot_writes_for(aggregate)?; let identity = StreamIdentity::new(A::aggregate_type(), aggregate.entity().id())?; let stream = StreamWrite::new(identity, aggregate.entity_mut()); @@ -65,51 +130,17 @@ where if let Some(version) = snapshot_version { aggregate.entity_mut().set_snapshot_version(version); } + + // Best-effort immediate publish. A failure leaves the claimed row for the + // polling worker and never fails the already-committed command. + if let (Some(config), Some(claimed)) = (publisher, claimed) { + let _ = config.hook.publish_claimed(claimed).await; + } + Ok(CommitReceipt { outbox_message_ids: vec![outbox_message_id], }) } - - /// Commit the aggregate and outbox message together, **claiming the outbox - /// row for publication in the same transaction**. - /// - /// The row inserts already `InFlight` under `worker_id`'s lease - /// (`attempts = 1`), so the caller can publish it immediately after commit - /// without a separate claim and without racing the polling worker. While the - /// lease is held the poller skips the row; if the caller never publishes - /// (e.g. a crash), the lease expires and the worker reclaims it. - /// - /// Returns the receipt plus a clone of the claimed message so the caller can - /// build the transport message and settle the claim (`complete` on success, - /// `record_failure` on a publish error). The publish itself happens after - /// this returns — a broker call is never held open inside the transaction. - pub async fn commit_claimed( - mut self, - aggregate: &mut A, - worker_id: &str, - lease: Duration, - ) -> Result<(CommitReceipt, OutboxMessage), RepositoryError> { - self.message.set_source(aggregate); - self.message.claim_at(worker_id, lease, SystemTime::now())?; - let claimed = self.message.clone(); - let outbox_message_id = self.message.id().to_string(); - let (snapshots, snapshot_version) = self.repo.snapshot_writes_for(aggregate)?; - let identity = StreamIdentity::new(A::aggregate_type(), aggregate.entity().id())?; - let stream = StreamWrite::new(identity, aggregate.entity_mut()); - let mut batch = CommitBatch::new(vec![stream]); - batch.outbox_messages.push(self.message); - batch.snapshots = snapshots; - self.repo.repo().commit_batch(batch).await?; - if let Some(version) = snapshot_version { - aggregate.entity_mut().set_snapshot_version(version); - } - Ok(( - CommitReceipt { - outbox_message_ids: vec![outbox_message_id], - }, - claimed, - )) - } } impl AggregateRepository { @@ -122,64 +153,6 @@ impl AggregateRepository { } } -/// A repository that can commit an aggregate together with an outbox message in -/// one transaction, staging whatever else the repository requires (for example a -/// snapshot, for snapshot-backed repositories). -/// -/// This is the abstraction -/// [`Context::commit_outbox`](crate::microsvc::Context::commit_outbox) binds to, -/// so the durable-enqueue command path works for **any** repository shape — a -/// plain [`AggregateRepository`] or a `SnapshotAggregateRepository` — not just -/// one. The underlying `CommitBatch`/`TransactionalCommit` boundary already -/// applies streams, outbox rows, read models, and snapshots in one transaction; -/// this trait exposes that to the ergonomic command path. -pub trait OutboxCommitting { - /// Commit the aggregate and outbox row (left `pending`) in one transaction. - /// The polling worker publishes the row later. - fn commit_outbox_pending( - &self, - aggregate: &mut A, - message: OutboxMessage, - ) -> impl core::future::Future> + Send; - - /// Commit the aggregate and outbox row, claiming the row in the same - /// transaction for immediate publication. Returns a clone of the claimed - /// message so the caller can build the transport message and settle the claim. - fn commit_outbox_claimed( - &self, - aggregate: &mut A, - message: OutboxMessage, - worker_id: &str, - lease: Duration, - ) -> impl core::future::Future> + Send; -} - -impl OutboxCommitting for AggregateRepository -where - R: TransactionalCommit + Send + Sync, - A: Aggregate + Send + Sync, -{ - async fn commit_outbox_pending( - &self, - aggregate: &mut A, - message: OutboxMessage, - ) -> Result { - self.outbox(message).commit(aggregate).await - } - - async fn commit_outbox_claimed( - &self, - aggregate: &mut A, - message: OutboxMessage, - worker_id: &str, - lease: Duration, - ) -> Result<(CommitReceipt, OutboxMessage), RepositoryError> { - self.outbox(message) - .commit_claimed(aggregate, worker_id, lease) - .await - } -} - #[cfg(test)] mod tests { use super::*; @@ -247,35 +220,6 @@ mod tests { assert_eq!(pending[0].id(), "msg-1"); } - #[tokio::test] - async fn commit_claimed_inserts_row_in_flight_under_lease() { - let repo = HashMapRepository::new().aggregate::(); - - let mut aggregate = Dummy::default(); - aggregate.touch().unwrap(); - - let event = OutboxMessage::create("msg-1", "DummyTouched", b"{}".to_vec()).unwrap(); - - let (receipt, claimed) = repo - .outbox(event) - .commit_claimed(&mut aggregate, "immediate:test", Duration::from_secs(5)) - .await - .unwrap(); - - // The committed row is claimed in the same transaction: in-flight, under - // this worker's lease, attempt 1. - assert_eq!(receipt.outbox_message_ids(), ["msg-1".to_string()]); - assert!(!claimed.is_pending()); - assert_eq!(claimed.attempts, 1); - - // A polling worker sees nothing claimable while the lease is held. - let pending = repo.repo().outbox_store().pending().unwrap(); - assert!( - pending.is_empty(), - "a row claimed in-transaction must not be claimable by the poller" - ); - } - #[tokio::test] async fn outbox_helper_failure_leaves_entities_uncommitted() { let repo = AggregateRepository::<_, Dummy>::new(FailingOutboxRepo::default()); diff --git a/src/outbox/mod.rs b/src/outbox/mod.rs index 1ad33e2..0592449 100644 --- a/src/outbox/mod.rs +++ b/src/outbox/mod.rs @@ -49,4 +49,4 @@ pub use table::{ }; // Commit helpers -pub use commit::{CommitReceipt, OutboxCommit, OutboxCommitting}; +pub use commit::{CommitReceipt, OutboxCommit, OutboxPublishHook, OutboxPublisherConfig}; diff --git a/src/outbox_worker/mod.rs b/src/outbox_worker/mod.rs index 88b5c45..5f8c889 100644 --- a/src/outbox_worker/mod.rs +++ b/src/outbox_worker/mod.rs @@ -40,6 +40,7 @@ mod bus_publisher; mod outbox_dispatch; mod outbox_source; +mod publish_hook; mod publisher; mod store; mod worker; @@ -62,6 +63,7 @@ pub use worker::{DrainResult, OutboxWorker, ProcessOneResult}; // Outbox -> bus bridge (moved out of the bus module; depends up on bus traits). pub use bus_publisher::BusPublisher; pub use outbox_dispatch::{OutboxDispatchOutcome, OutboxDispatcher, SOURCED_METADATA_PREFIX}; +pub use publish_hook::BusOutboxPublishHook; pub use outbox_source::{ OutboxSource, ReceivedOutboxMessage, DEFAULT_OUTBOX_SOURCE_BATCH, DEFAULT_OUTBOX_SOURCE_LEASE, }; diff --git a/src/outbox_worker/publish_hook.rs b/src/outbox_worker/publish_hook.rs new file mode 100644 index 0000000..3469d70 --- /dev/null +++ b/src/outbox_worker/publish_hook.rs @@ -0,0 +1,61 @@ +//! [`OutboxPublishHook`] backed by an outbox store + a message publisher. +//! +//! This is what makes `repo.outbox(msg).commit(agg)` publish: `Service::with_bus` +//! installs one of these on the repository, and `OutboxCommit::commit` hands it +//! the row it just committed-and-claimed. The hook publishes the row and settles +//! its claim — `complete` on success, `record_failure` (release/fail) on a +//! publish error so the row stays retryable for the polling worker. It never +//! re-claims: the row was already claimed in the commit transaction. + +use std::future::Future; +use std::pin::Pin; + +use crate::bus::{AsyncMessagePublisher, Message}; +use crate::outbox::{OutboxMessage, OutboxPublishHook}; +use crate::repository::RepositoryError; + +use super::{AsyncOutboxStore, OutboxClaimRef}; + +/// Publishes committed outbox rows through `publisher` and settles their claims +/// in `store`. The `store` must be the same outbox store the commit wrote to. +pub struct BusOutboxPublishHook { + store: S, + publisher: P, + max_attempts: u32, +} + +impl BusOutboxPublishHook { + /// Build the hook from the outbox store, a message publisher (e.g. a + /// `BusPublisher` over a `*Bus`), and the publish-failure ceiling. + pub fn new(store: S, publisher: P, max_attempts: u32) -> Self { + Self { + store, + publisher, + max_attempts, + } + } +} + +impl OutboxPublishHook for BusOutboxPublishHook +where + S: AsyncOutboxStore, + P: AsyncMessagePublisher, +{ + fn publish_claimed<'a>( + &'a self, + claimed: OutboxMessage, + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + let claim = OutboxClaimRef::from_message(&claimed)?; + let message = Message::from(&claimed); + match self.publisher.publish(message).await { + Ok(()) => self.store.complete_async(&claim).await, + Err(error) => self + .store + .record_failure_async(&claim, &error.to_string(), self.max_attempts) + .await + .map(|_action| ()), + } + }) + } +} diff --git a/tests/durable_enqueue_sqlite/main.rs b/tests/durable_enqueue_sqlite/main.rs index 8e728d1..68ea6b6 100644 --- a/tests/durable_enqueue_sqlite/main.rs +++ b/tests/durable_enqueue_sqlite/main.rs @@ -1,7 +1,8 @@ //! End-to-end durable-enqueue dispatch over a real SQL backend (in-memory -//! SQLite). Exercises `commit_outbox` (claim-in-transaction + immediate publish) -//! and the `with_bus` runtime against a persistent repository, not just the -//! in-memory `HashMapRepository` covered by the unit tests. +//! SQLite). Exercises `repo.outbox(msg).commit(agg)` (claim-in-transaction + +//! immediate publish, enabled by `with_bus`) and the `with_bus` runtime against a +//! persistent repository, not just the in-memory `HashMapRepository` covered by +//! the unit tests. #![cfg(feature = "sqlite")] @@ -36,7 +37,7 @@ async fn handle_touch(ctx: &Context<'_, Repo>) -> Result { let mut counter = Counter::default(); counter.touch("c1".to_string())?; let message = OutboxMessage::create("evt-c1", "counter.touched", b"{}".to_vec())?; - ctx.commit_outbox(&mut counter, message).await?; + ctx.repo().outbox(message).commit(&mut counter).await?; Ok(json!({ "value": counter.value })) } @@ -49,7 +50,7 @@ async fn service() -> Repo { } #[tokio::test] -async fn commit_outbox_publishes_immediately_over_sqlite() { +async fn commit_publishes_immediately_over_sqlite() { let microservice = Service::with_repo(service().await) .command("counter.touch") .handle(handle_touch) From bbe7405865dbef49246ad91449c9051412e5afa9 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 4 Jun 2026 21:19:42 -0500 Subject: [PATCH 13/24] refactor: with_bus is a Service builder step, not a separate type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fold Microservice back into Service. Attaching a bus no longer changes the type: with_bus(bus) returns the same Service and run() is a method on it, so the whole thing reads as one fluent builder — Service::with_repo(r).command(..).handle(..).with_bus(bus).run(opts) The bus's consume behavior is type-erased into a single closure field on the service (ServiceRunner), so Service stays single-param — message_router, the register_handlers! macro, and every existing Service call site are untouched. Removes the Microservice type and the speculative dispatcher() accessor (the backstop poll loop is a later, runtime-gated addition). Net simpler: one type, one builder, less code. Implements [[specs/durable-enqueue-outbox-dispatch]] Co-Authored-By: Claude Opus 4.8 (1M context) --- src/microsvc/mod.rs | 2 +- src/microsvc/runtime.rs | 368 ++++++++++----------------- src/microsvc/service.rs | 28 +- tests/durable_enqueue_sqlite/main.rs | 22 +- 4 files changed, 177 insertions(+), 243 deletions(-) diff --git a/src/microsvc/mod.rs b/src/microsvc/mod.rs index bbc1142..11c1439 100644 --- a/src/microsvc/mod.rs +++ b/src/microsvc/mod.rs @@ -68,7 +68,7 @@ pub use dependencies::{ ReadModelStoreDependencies, RepoDependencies, RepoReadModelDependencies, }; pub use error::HandlerError; -pub use runtime::{Microservice, DEFAULT_MAX_PUBLISH_ATTEMPTS, DEFAULT_PUBLISH_LEASE}; +pub use runtime::{DEFAULT_MAX_PUBLISH_ATTEMPTS, DEFAULT_PUBLISH_LEASE}; pub use service::{ CommandRequest, CommandResponse, DeliveryKind, HandlerBuilder, HandlerNames, HandlerSpec, Service, diff --git a/src/microsvc/runtime.rs b/src/microsvc/runtime.rs index 92994a0..a959b60 100644 --- a/src/microsvc/runtime.rs +++ b/src/microsvc/runtime.rs @@ -1,208 +1,133 @@ -//! Runtime that ties a [`Service`] to a bus. +//! Bus integration for [`Service`]: `with_bus` (a builder step) and `run`. //! -//! `register_handlers!` builds a [`Service`] that is purely a consumer. Attaching -//! a bus with [`Service::with_bus`] turns it into a [`Microservice`] that carries -//! the transport config for both sides: it can drain committed outbox rows to the -//! bus (produce) and — once `run` lands — derive listen/subscribe from the -//! registered handlers (consume). -//! -//! The producing side is a thin assembly over [`OutboxDispatcher`] and -//! [`BusPublisher`]: [`Microservice::dispatcher`] hands back a dispatcher whose -//! store is the service's own outbox store and whose publisher routes through the -//! attached bus by [`MessageKind`](crate::bus::MessageKind). The same dispatcher -//! backs immediate after-commit dispatch and a background poll loop. - +//! Attaching a bus does not change the service's type. `with_bus` is a plain +//! builder step on [`Service`] that (1) installs an outbox publisher on the +//! repository, so `repo.outbox(msg).commit(agg)` publishes immediately, and +//! (2) captures the bus's consume behavior as a type-erased closure on the +//! service. `run` drives that closure. There is no separate runtime type. + +use std::future::{poll_fn, Future}; +use std::pin::Pin; use std::sync::Arc; +use std::task::Poll; use std::time::Duration; -use super::dependencies::{ConfigurableOutboxPublisher, HasOutboxStore, HasRepo}; +use super::dependencies::{ConfigurableOutboxPublisher, HasOutboxStore}; use super::Service; use crate::bus::{Bus, BusConsumer, RunOptions, TransportError}; use crate::outbox::OutboxPublisherConfig; -use crate::outbox_worker::{BusOutboxPublishHook, BusPublisher, OutboxDispatcher}; +use crate::outbox_worker::{BusOutboxPublishHook, BusPublisher}; -/// Default lease for an immediate after-commit outbox publish. -/// -/// Short by design: it only needs to cover commit → publish, so a crash before -/// the publish completes hands the row back to the polling worker quickly. +/// Default lease for an immediate after-commit outbox publish. Short by design: +/// it only needs to cover commit → publish, so a crash before the publish +/// completes hands the row back to the polling worker quickly. pub const DEFAULT_PUBLISH_LEASE: Duration = Duration::from_secs(5); /// Default publish-failure ceiling before an outbox row is permanently failed. pub const DEFAULT_MAX_PUBLISH_ATTEMPTS: u32 = 5; -/// A [`Service`] bound to a bus. -/// -/// Holds the service and bus behind `Arc`s so the produce side (the dispatcher) -/// and the consume side (listen/subscribe) can share them. -pub struct Microservice { - service: Arc>, - bus: Arc, - worker_id: String, - publish_lease: Duration, - max_attempts: u32, -} - -impl Microservice { - /// Bind a service to a bus with default dispatch settings. - pub fn new(service: Arc>, bus: Arc) -> Self { - Self { - service, - bus, - worker_id: format!("microsvc-immediate:{}", std::process::id()), - publish_lease: DEFAULT_PUBLISH_LEASE, - max_attempts: DEFAULT_MAX_PUBLISH_ATTEMPTS, - } - } - - /// The bound service. - pub fn service(&self) -> &Arc> { - &self.service - } - - /// The bound bus. - pub fn bus(&self) -> &Arc { - &self.bus - } - - /// Set the worker id used to scope outbox claims (default - /// `microsvc-immediate:`). - pub fn with_worker_id(mut self, worker_id: impl Into) -> Self { - self.worker_id = worker_id.into(); - self - } +impl Service +where + D: Send + Sync + 'static + HasOutboxStore + ConfigurableOutboxPublisher, +{ + /// Attach a bus — a builder step that returns the same [`Service`]. + /// + /// Two effects, both composing with the rest of the builder: + /// - installs an outbox publisher on the repository, so + /// `repo.outbox(msg).commit(agg)` claims the row in the commit transaction + /// and publishes it immediately through this bus (the polling worker stays + /// the crash/retry backstop); + /// - captures how to consume, so [`run`](Self::run) listens for the + /// registered command names (competing) and subscribes to the event names + /// (fan-out). + pub fn with_bus(mut self, bus: B) -> Self + where + B: Bus + BusConsumer + 'static, + { + let bus = Arc::new(bus); - /// Set the lease taken when claiming an outbox row for publication. - pub fn with_publish_lease(mut self, lease: Duration) -> Self { - self.publish_lease = lease; - self - } + let hook = BusOutboxPublishHook::new( + self.dependencies().outbox_store(), + BusPublisher::new(Arc::clone(&bus)), + DEFAULT_MAX_PUBLISH_ATTEMPTS, + ); + self.dependencies_mut() + .configure_outbox_publisher(OutboxPublisherConfig::new( + Arc::new(hook), + format!("microsvc-immediate:{}", std::process::id()), + DEFAULT_PUBLISH_LEASE, + )); - /// Set the publish-failure ceiling before a row is permanently failed. - pub fn with_max_attempts(mut self, max_attempts: u32) -> Self { - self.max_attempts = max_attempts; + self.set_runner(Box::new( + move |service: Arc>, options: RunOptions| { + let bus = Arc::clone(&bus); + Box::pin(async move { run_consumers(&*bus, service, options).await }) + }, + )); self } } -impl Microservice -where - D: HasRepo + Send + Sync + 'static, - D::Repo: HasOutboxStore, - B: Bus, -{ - /// Build a dispatcher that drains committed outbox rows to the bus. +impl Service { + /// Run against the bus attached with [`with_bus`](Self::with_bus): consume + /// the registered command names (competing `listen`) and event names + /// (fan-out `subscribe`) concurrently on the caller's runtime. Returns when + /// the consumers stop (a pull source that drains, or the first error). /// - /// The store is the service's own outbox store; the publisher routes each - /// message to the bus by kind (commands point-to-point, events fan-out). The - /// same dispatcher is used by immediate after-commit dispatch - /// (`dispatch_ids`) and a background poll loop (`dispatch_batch`). - pub fn dispatcher( - &self, - ) -> OutboxDispatcher<::OutboxStore, BusPublisher> { - OutboxDispatcher::new( - self.service.repo().outbox_store(), - BusPublisher::new(Arc::clone(&self.bus)), - self.worker_id.clone(), - self.publish_lease, - self.max_attempts, - ) + /// Producing is handled on the commit path — `repo.outbox(msg).commit(agg)` + /// publishes immediately once a bus is attached. + /// + /// # Panics + /// If no bus was attached — call [`with_bus`](Self::with_bus) first. + pub async fn run(mut self, options: RunOptions) -> Result<(), TransportError> { + let runner = self + .take_runner() + .expect("Service::run requires a bus; call `with_bus` first"); + runner(Arc::new(self), options).await } } -impl Microservice +/// Drive a service's command/event consumers concurrently on the caller's +/// runtime — no spawn, no timer. Returns on the first error; finishes when all +/// consumers stop. +async fn run_consumers<'b, D, B>( + bus: &'b B, + service: Arc>, + options: RunOptions, +) -> Result<(), TransportError> where D: Send + Sync + 'static, B: Bus + BusConsumer, { - /// Run the service against the attached bus. - /// - /// Derives the consumers from the registered handlers: command handlers are - /// consumed with competing (point-to-point) `listen`, event handlers with - /// fan-out `subscribe`. Both run concurrently on the caller's runtime; - /// `run` returns when the consumers stop (a pull source that drains, or the - /// first error). - /// - /// Producing is handled separately: the primary path is immediate publish on - /// `repo.outbox(msg).commit(agg)` (enabled by `with_bus`); the background poll - /// loop (the crash backstop, which needs an async timer) is driven from - /// [`Self::dispatcher`] by a runtime that provides one. - pub async fn run(&self, options: RunOptions) -> Result<(), TransportError> { - use std::future::{poll_fn, Future}; - use std::pin::Pin; - use std::task::Poll; - - let plan = self.service.subscription_plan(); - let mut consumers: Vec< - Pin> + Send + '_>>, - > = Vec::new(); - if !plan.commands.is_empty() { - consumers.push(Box::pin( - self.bus.listen(Arc::clone(&self.service), options.clone()), - )); - } - if !plan.events.is_empty() { - consumers.push(Box::pin( - self.bus.subscribe(Arc::clone(&self.service), options), - )); - } + let plan = service.subscription_plan(); + let mut consumers: Vec> + Send + 'b>>> = + Vec::new(); + if !plan.commands.is_empty() { + consumers.push(Box::pin(bus.listen(Arc::clone(&service), options.clone()))); + } + if !plan.events.is_empty() { + consumers.push(Box::pin(bus.subscribe(Arc::clone(&service), options))); + } - // Drive every consumer concurrently on the caller's runtime — no spawn, - // no timer. Return on the first error; finish when all consumers stop. - poll_fn(move |cx| { - let mut index = 0; - while index < consumers.len() { - match consumers[index].as_mut().poll(cx) { - Poll::Ready(Ok(())) => { - // Drop the finished consumer future; nothing left to poll. - let _finished = consumers.remove(index); - } - Poll::Ready(Err(error)) => return Poll::Ready(Err(error)), - Poll::Pending => index += 1, + poll_fn(move |cx| { + let mut index = 0; + while index < consumers.len() { + match consumers[index].as_mut().poll(cx) { + Poll::Ready(Ok(())) => { + // Drop the finished consumer future; nothing left to poll. + let _finished = consumers.remove(index); } + Poll::Ready(Err(error)) => return Poll::Ready(Err(error)), + Poll::Pending => index += 1, } - if consumers.is_empty() { - Poll::Ready(Ok(())) - } else { - Poll::Pending - } - }) - .await - } -} - -impl Service -where - D: Send + Sync + 'static + HasOutboxStore + ConfigurableOutboxPublisher, -{ - /// Attach a bus, producing a [`Microservice`] that carries the transport - /// config for both producing and consuming. - /// - /// Attaching a bus installs an outbox publisher on the repository, so - /// `repo.outbox(msg).commit(agg)` (and `ctx.repo().outbox(...).commit(...)`) - /// claims the row in the commit transaction and publishes it immediately - /// through this bus — no separate call. The immediate path uses - /// [`DEFAULT_PUBLISH_LEASE`] and [`DEFAULT_MAX_PUBLISH_ATTEMPTS`]; the polling - /// worker remains the crash/retry backstop. - pub fn with_bus(mut self, bus: B) -> Microservice - where - B: Bus + 'static, - { - let bus = Arc::new(bus); - // Build the publish hook over the service's own outbox store + this bus, - // and install it on the repository so commits publish immediately. - let hook = BusOutboxPublishHook::new( - self.dependencies().outbox_store(), - BusPublisher::new(Arc::clone(&bus)), - DEFAULT_MAX_PUBLISH_ATTEMPTS, - ); - let config = OutboxPublisherConfig::new( - Arc::new(hook), - format!("microsvc-immediate:{}", std::process::id()), - DEFAULT_PUBLISH_LEASE, - ); - self.dependencies_mut().configure_outbox_publisher(config); - Microservice::new(Arc::new(self), bus) - } + } + if consumers.is_empty() { + Poll::Ready(Ok(())) + } else { + Poll::Pending + } + }) + .await } #[cfg(test)] @@ -233,30 +158,25 @@ mod tests { } #[tokio::test] - async fn commit_publishes_immediately_leaving_nothing_for_the_dispatcher() { - let microservice = Service::with_repo(HashMapRepository::new().queued().aggregate::()) + async fn plain_commit_publishes_immediately_when_bus_is_attached() { + let service = Service::with_repo(HashMapRepository::new().queued().aggregate::()) .with_bus(InMemoryBus::new()); + let store = service.repo().outbox_store(); // The plain commit API publishes immediately because a bus is attached. let mut dummy = Dummy::default(); dummy.touch().unwrap(); let message = OutboxMessage::create("evt-1", "dummy.touched", b"{}".to_vec()).unwrap(); - let receipt = microservice - .service() - .repo() - .outbox(message) - .commit(&mut dummy) - .await - .unwrap(); + let receipt = service.repo().outbox(message).commit(&mut dummy).await.unwrap(); assert_eq!(receipt.outbox_message_ids(), ["evt-1".to_string()]); - // The row was already published at commit time, so the backstop - // dispatcher (poll loop) finds nothing to drain. - let outcome = microservice.dispatcher().dispatch_batch(10).await.unwrap(); - assert_eq!(outcome.claimed, 0, "row was already published at commit"); - assert_eq!(outcome.published, 0); - assert_eq!(outcome.released, 0); - assert_eq!(outcome.failed, 0); + let published = store + .messages_by_status_async(OutboxMessageStatus::Published) + .await + .unwrap(); + assert_eq!(published.len(), 1, "row should be published at commit time"); + assert_eq!(published[0].id(), "evt-1"); + assert!(store.pending_async().await.unwrap().is_empty()); } type TouchRepo = AggregateRepository, Dummy>; @@ -272,54 +192,46 @@ mod tests { } #[tokio::test] - async fn commit_publishes_immediately_when_bus_is_attached() { + async fn dispatch_through_a_handler_publishes_immediately() { let service = Service::with_repo(HashMapRepository::new().queued().aggregate::()) .command("dummy.touch") - .handle(touch_and_publish); - let microservice = service.with_bus(InMemoryBus::new()); + .handle(touch_and_publish) + .with_bus(InMemoryBus::new()); - // Dispatching the command runs the handler, which calls `outbox().commit()`: - // claim-in-transaction, then immediate publish through the attached bus. - microservice - .service() + // The handler runs `outbox().commit()`: claim-in-transaction, then + // immediate publish through the attached bus. + service .dispatch("dummy.touch", json!({}), Session::new()) .await .unwrap(); - // The row was published immediately (claim-in-tx -> publish -> complete), - // so nothing is left pending for the poller. - let store = microservice.service().repo().outbox_store(); + let store = service.repo().outbox_store(); let published = store .messages_by_status_async(OutboxMessageStatus::Published) .await .unwrap(); assert_eq!(published.len(), 1, "row should be published immediately"); assert_eq!(published[0].id(), "evt-1"); - assert!( - store.pending_async().await.unwrap().is_empty(), - "no row should be left for the poller" - ); + assert!(store.pending_async().await.unwrap().is_empty()); } #[tokio::test] async fn run_consumes_registered_commands_from_the_bus() { + let bus = InMemoryBus::new(); let service = Service::with_repo(HashMapRepository::new().queued().aggregate::()) .command("dummy.touch") - .handle(touch_and_publish); - let microservice = service.with_bus(InMemoryBus::new()); + .handle(touch_and_publish) + .with_bus(bus.clone()); + // The store shares state with the repo, so it stays inspectable after + // `run` consumes the service. + let store = service.repo().outbox_store(); // Enqueue a command on the bus, then run: `listen` is derived from the - // registered command, drains the message, and the handler runs - // (commit publishes immediately). `run` returns once the queue is - // empty (InMemoryBus source yields `None`). - microservice - .bus() - .send("dummy.touch", b"{}".to_vec()) - .await - .unwrap(); - microservice.run(RunOptions::idempotent()).await.unwrap(); + // registered command, drains the message, and the handler publishes. + // `run` returns once the queue is empty (InMemoryBus yields `None`). + bus.send("dummy.touch", b"{}".to_vec()).await.unwrap(); + service.run(RunOptions::idempotent()).await.unwrap(); - let store = microservice.service().repo().outbox_store(); let published = store .messages_by_status_async(OutboxMessageStatus::Published) .await @@ -362,22 +274,22 @@ mod tests { // `outbox().commit()` must work for a snapshot-backed repository too: the // outbox row and the snapshot commit together in one transaction, then // the row publishes immediately. - let repo = HashMapRepository::new() - .queued() - .aggregate::() - .with_snapshots(1); - let microservice = Service::with_repo(repo) - .command("snap.touch") - .handle(touch_snap) - .with_bus(InMemoryBus::new()); + let service = Service::with_repo( + HashMapRepository::new() + .queued() + .aggregate::() + .with_snapshots(1), + ) + .command("snap.touch") + .handle(touch_snap) + .with_bus(InMemoryBus::new()); - microservice - .service() + service .dispatch("snap.touch", json!({}), Session::new()) .await .unwrap(); - let store = microservice.service().repo().outbox_store(); + let store = service.repo().outbox_store(); let published = store .messages_by_status_async(OutboxMessageStatus::Published) .await diff --git a/src/microsvc/service.rs b/src/microsvc/service.rs index 3858b08..23f47ef 100644 --- a/src/microsvc/service.rs +++ b/src/microsvc/service.rs @@ -30,7 +30,21 @@ use super::context::Context; use super::dependencies::{HasReadModelStore, HasRepo, RepoReadModelDependencies}; use super::error::HandlerError; use super::session::Session; -use crate::bus::{Message, MessageKind, SubscriptionPlan}; +use crate::bus::{Message, MessageKind, RunOptions, SubscriptionPlan, TransportError}; + +/// The bus run behavior captured by [`Service::with_bus`], type-erased so that +/// attaching a bus does not change the service's type. Given the service (as the +/// transport router) and run options, it consumes the registered command/event +/// names. Stored on the service so `with_bus` stays a plain builder step and +/// `run` can drive it. +pub(crate) type ServiceRunner = Box< + dyn Fn( + Arc>, + RunOptions, + ) -> Pin> + Send>> + + Send + + Sync, +>; type GuardFn = dyn Fn(&Context) -> bool + Send + Sync; type HandlerFuture<'a> = Pin> + Send + 'a>>; @@ -177,6 +191,7 @@ pub struct Service { dependencies: D, handlers: HashMap<(MessageKind, String), RegisteredHandler>, handler_specs: Vec, + runner: Option>, } impl Service { @@ -186,6 +201,7 @@ impl Service { dependencies, handlers: HashMap::new(), handler_specs: Vec::new(), + runner: None, } } @@ -195,6 +211,16 @@ impl Service { &mut self.dependencies } + /// Install the bus run behavior (used by `with_bus`). + pub(crate) fn set_runner(&mut self, runner: ServiceRunner) { + self.runner = Some(runner); + } + + /// Take the installed bus run behavior (used by `run`). + pub(crate) fn take_runner(&mut self) -> Option> { + self.runner.take() + } + /// Create a service whose dependency type is an aggregate repository. pub fn with_repo(repo: D) -> Self where diff --git a/tests/durable_enqueue_sqlite/main.rs b/tests/durable_enqueue_sqlite/main.rs index 68ea6b6..cc36ae2 100644 --- a/tests/durable_enqueue_sqlite/main.rs +++ b/tests/durable_enqueue_sqlite/main.rs @@ -51,20 +51,19 @@ async fn service() -> Repo { #[tokio::test] async fn commit_publishes_immediately_over_sqlite() { - let microservice = Service::with_repo(service().await) + let service = Service::with_repo(service().await) .command("counter.touch") .handle(handle_touch) .with_bus(InMemoryBus::new()); // The handler claims the outbox row in the SQL transaction, then publishes // it immediately through the attached bus. - microservice - .service() + service .dispatch("counter.touch", json!({}), Session::new()) .await .unwrap(); - let store = microservice.service().repo().outbox_store(); + let store = service.repo().outbox_store(); let published = store .messages_by_status_async(OutboxMessageStatus::Published) .await @@ -79,21 +78,18 @@ async fn commit_publishes_immediately_over_sqlite() { #[tokio::test] async fn run_consumes_command_and_publishes_over_sqlite() { - let microservice = Service::with_repo(service().await) + let bus = InMemoryBus::new(); + let service = Service::with_repo(service().await) .command("counter.touch") .handle(handle_touch) - .with_bus(InMemoryBus::new()); + .with_bus(bus.clone()); + let store = service.repo().outbox_store(); // Enqueue a command, then run: listen is derived from the registered command, // drains it, and the handler publishes its outbox row through the bus. - microservice - .bus() - .send("counter.touch", b"{}".to_vec()) - .await - .unwrap(); - microservice.run(RunOptions::idempotent()).await.unwrap(); + bus.send("counter.touch", b"{}".to_vec()).await.unwrap(); + service.run(RunOptions::idempotent()).await.unwrap(); - let store = microservice.service().repo().outbox_store(); let published = store .messages_by_status_async(OutboxMessageStatus::Published) .await From 25d58953cfe24359a22847c2148907e0c16525fa Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 4 Jun 2026 21:34:19 -0500 Subject: [PATCH 14/24] refactor: Service::new().with_repo().with_read_model_store() builder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the with_repo / with_read_model_store / with_repo_and_read_model_store constructors with one fluent builder: every service starts at Service::new() and chains dependency + bus steps — Service::new().with_repo(r).with_read_model_store(s).with_bus(bus) with_repo/with_read_model_store are type-state transitions that produce exactly the same D as before (Service, or RepoReadModelDependencies for both), so handler signatures are unchanged — only construction call sites move. Combined deps now delegate HasOutboxStore + ConfigurableOutboxPublisher to the repo so a repo+read-model service can also with_bus. Migrated all call sites + README. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 10 +- src/aggregate/repository.rs | 16 ++-- src/lib.rs | 4 +- src/microsvc/dependencies.rs | 15 +++ src/microsvc/grpc.rs | 2 +- src/microsvc/http.rs | 2 +- src/microsvc/mod.rs | 4 +- src/microsvc/runtime.rs | 44 +++++---- src/microsvc/service.rs | 93 +++++++++++++------ src/outbox_worker/mod.rs | 2 +- src/outbox_worker/outbox_source.rs | 4 +- .../checkout_saga_service/service.rs | 2 +- tests/distributed_read_model/main.rs | 2 +- .../projection_service/service.rs | 2 +- .../seat_inventory_service/service.rs | 2 +- .../board_service/service.rs | 2 +- .../projections_service/mod.rs | 2 +- tests/durable_enqueue_sqlite/main.rs | 8 +- tests/kafka_transport/main.rs | 6 +- tests/knative_cloudevents/main.rs | 2 +- tests/microsvc/basic.rs | 3 +- tests/microsvc/convention.rs | 12 +-- tests/microsvc/session.rs | 4 +- tests/microsvc/transport_grpc.rs | 2 +- tests/microsvc/transport_http.rs | 2 +- tests/microsvc/transport_listen.rs | 6 +- tests/microsvc/transport_subscribe.rs | 3 +- tests/nats_transport/main.rs | 6 +- tests/postgres_transport/main.rs | 4 +- tests/rabbitmq_transport/main.rs | 6 +- tests/sagas/microsvc_saga.rs | 18 ++-- tests/snapshots/main.rs | 77 +++------------ tests/transport_conformance/mod.rs | 2 +- 33 files changed, 192 insertions(+), 177 deletions(-) diff --git a/README.md b/README.md index b5228ee..680cde4 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ use serde_json::json; #[tokio::main] async fn main() -> Result<(), Box> { let service = Arc::new(distributed::register_handlers!( - Service::with_repo( + Service::new().with_repo( HashMapRepository::new() .queued() .aggregate::() @@ -165,7 +165,7 @@ default you replace with a durable adapter. // Persistence: HashMapRepository → durable SQL (features "postgres" / "sqlite") let repo = distributed::PostgresRepository::connect_and_migrate(database_url).await?; let service = Arc::new(distributed::register_handlers!( - Service::with_repo(repo.queued().aggregate::()), + Service::new().with_repo(repo.queued().aggregate::()), command handlers::todo_create, command handlers::todo_complete, )); @@ -827,7 +827,7 @@ The `microsvc` module provides a convention-based async command/event handler fr ### Defining a Service -A `Service` is generic over a dependency type `D` that handlers read via `ctx`. Use `Service::with_repo` for aggregate command handlers, `Service::with_read_model_store` for projection handlers, `Service::with_repo_and_read_model_store` when a handler needs both, or `Service::new(deps)` for an arbitrary dependency. +A `Service` is generic over a dependency type `D` that handlers read via `ctx`. Build one fluently from `Service::new()`: add `.with_repo(repo)` for aggregate command handlers, `.with_read_model_store(store)` for projection handlers (chain both when a handler needs both), and `.with_bus(bus)` to consume from / publish to a transport. Handlers are registered with a fluent builder. `.command(name)` / `.event(name)` start a registration; `.handle(closure)` adds an unguarded handler and `.guarded(guard, closure)` adds a guarded one. The handler closure receives `&Context` and returns a future: @@ -838,7 +838,7 @@ use distributed::{AggregateBuilder, HashMapRepository, Queueable}; use serde_json::json; let service = Arc::new( - Service::with_repo(HashMapRepository::new().queued().aggregate::()) + Service::new().with_repo(HashMapRepository::new().queued().aggregate::()) .command("counter.initialize") .handle(|ctx: &Context| { let input = ctx.input::(); @@ -927,7 +927,7 @@ Register them with the `register_handlers!` macro: ```rust let service = distributed::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::new().with_repo(HashMapRepository::new().queued().aggregate::()), command handlers::counter_create, command handlers::counter_increment, ); diff --git a/src/aggregate/repository.rs b/src/aggregate/repository.rs index b771d9d..454851f 100644 --- a/src/aggregate/repository.rs +++ b/src/aggregate/repository.rs @@ -45,11 +45,12 @@ pub(crate) struct SnapshotPolicy { hydrate: HydrateFn, } -type HydrateFn = for<'a> fn( - &'a R, - &'a StreamIdentity, - Entity, -) -> Pin> + Send + 'a>>; +type HydrateFn = + for<'a> fn( + &'a R, + &'a StreamIdentity, + Entity, + ) -> Pin> + Send + 'a>>; impl SnapshotPolicy { /// Construct a policy from its captured hooks. Called by `with_snapshots`, @@ -150,7 +151,10 @@ where }; let version = record.version; let identity = stream_identity_for::(aggregate.entity().id())?; - Ok((vec![SnapshotWrite::Save { identity, record }], Some(version))) + Ok(( + vec![SnapshotWrite::Save { identity, record }], + Some(version), + )) } /// Snapshot writes for `aggregate`, exposed to the outbox/read-model commit diff --git a/src/lib.rs b/src/lib.rs index f48617e..50f6ee0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -145,9 +145,7 @@ pub use commit_builder::{ }; // Snapshot: state snapshot payloads and rebuildable cache records for hydration -pub use snapshot::{ - hydrate_from_snapshot, InMemorySnapshotStore, SnapshotRecord, Snapshottable, -}; +pub use snapshot::{hydrate_from_snapshot, InMemorySnapshotStore, SnapshotRecord, Snapshottable}; // Re-export the EventEmitter from the event_emitter_rs crate (requires "emitter" feature) #[cfg(feature = "emitter")] diff --git a/src/microsvc/dependencies.rs b/src/microsvc/dependencies.rs index 7b44283..2de1b70 100644 --- a/src/microsvc/dependencies.rs +++ b/src/microsvc/dependencies.rs @@ -36,6 +36,21 @@ impl ConfigurableOutboxPublisher for AggregateRepository { } } +impl ConfigurableOutboxPublisher + for RepoReadModelDependencies +{ + fn configure_outbox_publisher(&mut self, config: OutboxPublisherConfig) { + self.repo.configure_outbox_publisher(config); + } +} + +impl HasOutboxStore for RepoReadModelDependencies { + type OutboxStore = R::OutboxStore; + fn outbox_store(&self) -> Self::OutboxStore { + self.repo().outbox_store() + } +} + /// Dependency capability for repositories that expose a durable outbox store. /// /// A runtime uses this to build an `OutboxDispatcher` that drains committed diff --git a/src/microsvc/grpc.rs b/src/microsvc/grpc.rs index 6325abc..957c0e0 100644 --- a/src/microsvc/grpc.rs +++ b/src/microsvc/grpc.rs @@ -15,7 +15,7 @@ //! use distributed::{microsvc, HashMapRepository}; //! //! let service = Arc::new( -//! microsvc::Service::with_repo(HashMapRepository::new()) +//! microsvc::Service::new().with_repo(HashMapRepository::new()) //! .command("counter.create") //! .handle(|ctx| { /* ... */ }) //! ); diff --git a/src/microsvc/http.rs b/src/microsvc/http.rs index 6b04503..936ca6d 100644 --- a/src/microsvc/http.rs +++ b/src/microsvc/http.rs @@ -14,7 +14,7 @@ //! use distributed::{microsvc, HashMapRepository}; //! //! let service = Arc::new( -//! microsvc::Service::with_repo(HashMapRepository::new()) +//! microsvc::Service::new().with_repo(HashMapRepository::new()) //! .command("counter.create") //! .handle(|ctx| { /* ... */ }) //! ); diff --git a/src/microsvc/mod.rs b/src/microsvc/mod.rs index 11c1439..347bbcf 100644 --- a/src/microsvc/mod.rs +++ b/src/microsvc/mod.rs @@ -12,7 +12,7 @@ //! use serde_json::json; //! //! let service = Arc::new( -//! microsvc::Service::with_repo(HashMapRepository::new()) +//! microsvc::Service::new().with_repo(HashMapRepository::new()) //! .command("order.create") //! .handle(|ctx| { //! let input = ctx.input::()?; @@ -111,7 +111,7 @@ pub use grpc::{grpc_server, serve_grpc, GrpcServeError}; /// # Example /// ```ignore /// let service = distributed::register_handlers!( -/// microsvc::Service::with_repo(HashMapRepository::new()), +/// microsvc::Service::new().with_repo(HashMapRepository::new()), /// command handlers::counter_create, /// command handlers::counter_increment, /// event handlers::counter_rebuilt, diff --git a/src/microsvc/runtime.rs b/src/microsvc/runtime.rs index a959b60..67de36a 100644 --- a/src/microsvc/runtime.rs +++ b/src/microsvc/runtime.rs @@ -87,6 +87,10 @@ impl Service { } } +/// A running transport consumer (a `listen` or `subscribe` loop), borrowing the +/// bus for `'b`. +type ConsumerFuture<'b> = Pin> + Send + 'b>>; + /// Drive a service's command/event consumers concurrently on the caller's /// runtime — no spawn, no timer. Returns on the first error; finishes when all /// consumers stop. @@ -100,8 +104,7 @@ where B: Bus + BusConsumer, { let plan = service.subscription_plan(); - let mut consumers: Vec> + Send + 'b>>> = - Vec::new(); + let mut consumers: Vec> = Vec::new(); if !plan.commands.is_empty() { consumers.push(Box::pin(bus.listen(Arc::clone(&service), options.clone()))); } @@ -139,7 +142,7 @@ mod tests { use crate::outbox_worker::AsyncOutboxStore; use crate::{ sourced, AggregateBuilder, AggregateRepository, Entity, HashMapRepository, OutboxMessage, - OutboxMessageStatus, QueuedRepository, Queueable, Snapshot, + OutboxMessageStatus, Queueable, QueuedRepository, Snapshot, }; #[derive(Default)] @@ -159,7 +162,8 @@ mod tests { #[tokio::test] async fn plain_commit_publishes_immediately_when_bus_is_attached() { - let service = Service::with_repo(HashMapRepository::new().queued().aggregate::()) + let service = Service::new() + .with_repo(HashMapRepository::new().queued().aggregate::()) .with_bus(InMemoryBus::new()); let store = service.repo().outbox_store(); @@ -167,7 +171,12 @@ mod tests { let mut dummy = Dummy::default(); dummy.touch().unwrap(); let message = OutboxMessage::create("evt-1", "dummy.touched", b"{}".to_vec()).unwrap(); - let receipt = service.repo().outbox(message).commit(&mut dummy).await.unwrap(); + let receipt = service + .repo() + .outbox(message) + .commit(&mut dummy) + .await + .unwrap(); assert_eq!(receipt.outbox_message_ids(), ["evt-1".to_string()]); let published = store @@ -193,7 +202,8 @@ mod tests { #[tokio::test] async fn dispatch_through_a_handler_publishes_immediately() { - let service = Service::with_repo(HashMapRepository::new().queued().aggregate::()) + let service = Service::new() + .with_repo(HashMapRepository::new().queued().aggregate::()) .command("dummy.touch") .handle(touch_and_publish) .with_bus(InMemoryBus::new()); @@ -218,7 +228,8 @@ mod tests { #[tokio::test] async fn run_consumes_registered_commands_from_the_bus() { let bus = InMemoryBus::new(); - let service = Service::with_repo(HashMapRepository::new().queued().aggregate::()) + let service = Service::new() + .with_repo(HashMapRepository::new().queued().aggregate::()) .command("dummy.touch") .handle(touch_and_publish) .with_bus(bus.clone()); @@ -274,15 +285,16 @@ mod tests { // `outbox().commit()` must work for a snapshot-backed repository too: the // outbox row and the snapshot commit together in one transaction, then // the row publishes immediately. - let service = Service::with_repo( - HashMapRepository::new() - .queued() - .aggregate::() - .with_snapshots(1), - ) - .command("snap.touch") - .handle(touch_snap) - .with_bus(InMemoryBus::new()); + let service = Service::new() + .with_repo( + HashMapRepository::new() + .queued() + .aggregate::() + .with_snapshots(1), + ) + .command("snap.touch") + .handle(touch_snap) + .with_bus(InMemoryBus::new()); service .dispatch("snap.touch", json!({}), Session::new()) diff --git a/src/microsvc/service.rs b/src/microsvc/service.rs index 23f47ef..d7c6d45 100644 --- a/src/microsvc/service.rs +++ b/src/microsvc/service.rs @@ -9,7 +9,7 @@ //! use distributed::microsvc; //! use serde_json::json; //! -//! let service = microsvc::Service::new(()) +//! let service = microsvc::Service::new() //! .command("order.create") //! .handle(|ctx| { //! let input = ctx.input::()?; @@ -184,9 +184,9 @@ impl HandlerBuilder { /// A microservice that routes commands to handler functions. /// -/// Generic over `D`, the service dependency type. Prefer -/// [`Service::with_repo`], [`Service::with_read_model_store`], or -/// [`Service::with_repo_and_read_model_store`] for common dependency shapes. +/// Generic over `D`, the service dependency type. Build one fluently from +/// [`Service::new`], adding dependencies and a bus with the `with_*` steps: +/// `Service::new().with_repo(repo).with_read_model_store(store).with_bus(bus)`. pub struct Service { dependencies: D, handlers: HashMap<(MessageKind, String), RegisteredHandler>, @@ -195,8 +195,8 @@ pub struct Service { } impl Service { - /// Create a new service with custom dependencies. - pub fn new(dependencies: D) -> Self { + /// Build a service around an already-assembled dependency value. + pub(crate) fn from_dependencies(dependencies: D) -> Self { Self { dependencies, handlers: HashMap::new(), @@ -221,22 +221,6 @@ impl Service { self.runner.take() } - /// Create a service whose dependency type is an aggregate repository. - pub fn with_repo(repo: D) -> Self - where - D: HasRepo, - { - Self::new(repo) - } - - /// Create a service whose dependency type is a read-model store. - pub fn with_read_model_store(read_model_store: D) -> Self - where - D: HasReadModelStore, - { - Self::new(read_model_store) - } - /// Start registering a command handler that consumes JSON payload input. pub fn command(self, name: &'static str) -> HandlerBuilder { self.handler(HandlerSpec::command(name)) @@ -447,11 +431,64 @@ impl Service { } } -impl Service> { - /// Create a service whose handlers need both an aggregate repository and a - /// read-model store. - pub fn with_repo_and_read_model_store(repo: R, read_model_store: S) -> Self { - Self::new(RepoReadModelDependencies::new(repo, read_model_store)) +// ============================================================================= +// Dependency builder: `Service::new().with_repo(..).with_read_model_store(..)` +// ============================================================================= + +impl Default for Service<()> { + fn default() -> Self { + Self::new() + } +} + +impl Service<()> { + /// Start building a service. Add dependencies and a bus with the `with_*` + /// builder steps, then register handlers: + /// + /// ```ignore + /// Service::new() + /// .with_repo(repo) + /// .with_read_model_store(store) + /// .with_bus(bus) + /// .command("x").handle(handler) + /// .run(opts).await?; + /// ``` + pub fn new() -> Self { + Self::from_dependencies(()) + } + + /// Use an aggregate repository as the service's dependency. + pub fn with_repo(self, repo: R) -> Service + where + R: HasRepo + Send + Sync + 'static, + { + Service::from_dependencies(repo) + } + + /// Use a read-model store as the service's dependency. + pub fn with_read_model_store(self, read_model_store: S) -> Service + where + S: HasReadModelStore + Send + Sync + 'static, + { + Service::from_dependencies(read_model_store) + } +} + +impl Service { + /// Add a read-model store alongside the aggregate repository, so handlers can + /// reach both via `ctx.repo()` and `ctx.read_model_store()`. Call after + /// `with_repo`. + pub fn with_read_model_store( + self, + read_model_store: S, + ) -> Service> + where + S: HasReadModelStore + Send + Sync + 'static, + { + Service::from_dependencies(RepoReadModelDependencies::new( + self.dependencies, + read_model_store, + )) } } @@ -501,7 +538,7 @@ mod tests { use serde_json::json; fn test_service() -> Service<()> { - Service::new(()) + Service::new() } #[tokio::test] diff --git a/src/outbox_worker/mod.rs b/src/outbox_worker/mod.rs index 5f8c889..498dcb9 100644 --- a/src/outbox_worker/mod.rs +++ b/src/outbox_worker/mod.rs @@ -63,7 +63,7 @@ pub use worker::{DrainResult, OutboxWorker, ProcessOneResult}; // Outbox -> bus bridge (moved out of the bus module; depends up on bus traits). pub use bus_publisher::BusPublisher; pub use outbox_dispatch::{OutboxDispatchOutcome, OutboxDispatcher, SOURCED_METADATA_PREFIX}; -pub use publish_hook::BusOutboxPublishHook; pub use outbox_source::{ OutboxSource, ReceivedOutboxMessage, DEFAULT_OUTBOX_SOURCE_BATCH, DEFAULT_OUTBOX_SOURCE_LEASE, }; +pub use publish_hook::BusOutboxPublishHook; diff --git a/src/outbox_worker/outbox_source.rs b/src/outbox_worker/outbox_source.rs index e8eb81a..e7745ef 100644 --- a/src/outbox_worker/outbox_source.rs +++ b/src/outbox_worker/outbox_source.rs @@ -303,7 +303,7 @@ mod tests { let handled = Arc::new(std::sync::Mutex::new(Vec::::new())); let h = handled.clone(); - let service = Arc::new(Service::new(()).event("evt").handle( + let service = Arc::new(Service::new().event("evt").handle( move |ctx: &crate::microsvc::Context<()>| { let h = h.clone(); let id = ctx.message().id().unwrap_or_default().to_string(); @@ -330,7 +330,7 @@ mod tests { // Service handles a different event; the unrelated row is acked-ignored, // i.e. completed, so it does not loop forever. let service: Arc> = Arc::new( - Service::new(()) + Service::new() .event("evt") .handle(|_: &crate::microsvc::Context<()>| async move { Ok(json!({})) }), ); diff --git a/tests/distributed_read_model/checkout_saga_service/service.rs b/tests/distributed_read_model/checkout_saga_service/service.rs index 2dc1213..ac1c20c 100644 --- a/tests/distributed_read_model/checkout_saga_service/service.rs +++ b/tests/distributed_read_model/checkout_saga_service/service.rs @@ -6,7 +6,7 @@ use super::{handlers, CheckoutRepo}; pub fn service(repo: CheckoutRepo) -> Arc> { Arc::new(distributed::register_handlers!( - Service::with_repo(repo), + Service::new().with_repo(repo), command handlers::start, event handlers::record_seat_reserved, )) diff --git a/tests/distributed_read_model/main.rs b/tests/distributed_read_model/main.rs index a7ee5b2..80bb559 100644 --- a/tests/distributed_read_model/main.rs +++ b/tests/distributed_read_model/main.rs @@ -800,7 +800,7 @@ fn build_collector() -> (StdArc>, Collected) { collected.clone(), collected.clone(), ); - let service = Service::new(()) + let service = Service::new() .event(seat_event::ADDED) .handle(move |ctx: &Context<()>| { record_message(&c1, ctx.message()); diff --git a/tests/distributed_read_model/projection_service/service.rs b/tests/distributed_read_model/projection_service/service.rs index ac57432..d0a754b 100644 --- a/tests/distributed_read_model/projection_service/service.rs +++ b/tests/distributed_read_model/projection_service/service.rs @@ -9,7 +9,7 @@ pub type ProjectionDependencies = InMemoryReadModelStore; pub fn service(store: InMemoryReadModelStore) -> Arc> { Arc::new(distributed::register_handlers!( - Service::with_read_model_store(store), + Service::new().with_read_model_store(store), events handlers::checkout, events handlers::seat, )) diff --git a/tests/distributed_read_model/seat_inventory_service/service.rs b/tests/distributed_read_model/seat_inventory_service/service.rs index f924b19..4cc7940 100644 --- a/tests/distributed_read_model/seat_inventory_service/service.rs +++ b/tests/distributed_read_model/seat_inventory_service/service.rs @@ -6,7 +6,7 @@ use super::{handlers, SeatRepo}; pub fn service(repo: SeatRepo) -> Arc> { Arc::new(distributed::register_handlers!( - Service::with_repo(repo), + Service::new().with_repo(repo), command handlers::add, event handlers::reserve_started_checkout_seat, )) diff --git a/tests/distributed_read_model_board/board_service/service.rs b/tests/distributed_read_model_board/board_service/service.rs index e4ca6dc..21c3345 100644 --- a/tests/distributed_read_model_board/board_service/service.rs +++ b/tests/distributed_read_model_board/board_service/service.rs @@ -6,7 +6,7 @@ use super::{handlers, BoardRepo}; pub fn model_service(repo: BoardRepo) -> Arc> { Arc::new(distributed::register_handlers!( - Service::with_repo(repo), + Service::new().with_repo(repo), command handlers::board_open, command handlers::board_add_card, command handlers::board_move_card, diff --git a/tests/distributed_read_model_board/projections_service/mod.rs b/tests/distributed_read_model_board/projections_service/mod.rs index c1dbd01..dc67f3a 100644 --- a/tests/distributed_read_model_board/projections_service/mod.rs +++ b/tests/distributed_read_model_board/projections_service/mod.rs @@ -14,7 +14,7 @@ pub type ProjectionDependencies = InMemoryReadModelStore; pub fn service(store: InMemoryReadModelStore) -> Arc> { Arc::new(distributed::register_handlers!( - Service::with_read_model_store(store), + Service::new().with_read_model_store(store), events handlers::board, )) } diff --git a/tests/durable_enqueue_sqlite/main.rs b/tests/durable_enqueue_sqlite/main.rs index cc36ae2..73ba160 100644 --- a/tests/durable_enqueue_sqlite/main.rs +++ b/tests/durable_enqueue_sqlite/main.rs @@ -12,7 +12,7 @@ use distributed::bus::{Bus, InMemoryBus, RunOptions}; use distributed::microsvc::{Context, HandlerError, HasOutboxStore, Service, Session}; use distributed::{ sourced, AggregateBuilder, AggregateRepository, AsyncOutboxStore, Entity, OutboxMessage, - OutboxMessageStatus, QueuedRepository, Queueable, SqliteRepository, + OutboxMessageStatus, Queueable, QueuedRepository, SqliteRepository, }; #[derive(Default)] @@ -51,7 +51,8 @@ async fn service() -> Repo { #[tokio::test] async fn commit_publishes_immediately_over_sqlite() { - let service = Service::with_repo(service().await) + let service = Service::new() + .with_repo(service().await) .command("counter.touch") .handle(handle_touch) .with_bus(InMemoryBus::new()); @@ -79,7 +80,8 @@ async fn commit_publishes_immediately_over_sqlite() { #[tokio::test] async fn run_consumes_command_and_publishes_over_sqlite() { let bus = InMemoryBus::new(); - let service = Service::with_repo(service().await) + let service = Service::new() + .with_repo(service().await) .command("counter.touch") .handle(handle_touch) .with_bus(bus.clone()); diff --git a/tests/kafka_transport/main.rs b/tests/kafka_transport/main.rs index dd95868..7e3a273 100644 --- a/tests/kafka_transport/main.rs +++ b/tests/kafka_transport/main.rs @@ -22,7 +22,7 @@ static SEQ: AtomicU64 = AtomicU64::new(1); /// event registration. fn recording_for(name: &str, kind: MessageKind, rec: Arc>>) -> Arc> { let leaked: &'static str = Box::leak(name.to_string().into_boxed_str()); - let builder = Service::new(()); + let builder = Service::new(); let registered = match kind { MessageKind::Command => builder.command(leaked), MessageKind::Event => builder.event(leaked), @@ -78,7 +78,7 @@ async fn publish_then_consume_round_trips_through_kafka() { let handled = Arc::new(Mutex::new(Vec::::new())); let h = handled.clone(); let service = Arc::new( - Service::new(()) + Service::new() .event(Box::leak(topic.clone().into_boxed_str())) .handle(move |ctx: &Context<()>| { h.lock() @@ -123,7 +123,7 @@ async fn message_id_and_metadata_survive_the_round_trip() { let observed = Arc::new(Mutex::new(None)); let o = observed.clone(); let service = Arc::new( - Service::new(()) + Service::new() .event(Box::leak(topic.clone().into_boxed_str())) .handle(move |ctx: &Context<()>| { let m = ctx.message(); diff --git a/tests/knative_cloudevents/main.rs b/tests/knative_cloudevents/main.rs index f3616d7..d7ac06d 100644 --- a/tests/knative_cloudevents/main.rs +++ b/tests/knative_cloudevents/main.rs @@ -18,7 +18,7 @@ async fn spawn_server() -> (String, Arc>>) { let handled = Arc::new(Mutex::new(Vec::::new())); let h = handled.clone(); let service = Arc::new( - Service::new(()) + Service::new() .event("order.initialized") .handle(move |ctx: &Context<()>| { h.lock() diff --git a/tests/microsvc/basic.rs b/tests/microsvc/basic.rs index f692eab..12e0f57 100644 --- a/tests/microsvc/basic.rs +++ b/tests/microsvc/basic.rs @@ -8,7 +8,8 @@ use crate::models::counter::{Counter, CreateCounter, DecrementCounter, Increment #[tokio::test] async fn full_lifecycle() { - let service = Service::with_repo(HashMapRepository::new()) + let service = Service::new() + .with_repo(HashMapRepository::new()) .command("counter.initialize") .handle(|ctx: &Context| { let input = ctx.input::(); diff --git a/tests/microsvc/convention.rs b/tests/microsvc/convention.rs index f70ac40..d6902b6 100644 --- a/tests/microsvc/convention.rs +++ b/tests/microsvc/convention.rs @@ -21,7 +21,7 @@ use crate::models::counter::Counter; #[tokio::test] async fn register_handlers_and_dispatch() { let service = distributed::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::new().with_repo(HashMapRepository::new().queued().aggregate::()), command handlers::counter_create, command handlers::counter_increment, ); @@ -56,7 +56,7 @@ async fn register_handlers_and_dispatch() { #[tokio::test] async fn guard_rejects_bad_input() { let service = distributed::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::new().with_repo(HashMapRepository::new().queued().aggregate::()), command handlers::counter_create, ); @@ -69,7 +69,7 @@ async fn guard_rejects_bad_input() { #[tokio::test] async fn handler_rejects_duplicate_create() { let service = distributed::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::new().with_repo(HashMapRepository::new().queued().aggregate::()), command handlers::counter_create, ); @@ -91,7 +91,7 @@ async fn handler_rejects_duplicate_create() { #[tokio::test] async fn create_persists_outbox_message() { let service = distributed::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::new().with_repo(HashMapRepository::new().queued().aggregate::()), command handlers::counter_create, ); @@ -116,7 +116,7 @@ async fn create_persists_outbox_message() { #[tokio::test] async fn duplicate_create_leaves_single_outbox_message() { let service = distributed::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::new().with_repo(HashMapRepository::new().queued().aggregate::()), command handlers::counter_create, ); @@ -144,7 +144,7 @@ async fn duplicate_create_leaves_single_outbox_message() { #[tokio::test] async fn increment_persists_outbox_message() { let service = distributed::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::new().with_repo(HashMapRepository::new().queued().aggregate::()), command handlers::counter_create, command handlers::counter_increment, ); diff --git a/tests/microsvc/session.rs b/tests/microsvc/session.rs index 6e4c439..3c59e7b 100644 --- a/tests/microsvc/session.rs +++ b/tests/microsvc/session.rs @@ -6,7 +6,7 @@ use std::collections::HashMap; #[tokio::test] async fn handler_accesses_user_id() { - let service = Service::new(()) + let service = Service::new() .command("session.identify") .handle(|ctx: &Context<()>| { let user_id = ctx.user_id().map(|id| id.to_string()); @@ -29,7 +29,7 @@ async fn handler_accesses_user_id() { #[tokio::test] async fn missing_user_id_returns_unauthorized() { - let service = Service::new(()) + let service = Service::new() .command("session.identify") .handle(|ctx: &Context<()>| { let user_id = ctx.user_id().map(|id| id.to_string()); diff --git a/tests/microsvc/transport_grpc.rs b/tests/microsvc/transport_grpc.rs index f4805f3..986ee2f 100644 --- a/tests/microsvc/transport_grpc.rs +++ b/tests/microsvc/transport_grpc.rs @@ -19,7 +19,7 @@ use crate::models::counter::Counter; fn counter_service() -> Arc> { Arc::new(distributed::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::new().with_repo(HashMapRepository::new().queued().aggregate::()), command handlers::counter_create, command handlers::counter_increment, command handlers::whoami, diff --git a/tests/microsvc/transport_http.rs b/tests/microsvc/transport_http.rs index 01b3d31..9cedfac 100644 --- a/tests/microsvc/transport_http.rs +++ b/tests/microsvc/transport_http.rs @@ -14,7 +14,7 @@ use crate::models::counter::Counter; fn counter_service() -> Arc> { Arc::new(distributed::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::new().with_repo(HashMapRepository::new().queued().aggregate::()), command handlers::counter_create, command handlers::counter_increment, command handlers::whoami, diff --git a/tests/microsvc/transport_listen.rs b/tests/microsvc/transport_listen.rs index ae74d26..916ac32 100644 --- a/tests/microsvc/transport_listen.rs +++ b/tests/microsvc/transport_listen.rs @@ -20,7 +20,7 @@ use crate::models::counter::Counter; fn counter_service() -> Arc> { Arc::new(distributed::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::new().with_repo(HashMapRepository::new().queued().aggregate::()), command handlers::counter_create, command handlers::counter_increment, command handlers::whoami, @@ -169,11 +169,11 @@ async fn multiple_services_on_different_queues() { let store = HashMapRepository::new(); let service_a = Arc::new(distributed::register_handlers!( - Service::with_repo(store.clone().queued().aggregate::()), + Service::new().with_repo(store.clone().queued().aggregate::()), command handlers::counter_create, )); let service_b = Arc::new(distributed::register_handlers!( - Service::with_repo(store.queued().aggregate::()), + Service::new().with_repo(store.queued().aggregate::()), command handlers::counter_increment, )); diff --git a/tests/microsvc/transport_subscribe.rs b/tests/microsvc/transport_subscribe.rs index d9149dc..8d60728 100644 --- a/tests/microsvc/transport_subscribe.rs +++ b/tests/microsvc/transport_subscribe.rs @@ -15,7 +15,8 @@ use crate::models::counter::Counter; fn counter_service() -> Arc> { Arc::new( - Service::with_repo(HashMapRepository::new().queued().aggregate::()) + Service::new() + .with_repo(HashMapRepository::new().queued().aggregate::()) .event(handlers::counter_create::COMMAND) .guarded( handlers::counter_create::guard, diff --git a/tests/nats_transport/main.rs b/tests/nats_transport/main.rs index cb510d1..186aa20 100644 --- a/tests/nats_transport/main.rs +++ b/tests/nats_transport/main.rs @@ -78,7 +78,7 @@ async fn publish_then_consume_round_trips_through_jetstream() { let h = handled.clone(); let subject_for_handler = subject.clone(); let service = Arc::new( - Service::new(()) + Service::new() .event(Box::leak(subject.clone().into_boxed_str())) .handle(move |ctx: &Context<()>| { assert_eq!(ctx.message().name(), subject_for_handler); @@ -124,7 +124,7 @@ async fn message_id_and_metadata_survive_the_round_trip() { let observed = Arc::new(Mutex::new(None)); let o = observed.clone(); let service = Arc::new( - Service::new(()) + Service::new() .event(Box::leak(subject.clone().into_boxed_str())) .handle(move |ctx: &Context<()>| { let m = ctx.message(); @@ -155,7 +155,7 @@ fn recording_service( rec: Arc>>, ) -> Arc> { let leaked: &'static str = Box::leak(name.to_string().into_boxed_str()); - let builder = Service::new(()); + let builder = Service::new(); let registered = match kind { MessageKind::Command => builder.command(leaked), MessageKind::Event => builder.event(leaked), diff --git a/tests/postgres_transport/main.rs b/tests/postgres_transport/main.rs index bfae459..b7cd343 100644 --- a/tests/postgres_transport/main.rs +++ b/tests/postgres_transport/main.rs @@ -56,7 +56,7 @@ async fn status(store: &PostgresOutboxStore, id: &str) -> Option>>) -> Arc> { Arc::new( - Service::new(()) + Service::new() .event("order.initialized") .handle(move |ctx: &Context<()>| { handled @@ -195,7 +195,7 @@ async fn dead_letter_marks_row_failed() { /// event registration. fn recording_for(name: &str, kind: MessageKind, rec: Arc>>) -> Arc> { let leaked: &'static str = Box::leak(name.to_string().into_boxed_str()); - let builder = Service::new(()); + let builder = Service::new(); let registered = match kind { MessageKind::Command => builder.command(leaked), MessageKind::Event => builder.event(leaked), diff --git a/tests/rabbitmq_transport/main.rs b/tests/rabbitmq_transport/main.rs index 9e75f18..8c0390b 100644 --- a/tests/rabbitmq_transport/main.rs +++ b/tests/rabbitmq_transport/main.rs @@ -49,7 +49,7 @@ fn unique(prefix: &str) -> String { /// event registration. fn recording_for(name: &str, kind: MessageKind, rec: Arc>>) -> Arc> { let leaked: &'static str = Box::leak(name.to_string().into_boxed_str()); - let builder = Service::new(()); + let builder = Service::new(); let registered = match kind { MessageKind::Command => builder.command(leaked), MessageKind::Event => builder.event(leaked), @@ -84,7 +84,7 @@ async fn publish_then_consume_round_trips_through_rabbitmq() { let handled = Arc::new(Mutex::new(Vec::::new())); let h = handled.clone(); let service = Arc::new( - Service::new(()) + Service::new() .event(Box::leak(queue.clone().into_boxed_str())) .handle(move |ctx: &Context<()>| { h.lock() @@ -127,7 +127,7 @@ async fn message_id_and_metadata_survive_the_round_trip() { let observed = Arc::new(Mutex::new(None)); let o = observed.clone(); let service = Arc::new( - Service::new(()) + Service::new() .event(Box::leak(queue.clone().into_boxed_str())) .handle(move |ctx: &Context<()>| { let m = ctx.message(); diff --git a/tests/sagas/microsvc_saga.rs b/tests/sagas/microsvc_saga.rs index 2f8b214..ca91114 100644 --- a/tests/sagas/microsvc_saga.rs +++ b/tests/sagas/microsvc_saga.rs @@ -4,7 +4,7 @@ //! organized by service domain under `handlers/`. //! //! Each service is typed to a specific aggregate via -//! `Service::with_repo(repo.queued().aggregate::())`, so handlers access +//! `Service::new().with_repo(repo.queued().aggregate::())`, so handlers access //! `ctx.repo().get()`, `ctx.repo().commit()`, etc. directly. //! //! Two tests: @@ -54,7 +54,7 @@ fn event_message(name: &str, input: serde_json::Value) -> Message { #[tokio::test] async fn saga_orchestrated() { let saga_svc = distributed::register_handlers!( - Service::with_repo( + Service::new().with_repo( HashMapRepository::new() .queued() .aggregate::() @@ -67,19 +67,19 @@ async fn saga_orchestrated() { ); let order_svc = distributed::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::new().with_repo(HashMapRepository::new().queued().aggregate::()), command handlers::orders::create, command handlers::orders::complete, ); let inventory_svc = distributed::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::new().with_repo(HashMapRepository::new().queued().aggregate::()), command handlers::inventory::init, command handlers::inventory::reserve, ); let payment_svc = distributed::register_handlers!( - Service::with_repo(HashMapRepository::new().queued().aggregate::()), + Service::new().with_repo(HashMapRepository::new().queued().aggregate::()), command handlers::payments::process, ); @@ -291,7 +291,7 @@ async fn saga_distributed() { let saga_repo = HashMapRepository::new(); let saga_outbox = saga_repo.outbox_store(); let saga_svc = Arc::new(distributed::register_handlers!( - Service::with_repo(saga_repo.queued().aggregate::()), + Service::new().with_repo(saga_repo.queued().aggregate::()), command handlers::saga::start, event handlers::saga::on_order_created, event handlers::saga::on_inventory_reserved, @@ -303,7 +303,7 @@ async fn saga_distributed() { let order_repo = HashMapRepository::new(); let order_outbox = order_repo.outbox_store(); let order_svc = Arc::new(distributed::register_handlers!( - Service::with_repo(order_repo.queued().aggregate::()), + Service::new().with_repo(order_repo.queued().aggregate::()), command handlers::orders::create, command handlers::orders::complete, )); @@ -318,7 +318,7 @@ async fn saga_distributed() { tmp.commit(&mut inv).await.unwrap(); } let inventory_svc = Arc::new(distributed::register_handlers!( - Service::with_repo(inventory_repo.queued().aggregate::()), + Service::new().with_repo(inventory_repo.queued().aggregate::()), command handlers::inventory::init, command handlers::inventory::reserve, )); @@ -327,7 +327,7 @@ async fn saga_distributed() { let payment_repo = HashMapRepository::new(); let payment_outbox = payment_repo.outbox_store(); let payment_svc = Arc::new(distributed::register_handlers!( - Service::with_repo(payment_repo.queued().aggregate::()), + Service::new().with_repo(payment_repo.queued().aggregate::()), command handlers::payments::process, )); diff --git a/tests/snapshots/main.rs b/tests/snapshots/main.rs index a51573c..01e7544 100644 --- a/tests/snapshots/main.rs +++ b/tests/snapshots/main.rs @@ -60,12 +60,7 @@ async fn snapshot_created_at_frequency_threshold() { let identity = StreamIdentity::new(Todo::aggregate_type(), "t1").unwrap(); // Version 1 — below threshold of 2, no snapshot yet - assert!(repo - .repo() - .get_snapshot(&identity) - .await - .unwrap() - .is_none()); + assert!(repo.repo().get_snapshot(&identity).await.unwrap().is_none()); // Load, add another event to reach version 2 let mut todo = repo.get("t1").await.unwrap().unwrap(); @@ -106,12 +101,7 @@ async fn no_snapshot_before_threshold() { let identity = StreamIdentity::new(Todo::aggregate_type(), "t1").unwrap(); // Only 1 event, threshold is 5 - assert!(repo - .repo() - .get_snapshot(&identity) - .await - .unwrap() - .is_none()); + assert!(repo.repo().get_snapshot(&identity).await.unwrap().is_none()); } #[tokio::test] @@ -129,12 +119,7 @@ async fn load_from_snapshot_produces_correct_state() { let identity = StreamIdentity::new(Todo::aggregate_type(), "t1").unwrap(); // Snapshot at version 2 - assert!(repo - .repo() - .get_snapshot(&identity) - .await - .unwrap() - .is_some()); + assert!(repo.repo().get_snapshot(&identity).await.unwrap().is_some()); // Reload — should use snapshot let loaded = repo.get("t1").await.unwrap().unwrap(); @@ -164,12 +149,7 @@ async fn snapshot_plus_newer_events() { let identity = StreamIdentity::new(Todo::aggregate_type(), "t1").unwrap(); // Snapshot exists at version 2, completed = true - let snap = repo - .repo() - .get_snapshot(&identity) - .await - .unwrap() - .unwrap(); + let snap = repo.repo().get_snapshot(&identity).await.unwrap().unwrap(); assert_eq!(snap.version, 2); // Now create a second todo to verify snapshot + partial replay works. @@ -260,21 +240,11 @@ async fn no_snapshot_falls_back_to_full_replay() { let identity = StreamIdentity::new(Todo::aggregate_type(), "t1").unwrap(); // Snapshot exists - assert!(repo - .repo() - .get_snapshot(&identity) - .await - .unwrap() - .is_some()); + assert!(repo.repo().get_snapshot(&identity).await.unwrap().is_some()); // Delete the snapshot repo.repo().delete_snapshot(&identity).await.unwrap(); - assert!(repo - .repo() - .get_snapshot(&identity) - .await - .unwrap() - .is_none()); + assert!(repo.repo().get_snapshot(&identity).await.unwrap().is_none()); // Loading should still work via full replay let loaded = repo.get("t1").await.unwrap().unwrap(); @@ -298,12 +268,7 @@ async fn snapshot_version_advances_on_second_snapshot() { let identity = StreamIdentity::new(Todo::aggregate_type(), "t1").unwrap(); // First snapshot at version 1 - let snap = repo - .repo() - .get_snapshot(&identity) - .await - .unwrap() - .unwrap(); + let snap = repo.repo().get_snapshot(&identity).await.unwrap().unwrap(); assert_eq!(snap.version, 1); // Add another event @@ -312,12 +277,7 @@ async fn snapshot_version_advances_on_second_snapshot() { repo.commit(&mut todo).await.unwrap(); // Second snapshot at version 2 - let snap = repo - .repo() - .get_snapshot(&identity) - .await - .unwrap() - .unwrap(); + let snap = repo.repo().get_snapshot(&identity).await.unwrap().unwrap(); assert_eq!(snap.version, 2); // Verify the loaded aggregate has correct snapshot_version @@ -345,12 +305,7 @@ async fn with_queued_repo() { let identity = StreamIdentity::new(Todo::aggregate_type(), "t1").unwrap(); // Snapshot should exist through the queued + snapshot chain - let snap = repo - .repo() - .inner() - .get_snapshot(&identity) - .await - .unwrap(); + let snap = repo.repo().inner().get_snapshot(&identity).await.unwrap(); assert!(snap.is_some()); assert_eq!(snap.unwrap().version, 2); @@ -416,18 +371,8 @@ async fn commit_all_with_snapshots() { let identity2 = StreamIdentity::new(Todo::aggregate_type(), "t2").unwrap(); // Both should have snapshots at version 2 - let snap1 = repo - .repo() - .get_snapshot(&identity1) - .await - .unwrap() - .unwrap(); + let snap1 = repo.repo().get_snapshot(&identity1).await.unwrap().unwrap(); assert_eq!(snap1.version, 2); - let snap2 = repo - .repo() - .get_snapshot(&identity2) - .await - .unwrap() - .unwrap(); + let snap2 = repo.repo().get_snapshot(&identity2).await.unwrap().unwrap(); assert_eq!(snap2.version, 2); } diff --git a/tests/transport_conformance/mod.rs b/tests/transport_conformance/mod.rs index 73d1d96..4674cf2 100644 --- a/tests/transport_conformance/mod.rs +++ b/tests/transport_conformance/mod.rs @@ -182,7 +182,7 @@ pub fn recording_service(recorder: &Arc) -> Arc> { let retryable = recorder.clone(); let permanent = recorder.clone(); Arc::new( - Service::new(()) + Service::new() .event("delivery.succeeded") .handle(move |ctx: &Context<()>| { ok.push(Event::Handled(ctx.message().name().to_string())); From 3bb37dfdd442c199b1511a3b722e435a699bf006 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 4 Jun 2026 21:47:15 -0500 Subject: [PATCH 15/24] docs: README durable-enqueue framing + close the Quick Start produce loop Update the Quick Start to the Service::new().with_repo(..).with_bus(bus).run() builder and make the produce loop explicit: step 2 commits an outbox row, step 3 attaches a bus so that commit publishes on commit. Rewrite Draining the Outbox as Publishing the Outbox (immediate-on-commit vs pending+worker), and document the backstop poll loop as the composable OutboxDispatcher + your timer. Note the with_bus().run() convenience alongside the lower-level listen/subscribe facade. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 96 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 66 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 680cde4..f40316c 100644 --- a/README.md +++ b/README.md @@ -104,8 +104,9 @@ pub async fn handle(ctx: &Context<'_, Repo>) -> Result { let mut todo = Todo::default(); todo.initialize(input.id.clone(), input.user_id, input.task)?; - // Publish a fact for other services. The outbox row commits atomically - // with the aggregate's events. + // Record a fact for other services. The outbox row commits atomically with + // the aggregate's events. Once a bus is attached (step 3) this `commit` + // publishes the row immediately; with no bus it stays pending for a worker. let message = OutboxMessage::domain_event("todo.initialized", &todo)?; ctx.repo().outbox(message).commit(&mut todo).await?; @@ -115,19 +116,20 @@ pub async fn handle(ctx: &Context<'_, Repo>) -> Result { ### 3. Serve it -Register your handlers on a `microsvc::Service` with `register_handlers!`, then -expose the exact same service over direct dispatch, HTTP, gRPC, or the bus. Handlers -are written once and are transport-agnostic. +Build the service fluently from `Service::new()`, register handlers with +`register_handlers!`, then expose the exact same service over direct dispatch, +HTTP, gRPC, or the bus. Handlers are written once and are transport-agnostic. ```rust use std::sync::Arc; use distributed::microsvc::{self, Service, Session}; +use distributed::bus::{InMemoryBus, RunOptions}; use distributed::{AggregateBuilder, HashMapRepository, Queueable}; use serde_json::json; #[tokio::main] async fn main() -> Result<(), Box> { - let service = Arc::new(distributed::register_handlers!( + let service = distributed::register_handlers!( Service::new().with_repo( HashMapRepository::new() .queued() @@ -135,21 +137,20 @@ async fn main() -> Result<(), Box> { ), command handlers::todo_create, command handlers::todo_complete, - )); + ); - // Direct, in-process dispatch + // Attach a bus and run. `with_bus` closes the loop from step 2: that + // `outbox(..).commit(..)` now publishes on commit, and `run` consumes the + // registered commands (and events). Same handlers, one line of wiring. service - .dispatch( - "todo.initialize", - json!({ "id": "todo-1", "user_id": "alice", "task": "Ship it" }), - Session::new(), - ) + .with_bus(InMemoryBus::new()) + .run(RunOptions::idempotent()) .await?; - // ...or expose it over the network / a broker — pick any, they share handlers: - // microsvc::serve(service, "0.0.0.0:3000").await?; // HTTP (feature = "http") - // microsvc::serve_grpc(service, "[::1]:50051").await?; // gRPC (feature = "grpc") - // InMemoryBus::new().listen(service, RunOptions::idempotent()).await?; // bus + // Alternatives that share the same handlers: + // service.dispatch("todo.initialize", json!({ "id": "todo-1", .. }), Session::new()).await?; // in-process + // microsvc::serve(Arc::new(service), "0.0.0.0:3000").await?; // HTTP (feature = "http") + // microsvc::serve_grpc(Arc::new(service), "[::1]:50051").await?; // gRPC (feature = "grpc") Ok(()) } @@ -164,19 +165,19 @@ default you replace with a durable adapter. ```rust // Persistence: HashMapRepository → durable SQL (features "postgres" / "sqlite") let repo = distributed::PostgresRepository::connect_and_migrate(database_url).await?; -let service = Arc::new(distributed::register_handlers!( +let service = distributed::register_handlers!( Service::new().with_repo(repo.queued().aggregate::()), command handlers::todo_create, command handlers::todo_complete, -)); +); -// Transport: InMemoryBus → a real broker. send/listen/publish/subscribe + the -// handlers are unchanged; only this line differs. +// Transport: InMemoryBus → a real broker. The handlers and the +// `with_bus(..).run(..)` wiring are unchanged; only this constructor line differs. // let bus = NatsBus::connect("nats://localhost:4222", "todos", "app").await?; // let bus = PostgresBus::new(pool, "todos"); // let bus = RabbitBus::connect("amqp://localhost:5672/%2f", "todos", "app").await?; // let bus = KafkaBus::connect("localhost:9092", "todos", "app").await?; -bus.listen(service, RunOptions::idempotent()).await?; +service.with_bus(bus).run(RunOptions::idempotent()).await?; ``` | Concern | In-memory default | Swap in for production | @@ -732,20 +733,43 @@ let message = OutboxMessage::encode_for_entity( )?; ``` -### Draining the Outbox +### Publishing the Outbox + +How a committed row reaches the bus depends on whether a bus is attached to the +service: + +- **Bus attached (`service.with_bus(bus)`)** — `repo.outbox(msg).commit(agg)` + claims the row in the commit transaction and publishes it **immediately** after + commit. A crash before the publish, or a publish failure, leaves the row + claimed under a short lease; when the lease expires the polling worker takes it. +- **No bus** — the row is committed `pending` and a worker publishes it. -`OutboxDispatcher` bridges durable outbox rows to a transport publisher, sharing one -claim → publish → complete path between background polling (`dispatch_batch`) and -after-commit immediate dispatch (`dispatch_ids`): +The polling worker is the durable backstop in both cases. It is the same +`OutboxDispatcher` primitive composed with your runtime's timer — run it in the +service process or as a separate worker, against the same outbox store: ```rust -let dispatcher = OutboxDispatcher::new(store, publisher, "worker-1", lease, max_attempts); -let outcome = dispatcher.dispatch_ids(&committed_ids).await?; // claim-before-publish +use distributed::{BusPublisher, OutboxDispatcher}; +use std::{sync::Arc, time::Duration}; + +let dispatcher = OutboxDispatcher::new( + repo.outbox_store(), + BusPublisher::new(Arc::new(bus)), // routes commands/events by kind + "outbox-worker-1", + Duration::from_secs(30), // claim lease + 5, // max publish attempts +); + +loop { + dispatcher.dispatch_batch(100).await?; // claim → publish → complete + tokio::time::sleep(Duration::from_secs(1)).await; +} ``` A row completes only after `publish()` resolves `Ok`; an unknown or failed publish leaves it retryable (released until the attempt ceiling, then moved to `Failed`). -Claims use leases, so competing workers never publish the same row concurrently. +Claims use leases, so the immediate path and competing workers never publish the +same row concurrently. ## Service Bus @@ -780,6 +804,12 @@ bus.subscribe(service.clone(), RunOptions::idempotent()).await?; // fan-out // let bus = KafkaBus::connect("localhost:9092", "orders", "app").await?; ``` +This is the low-level facade. For a `microsvc::Service`, the one-call convenience +is `service.with_bus(bus).run(opts)`: it derives the command names to `listen` +and the event names to `subscribe` from the registered handlers, and makes +`repo.outbox(msg).commit(agg)` publish on commit. Drop to `listen` / `subscribe` +/ `send` / `publish` directly when you need finer control. + Point-to-point vs fan-out is consistently a **consumer-group/identity** choice in each transport's native topology — the same `group` competes, different `group`s fan out: @@ -987,7 +1017,13 @@ Session handling mirrors HTTP — gRPC metadata headers are merged with payload ### Bus Transport -Drive a service from the bus with `listen` (point-to-point) or `subscribe` (fan-out). The same `Service` can handle commands from multiple transports simultaneously — HTTP, gRPC, bus, and direct dispatch all share the same handlers and repository. See [Service Bus](#service-bus) above. +Attach a bus with `service.with_bus(bus)` and drive it with `run(opts)`: it +derives `listen` (point-to-point commands) and `subscribe` (fan-out events) from +the registered handlers, and makes `repo.outbox(msg).commit(agg)` publish on +commit. The same `Service` can handle commands from multiple transports +simultaneously — HTTP, gRPC, bus, and direct dispatch all share the same handlers +and repository. For finer-grained control, call the `listen` / `subscribe` facade +methods directly. See [Service Bus](#service-bus) above. ### Error Handling From 3ad2b17e86469ab0d49865d91985ab5f9aaded3e Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 4 Jun 2026 21:58:59 -0500 Subject: [PATCH 16/24] feat: compose read-models + snapshots in one aggregate commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Close the last 'all features compose' gap. The AggregateRepository-level commit (formerly OutboxCommit, now AggregateCommit) carries outbox rows AND read-model write plans, and stages a snapshot from the repo's policy — all in one CommitBatch. New entry repo.read_models(plan) mirrors repo.outbox(msg); both chain (.outbox(..).read_models(..)) and end in .commit(agg), which also publishes the outbox rows on commit when a bus is attached. Previously read-model commits ran at the raw-repo level (CommitBuilder, no snapshot policy) so snapshots and read-models could not compose. Now a snapshot-backed repo commits streams + outbox + read-models + snapshot atomically. Test proves read-model row + snapshot land in one transaction. The raw-repo CommitBuilder (repo: &R) is unchanged for non-aggregate-repo use. Implements [[specs/durable-enqueue-outbox-dispatch]] [[tasks/snapshot-readmodel-commit-compose]] Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 2 +- src/lib.rs | 2 +- src/outbox/commit.rs | 207 ++++++++++++++++++++++++++++++++++--------- src/outbox/mod.rs | 2 +- 4 files changed, 166 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index f40316c..ecc0481 100644 --- a/README.md +++ b/README.md @@ -709,7 +709,7 @@ read models, the outbox, **and** the durable transport (`PostgresBus`). See Each outbox message is a durable delivery row committed alongside your domain entity. Aggregate event records are write-side replay history; they become domain events, integration events, commands, or transport messages only when application code creates an `OutboxMessage` for that purpose. ```rust -use distributed::{OutboxCommit, OutboxMessage}; +use distributed::OutboxMessage; let mut todo = Todo::default(); todo.entity.set_correlation_id("req-abc"); diff --git a/src/lib.rs b/src/lib.rs index 50f6ee0..eac232f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -67,7 +67,7 @@ pub use lock::{SqliteLock, SqliteLockManager}; // Outbox: commit concerns (aggregate + outbox in one commit) pub use outbox::{ outbox_message_insert_plan, outbox_message_key, outbox_message_row_values, - outbox_message_schema, CommitReceipt, OutboxCommit, OutboxMessage, OutboxMessageStatus, + outbox_message_schema, AggregateCommit, CommitReceipt, OutboxMessage, OutboxMessageStatus, OutboxPublishHook, OutboxPublisherConfig, OUTBOX_MESSAGES_TABLE, }; diff --git a/src/outbox/commit.rs b/src/outbox/commit.rs index c494ea9..d76c60e 100644 --- a/src/outbox/commit.rs +++ b/src/outbox/commit.rs @@ -5,6 +5,7 @@ use std::time::{Duration, SystemTime}; use crate::aggregate::{Aggregate, AggregateRepository}; use crate::outbox::OutboxMessage; +use crate::read_model::{ReadModelWritePlan, ReadModelWritePlanBuilder}; use crate::repository::{ CommitBatch, RepositoryError, StreamIdentity, StreamWrite, TransactionalCommit, }; @@ -75,81 +76,132 @@ impl CommitReceipt { } } -/// Helper returned by [`AggregateRepository::outbox`] to commit an aggregate -/// and an outbox row in the same async transactional batch. +/// Builder returned by [`AggregateRepository::outbox`] and +/// [`AggregateRepository::read_models`] that commits an aggregate together with +/// outbox rows, relational read-model writes, and (when the repository has +/// snapshots configured) a snapshot — all in one async transactional batch. /// /// Borrows the repository so it can be called through `ctx.repo()` inside async -/// handlers. -pub struct OutboxCommit<'a, R, A> { +/// handlers. Chain `.outbox(..)` / `.read_models(..)` to stage more, then +/// `.commit(&mut aggregate)`. +pub struct AggregateCommit<'a, R, A> { repo: &'a AggregateRepository, - message: OutboxMessage, + outbox_messages: Vec, + read_model_plans: Vec, + error: Option, } -impl OutboxCommit<'_, R, A> +impl<'a, R, A> AggregateCommit<'a, R, A> { + fn empty(repo: &'a AggregateRepository) -> Self { + Self { + repo, + outbox_messages: Vec::new(), + read_model_plans: Vec::new(), + error: None, + } + } + + /// Stage an outbox message to publish/enqueue with the commit. + pub fn outbox(mut self, message: OutboxMessage) -> Self { + self.outbox_messages.push(message); + self + } + + /// Stage relational read-model writes to apply in the same transaction. + pub fn read_models(mut self, read_models: ReadModelWritePlanBuilder) -> Self { + if self.error.is_none() { + match read_models.into_write_plan() { + Ok(plan) => self.read_model_plans.push(plan), + Err(err) => self.error = Some(err.into()), + } + } + self + } +} + +impl AggregateCommit<'_, R, A> where R: TransactionalCommit, A: Aggregate + Send, { - /// Commit the aggregate and outbox message together, and — when the - /// repository has a bus configured (via `Service::with_bus`) — publish the - /// row immediately. + /// Commit the aggregate together with the staged outbox rows, read-model + /// writes, and a snapshot (when due) in one transaction — and, when the + /// repository has a bus configured (via `Service::with_bus`), publish the + /// outbox rows immediately. /// - /// With a bus configured, the row is **claimed in this same transaction** - /// (born `InFlight` under a short lease) and published right after commit, so - /// publication needs no separate claim and cannot race the polling worker; a - /// crash before publish hands the row back to the worker at lease expiry, and - /// a publish failure leaves it retryable. Without a bus, the row is committed - /// `pending` for the polling worker to publish. A snapshot is also staged when - /// the repository has snapshots configured and one is due — all in the one - /// transaction. + /// With a bus configured, each outbox row is **claimed in this same + /// transaction** (born `InFlight` under a short lease) and published right + /// after commit, so publication needs no separate claim and cannot race the + /// polling worker; a crash before publish hands the row back to the worker at + /// lease expiry, and a publish failure leaves it retryable. Without a bus, + /// rows are committed `pending` for the worker to publish. /// - /// Returns a [`CommitReceipt`] carrying the inserted outbox message id. + /// Returns a [`CommitReceipt`] carrying the inserted outbox message ids. pub async fn commit(mut self, aggregate: &mut A) -> Result { - self.message.set_source(aggregate); - let outbox_message_id = self.message.id().to_string(); + if let Some(err) = self.error.take() { + return Err(err); + } + for message in &mut self.outbox_messages { + message.set_source(aggregate); + } + let outbox_message_ids: Vec = self + .outbox_messages + .iter() + .map(|message| message.id().to_string()) + .collect(); - // When a bus is configured, claim the row in this transaction so it can - // be published immediately after commit; otherwise leave it `pending`. + // When a bus is configured, claim the rows in this transaction so they + // can be published immediately after commit; otherwise leave them + // `pending`. let publisher = self.repo.outbox_publisher(); - let claimed = match publisher { - Some(config) => { - self.message - .claim_at(&config.worker_id, config.lease, SystemTime::now())?; - Some(self.message.clone()) + let mut claimed = Vec::new(); + if let Some(config) = publisher { + let now = SystemTime::now(); + for message in &mut self.outbox_messages { + message.claim_at(&config.worker_id, config.lease, now)?; + claimed.push(message.clone()); } - None => None, - }; + } let (snapshots, snapshot_version) = self.repo.snapshot_writes_for(aggregate)?; let identity = StreamIdentity::new(A::aggregate_type(), aggregate.entity().id())?; let stream = StreamWrite::new(identity, aggregate.entity_mut()); - let mut batch = CommitBatch::new(vec![stream]); - batch.outbox_messages.push(self.message); - batch.snapshots = snapshots; - self.repo.repo().commit_batch(batch).await?; + self.repo + .repo() + .commit_batch(CommitBatch { + streams: vec![stream], + outbox_messages: self.outbox_messages, + read_model_plans: self.read_model_plans, + snapshots, + inbox_receipts: Vec::new(), + }) + .await?; if let Some(version) = snapshot_version { aggregate.entity_mut().set_snapshot_version(version); } // Best-effort immediate publish. A failure leaves the claimed row for the // polling worker and never fails the already-committed command. - if let (Some(config), Some(claimed)) = (publisher, claimed) { - let _ = config.hook.publish_claimed(claimed).await; + if let Some(config) = publisher { + for message in claimed { + let _ = config.hook.publish_claimed(message).await; + } } - Ok(CommitReceipt { - outbox_message_ids: vec![outbox_message_id], - }) + Ok(CommitReceipt { outbox_message_ids }) } } impl AggregateRepository { - /// Attach an outbox message to be committed with the aggregate. - pub fn outbox(&self, message: OutboxMessage) -> OutboxCommit<'_, R, A> { - OutboxCommit { - repo: self, - message, - } + /// Start a commit with an outbox message attached. + pub fn outbox(&self, message: OutboxMessage) -> AggregateCommit<'_, R, A> { + AggregateCommit::empty(self).outbox(message) + } + + /// Start a commit with relational read-model writes attached. Composes with + /// `.outbox(..)`, the aggregate's events, and snapshots in one transaction. + pub fn read_models(&self, read_models: ReadModelWritePlanBuilder) -> AggregateCommit<'_, R, A> { + AggregateCommit::empty(self).read_models(read_models) } } @@ -239,4 +291,71 @@ mod tests { &["dummy-1".to_string(), "msg-fail".to_string()] ); } + + #[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize, crate::ReadModel)] + #[table("agg_commit_views")] + struct ComposeView { + #[id] + id: String, + n: i32, + } + + #[derive(Default, crate::Snapshot)] + struct ComposeCounter { + entity: Entity, + value: i64, + } + + #[sourced(entity, aggregate_type = "agg_commit_counter")] + impl ComposeCounter { + #[event("bumped")] + fn bump(&mut self, id: String) { + self.entity.set_id(&id); + self.value += 1; + } + } + + #[tokio::test] + async fn read_models_and_snapshot_commit_in_one_transaction() { + use crate::{ + Aggregate, ReadModelWorkspaceExt, ReadModelWritePlanBuilder, RowKey, RowValue, + SnapshotStore, StreamIdentity, + }; + + let repo = HashMapRepository::new() + .aggregate::() + .with_snapshots(1); + + let mut counter = ComposeCounter::default(); + counter.bump("c1".to_string()).unwrap(); + + let mut plan = ReadModelWritePlanBuilder::new(); + plan.upsert(&ComposeView { + id: "c1".into(), + n: 1, + }) + .unwrap(); + + // Read-model writes + the aggregate's events + a snapshot — one commit. + repo.read_models(plan).commit(&mut counter).await.unwrap(); + + // The read-model row is committed... + let loaded = repo + .repo() + .model_store() + .workspace() + .load::(RowKey::new([("id", RowValue::String("c1".into()))])) + .one() + .await + .unwrap(); + assert!(loaded.is_some(), "read-model row should be committed"); + + // ...and a snapshot was staged in the same transaction (frequency 1). + let identity = StreamIdentity::new(ComposeCounter::aggregate_type(), "c1").unwrap(); + let snapshot = repo.repo().get_snapshot(&identity).await.unwrap(); + assert!( + snapshot.is_some(), + "snapshot should be staged alongside the read-model commit" + ); + } } diff --git a/src/outbox/mod.rs b/src/outbox/mod.rs index 0592449..8101131 100644 --- a/src/outbox/mod.rs +++ b/src/outbox/mod.rs @@ -49,4 +49,4 @@ pub use table::{ }; // Commit helpers -pub use commit::{CommitReceipt, OutboxCommit, OutboxPublishHook, OutboxPublisherConfig}; +pub use commit::{AggregateCommit, CommitReceipt, OutboxPublishHook, OutboxPublisherConfig}; From 9879c6c9859e0d73d041485547fcee2daba6ab4b Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 4 Jun 2026 22:03:31 -0500 Subject: [PATCH 17/24] test: aggregate + outbox + read-model + snapshot in one transaction Single test exercising all four staged in one commit (repo.outbox(msg).read_models(plan).commit(agg) on a with_snapshots(1) repo): asserts the aggregate stream, outbox row, read-model row, and snapshot all land together. [[tasks/snapshot-readmodel-commit-compose]] Co-Authored-By: Claude Opus 4.8 (1M context) --- src/outbox/commit.rs | 64 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/src/outbox/commit.rs b/src/outbox/commit.rs index d76c60e..c9f225f 100644 --- a/src/outbox/commit.rs +++ b/src/outbox/commit.rs @@ -358,4 +358,68 @@ mod tests { "snapshot should be staged alongside the read-model commit" ); } + + #[tokio::test] + async fn aggregate_outbox_read_model_and_snapshot_commit_in_one_transaction() { + use crate::{ + Aggregate, GetStream, ReadModelWorkspaceExt, ReadModelWritePlanBuilder, RowKey, + RowValue, SnapshotStore, StreamIdentity, + }; + + let repo = HashMapRepository::new() + .aggregate::() + .with_snapshots(1); + + let mut counter = ComposeCounter::default(); + counter.bump("c1".to_string()).unwrap(); + + let mut plan = ReadModelWritePlanBuilder::new(); + plan.upsert(&ComposeView { + id: "c1".into(), + n: 1, + }) + .unwrap(); + + let message = OutboxMessage::create("evt-c1", "counter.bumped", b"{}".to_vec()).unwrap(); + + // All four in one commit: aggregate events + outbox row + read-model + // write + snapshot. + let receipt = repo + .outbox(message) + .read_models(plan) + .commit(&mut counter) + .await + .unwrap(); + assert_eq!(receipt.outbox_message_ids(), ["evt-c1".to_string()]); + + let identity = StreamIdentity::new(ComposeCounter::aggregate_type(), "c1").unwrap(); + + // 1) aggregate stream committed + assert!( + repo.repo().get_stream(&identity).await.unwrap().is_some(), + "aggregate stream should be committed" + ); + + // 2) outbox row present (pending — no bus attached here) + let pending = repo.repo().outbox_store().pending().unwrap(); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].id(), "evt-c1"); + + // 3) read-model row written + let loaded = repo + .repo() + .model_store() + .workspace() + .load::(RowKey::new([("id", RowValue::String("c1".into()))])) + .one() + .await + .unwrap(); + assert!(loaded.is_some(), "read-model row should be committed"); + + // 4) snapshot staged + assert!( + repo.repo().get_snapshot(&identity).await.unwrap().is_some(), + "snapshot should be staged" + ); + } } From dd6cb0935663d0c3ec71d2a90f4261705f52ced8 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 4 Jun 2026 22:25:16 -0500 Subject: [PATCH 18/24] =?UTF-8?q?feat:=20distributed=5Ftooling=20crate=20?= =?UTF-8?q?=E2=80=94=20pure=20service=20scaffold=20generation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New workspace crate implementing specs/distributed-service-scaffold-tooling: a pure ServiceScaffoldSpec -> GeneratedProject API (no fs, network, or CLI). Owns the deterministic generation rules — name/message normalization + validation, GitHub repo parsing, and the core service-crate templates (Cargo.toml, lib/main/ manifest/service/models/handlers/read_models). Returns GeneratedFile list + warnings + PostCreateAction (EnsureGithubRepository) for the caller to act on. Generated service.rs uses the new Service::new().with_repo(repo) builder. The public API includes the gitops/github spec fields; those artifact templates + the hops-cli rewire are the next slices. 7 tests green; workspace builds; clippy clean. Implements [[specs/distributed-service-scaffold-tooling]] [[tasks/distributed-tooling-crate-extraction]] Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.toml | 2 +- distributed_tooling/Cargo.toml | 10 + distributed_tooling/src/generate.rs | 939 ++++++++++++++++++++++++++++ distributed_tooling/src/lib.rs | 194 ++++++ 4 files changed, 1144 insertions(+), 1 deletion(-) create mode 100644 distributed_tooling/Cargo.toml create mode 100644 distributed_tooling/src/generate.rs create mode 100644 distributed_tooling/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 104e55f..1d51d81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["distributed_macros"] +members = ["distributed_macros", "distributed_tooling"] resolver = "2" [workspace.package] diff --git a/distributed_tooling/Cargo.toml b/distributed_tooling/Cargo.toml new file mode 100644 index 0000000..0e5d1c3 --- /dev/null +++ b/distributed_tooling/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "distributed_tooling" +description = "Deterministic service-scaffold and artifact generation for Distributed services. Pure (no filesystem, network, or CLI): a ServiceScaffoldSpec in, a GeneratedProject out." +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +serde_json = "1" diff --git a/distributed_tooling/src/generate.rs b/distributed_tooling/src/generate.rs new file mode 100644 index 0000000..92d3cb4 --- /dev/null +++ b/distributed_tooling/src/generate.rs @@ -0,0 +1,939 @@ +//! Internal generation: build a normalized `Scaffold` from a spec and render the +//! project files. Pure — every method returns `String`s; nothing touches the +//! filesystem. Ported from the original `hops-cli service scaffold` command. + +use std::collections::BTreeSet; + +use crate::{ + BusTarget, GeneratedFile, GeneratedProject, GithubRepo, GithubScaffoldSpec, + GitopsPromoteTarget, PostCreateAction, ScaffoldError, ServiceScaffoldSpec, ServiceTransport, + StoreTarget, +}; + +/// Generate a Distributed service project from a spec. The public entry point. +pub fn generate_service_scaffold( + spec: ServiceScaffoldSpec, +) -> Result { + Ok(Scaffold::from_spec(spec)?.generate()) +} + +/// Parse an `owner/repo` string (used by [`GithubRepo::parse`]). +pub(crate) fn parse_github_repo(raw: &str) -> Result { + let trimmed = raw.trim(); + let Some((owner, repo)) = trimmed.split_once('/') else { + return Err(ScaffoldError::new("repository must be in OWNER/REPO form")); + }; + if owner.is_empty() || repo.is_empty() || repo.contains('/') { + return Err(ScaffoldError::new("repository must be in OWNER/REPO form")); + } + let valid = [owner, repo].into_iter().all(|part| { + part.chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.') + }); + if !valid { + return Err(ScaffoldError::new( + "repository contains unsupported GitHub characters", + )); + } + Ok(GithubRepo { + owner: owner.to_string(), + repo: repo.to_string(), + }) +} + +struct Scaffold { + names: ScaffoldNames, + distributed_dependency_path: String, + transport: ServiceTransport, + store: StoreTarget, + #[allow(dead_code)] // used by GitOps/Knative generation (follow-up slice) + bus: Option, + include_read_models: bool, + gitops: bool, + gitops_promote: Option, + github: Option, + models: Vec, + read_models: Vec, + commands: Vec, + events: Vec, +} + +impl Scaffold { + fn from_spec(spec: ServiceScaffoldSpec) -> Result { + let names = ScaffoldNames::new(&spec.name)?; + let models = model_scaffolds(&spec.models)?; + let read_models = if spec.read_models { + if models.is_empty() { + vec![ModelScaffold::new(&names.package_name)?] + } else { + models.clone() + } + } else { + Vec::new() + }; + let mut module_idents = BTreeSet::new(); + let commands = message_handlers_with_modules( + if spec.commands.is_empty() { + vec![default_command_name(&names, &models)] + } else { + spec.commands.clone() + }, + "command", + &mut module_idents, + )?; + let events = + message_handlers_with_modules(spec.events.clone(), "event", &mut module_idents)?; + Ok(Self { + names, + distributed_dependency_path: spec.distributed_dependency_path, + transport: spec.transport, + store: spec.store, + bus: spec.bus, + include_read_models: spec.read_models, + gitops: spec.gitops, + gitops_promote: spec.gitops_promote, + github: spec.github, + models, + read_models, + commands, + events, + }) + } + + fn generate(self) -> GeneratedProject { + let mut files = Vec::new(); + let mut warnings = Vec::new(); + let mut post_create_actions = Vec::new(); + + files.push(file("Cargo.toml", self.cargo_toml())); + files.push(file("src/lib.rs", self.lib_rs())); + files.push(file("src/main.rs", self.main_rs())); + files.push(file("src/manifest.rs", self.manifest_rs())); + files.push(file("src/service.rs", self.service_rs())); + if !self.models.is_empty() { + files.push(file("src/models/mod.rs", self.models_mod_rs())); + for model in &self.models { + files.push(file( + &format!("src/models/{}.rs", model.module_ident), + self.model_rs(model), + )); + } + } + files.push(file("src/handlers/mod.rs", self.handlers_mod_rs())); + for command in &self.commands { + files.push(file( + &format!("src/handlers/{}.rs", command.module_ident), + self.command_handler_rs(command), + )); + } + for event in &self.events { + files.push(file( + &format!("src/handlers/{}.rs", event.module_ident), + self.event_handler_rs(event), + )); + } + if self.include_read_models { + files.push(file("src/read_models/mod.rs", self.read_models_mod_rs())); + } + + // GitOps deploy/promote charts and GitHub workflow files are a follow-up + // slice; the spec fields are accepted so the API is stable. Until then, + // surface a warning so the caller knows those artifacts were not emitted. + if self.gitops || self.gitops_promote.is_some() || self.github.is_some() { + warnings.push( + "GitOps and GitHub workflow generation are not yet ported into \ + distributed_tooling; those artifacts were not generated" + .to_string(), + ); + } + if let Some(github) = &self.github { + post_create_actions.push(PostCreateAction::EnsureGithubRepository { + repo: github.repository.clone(), + }); + } + + GeneratedProject { + files, + warnings, + post_create_actions, + } + } + + fn cargo_toml(&self) -> String { + let distributed_path = toml_string(&self.distributed_dependency_path); + let features = self + .distributed_features() + .into_iter() + .map(toml_string) + .collect::>() + .join(", "); + let axum = if self.transport == ServiceTransport::Knative { + "axum = \"0.7\"\n" + } else { + "" + }; + + format!( + r#"[package] +name = {package_name} +version = "0.1.0" +edition = "2021" + +[workspace] + +[dependencies] +distributed = {{ path = {distributed_path}, features = [{features}] }} +{axum}serde = {{ version = "1", features = ["derive"] }} +serde_json = "1" +tokio = {{ version = "1", features = ["macros", "net", "rt-multi-thread"] }} +"#, + package_name = toml_string(&self.names.package_name), + ) + } + + fn distributed_features(&self) -> Vec<&'static str> { + let mut features = Vec::new(); + match self.transport { + ServiceTransport::Http => features.push("http"), + ServiceTransport::Knative => features.push("http"), + } + match self.store { + StoreTarget::Postgres => features.push("postgres"), + StoreTarget::Sqlite => features.push("sqlite"), + StoreTarget::InMemory => {} + } + features + } + + fn lib_rs(&self) -> String { + let models = if !self.models.is_empty() { + "pub mod models;\n" + } else { + "" + }; + let read_models = if self.include_read_models { + "pub mod read_models;\n" + } else { + "" + }; + format!( + r#"pub mod handlers; +pub mod manifest; +{models}{read_models}pub mod service; + +pub use manifest::distributed_manifest; +"# + ) + } + + fn main_rs(&self) -> String { + match self.transport { + ServiceTransport::Http => format!( + r#"#[tokio::main] +async fn main() -> Result<(), Box> {{ + let addr = std::env::var("BIND_ADDR").unwrap_or_else(|_| "127.0.0.1:3000".to_string()); + let service = {crate_ident}::service::in_memory(); + distributed::microsvc::serve(service, &addr).await?; + Ok(()) +}} +"#, + crate_ident = self.names.crate_ident, + ), + ServiceTransport::Knative => format!( + r#"#[tokio::main] +async fn main() -> Result<(), Box> {{ + let addr = std::env::var("BIND_ADDR").unwrap_or_else(|_| "127.0.0.1:3000".to_string()); + let service = {crate_ident}::service::in_memory(); + let listener = tokio::net::TcpListener::bind(&addr).await?; + let app = distributed::microsvc::cloud_events_router(service); + axum::serve(listener, app).await?; + Ok(()) +}} +"#, + crate_ident = self.names.crate_ident, + ), + } + } + + fn manifest_rs(&self) -> String { + let read_model_import = if self.include_read_models && !self.read_models.is_empty() { + format!( + "use crate::read_models::{{{}}};\n\n", + self.read_models + .iter() + .map(|model| model.view_ident.as_str()) + .collect::>() + .join(", ") + ) + } else { + String::new() + }; + let read_model_registration = self + .read_models + .iter() + .map(|model| format!(" .read_model::<{}>()\n", model.view_ident)) + .collect::(); + format!( + r#"use distributed::{{ + DistributedProjectManifest, ServiceManifest, +}}; + +{read_model_import}pub fn distributed_manifest() -> DistributedProjectManifest {{ + DistributedProjectManifest::new({project_name}) +{read_model_registration} .service(crate::service::manifest()) +}} + +pub fn service_manifest() -> ServiceManifest {{ + crate::service::manifest() +}} +"#, + project_name = rust_string(&self.names.package_name), + ) + } + + fn service_rs(&self) -> String { + let registrations = self + .commands + .iter() + .map(|handler| format!(" command handlers::{},\n", handler.module_ident)) + .chain( + self.events + .iter() + .map(|handler| format!(" event handlers::{},\n", handler.module_ident)), + ) + .collect::(); + let manifest_commands = self + .commands + .iter() + .map(|handler| { + format!( + " .command(handlers::{}::COMMAND)\n", + handler.module_ident + ) + }) + .collect::(); + let manifest_events = self + .events + .iter() + .map(|handler| { + format!( + " .event(handlers::{}::EVENT)\n", + handler.module_ident + ) + }) + .collect::(); + let transport = match self.transport { + ServiceTransport::Http => "http", + ServiceTransport::Knative => "knative", + }; + + format!( + r#"use std::sync::Arc; + +use distributed::{{microsvc::Service, HashMapRepository, ServiceManifest}}; + +use crate::handlers; + +pub type ServiceRepo = HashMapRepository; + +pub fn in_memory() -> Arc> {{ + build(HashMapRepository::new()) +}} + +pub fn build(repo: ServiceRepo) -> Arc> {{ + Arc::new(distributed::register_handlers!( + Service::new().with_repo(repo), +{registrations} )) +}} + +pub fn manifest() -> ServiceManifest {{ + ServiceManifest::new({service_name}) +{manifest_commands}{manifest_events} .transport({transport}) +}} +"#, + service_name = rust_string(&self.names.package_name), + transport = rust_string(transport), + ) + } + + fn models_mod_rs(&self) -> String { + let modules = self + .models + .iter() + .map(|model| { + format!( + "pub mod {module_ident};\npub use {module_ident}::{type_ident};\n", + module_ident = model.module_ident, + type_ident = model.type_ident, + ) + }) + .collect::>() + .join(""); + + format!( + r#"{modules} +use serde::{{Deserialize, Serialize}}; + + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct CommandInput {{ + pub id: String, + #[serde(default)] + pub name: Option, +}} +"# + ) + } + + fn model_rs(&self, model: &ModelScaffold) -> String { + format!( + r#"use distributed::{{sourced, Entity, Snapshot}}; + +#[derive(Default, Snapshot)] +pub struct {model_struct} {{ + pub entity: Entity, + pub name: Option, + pub status: String, +}} + +#[sourced(entity)] +impl {model_struct} {{ + #[event({command_recorded_event})] + pub fn record_command(&mut self, command: String, id: String, name: Option) {{ + self.entity.set_id(&id); + if let Some(name) = name {{ + self.name = Some(name); + }} + self.status = command; + }} +}} +"#, + model_struct = model.type_ident, + command_recorded_event = + rust_string(&format!("{}.command_recorded", model.message_prefix)), + ) + } + + fn handlers_mod_rs(&self) -> String { + self.commands + .iter() + .chain(self.events.iter()) + .map(|handler| format!("pub mod {};\n", handler.module_ident)) + .collect() + } + + fn command_handler_rs(&self, handler: &MessageHandler) -> String { + if let Some(model) = self.command_model(handler) { + format!( + r#"use distributed::{{ + microsvc::{{Context, HandlerError}}, Aggregate, CommitBatch, StreamIdentity, StreamWrite, + TransactionalCommit, +}}; +use serde_json::{{json, Value}}; + +use crate::models::{{CommandInput, {model_type}}}; +use crate::service::ServiceRepo; + +pub const COMMAND: &str = {message_name}; +pub const MODEL: &str = {model_name}; + +pub fn guard(ctx: &Context) -> bool {{ + ctx.has_fields(&["id"]) +}} + +pub async fn handle(ctx: &Context<'_, ServiceRepo>) -> Result {{ + let input = ctx.input::()?; + let mut aggregate = {model_type}::default(); + aggregate.record_command(COMMAND.to_string(), input.id.clone(), input.name.clone())?; + let identity = StreamIdentity::new({model_type}::aggregate_type(), aggregate.entity().id())?; + let stream = StreamWrite::new(identity, aggregate.entity_mut()); + ctx.repo().commit_batch(CommitBatch::new(vec![stream])).await?; + Ok(json!({{ "command": COMMAND, "id": input.id, "model": MODEL, "name": input.name }})) +}} +"#, + model_type = model.type_ident, + message_name = rust_string(&handler.message_name), + model_name = rust_string(&model.name), + ) + } else { + format!( + r#"use distributed::microsvc::{{Context, HandlerError}}; +use serde_json::{{json, Value}}; + +use crate::service::ServiceRepo; + +pub const COMMAND: &str = {message_name}; + +pub fn guard(_ctx: &Context) -> bool {{ + true +}} + +pub async fn handle(ctx: &Context<'_, ServiceRepo>) -> Result {{ + let input = ctx.input::()?; + Ok(json!({{ "command": COMMAND, "input": input }})) +}} +"#, + message_name = rust_string(&handler.message_name), + ) + } + } + + fn event_handler_rs(&self, handler: &MessageHandler) -> String { + format!( + r#"use distributed::microsvc::{{Context, HandlerError}}; +use serde_json::{{json, Value}}; + +use crate::service::ServiceRepo; + +pub const EVENT: &str = {message_name}; + +pub fn guard(_ctx: &Context) -> bool {{ + true +}} + +pub async fn handle(ctx: &Context<'_, ServiceRepo>) -> Result {{ + let input = ctx.input::()?; + Ok(json!({{ "event": EVENT, "input": input }})) +}} +"#, + message_name = rust_string(&handler.message_name), + ) + } + + fn read_models_mod_rs(&self) -> String { + let views = self + .read_models + .iter() + .map(|model| { + format!( + r#"#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, ReadModel)] +#[table({table_name})] +pub struct {view_struct} {{ + #[id("id")] + pub id: String, + pub name: String, + pub status: String, +}} +"#, + table_name = rust_string(&model.table_name), + view_struct = model.view_ident, + ) + }) + .collect::>() + .join("\n"); + + format!( + r#"use distributed::ReadModel; +use serde::{{Deserialize, Serialize}}; + +{views} +"# + ) + } + + fn command_model(&self, handler: &MessageHandler) -> Option<&ModelScaffold> { + if self.models.is_empty() { + return None; + } + let message_model = message_owner(&handler.message_name); + self.models + .iter() + .find(|model| model.name == message_model) + .or_else(|| self.models.first()) + } +} + +fn file(path: &str, contents: String) -> GeneratedFile { + GeneratedFile { + path: path.to_string(), + contents, + mode: None, + } +} + +// --------------------------------------------------------------------------- +// Normalization helpers (the pure rules this crate owns) +// --------------------------------------------------------------------------- + +struct ScaffoldNames { + package_name: String, + crate_ident: String, + command_name: String, +} + +impl ScaffoldNames { + fn new(input: &str) -> Result { + let package_name = to_kebab_case(input); + if package_name.is_empty() { + return Err(ScaffoldError::new( + "service name must contain at least one ASCII letter or digit", + )); + } + let crate_ident = package_name.replace('-', "_"); + let command_name = format!("{crate_ident}.create"); + Ok(Self { + package_name, + crate_ident, + command_name, + }) + } +} + +#[derive(Clone, Debug)] +struct ModelScaffold { + name: String, + message_prefix: String, + module_ident: String, + type_ident: String, + view_ident: String, + table_name: String, +} + +impl ModelScaffold { + fn new(raw_name: &str) -> Result { + let name = to_kebab_case(raw_name); + if name.is_empty() { + return Err(ScaffoldError::new( + "model name must contain at least one ASCII letter or digit", + )); + } + let ident = name.replace('-', "_"); + let type_ident = to_pascal_case(&name); + let view_ident = format!("{type_ident}View"); + Ok(Self { + name: name.clone(), + message_prefix: name.clone(), + module_ident: ident.clone(), + type_ident, + view_ident, + table_name: format!("{ident}_views"), + }) + } +} + +fn model_scaffolds(raw_models: &[String]) -> Result, ScaffoldError> { + let mut seen = BTreeSet::new(); + let mut models = Vec::new(); + for raw_model in raw_models { + let model = ModelScaffold::new(raw_model)?; + if !seen.insert(model.name.clone()) { + return Err(ScaffoldError::new(format!( + "duplicate model `{}`", + model.name + ))); + } + models.push(model); + } + Ok(models) +} + +fn default_command_name(names: &ScaffoldNames, models: &[ModelScaffold]) -> String { + models + .first() + .map(|model| format!("{}.create", model.name)) + .unwrap_or_else(|| names.command_name.clone()) +} + +#[derive(Clone, Debug)] +struct MessageHandler { + message_name: String, + module_ident: String, +} + +fn message_handlers_with_modules( + names: Vec, + fallback_prefix: &str, + seen_modules: &mut BTreeSet, +) -> Result, ScaffoldError> { + let mut seen_names = BTreeSet::new(); + let mut handlers = Vec::new(); + for raw_name in names { + let message_name = raw_name.trim(); + validate_message_name(message_name, fallback_prefix)?; + if !seen_names.insert(message_name.to_string()) { + return Err(ScaffoldError::new(format!( + "duplicate {fallback_prefix} `{message_name}`" + ))); + } + let base_module = to_rust_ident(message_name, fallback_prefix); + let mut module_ident = base_module.clone(); + let mut suffix = 2; + while !seen_modules.insert(module_ident.clone()) { + module_ident = format!("{base_module}_{suffix}"); + suffix += 1; + } + handlers.push(MessageHandler { + message_name: message_name.to_string(), + module_ident, + }); + } + Ok(handlers) +} + +fn validate_message_name(name: &str, kind: &str) -> Result<(), ScaffoldError> { + if name.is_empty() { + return Err(ScaffoldError::new(format!("{kind} name cannot be empty"))); + } + if name.chars().any(char::is_control) { + return Err(ScaffoldError::new(format!( + "{kind} `{name}` contains a control character" + ))); + } + Ok(()) +} + +fn message_owner(message_name: &str) -> String { + message_name + .split('.') + .find(|part| !part.is_empty()) + .map(k8s_name) + .unwrap_or_else(|| "message".to_string()) +} + +fn k8s_name(value: &str) -> String { + let name = to_kebab_case(value); + if name.is_empty() { + "generated".to_string() + } else { + name + } +} + +fn to_rust_ident(value: &str, fallback_prefix: &str) -> String { + let mut ident = String::new(); + let mut last_was_separator = false; + for char in value.chars() { + if char.is_ascii_alphanumeric() { + ident.push(char.to_ascii_lowercase()); + last_was_separator = false; + } else if !last_was_separator { + ident.push('_'); + last_was_separator = true; + } + } + while ident.ends_with('_') { + ident.pop(); + } + while ident.starts_with('_') { + ident.remove(0); + } + if ident.is_empty() { + ident = fallback_prefix.to_string(); + } + if ident + .chars() + .next() + .is_some_and(|char| char.is_ascii_digit()) + || is_rust_keyword(&ident) + { + ident = format!("{fallback_prefix}_{ident}"); + } + ident +} + +fn is_rust_keyword(value: &str) -> bool { + matches!( + value, + "as" | "break" + | "const" + | "continue" + | "crate" + | "else" + | "enum" + | "extern" + | "false" + | "fn" + | "for" + | "if" + | "impl" + | "in" + | "let" + | "loop" + | "match" + | "mod" + | "move" + | "mut" + | "pub" + | "ref" + | "return" + | "self" + | "Self" + | "static" + | "struct" + | "super" + | "trait" + | "true" + | "type" + | "unsafe" + | "use" + | "where" + | "while" + | "async" + | "await" + | "dyn" + ) +} + +fn to_kebab_case(input: &str) -> String { + let mut out = String::new(); + let mut last_was_separator = true; + for char in input.chars() { + if char.is_ascii_alphanumeric() { + out.push(char.to_ascii_lowercase()); + last_was_separator = false; + } else if !last_was_separator { + out.push('-'); + last_was_separator = true; + } + } + while out.ends_with('-') { + out.pop(); + } + out +} + +fn to_pascal_case(input: &str) -> String { + input + .split(['-', '_']) + .filter(|part| !part.is_empty()) + .map(|part| { + let mut chars = part.chars(); + let Some(first) = chars.next() else { + return String::new(); + }; + let mut out = String::new(); + out.push(first.to_ascii_uppercase()); + out.extend(chars); + out + }) + .collect() +} + +fn toml_string(value: impl AsRef) -> String { + serde_json::to_string(value.as_ref()).expect("string serialization should succeed") +} + +fn rust_string(value: &str) -> String { + toml_string(value) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn spec(name: &str) -> ServiceScaffoldSpec { + ServiceScaffoldSpec { + name: name.to_string(), + transport: ServiceTransport::Http, + store: StoreTarget::Postgres, + bus: None, + models: Vec::new(), + read_models: false, + commands: Vec::new(), + events: Vec::new(), + distributed_dependency_path: "../distributed".to_string(), + gitops: false, + gitops_promote: None, + github: None, + } + } + + fn paths(project: &GeneratedProject) -> Vec<&str> { + project.files.iter().map(|f| f.path.as_str()).collect() + } + + fn contents<'a>(project: &'a GeneratedProject, path: &str) -> &'a str { + project + .files + .iter() + .find(|f| f.path == path) + .map(|f| f.contents.as_str()) + .unwrap_or_else(|| panic!("missing file {path}")) + } + + #[test] + fn generates_the_core_service_crate() { + let project = generate_service_scaffold(spec("orders")).unwrap(); + let paths = paths(&project); + for expected in [ + "Cargo.toml", + "src/lib.rs", + "src/main.rs", + "src/manifest.rs", + "src/service.rs", + "src/handlers/mod.rs", + ] { + assert!(paths.contains(&expected), "missing {expected} in {paths:?}"); + } + // A default command handler is derived when none is given. + assert!(paths + .iter() + .any(|p| p.starts_with("src/handlers/") && *p != "src/handlers/mod.rs")); + } + + #[test] + fn service_uses_the_new_builder_api() { + let project = generate_service_scaffold(spec("orders")).unwrap(); + let service = contents(&project, "src/service.rs"); + assert!( + service.contains("Service::new().with_repo(repo)"), + "service.rs should use the new builder API:\n{service}" + ); + assert!(!service.contains("Service::with_repo(")); + } + + #[test] + fn cargo_features_track_transport_and_store() { + let mut s = spec("orders"); + s.store = StoreTarget::Sqlite; + let project = generate_service_scaffold(s).unwrap(); + let cargo = contents(&project, "Cargo.toml"); + assert!(cargo.contains("\"http\"")); + assert!(cargo.contains("\"sqlite\"")); + } + + #[test] + fn read_models_and_models_emit_modules() { + let mut s = spec("orders"); + s.models = vec!["order".to_string()]; + s.read_models = true; + let project = generate_service_scaffold(s).unwrap(); + let paths = paths(&project); + assert!(paths.contains(&"src/models/mod.rs")); + assert!(paths.contains(&"src/models/order.rs")); + assert!(paths.contains(&"src/read_models/mod.rs")); + } + + #[test] + fn github_yields_a_post_create_action_and_warning() { + let mut s = spec("orders"); + s.github = Some(GithubScaffoldSpec { + repository: GithubRepo::parse("hops-ops/orders").unwrap(), + preview_environment_repository: None, + promote_environment_repository: None, + }); + let project = generate_service_scaffold(s).unwrap(); + assert_eq!( + project.post_create_actions, + vec![PostCreateAction::EnsureGithubRepository { + repo: GithubRepo { + owner: "hops-ops".to_string(), + repo: "orders".to_string(), + }, + }] + ); + assert!(!project.warnings.is_empty()); + } + + #[test] + fn invalid_name_is_an_error() { + assert!(generate_service_scaffold(spec(" ")).is_err()); + } + + #[test] + fn github_repo_parse_rejects_bad_input() { + assert!(GithubRepo::parse("no-slash").is_err()); + assert!(GithubRepo::parse("a/b/c").is_err()); + assert_eq!(GithubRepo::parse("o/r").unwrap().slug(), "o/r"); + } +} diff --git a/distributed_tooling/src/lib.rs b/distributed_tooling/src/lib.rs new file mode 100644 index 0000000..5740c85 --- /dev/null +++ b/distributed_tooling/src/lib.rs @@ -0,0 +1,194 @@ +//! Deterministic service-scaffold generation for Distributed services. +//! +//! This crate owns the *pure* generation rules for a Distributed service project: +//! Cargo layout, Rust source templates, manifest wiring, read-model/handler +//! defaults, GitOps/Knative inference, and GitHub workflow contents. It performs +//! **no** filesystem, network, or CLI side effects — [`generate_service_scaffold`] +//! takes a [`ServiceScaffoldSpec`] and returns a [`GeneratedProject`] describing +//! the files to write and any follow-up actions to perform. +//! +//! A CLI such as `hops-cli` maps its flags to a [`ServiceScaffoldSpec`], calls +//! this crate, then decides where to write files, whether to overwrite, and +//! whether to run the [`PostCreateAction`]s (e.g. `gh repo create`). + +mod generate; + +pub use generate::generate_service_scaffold; + +/// What to scaffold. The pure input to [`generate_service_scaffold`]. +/// +/// `name` and the raw `models`/`commands`/`events` strings are normalized by the +/// generator (kebab/pascal/ident casing, validation, dedup) — that normalization +/// is part of the rules this crate owns. +#[derive(Clone, Debug)] +pub struct ServiceScaffoldSpec { + /// Service / package name (free-form; normalized to a kebab package name). + pub name: String, + /// Runtime transport to scaffold. + pub transport: ServiceTransport, + /// Read-model / schema storage target. + pub store: StoreTarget, + /// Optional message bus backend. + pub bus: Option, + /// Aggregate model names to scaffold (raw; may be empty). + pub models: Vec, + /// Generate placeholder read-model modules and register them in the manifest. + pub read_models: bool, + /// Command handler message names (raw; empty → a default command is derived). + pub commands: Vec, + /// Event handler message names (raw; may be empty). + pub events: Vec, + /// Relative path (from the generated project dir) to the local `distributed` + /// crate, used in the generated `Cargo.toml` dependency. + pub distributed_dependency_path: String, + /// Generate a Helm deploy chart under `.gitops/deploy`. + pub gitops: bool, + /// Generate a GitOps promotion chart for Argo CD or Flux. + pub gitops_promote: Option, + /// GitHub repository + release/GitOps workflow scaffolding. + pub github: Option, +} + +/// Runtime transport for the scaffolded service. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ServiceTransport { + /// Axum HTTP transport (`microsvc::serve`). + Http, + /// Knative / CloudEvents HTTP ingress (`cloud_events_router`). + Knative, +} + +/// Read-model / schema storage target. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StoreTarget { + /// Postgres-backed persistence (`postgres` feature). + Postgres, + /// SQLite-backed persistence (`sqlite` feature). + Sqlite, + /// In-memory only (no persistence feature). + InMemory, +} + +/// Message bus backend to scaffold. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BusTarget { + /// RabbitMQ. + Rabbitmq, + /// Kafka. + Kafka, + /// Postgres-backed bus. + Psql, + /// NATS JetStream. + Nats, +} + +impl BusTarget { + /// The lowercase kind string used in generated env/manifest values. + pub fn kind(self) -> &'static str { + match self { + BusTarget::Rabbitmq => "rabbitmq", + BusTarget::Kafka => "kafka", + BusTarget::Psql => "psql", + BusTarget::Nats => "nats", + } + } +} + +/// GitOps promotion flavor. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum GitopsPromoteTarget { + /// Argo CD `Application`. + Argo, + /// Flux `HelmRelease`. + Flux, +} + +/// GitHub repository + workflow scaffolding inputs. +#[derive(Clone, Debug)] +pub struct GithubScaffoldSpec { + /// The service's own GitHub repository. + pub repository: GithubRepo, + /// Optional preview-environment GitOps repository. + pub preview_environment_repository: Option, + /// Optional permanent-environment GitOps repository. + pub promote_environment_repository: Option, +} + +/// An `owner/repo` GitHub identifier. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GithubRepo { + /// Repository owner (user or org). + pub owner: String, + /// Repository name. + pub repo: String, +} + +impl GithubRepo { + /// Parse an `owner/repo` string, validating both halves. + pub fn parse(raw: &str) -> Result { + generate::parse_github_repo(raw) + } + + /// `owner/repo`. + pub fn slug(&self) -> String { + format!("{}/{}", self.owner, self.repo) + } +} + +/// The result of generating a scaffold: the files to write, advisory warnings, +/// and side effects for the caller to perform. Filesystem-agnostic. +#[derive(Clone, Debug, Default)] +pub struct GeneratedProject { + /// Files to write, with paths relative to the project directory. + pub files: Vec, + /// Non-fatal advisory messages (e.g. a requested feature not yet generated). + pub warnings: Vec, + /// Side effects the caller should perform after writing files. + pub post_create_actions: Vec, +} + +/// A single generated file: a relative path and its contents. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GeneratedFile { + /// Path relative to the project directory (forward slashes). + pub path: String, + /// File contents. + pub contents: String, + /// Optional file mode hint (e.g. executable). `None` = default text file. + pub mode: Option, +} + +/// File mode hint for a [`GeneratedFile`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FileMode { + /// The file should be marked executable. + Executable, +} + +/// A side effect the caller should perform after writing the generated files. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum PostCreateAction { + /// Ensure the GitHub repository exists (e.g. `gh repo view` / `gh repo create`). + EnsureGithubRepository { + /// The repository to ensure. + repo: GithubRepo, + }, +} + +/// A scaffold generation error (bad spec value). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScaffoldError(pub String); + +impl std::fmt::Display for ScaffoldError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for ScaffoldError {} + +impl ScaffoldError { + pub(crate) fn new(message: impl Into) -> Self { + Self(message.into()) + } +} From b79abb0da7c70d854053b34a9ac1ecffde868fd1 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 4 Jun 2026 22:35:00 -0500 Subject: [PATCH 19/24] refactor: split distributed_tooling generate.rs into focused modules generate/ becomes a module: mod.rs (Scaffold + orchestration + entry + tests), names.rs (name/message normalization + validation), service_crate.rs (the Rust templates as impl Scaffold), github.rs (repo parsing). Sets up gitops.rs and the GitHub workflow templates as their own files for the next slice. No behavior change; 7 tests green. Implements [[specs/distributed-service-scaffold-tooling]] Co-Authored-By: Claude Opus 4.8 (1M context) --- distributed_tooling/src/generate.rs | 939 ------------------ distributed_tooling/src/generate/github.rs | 28 + distributed_tooling/src/generate/mod.rs | 284 ++++++ distributed_tooling/src/generate/names.rs | 276 +++++ .../src/generate/service_crate.rs | 392 ++++++++ 5 files changed, 980 insertions(+), 939 deletions(-) delete mode 100644 distributed_tooling/src/generate.rs create mode 100644 distributed_tooling/src/generate/github.rs create mode 100644 distributed_tooling/src/generate/mod.rs create mode 100644 distributed_tooling/src/generate/names.rs create mode 100644 distributed_tooling/src/generate/service_crate.rs diff --git a/distributed_tooling/src/generate.rs b/distributed_tooling/src/generate.rs deleted file mode 100644 index 92d3cb4..0000000 --- a/distributed_tooling/src/generate.rs +++ /dev/null @@ -1,939 +0,0 @@ -//! Internal generation: build a normalized `Scaffold` from a spec and render the -//! project files. Pure — every method returns `String`s; nothing touches the -//! filesystem. Ported from the original `hops-cli service scaffold` command. - -use std::collections::BTreeSet; - -use crate::{ - BusTarget, GeneratedFile, GeneratedProject, GithubRepo, GithubScaffoldSpec, - GitopsPromoteTarget, PostCreateAction, ScaffoldError, ServiceScaffoldSpec, ServiceTransport, - StoreTarget, -}; - -/// Generate a Distributed service project from a spec. The public entry point. -pub fn generate_service_scaffold( - spec: ServiceScaffoldSpec, -) -> Result { - Ok(Scaffold::from_spec(spec)?.generate()) -} - -/// Parse an `owner/repo` string (used by [`GithubRepo::parse`]). -pub(crate) fn parse_github_repo(raw: &str) -> Result { - let trimmed = raw.trim(); - let Some((owner, repo)) = trimmed.split_once('/') else { - return Err(ScaffoldError::new("repository must be in OWNER/REPO form")); - }; - if owner.is_empty() || repo.is_empty() || repo.contains('/') { - return Err(ScaffoldError::new("repository must be in OWNER/REPO form")); - } - let valid = [owner, repo].into_iter().all(|part| { - part.chars() - .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.') - }); - if !valid { - return Err(ScaffoldError::new( - "repository contains unsupported GitHub characters", - )); - } - Ok(GithubRepo { - owner: owner.to_string(), - repo: repo.to_string(), - }) -} - -struct Scaffold { - names: ScaffoldNames, - distributed_dependency_path: String, - transport: ServiceTransport, - store: StoreTarget, - #[allow(dead_code)] // used by GitOps/Knative generation (follow-up slice) - bus: Option, - include_read_models: bool, - gitops: bool, - gitops_promote: Option, - github: Option, - models: Vec, - read_models: Vec, - commands: Vec, - events: Vec, -} - -impl Scaffold { - fn from_spec(spec: ServiceScaffoldSpec) -> Result { - let names = ScaffoldNames::new(&spec.name)?; - let models = model_scaffolds(&spec.models)?; - let read_models = if spec.read_models { - if models.is_empty() { - vec![ModelScaffold::new(&names.package_name)?] - } else { - models.clone() - } - } else { - Vec::new() - }; - let mut module_idents = BTreeSet::new(); - let commands = message_handlers_with_modules( - if spec.commands.is_empty() { - vec![default_command_name(&names, &models)] - } else { - spec.commands.clone() - }, - "command", - &mut module_idents, - )?; - let events = - message_handlers_with_modules(spec.events.clone(), "event", &mut module_idents)?; - Ok(Self { - names, - distributed_dependency_path: spec.distributed_dependency_path, - transport: spec.transport, - store: spec.store, - bus: spec.bus, - include_read_models: spec.read_models, - gitops: spec.gitops, - gitops_promote: spec.gitops_promote, - github: spec.github, - models, - read_models, - commands, - events, - }) - } - - fn generate(self) -> GeneratedProject { - let mut files = Vec::new(); - let mut warnings = Vec::new(); - let mut post_create_actions = Vec::new(); - - files.push(file("Cargo.toml", self.cargo_toml())); - files.push(file("src/lib.rs", self.lib_rs())); - files.push(file("src/main.rs", self.main_rs())); - files.push(file("src/manifest.rs", self.manifest_rs())); - files.push(file("src/service.rs", self.service_rs())); - if !self.models.is_empty() { - files.push(file("src/models/mod.rs", self.models_mod_rs())); - for model in &self.models { - files.push(file( - &format!("src/models/{}.rs", model.module_ident), - self.model_rs(model), - )); - } - } - files.push(file("src/handlers/mod.rs", self.handlers_mod_rs())); - for command in &self.commands { - files.push(file( - &format!("src/handlers/{}.rs", command.module_ident), - self.command_handler_rs(command), - )); - } - for event in &self.events { - files.push(file( - &format!("src/handlers/{}.rs", event.module_ident), - self.event_handler_rs(event), - )); - } - if self.include_read_models { - files.push(file("src/read_models/mod.rs", self.read_models_mod_rs())); - } - - // GitOps deploy/promote charts and GitHub workflow files are a follow-up - // slice; the spec fields are accepted so the API is stable. Until then, - // surface a warning so the caller knows those artifacts were not emitted. - if self.gitops || self.gitops_promote.is_some() || self.github.is_some() { - warnings.push( - "GitOps and GitHub workflow generation are not yet ported into \ - distributed_tooling; those artifacts were not generated" - .to_string(), - ); - } - if let Some(github) = &self.github { - post_create_actions.push(PostCreateAction::EnsureGithubRepository { - repo: github.repository.clone(), - }); - } - - GeneratedProject { - files, - warnings, - post_create_actions, - } - } - - fn cargo_toml(&self) -> String { - let distributed_path = toml_string(&self.distributed_dependency_path); - let features = self - .distributed_features() - .into_iter() - .map(toml_string) - .collect::>() - .join(", "); - let axum = if self.transport == ServiceTransport::Knative { - "axum = \"0.7\"\n" - } else { - "" - }; - - format!( - r#"[package] -name = {package_name} -version = "0.1.0" -edition = "2021" - -[workspace] - -[dependencies] -distributed = {{ path = {distributed_path}, features = [{features}] }} -{axum}serde = {{ version = "1", features = ["derive"] }} -serde_json = "1" -tokio = {{ version = "1", features = ["macros", "net", "rt-multi-thread"] }} -"#, - package_name = toml_string(&self.names.package_name), - ) - } - - fn distributed_features(&self) -> Vec<&'static str> { - let mut features = Vec::new(); - match self.transport { - ServiceTransport::Http => features.push("http"), - ServiceTransport::Knative => features.push("http"), - } - match self.store { - StoreTarget::Postgres => features.push("postgres"), - StoreTarget::Sqlite => features.push("sqlite"), - StoreTarget::InMemory => {} - } - features - } - - fn lib_rs(&self) -> String { - let models = if !self.models.is_empty() { - "pub mod models;\n" - } else { - "" - }; - let read_models = if self.include_read_models { - "pub mod read_models;\n" - } else { - "" - }; - format!( - r#"pub mod handlers; -pub mod manifest; -{models}{read_models}pub mod service; - -pub use manifest::distributed_manifest; -"# - ) - } - - fn main_rs(&self) -> String { - match self.transport { - ServiceTransport::Http => format!( - r#"#[tokio::main] -async fn main() -> Result<(), Box> {{ - let addr = std::env::var("BIND_ADDR").unwrap_or_else(|_| "127.0.0.1:3000".to_string()); - let service = {crate_ident}::service::in_memory(); - distributed::microsvc::serve(service, &addr).await?; - Ok(()) -}} -"#, - crate_ident = self.names.crate_ident, - ), - ServiceTransport::Knative => format!( - r#"#[tokio::main] -async fn main() -> Result<(), Box> {{ - let addr = std::env::var("BIND_ADDR").unwrap_or_else(|_| "127.0.0.1:3000".to_string()); - let service = {crate_ident}::service::in_memory(); - let listener = tokio::net::TcpListener::bind(&addr).await?; - let app = distributed::microsvc::cloud_events_router(service); - axum::serve(listener, app).await?; - Ok(()) -}} -"#, - crate_ident = self.names.crate_ident, - ), - } - } - - fn manifest_rs(&self) -> String { - let read_model_import = if self.include_read_models && !self.read_models.is_empty() { - format!( - "use crate::read_models::{{{}}};\n\n", - self.read_models - .iter() - .map(|model| model.view_ident.as_str()) - .collect::>() - .join(", ") - ) - } else { - String::new() - }; - let read_model_registration = self - .read_models - .iter() - .map(|model| format!(" .read_model::<{}>()\n", model.view_ident)) - .collect::(); - format!( - r#"use distributed::{{ - DistributedProjectManifest, ServiceManifest, -}}; - -{read_model_import}pub fn distributed_manifest() -> DistributedProjectManifest {{ - DistributedProjectManifest::new({project_name}) -{read_model_registration} .service(crate::service::manifest()) -}} - -pub fn service_manifest() -> ServiceManifest {{ - crate::service::manifest() -}} -"#, - project_name = rust_string(&self.names.package_name), - ) - } - - fn service_rs(&self) -> String { - let registrations = self - .commands - .iter() - .map(|handler| format!(" command handlers::{},\n", handler.module_ident)) - .chain( - self.events - .iter() - .map(|handler| format!(" event handlers::{},\n", handler.module_ident)), - ) - .collect::(); - let manifest_commands = self - .commands - .iter() - .map(|handler| { - format!( - " .command(handlers::{}::COMMAND)\n", - handler.module_ident - ) - }) - .collect::(); - let manifest_events = self - .events - .iter() - .map(|handler| { - format!( - " .event(handlers::{}::EVENT)\n", - handler.module_ident - ) - }) - .collect::(); - let transport = match self.transport { - ServiceTransport::Http => "http", - ServiceTransport::Knative => "knative", - }; - - format!( - r#"use std::sync::Arc; - -use distributed::{{microsvc::Service, HashMapRepository, ServiceManifest}}; - -use crate::handlers; - -pub type ServiceRepo = HashMapRepository; - -pub fn in_memory() -> Arc> {{ - build(HashMapRepository::new()) -}} - -pub fn build(repo: ServiceRepo) -> Arc> {{ - Arc::new(distributed::register_handlers!( - Service::new().with_repo(repo), -{registrations} )) -}} - -pub fn manifest() -> ServiceManifest {{ - ServiceManifest::new({service_name}) -{manifest_commands}{manifest_events} .transport({transport}) -}} -"#, - service_name = rust_string(&self.names.package_name), - transport = rust_string(transport), - ) - } - - fn models_mod_rs(&self) -> String { - let modules = self - .models - .iter() - .map(|model| { - format!( - "pub mod {module_ident};\npub use {module_ident}::{type_ident};\n", - module_ident = model.module_ident, - type_ident = model.type_ident, - ) - }) - .collect::>() - .join(""); - - format!( - r#"{modules} -use serde::{{Deserialize, Serialize}}; - - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct CommandInput {{ - pub id: String, - #[serde(default)] - pub name: Option, -}} -"# - ) - } - - fn model_rs(&self, model: &ModelScaffold) -> String { - format!( - r#"use distributed::{{sourced, Entity, Snapshot}}; - -#[derive(Default, Snapshot)] -pub struct {model_struct} {{ - pub entity: Entity, - pub name: Option, - pub status: String, -}} - -#[sourced(entity)] -impl {model_struct} {{ - #[event({command_recorded_event})] - pub fn record_command(&mut self, command: String, id: String, name: Option) {{ - self.entity.set_id(&id); - if let Some(name) = name {{ - self.name = Some(name); - }} - self.status = command; - }} -}} -"#, - model_struct = model.type_ident, - command_recorded_event = - rust_string(&format!("{}.command_recorded", model.message_prefix)), - ) - } - - fn handlers_mod_rs(&self) -> String { - self.commands - .iter() - .chain(self.events.iter()) - .map(|handler| format!("pub mod {};\n", handler.module_ident)) - .collect() - } - - fn command_handler_rs(&self, handler: &MessageHandler) -> String { - if let Some(model) = self.command_model(handler) { - format!( - r#"use distributed::{{ - microsvc::{{Context, HandlerError}}, Aggregate, CommitBatch, StreamIdentity, StreamWrite, - TransactionalCommit, -}}; -use serde_json::{{json, Value}}; - -use crate::models::{{CommandInput, {model_type}}}; -use crate::service::ServiceRepo; - -pub const COMMAND: &str = {message_name}; -pub const MODEL: &str = {model_name}; - -pub fn guard(ctx: &Context) -> bool {{ - ctx.has_fields(&["id"]) -}} - -pub async fn handle(ctx: &Context<'_, ServiceRepo>) -> Result {{ - let input = ctx.input::()?; - let mut aggregate = {model_type}::default(); - aggregate.record_command(COMMAND.to_string(), input.id.clone(), input.name.clone())?; - let identity = StreamIdentity::new({model_type}::aggregate_type(), aggregate.entity().id())?; - let stream = StreamWrite::new(identity, aggregate.entity_mut()); - ctx.repo().commit_batch(CommitBatch::new(vec![stream])).await?; - Ok(json!({{ "command": COMMAND, "id": input.id, "model": MODEL, "name": input.name }})) -}} -"#, - model_type = model.type_ident, - message_name = rust_string(&handler.message_name), - model_name = rust_string(&model.name), - ) - } else { - format!( - r#"use distributed::microsvc::{{Context, HandlerError}}; -use serde_json::{{json, Value}}; - -use crate::service::ServiceRepo; - -pub const COMMAND: &str = {message_name}; - -pub fn guard(_ctx: &Context) -> bool {{ - true -}} - -pub async fn handle(ctx: &Context<'_, ServiceRepo>) -> Result {{ - let input = ctx.input::()?; - Ok(json!({{ "command": COMMAND, "input": input }})) -}} -"#, - message_name = rust_string(&handler.message_name), - ) - } - } - - fn event_handler_rs(&self, handler: &MessageHandler) -> String { - format!( - r#"use distributed::microsvc::{{Context, HandlerError}}; -use serde_json::{{json, Value}}; - -use crate::service::ServiceRepo; - -pub const EVENT: &str = {message_name}; - -pub fn guard(_ctx: &Context) -> bool {{ - true -}} - -pub async fn handle(ctx: &Context<'_, ServiceRepo>) -> Result {{ - let input = ctx.input::()?; - Ok(json!({{ "event": EVENT, "input": input }})) -}} -"#, - message_name = rust_string(&handler.message_name), - ) - } - - fn read_models_mod_rs(&self) -> String { - let views = self - .read_models - .iter() - .map(|model| { - format!( - r#"#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, ReadModel)] -#[table({table_name})] -pub struct {view_struct} {{ - #[id("id")] - pub id: String, - pub name: String, - pub status: String, -}} -"#, - table_name = rust_string(&model.table_name), - view_struct = model.view_ident, - ) - }) - .collect::>() - .join("\n"); - - format!( - r#"use distributed::ReadModel; -use serde::{{Deserialize, Serialize}}; - -{views} -"# - ) - } - - fn command_model(&self, handler: &MessageHandler) -> Option<&ModelScaffold> { - if self.models.is_empty() { - return None; - } - let message_model = message_owner(&handler.message_name); - self.models - .iter() - .find(|model| model.name == message_model) - .or_else(|| self.models.first()) - } -} - -fn file(path: &str, contents: String) -> GeneratedFile { - GeneratedFile { - path: path.to_string(), - contents, - mode: None, - } -} - -// --------------------------------------------------------------------------- -// Normalization helpers (the pure rules this crate owns) -// --------------------------------------------------------------------------- - -struct ScaffoldNames { - package_name: String, - crate_ident: String, - command_name: String, -} - -impl ScaffoldNames { - fn new(input: &str) -> Result { - let package_name = to_kebab_case(input); - if package_name.is_empty() { - return Err(ScaffoldError::new( - "service name must contain at least one ASCII letter or digit", - )); - } - let crate_ident = package_name.replace('-', "_"); - let command_name = format!("{crate_ident}.create"); - Ok(Self { - package_name, - crate_ident, - command_name, - }) - } -} - -#[derive(Clone, Debug)] -struct ModelScaffold { - name: String, - message_prefix: String, - module_ident: String, - type_ident: String, - view_ident: String, - table_name: String, -} - -impl ModelScaffold { - fn new(raw_name: &str) -> Result { - let name = to_kebab_case(raw_name); - if name.is_empty() { - return Err(ScaffoldError::new( - "model name must contain at least one ASCII letter or digit", - )); - } - let ident = name.replace('-', "_"); - let type_ident = to_pascal_case(&name); - let view_ident = format!("{type_ident}View"); - Ok(Self { - name: name.clone(), - message_prefix: name.clone(), - module_ident: ident.clone(), - type_ident, - view_ident, - table_name: format!("{ident}_views"), - }) - } -} - -fn model_scaffolds(raw_models: &[String]) -> Result, ScaffoldError> { - let mut seen = BTreeSet::new(); - let mut models = Vec::new(); - for raw_model in raw_models { - let model = ModelScaffold::new(raw_model)?; - if !seen.insert(model.name.clone()) { - return Err(ScaffoldError::new(format!( - "duplicate model `{}`", - model.name - ))); - } - models.push(model); - } - Ok(models) -} - -fn default_command_name(names: &ScaffoldNames, models: &[ModelScaffold]) -> String { - models - .first() - .map(|model| format!("{}.create", model.name)) - .unwrap_or_else(|| names.command_name.clone()) -} - -#[derive(Clone, Debug)] -struct MessageHandler { - message_name: String, - module_ident: String, -} - -fn message_handlers_with_modules( - names: Vec, - fallback_prefix: &str, - seen_modules: &mut BTreeSet, -) -> Result, ScaffoldError> { - let mut seen_names = BTreeSet::new(); - let mut handlers = Vec::new(); - for raw_name in names { - let message_name = raw_name.trim(); - validate_message_name(message_name, fallback_prefix)?; - if !seen_names.insert(message_name.to_string()) { - return Err(ScaffoldError::new(format!( - "duplicate {fallback_prefix} `{message_name}`" - ))); - } - let base_module = to_rust_ident(message_name, fallback_prefix); - let mut module_ident = base_module.clone(); - let mut suffix = 2; - while !seen_modules.insert(module_ident.clone()) { - module_ident = format!("{base_module}_{suffix}"); - suffix += 1; - } - handlers.push(MessageHandler { - message_name: message_name.to_string(), - module_ident, - }); - } - Ok(handlers) -} - -fn validate_message_name(name: &str, kind: &str) -> Result<(), ScaffoldError> { - if name.is_empty() { - return Err(ScaffoldError::new(format!("{kind} name cannot be empty"))); - } - if name.chars().any(char::is_control) { - return Err(ScaffoldError::new(format!( - "{kind} `{name}` contains a control character" - ))); - } - Ok(()) -} - -fn message_owner(message_name: &str) -> String { - message_name - .split('.') - .find(|part| !part.is_empty()) - .map(k8s_name) - .unwrap_or_else(|| "message".to_string()) -} - -fn k8s_name(value: &str) -> String { - let name = to_kebab_case(value); - if name.is_empty() { - "generated".to_string() - } else { - name - } -} - -fn to_rust_ident(value: &str, fallback_prefix: &str) -> String { - let mut ident = String::new(); - let mut last_was_separator = false; - for char in value.chars() { - if char.is_ascii_alphanumeric() { - ident.push(char.to_ascii_lowercase()); - last_was_separator = false; - } else if !last_was_separator { - ident.push('_'); - last_was_separator = true; - } - } - while ident.ends_with('_') { - ident.pop(); - } - while ident.starts_with('_') { - ident.remove(0); - } - if ident.is_empty() { - ident = fallback_prefix.to_string(); - } - if ident - .chars() - .next() - .is_some_and(|char| char.is_ascii_digit()) - || is_rust_keyword(&ident) - { - ident = format!("{fallback_prefix}_{ident}"); - } - ident -} - -fn is_rust_keyword(value: &str) -> bool { - matches!( - value, - "as" | "break" - | "const" - | "continue" - | "crate" - | "else" - | "enum" - | "extern" - | "false" - | "fn" - | "for" - | "if" - | "impl" - | "in" - | "let" - | "loop" - | "match" - | "mod" - | "move" - | "mut" - | "pub" - | "ref" - | "return" - | "self" - | "Self" - | "static" - | "struct" - | "super" - | "trait" - | "true" - | "type" - | "unsafe" - | "use" - | "where" - | "while" - | "async" - | "await" - | "dyn" - ) -} - -fn to_kebab_case(input: &str) -> String { - let mut out = String::new(); - let mut last_was_separator = true; - for char in input.chars() { - if char.is_ascii_alphanumeric() { - out.push(char.to_ascii_lowercase()); - last_was_separator = false; - } else if !last_was_separator { - out.push('-'); - last_was_separator = true; - } - } - while out.ends_with('-') { - out.pop(); - } - out -} - -fn to_pascal_case(input: &str) -> String { - input - .split(['-', '_']) - .filter(|part| !part.is_empty()) - .map(|part| { - let mut chars = part.chars(); - let Some(first) = chars.next() else { - return String::new(); - }; - let mut out = String::new(); - out.push(first.to_ascii_uppercase()); - out.extend(chars); - out - }) - .collect() -} - -fn toml_string(value: impl AsRef) -> String { - serde_json::to_string(value.as_ref()).expect("string serialization should succeed") -} - -fn rust_string(value: &str) -> String { - toml_string(value) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn spec(name: &str) -> ServiceScaffoldSpec { - ServiceScaffoldSpec { - name: name.to_string(), - transport: ServiceTransport::Http, - store: StoreTarget::Postgres, - bus: None, - models: Vec::new(), - read_models: false, - commands: Vec::new(), - events: Vec::new(), - distributed_dependency_path: "../distributed".to_string(), - gitops: false, - gitops_promote: None, - github: None, - } - } - - fn paths(project: &GeneratedProject) -> Vec<&str> { - project.files.iter().map(|f| f.path.as_str()).collect() - } - - fn contents<'a>(project: &'a GeneratedProject, path: &str) -> &'a str { - project - .files - .iter() - .find(|f| f.path == path) - .map(|f| f.contents.as_str()) - .unwrap_or_else(|| panic!("missing file {path}")) - } - - #[test] - fn generates_the_core_service_crate() { - let project = generate_service_scaffold(spec("orders")).unwrap(); - let paths = paths(&project); - for expected in [ - "Cargo.toml", - "src/lib.rs", - "src/main.rs", - "src/manifest.rs", - "src/service.rs", - "src/handlers/mod.rs", - ] { - assert!(paths.contains(&expected), "missing {expected} in {paths:?}"); - } - // A default command handler is derived when none is given. - assert!(paths - .iter() - .any(|p| p.starts_with("src/handlers/") && *p != "src/handlers/mod.rs")); - } - - #[test] - fn service_uses_the_new_builder_api() { - let project = generate_service_scaffold(spec("orders")).unwrap(); - let service = contents(&project, "src/service.rs"); - assert!( - service.contains("Service::new().with_repo(repo)"), - "service.rs should use the new builder API:\n{service}" - ); - assert!(!service.contains("Service::with_repo(")); - } - - #[test] - fn cargo_features_track_transport_and_store() { - let mut s = spec("orders"); - s.store = StoreTarget::Sqlite; - let project = generate_service_scaffold(s).unwrap(); - let cargo = contents(&project, "Cargo.toml"); - assert!(cargo.contains("\"http\"")); - assert!(cargo.contains("\"sqlite\"")); - } - - #[test] - fn read_models_and_models_emit_modules() { - let mut s = spec("orders"); - s.models = vec!["order".to_string()]; - s.read_models = true; - let project = generate_service_scaffold(s).unwrap(); - let paths = paths(&project); - assert!(paths.contains(&"src/models/mod.rs")); - assert!(paths.contains(&"src/models/order.rs")); - assert!(paths.contains(&"src/read_models/mod.rs")); - } - - #[test] - fn github_yields_a_post_create_action_and_warning() { - let mut s = spec("orders"); - s.github = Some(GithubScaffoldSpec { - repository: GithubRepo::parse("hops-ops/orders").unwrap(), - preview_environment_repository: None, - promote_environment_repository: None, - }); - let project = generate_service_scaffold(s).unwrap(); - assert_eq!( - project.post_create_actions, - vec![PostCreateAction::EnsureGithubRepository { - repo: GithubRepo { - owner: "hops-ops".to_string(), - repo: "orders".to_string(), - }, - }] - ); - assert!(!project.warnings.is_empty()); - } - - #[test] - fn invalid_name_is_an_error() { - assert!(generate_service_scaffold(spec(" ")).is_err()); - } - - #[test] - fn github_repo_parse_rejects_bad_input() { - assert!(GithubRepo::parse("no-slash").is_err()); - assert!(GithubRepo::parse("a/b/c").is_err()); - assert_eq!(GithubRepo::parse("o/r").unwrap().slug(), "o/r"); - } -} diff --git a/distributed_tooling/src/generate/github.rs b/distributed_tooling/src/generate/github.rs new file mode 100644 index 0000000..52f7e9e --- /dev/null +++ b/distributed_tooling/src/generate/github.rs @@ -0,0 +1,28 @@ +//! GitHub repository parsing (and, in a follow-up slice, the GitHub Actions +//! workflow + GitOps promotion-chart templates). + +use crate::{GithubRepo, ScaffoldError}; + +/// Parse an `owner/repo` string, validating both halves. +pub(crate) fn parse_github_repo(raw: &str) -> Result { + let trimmed = raw.trim(); + let Some((owner, repo)) = trimmed.split_once('/') else { + return Err(ScaffoldError::new("repository must be in OWNER/REPO form")); + }; + if owner.is_empty() || repo.is_empty() || repo.contains('/') { + return Err(ScaffoldError::new("repository must be in OWNER/REPO form")); + } + let valid = [owner, repo].into_iter().all(|part| { + part.chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.') + }); + if !valid { + return Err(ScaffoldError::new( + "repository contains unsupported GitHub characters", + )); + } + Ok(GithubRepo { + owner: owner.to_string(), + repo: repo.to_string(), + }) +} diff --git a/distributed_tooling/src/generate/mod.rs b/distributed_tooling/src/generate/mod.rs new file mode 100644 index 0000000..8d370a0 --- /dev/null +++ b/distributed_tooling/src/generate/mod.rs @@ -0,0 +1,284 @@ +//! Scaffold generation: build a normalized [`Scaffold`] from a spec, then render +//! the project files. Pure — no filesystem, network, or CLI. The template bodies +//! live in submodules: +//! +//! - [`names`] — name/message normalization + validation (the portable rules); +//! - [`service_crate`] — the Rust service-crate templates; +//! - [`github`] — GitHub repo parsing (+ workflow templates, follow-up slice). + +mod github; +mod names; +mod service_crate; + +pub(crate) use github::parse_github_repo; + +use std::collections::BTreeSet; + +use names::{ + default_command_name, message_handlers_with_modules, model_scaffolds, MessageHandler, + ModelScaffold, ScaffoldNames, +}; + +use crate::{ + BusTarget, GeneratedFile, GeneratedProject, GithubScaffoldSpec, GitopsPromoteTarget, + PostCreateAction, ScaffoldError, ServiceScaffoldSpec, ServiceTransport, StoreTarget, +}; + +/// Generate a Distributed service project from a spec. The public entry point. +pub fn generate_service_scaffold( + spec: ServiceScaffoldSpec, +) -> Result { + Ok(Scaffold::from_spec(spec)?.generate()) +} + +/// The normalized scaffold: spec values resolved into derived names, models, and +/// handlers. Template methods are implemented in the `service_crate` submodule. +pub(crate) struct Scaffold { + pub(crate) names: ScaffoldNames, + pub(crate) distributed_dependency_path: String, + pub(crate) transport: ServiceTransport, + pub(crate) store: StoreTarget, + #[allow(dead_code)] // consumed by GitOps/Knative generation (follow-up slice) + pub(crate) bus: Option, + pub(crate) include_read_models: bool, + pub(crate) gitops: bool, + pub(crate) gitops_promote: Option, + pub(crate) github: Option, + pub(crate) models: Vec, + pub(crate) read_models: Vec, + pub(crate) commands: Vec, + pub(crate) events: Vec, +} + +impl Scaffold { + fn from_spec(spec: ServiceScaffoldSpec) -> Result { + let names = ScaffoldNames::new(&spec.name)?; + let models = model_scaffolds(&spec.models)?; + let read_models = if spec.read_models { + if models.is_empty() { + vec![ModelScaffold::new(&names.package_name)?] + } else { + models.clone() + } + } else { + Vec::new() + }; + let mut module_idents = BTreeSet::new(); + let commands = message_handlers_with_modules( + if spec.commands.is_empty() { + vec![default_command_name(&names, &models)] + } else { + spec.commands.clone() + }, + "command", + &mut module_idents, + )?; + let events = + message_handlers_with_modules(spec.events.clone(), "event", &mut module_idents)?; + Ok(Self { + names, + distributed_dependency_path: spec.distributed_dependency_path, + transport: spec.transport, + store: spec.store, + bus: spec.bus, + include_read_models: spec.read_models, + gitops: spec.gitops, + gitops_promote: spec.gitops_promote, + github: spec.github, + models, + read_models, + commands, + events, + }) + } + + fn generate(self) -> GeneratedProject { + let mut files = Vec::new(); + let mut warnings = Vec::new(); + let mut post_create_actions = Vec::new(); + + files.push(file("Cargo.toml", self.cargo_toml())); + files.push(file("src/lib.rs", self.lib_rs())); + files.push(file("src/main.rs", self.main_rs())); + files.push(file("src/manifest.rs", self.manifest_rs())); + files.push(file("src/service.rs", self.service_rs())); + if !self.models.is_empty() { + files.push(file("src/models/mod.rs", self.models_mod_rs())); + for model in &self.models { + files.push(file( + &format!("src/models/{}.rs", model.module_ident), + self.model_rs(model), + )); + } + } + files.push(file("src/handlers/mod.rs", self.handlers_mod_rs())); + for command in &self.commands { + files.push(file( + &format!("src/handlers/{}.rs", command.module_ident), + self.command_handler_rs(command), + )); + } + for event in &self.events { + files.push(file( + &format!("src/handlers/{}.rs", event.module_ident), + self.event_handler_rs(event), + )); + } + if self.include_read_models { + files.push(file("src/read_models/mod.rs", self.read_models_mod_rs())); + } + + // GitOps deploy/promote charts and GitHub workflow files are a follow-up + // slice; the spec fields are accepted so the API is stable. Until then, + // surface a warning so the caller knows those artifacts were not emitted. + if self.gitops || self.gitops_promote.is_some() || self.github.is_some() { + warnings.push( + "GitOps and GitHub workflow generation are not yet ported into \ + distributed_tooling; those artifacts were not generated" + .to_string(), + ); + } + if let Some(github) = &self.github { + post_create_actions.push(PostCreateAction::EnsureGithubRepository { + repo: github.repository.clone(), + }); + } + + GeneratedProject { + files, + warnings, + post_create_actions, + } + } +} + +fn file(path: &str, contents: String) -> GeneratedFile { + GeneratedFile { + path: path.to_string(), + contents, + mode: None, + } +} + +#[cfg(test)] +mod tests { + use crate::{ + generate_service_scaffold, GeneratedProject, GithubRepo, GithubScaffoldSpec, + PostCreateAction, ServiceScaffoldSpec, ServiceTransport, StoreTarget, + }; + + fn spec(name: &str) -> ServiceScaffoldSpec { + ServiceScaffoldSpec { + name: name.to_string(), + transport: ServiceTransport::Http, + store: StoreTarget::Postgres, + bus: None, + models: Vec::new(), + read_models: false, + commands: Vec::new(), + events: Vec::new(), + distributed_dependency_path: "../distributed".to_string(), + gitops: false, + gitops_promote: None, + github: None, + } + } + + fn paths(project: &GeneratedProject) -> Vec<&str> { + project.files.iter().map(|f| f.path.as_str()).collect() + } + + fn contents<'a>(project: &'a GeneratedProject, path: &str) -> &'a str { + project + .files + .iter() + .find(|f| f.path == path) + .map(|f| f.contents.as_str()) + .unwrap_or_else(|| panic!("missing file {path}")) + } + + #[test] + fn generates_the_core_service_crate() { + let project = generate_service_scaffold(spec("orders")).unwrap(); + let paths = paths(&project); + for expected in [ + "Cargo.toml", + "src/lib.rs", + "src/main.rs", + "src/manifest.rs", + "src/service.rs", + "src/handlers/mod.rs", + ] { + assert!(paths.contains(&expected), "missing {expected} in {paths:?}"); + } + assert!(paths + .iter() + .any(|p| p.starts_with("src/handlers/") && *p != "src/handlers/mod.rs")); + } + + #[test] + fn service_uses_the_new_builder_api() { + let project = generate_service_scaffold(spec("orders")).unwrap(); + let service = contents(&project, "src/service.rs"); + assert!( + service.contains("Service::new().with_repo(repo)"), + "service.rs should use the new builder API:\n{service}" + ); + assert!(!service.contains("Service::with_repo(")); + } + + #[test] + fn cargo_features_track_transport_and_store() { + let mut s = spec("orders"); + s.store = StoreTarget::Sqlite; + let project = generate_service_scaffold(s).unwrap(); + let cargo = contents(&project, "Cargo.toml"); + assert!(cargo.contains("\"http\"")); + assert!(cargo.contains("\"sqlite\"")); + } + + #[test] + fn read_models_and_models_emit_modules() { + let mut s = spec("orders"); + s.models = vec!["order".to_string()]; + s.read_models = true; + let project = generate_service_scaffold(s).unwrap(); + let paths = paths(&project); + assert!(paths.contains(&"src/models/mod.rs")); + assert!(paths.contains(&"src/models/order.rs")); + assert!(paths.contains(&"src/read_models/mod.rs")); + } + + #[test] + fn github_yields_a_post_create_action_and_warning() { + let mut s = spec("orders"); + s.github = Some(GithubScaffoldSpec { + repository: GithubRepo::parse("hops-ops/orders").unwrap(), + preview_environment_repository: None, + promote_environment_repository: None, + }); + let project = generate_service_scaffold(s).unwrap(); + assert_eq!( + project.post_create_actions, + vec![PostCreateAction::EnsureGithubRepository { + repo: GithubRepo { + owner: "hops-ops".to_string(), + repo: "orders".to_string(), + }, + }] + ); + assert!(!project.warnings.is_empty()); + } + + #[test] + fn invalid_name_is_an_error() { + assert!(generate_service_scaffold(spec(" ")).is_err()); + } + + #[test] + fn github_repo_parse_rejects_bad_input() { + assert!(GithubRepo::parse("no-slash").is_err()); + assert!(GithubRepo::parse("a/b/c").is_err()); + assert_eq!(GithubRepo::parse("o/r").unwrap().slug(), "o/r"); + } +} diff --git a/distributed_tooling/src/generate/names.rs b/distributed_tooling/src/generate/names.rs new file mode 100644 index 0000000..267e6cf --- /dev/null +++ b/distributed_tooling/src/generate/names.rs @@ -0,0 +1,276 @@ +//! Name / message normalization and validation — the portable spec-value rules +//! this crate owns (kebab/pascal/ident casing, dedup, keyword avoidance). + +use std::collections::BTreeSet; + +use crate::ScaffoldError; + +/// Derived names for the generated service crate. +pub(crate) struct ScaffoldNames { + pub(crate) package_name: String, + pub(crate) crate_ident: String, + pub(crate) command_name: String, +} + +impl ScaffoldNames { + pub(crate) fn new(input: &str) -> Result { + let package_name = to_kebab_case(input); + if package_name.is_empty() { + return Err(ScaffoldError::new( + "service name must contain at least one ASCII letter or digit", + )); + } + let crate_ident = package_name.replace('-', "_"); + let command_name = format!("{crate_ident}.create"); + Ok(Self { + package_name, + crate_ident, + command_name, + }) + } +} + +/// A scaffolded aggregate model and its derived identifiers. +#[derive(Clone, Debug)] +pub(crate) struct ModelScaffold { + pub(crate) name: String, + pub(crate) message_prefix: String, + pub(crate) module_ident: String, + pub(crate) type_ident: String, + pub(crate) view_ident: String, + pub(crate) table_name: String, +} + +impl ModelScaffold { + pub(crate) fn new(raw_name: &str) -> Result { + let name = to_kebab_case(raw_name); + if name.is_empty() { + return Err(ScaffoldError::new( + "model name must contain at least one ASCII letter or digit", + )); + } + let ident = name.replace('-', "_"); + let type_ident = to_pascal_case(&name); + let view_ident = format!("{type_ident}View"); + Ok(Self { + name: name.clone(), + message_prefix: name.clone(), + module_ident: ident.clone(), + type_ident, + view_ident, + table_name: format!("{ident}_views"), + }) + } +} + +pub(crate) fn model_scaffolds(raw_models: &[String]) -> Result, ScaffoldError> { + let mut seen = BTreeSet::new(); + let mut models = Vec::new(); + for raw_model in raw_models { + let model = ModelScaffold::new(raw_model)?; + if !seen.insert(model.name.clone()) { + return Err(ScaffoldError::new(format!( + "duplicate model `{}`", + model.name + ))); + } + models.push(model); + } + Ok(models) +} + +pub(crate) fn default_command_name(names: &ScaffoldNames, models: &[ModelScaffold]) -> String { + models + .first() + .map(|model| format!("{}.create", model.name)) + .unwrap_or_else(|| names.command_name.clone()) +} + +/// A scaffolded command/event handler and its module identifier. +#[derive(Clone, Debug)] +pub(crate) struct MessageHandler { + pub(crate) message_name: String, + pub(crate) module_ident: String, +} + +pub(crate) fn message_handlers_with_modules( + names: Vec, + fallback_prefix: &str, + seen_modules: &mut BTreeSet, +) -> Result, ScaffoldError> { + let mut seen_names = BTreeSet::new(); + let mut handlers = Vec::new(); + for raw_name in names { + let message_name = raw_name.trim(); + validate_message_name(message_name, fallback_prefix)?; + if !seen_names.insert(message_name.to_string()) { + return Err(ScaffoldError::new(format!( + "duplicate {fallback_prefix} `{message_name}`" + ))); + } + let base_module = to_rust_ident(message_name, fallback_prefix); + let mut module_ident = base_module.clone(); + let mut suffix = 2; + while !seen_modules.insert(module_ident.clone()) { + module_ident = format!("{base_module}_{suffix}"); + suffix += 1; + } + handlers.push(MessageHandler { + message_name: message_name.to_string(), + module_ident, + }); + } + Ok(handlers) +} + +fn validate_message_name(name: &str, kind: &str) -> Result<(), ScaffoldError> { + if name.is_empty() { + return Err(ScaffoldError::new(format!("{kind} name cannot be empty"))); + } + if name.chars().any(char::is_control) { + return Err(ScaffoldError::new(format!( + "{kind} `{name}` contains a control character" + ))); + } + Ok(()) +} + +/// The owning segment of a dotted message name (e.g. `orders` in +/// `orders.create`), normalized to a k8s-safe name. Used to match a command to +/// its model and to derive broker names. +pub(crate) fn message_owner(message_name: &str) -> String { + message_name + .split('.') + .find(|part| !part.is_empty()) + .map(k8s_name) + .unwrap_or_else(|| "message".to_string()) +} + +pub(crate) fn k8s_name(value: &str) -> String { + let name = to_kebab_case(value); + if name.is_empty() { + "generated".to_string() + } else { + name + } +} + +fn to_rust_ident(value: &str, fallback_prefix: &str) -> String { + let mut ident = String::new(); + let mut last_was_separator = false; + for char in value.chars() { + if char.is_ascii_alphanumeric() { + ident.push(char.to_ascii_lowercase()); + last_was_separator = false; + } else if !last_was_separator { + ident.push('_'); + last_was_separator = true; + } + } + while ident.ends_with('_') { + ident.pop(); + } + while ident.starts_with('_') { + ident.remove(0); + } + if ident.is_empty() { + ident = fallback_prefix.to_string(); + } + if ident + .chars() + .next() + .is_some_and(|char| char.is_ascii_digit()) + || is_rust_keyword(&ident) + { + ident = format!("{fallback_prefix}_{ident}"); + } + ident +} + +fn is_rust_keyword(value: &str) -> bool { + matches!( + value, + "as" | "break" + | "const" + | "continue" + | "crate" + | "else" + | "enum" + | "extern" + | "false" + | "fn" + | "for" + | "if" + | "impl" + | "in" + | "let" + | "loop" + | "match" + | "mod" + | "move" + | "mut" + | "pub" + | "ref" + | "return" + | "self" + | "Self" + | "static" + | "struct" + | "super" + | "trait" + | "true" + | "type" + | "unsafe" + | "use" + | "where" + | "while" + | "async" + | "await" + | "dyn" + ) +} + +fn to_kebab_case(input: &str) -> String { + let mut out = String::new(); + let mut last_was_separator = true; + for char in input.chars() { + if char.is_ascii_alphanumeric() { + out.push(char.to_ascii_lowercase()); + last_was_separator = false; + } else if !last_was_separator { + out.push('-'); + last_was_separator = true; + } + } + while out.ends_with('-') { + out.pop(); + } + out +} + +fn to_pascal_case(input: &str) -> String { + input + .split(['-', '_']) + .filter(|part| !part.is_empty()) + .map(|part| { + let mut chars = part.chars(); + let Some(first) = chars.next() else { + return String::new(); + }; + let mut out = String::new(); + out.push(first.to_ascii_uppercase()); + out.extend(chars); + out + }) + .collect() +} + +/// Escape a string as a Rust/TOML string literal. +pub(crate) fn rust_string(value: &str) -> String { + toml_string(value) +} + +/// Escape a string as a TOML string literal. +pub(crate) fn toml_string(value: impl AsRef) -> String { + serde_json::to_string(value.as_ref()).expect("string serialization should succeed") +} diff --git a/distributed_tooling/src/generate/service_crate.rs b/distributed_tooling/src/generate/service_crate.rs new file mode 100644 index 0000000..708e556 --- /dev/null +++ b/distributed_tooling/src/generate/service_crate.rs @@ -0,0 +1,392 @@ +//! Rust service-crate templates: `Cargo.toml` and the `src/**` sources +//! (lib/main/manifest/service/models/handlers/read_models). Pure — each method +//! returns the file contents as a `String`. + +use super::names::{message_owner, rust_string, toml_string, MessageHandler, ModelScaffold}; +use super::Scaffold; +use crate::{ServiceTransport, StoreTarget}; + +impl Scaffold { + pub(super) fn cargo_toml(&self) -> String { + let distributed_path = toml_string(&self.distributed_dependency_path); + let features = self + .distributed_features() + .into_iter() + .map(toml_string) + .collect::>() + .join(", "); + let axum = if self.transport == ServiceTransport::Knative { + "axum = \"0.7\"\n" + } else { + "" + }; + + format!( + r#"[package] +name = {package_name} +version = "0.1.0" +edition = "2021" + +[workspace] + +[dependencies] +distributed = {{ path = {distributed_path}, features = [{features}] }} +{axum}serde = {{ version = "1", features = ["derive"] }} +serde_json = "1" +tokio = {{ version = "1", features = ["macros", "net", "rt-multi-thread"] }} +"#, + package_name = toml_string(&self.names.package_name), + ) + } + + fn distributed_features(&self) -> Vec<&'static str> { + let mut features = Vec::new(); + match self.transport { + ServiceTransport::Http => features.push("http"), + ServiceTransport::Knative => features.push("http"), + } + match self.store { + StoreTarget::Postgres => features.push("postgres"), + StoreTarget::Sqlite => features.push("sqlite"), + StoreTarget::InMemory => {} + } + features + } + + pub(super) fn lib_rs(&self) -> String { + let models = if !self.models.is_empty() { + "pub mod models;\n" + } else { + "" + }; + let read_models = if self.include_read_models { + "pub mod read_models;\n" + } else { + "" + }; + format!( + r#"pub mod handlers; +pub mod manifest; +{models}{read_models}pub mod service; + +pub use manifest::distributed_manifest; +"# + ) + } + + pub(super) fn main_rs(&self) -> String { + match self.transport { + ServiceTransport::Http => format!( + r#"#[tokio::main] +async fn main() -> Result<(), Box> {{ + let addr = std::env::var("BIND_ADDR").unwrap_or_else(|_| "127.0.0.1:3000".to_string()); + let service = {crate_ident}::service::in_memory(); + distributed::microsvc::serve(service, &addr).await?; + Ok(()) +}} +"#, + crate_ident = self.names.crate_ident, + ), + ServiceTransport::Knative => format!( + r#"#[tokio::main] +async fn main() -> Result<(), Box> {{ + let addr = std::env::var("BIND_ADDR").unwrap_or_else(|_| "127.0.0.1:3000".to_string()); + let service = {crate_ident}::service::in_memory(); + let listener = tokio::net::TcpListener::bind(&addr).await?; + let app = distributed::microsvc::cloud_events_router(service); + axum::serve(listener, app).await?; + Ok(()) +}} +"#, + crate_ident = self.names.crate_ident, + ), + } + } + + pub(super) fn manifest_rs(&self) -> String { + let read_model_import = if self.include_read_models && !self.read_models.is_empty() { + format!( + "use crate::read_models::{{{}}};\n\n", + self.read_models + .iter() + .map(|model| model.view_ident.as_str()) + .collect::>() + .join(", ") + ) + } else { + String::new() + }; + let read_model_registration = self + .read_models + .iter() + .map(|model| format!(" .read_model::<{}>()\n", model.view_ident)) + .collect::(); + format!( + r#"use distributed::{{ + DistributedProjectManifest, ServiceManifest, +}}; + +{read_model_import}pub fn distributed_manifest() -> DistributedProjectManifest {{ + DistributedProjectManifest::new({project_name}) +{read_model_registration} .service(crate::service::manifest()) +}} + +pub fn service_manifest() -> ServiceManifest {{ + crate::service::manifest() +}} +"#, + project_name = rust_string(&self.names.package_name), + ) + } + + pub(super) fn service_rs(&self) -> String { + let registrations = self + .commands + .iter() + .map(|handler| format!(" command handlers::{},\n", handler.module_ident)) + .chain( + self.events + .iter() + .map(|handler| format!(" event handlers::{},\n", handler.module_ident)), + ) + .collect::(); + let manifest_commands = self + .commands + .iter() + .map(|handler| { + format!( + " .command(handlers::{}::COMMAND)\n", + handler.module_ident + ) + }) + .collect::(); + let manifest_events = self + .events + .iter() + .map(|handler| { + format!( + " .event(handlers::{}::EVENT)\n", + handler.module_ident + ) + }) + .collect::(); + let transport = match self.transport { + ServiceTransport::Http => "http", + ServiceTransport::Knative => "knative", + }; + + format!( + r#"use std::sync::Arc; + +use distributed::{{microsvc::Service, HashMapRepository, ServiceManifest}}; + +use crate::handlers; + +pub type ServiceRepo = HashMapRepository; + +pub fn in_memory() -> Arc> {{ + build(HashMapRepository::new()) +}} + +pub fn build(repo: ServiceRepo) -> Arc> {{ + Arc::new(distributed::register_handlers!( + Service::new().with_repo(repo), +{registrations} )) +}} + +pub fn manifest() -> ServiceManifest {{ + ServiceManifest::new({service_name}) +{manifest_commands}{manifest_events} .transport({transport}) +}} +"#, + service_name = rust_string(&self.names.package_name), + transport = rust_string(transport), + ) + } + + pub(super) fn models_mod_rs(&self) -> String { + let modules = self + .models + .iter() + .map(|model| { + format!( + "pub mod {module_ident};\npub use {module_ident}::{type_ident};\n", + module_ident = model.module_ident, + type_ident = model.type_ident, + ) + }) + .collect::>() + .join(""); + + format!( + r#"{modules} +use serde::{{Deserialize, Serialize}}; + + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct CommandInput {{ + pub id: String, + #[serde(default)] + pub name: Option, +}} +"# + ) + } + + pub(super) fn model_rs(&self, model: &ModelScaffold) -> String { + format!( + r#"use distributed::{{sourced, Entity, Snapshot}}; + +#[derive(Default, Snapshot)] +pub struct {model_struct} {{ + pub entity: Entity, + pub name: Option, + pub status: String, +}} + +#[sourced(entity)] +impl {model_struct} {{ + #[event({command_recorded_event})] + pub fn record_command(&mut self, command: String, id: String, name: Option) {{ + self.entity.set_id(&id); + if let Some(name) = name {{ + self.name = Some(name); + }} + self.status = command; + }} +}} +"#, + model_struct = model.type_ident, + command_recorded_event = + rust_string(&format!("{}.command_recorded", model.message_prefix)), + ) + } + + pub(super) fn handlers_mod_rs(&self) -> String { + self.commands + .iter() + .chain(self.events.iter()) + .map(|handler| format!("pub mod {};\n", handler.module_ident)) + .collect() + } + + pub(super) fn command_handler_rs(&self, handler: &MessageHandler) -> String { + if let Some(model) = self.command_model(handler) { + format!( + r#"use distributed::{{ + microsvc::{{Context, HandlerError}}, Aggregate, CommitBatch, StreamIdentity, StreamWrite, + TransactionalCommit, +}}; +use serde_json::{{json, Value}}; + +use crate::models::{{CommandInput, {model_type}}}; +use crate::service::ServiceRepo; + +pub const COMMAND: &str = {message_name}; +pub const MODEL: &str = {model_name}; + +pub fn guard(ctx: &Context) -> bool {{ + ctx.has_fields(&["id"]) +}} + +pub async fn handle(ctx: &Context<'_, ServiceRepo>) -> Result {{ + let input = ctx.input::()?; + let mut aggregate = {model_type}::default(); + aggregate.record_command(COMMAND.to_string(), input.id.clone(), input.name.clone())?; + let identity = StreamIdentity::new({model_type}::aggregate_type(), aggregate.entity().id())?; + let stream = StreamWrite::new(identity, aggregate.entity_mut()); + ctx.repo().commit_batch(CommitBatch::new(vec![stream])).await?; + Ok(json!({{ "command": COMMAND, "id": input.id, "model": MODEL, "name": input.name }})) +}} +"#, + model_type = model.type_ident, + message_name = rust_string(&handler.message_name), + model_name = rust_string(&model.name), + ) + } else { + format!( + r#"use distributed::microsvc::{{Context, HandlerError}}; +use serde_json::{{json, Value}}; + +use crate::service::ServiceRepo; + +pub const COMMAND: &str = {message_name}; + +pub fn guard(_ctx: &Context) -> bool {{ + true +}} + +pub async fn handle(ctx: &Context<'_, ServiceRepo>) -> Result {{ + let input = ctx.input::()?; + Ok(json!({{ "command": COMMAND, "input": input }})) +}} +"#, + message_name = rust_string(&handler.message_name), + ) + } + } + + pub(super) fn event_handler_rs(&self, handler: &MessageHandler) -> String { + format!( + r#"use distributed::microsvc::{{Context, HandlerError}}; +use serde_json::{{json, Value}}; + +use crate::service::ServiceRepo; + +pub const EVENT: &str = {message_name}; + +pub fn guard(_ctx: &Context) -> bool {{ + true +}} + +pub async fn handle(ctx: &Context<'_, ServiceRepo>) -> Result {{ + let input = ctx.input::()?; + Ok(json!({{ "event": EVENT, "input": input }})) +}} +"#, + message_name = rust_string(&handler.message_name), + ) + } + + pub(super) fn read_models_mod_rs(&self) -> String { + let views = self + .read_models + .iter() + .map(|model| { + format!( + r#"#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, ReadModel)] +#[table({table_name})] +pub struct {view_struct} {{ + #[id("id")] + pub id: String, + pub name: String, + pub status: String, +}} +"#, + table_name = rust_string(&model.table_name), + view_struct = model.view_ident, + ) + }) + .collect::>() + .join("\n"); + + format!( + r#"use distributed::ReadModel; +use serde::{{Deserialize, Serialize}}; + +{views} +"# + ) + } + + fn command_model(&self, handler: &MessageHandler) -> Option<&ModelScaffold> { + if self.models.is_empty() { + return None; + } + let message_model = message_owner(&handler.message_name); + self.models + .iter() + .find(|model| model.name == message_model) + .or_else(|| self.models.first()) + } +} From 8ff658f407b3ae842469dd996f2699f44dea1974 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 4 Jun 2026 22:41:57 -0500 Subject: [PATCH 20/24] feat: port GitOps + GitHub workflow generation into distributed_tooling generate/gitops.rs: .gitops/deploy Helm chart (HTTP Deployment+Service or Knative Service+Brokers+Triggers) + optional .gitops/promote Argo/Flux chart, with Knative broker/trigger inference and image-repo selection. generate/github.rs gains the release/preview/promote workflow templates + the Argo CD promotion chart. generate() now emits these (deploy chart whenever any gitops/github option is set, matching the original); the placeholder warning is gone. 10 crate tests (added GitOps HTTP, Knative brokers/triggers, Flux promote, and full GitHub workflow coverage). Next: the hops-cli rewire. Implements [[specs/distributed-service-scaffold-tooling]] [[tasks/distributed-tooling-crate-extraction]] Co-Authored-By: Claude Opus 4.8 (1M context) --- distributed_tooling/src/generate/github.rs | 282 ++++++++++++- distributed_tooling/src/generate/gitops.rs | 381 ++++++++++++++++++ distributed_tooling/src/generate/mod.rs | 80 +++- distributed_tooling/src/generate/names.rs | 42 ++ .../src/generate/service_crate.rs | 2 +- 5 files changed, 766 insertions(+), 21 deletions(-) create mode 100644 distributed_tooling/src/generate/gitops.rs diff --git a/distributed_tooling/src/generate/github.rs b/distributed_tooling/src/generate/github.rs index 52f7e9e..4e54898 100644 --- a/distributed_tooling/src/generate/github.rs +++ b/distributed_tooling/src/generate/github.rs @@ -1,7 +1,10 @@ -//! GitHub repository parsing (and, in a follow-up slice, the GitHub Actions -//! workflow + GitOps promotion-chart templates). +//! GitHub repository parsing, the GitHub Actions release/preview/promote +//! workflow templates, and the Argo CD promotion chart used by those workflows. +//! Pure — produces `GeneratedFile`s and string contents. -use crate::{GithubRepo, ScaffoldError}; +use super::names::k8s_name; +use super::{file, Scaffold}; +use crate::{GeneratedFile, GithubRepo, ScaffoldError}; /// Parse an `owner/repo` string, validating both halves. pub(crate) fn parse_github_repo(raw: &str) -> Result { @@ -26,3 +29,276 @@ pub(crate) fn parse_github_repo(raw: &str) -> Result repo: repo.to_string(), }) } + +impl Scaffold { + /// The `.github/workflows/*` files and (when preview/promote repos are set) + /// the `.gitops/{preview,promote}/helm` promotion charts. + pub(super) fn github_files(&self) -> Vec { + let mut files = Vec::new(); + let Some(github) = &self.github else { + return files; + }; + + files.push(file( + ".github/workflows/version.yaml", + github_version_workflow_yaml(), + )); + files.push(file( + ".github/workflows/release.yaml", + github_release_workflow_yaml(), + )); + + if let Some(preview) = &github.preview_environment_repository { + files.push(file( + ".github/workflows/preview.yaml", + self.github_preview_workflow_yaml(preview), + )); + files.extend(self.github_promotion_chart(".gitops/preview/helm")); + } + if let Some(promote) = &github.promote_environment_repository { + files.push(file( + ".github/workflows/promote.yaml", + self.github_promote_workflow_yaml(promote), + )); + files.extend(self.github_promotion_chart(".gitops/promote/helm")); + } + + files + } + + fn github_promotion_chart(&self, path: &str) -> Vec { + vec![ + file( + &format!("{path}/Chart.yaml"), + self.github_promotion_chart_yaml(), + ), + file( + &format!("{path}/values.yaml"), + self.github_promotion_values_yaml(), + ), + file( + &format!("{path}/templates/application.yaml"), + github_promotion_application_yaml(), + ), + ] + } + + fn github_preview_workflow_yaml(&self, preview: &GithubRepo) -> String { + let environment_name = k8s_name(&preview.repo); + format!( + r#"name: Preview + +on: + pull_request: + branches: + - main + types: + - labeled + - opened + - reopened + - synchronize + +permissions: + contents: write + issues: write + packages: write + pull-requests: write + +jobs: + preview: + name: Preview Promotion PR + if: contains(github.event.pull_request.labels.*.name, 'preview') + uses: unbounded-tech/workflows-gitops/.github/workflows/argocd-promote-helm.yaml@v1 + secrets: + GH_PAT: ${{{{ secrets.GH_ORG_ACTIONS_REPO_WRITE_PACKAGES }}}} + with: + promotion_chart_path: .gitops/preview/helm + environment_repository: {environment_repository} + environment_name: {environment_name} + project: {environment_name} + name: ${{{{ github.event.repository.name }}}} + preview: true + promotion_pr: true + values: | + image: + repository: {image_repository} + comment: | + Preview promoted for `${{{{ github.event.repository.name }}}}`. + The current tag is: `pr-${{{{ github.event.pull_request.number }}}}-${{{{ github.event.pull_request.head.sha }}}}` +"#, + environment_repository = preview.slug(), + image_repository = self.image_repository(), + ) + } + + fn github_promote_workflow_yaml(&self, promote: &GithubRepo) -> String { + let environment_name = k8s_name(&promote.repo); + format!( + r#"name: Promote + +on: + push: + tags: + - "v*.*.*" + +permissions: + contents: write + issues: write + packages: write + pull-requests: write + +jobs: + promote: + name: Release Promotion + uses: unbounded-tech/workflows-gitops/.github/workflows/argocd-promote-helm.yaml@v1 + secrets: + GH_PAT: ${{{{ secrets.GH_ORG_ACTIONS_REPO_WRITE_PACKAGES }}}} + with: + promotion_chart_path: .gitops/promote/helm + destination_path: .gitops/deploy + environment_repository: {environment_repository} + environment_name: {environment_name} + project: {environment_name} + name: {application_name} + values: | + image: + repository: {image_repository} + tag: ${{{{ github.ref_name }}}} +"#, + environment_repository = promote.slug(), + application_name = self.names.package_name, + image_repository = self.image_repository(), + ) + } + + fn github_promotion_chart_yaml(&self) -> String { + format!( + r#"apiVersion: v2 +name: {chart_name}-promotion +description: Argo CD promotion chart for {service_name} +type: application +version: 0.1.0 +appVersion: "0.1.0" +"#, + chart_name = k8s_name(&self.names.package_name), + service_name = self.names.package_name, + ) + } + + fn github_promotion_values_yaml(&self) -> String { + format!( + r#"application: + name: {service_name} + repository: https://github.com/{github_repository}.git + targetRevision: main + path: .gitops/deploy + values: "" + destination: + namespace: default + server: https://kubernetes.default.svc + image: + tag: latest + +project: default +preview: false +"#, + service_name = self.names.package_name, + github_repository = self + .github + .as_ref() + .map(|g| g.repository.slug()) + .unwrap_or_else(|| "OWNER/REPO".to_string()), + ) + } +} + +fn github_version_workflow_yaml() -> String { + r#"name: Version and Tag + +on: + push: + branches: + - main + +permissions: + contents: write + +jobs: + version-and-tag: + name: Version and Tag + uses: unbounded-tech/workflow-vnext-tag/.github/workflows/workflow.yaml@v1.21.3 + secrets: inherit + with: + useDeployKey: true + rust: true + yqPatches: | + patches: + - filePath: .gitops/deploy/values.yaml + selector: .image.tag + valuePrefix: v + - filePath: .gitops/deploy/Chart.yaml + selector: .version + valuePrefix: "" + - filePath: .gitops/deploy/Chart.yaml + selector: .appVersion + valuePrefix: v +"# + .to_string() +} + +fn github_release_workflow_yaml() -> String { + r#"name: Release + +on: + push: + tags: + - "v*.*.*" + +permissions: + contents: write + +jobs: + release: + name: GitHub Release + uses: unbounded-tech/workflow-simple-release/.github/workflows/workflow.yaml@v2.1.3 + with: + tag: ${{ github.ref_name }} + name: ${{ github.ref_name }} +"# + .to_string() +} + +fn github_promotion_application_yaml() -> String { + r#"apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: {{ .Values.project }}-{{ .Values.application.name }} + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: {{ .Values.project }} + source: + path: {{ .Values.application.path }} + repoURL: {{ .Values.application.repository }} + targetRevision: {{ .Values.application.targetRevision }} + helm: + version: v3 + values: | + {{- if .Values.preview }} + image: + tag: {{ .Values.application.image.tag }} + {{- end }} + {{- if .Values.application.values }} + {{ .Values.application.values | nindent 8 }} + {{- end }} + destination: + namespace: {{ .Values.application.destination.namespace }} + server: {{ .Values.application.destination.server }} + syncPolicy: + automated: + selfHeal: true + prune: true +"# + .to_string() +} diff --git a/distributed_tooling/src/generate/gitops.rs b/distributed_tooling/src/generate/gitops.rs new file mode 100644 index 0000000..cc59f41 --- /dev/null +++ b/distributed_tooling/src/generate/gitops.rs @@ -0,0 +1,381 @@ +//! GitOps artifact templates: the `.gitops/deploy` Helm chart (HTTP Deployment + +//! Service, or Knative Service + Brokers + Triggers) and the `.gitops/promote` +//! Argo CD / Flux promotion chart. Pure — produces `GeneratedFile`s. + +use std::collections::BTreeSet; + +use super::names::{ + command_broker_for_message, event_broker_for_message, k8s_name, KnativeTrigger, +}; +use super::{file, Scaffold}; +use crate::{GeneratedFile, GitopsPromoteTarget, ServiceTransport}; + +impl Scaffold { + /// The `.gitops/deploy` (and optional `.gitops/promote`) files. The deploy + /// chart is emitted whenever any GitOps/GitHub option is set, because the + /// promotion charts target `.gitops/deploy`. + pub(super) fn gitops_files(&self) -> Vec { + let mut files = Vec::new(); + let want_deploy = self.gitops || self.gitops_promote.is_some() || self.github.is_some(); + if want_deploy { + files.push(file( + ".gitops/deploy/Chart.yaml", + self.gitops_deploy_chart_yaml(), + )); + files.push(file( + ".gitops/deploy/values.yaml", + self.gitops_deploy_values_yaml(), + )); + match self.transport { + ServiceTransport::Http => { + files.push(file( + ".gitops/deploy/templates/deployment.yaml", + self.gitops_http_deployment_yaml(), + )); + files.push(file( + ".gitops/deploy/templates/service.yaml", + self.gitops_http_service_yaml(), + )); + } + ServiceTransport::Knative => { + files.push(file( + ".gitops/deploy/templates/knative-service.yaml", + self.gitops_knative_service_yaml(), + )); + files.push(file( + ".gitops/deploy/templates/knative-brokers.yaml", + self.gitops_knative_brokers_yaml(), + )); + files.push(file( + ".gitops/deploy/templates/knative-triggers.yaml", + self.gitops_knative_triggers_yaml(), + )); + } + } + } + + if let Some(promote) = self.gitops_promote { + files.push(file( + ".gitops/promote/Chart.yaml", + self.gitops_promote_chart_yaml(promote), + )); + files.push(file( + ".gitops/promote/values.yaml", + self.gitops_promote_values_yaml(), + )); + match promote { + GitopsPromoteTarget::Argo => files.push(file( + ".gitops/promote/templates/application.yaml", + self.gitops_argo_application_yaml(), + )), + GitopsPromoteTarget::Flux => files.push(file( + ".gitops/promote/templates/helmrelease.yaml", + self.gitops_flux_helmrelease_yaml(), + )), + } + } + + files + } + + /// The container image repository: `ghcr.io//` when a + /// GitHub repo is configured, else a default under `hops-ops`. Shared with + /// the GitHub workflow templates. + pub(super) fn image_repository(&self) -> String { + self.github + .as_ref() + .map(|g| format!("ghcr.io/{}", g.repository.slug().to_ascii_lowercase())) + .unwrap_or_else(|| format!("ghcr.io/hops-ops/{}", self.names.package_name)) + } + + fn bus_env_yaml(&self) -> String { + self.bus + .map(|bus| { + format!( + r#" - name: HOPS_BUS + value: {} +"#, + bus.kind() + ) + }) + .unwrap_or_default() + } + + fn knative_broker_names(&self) -> Vec { + let mut brokers = BTreeSet::new(); + for model in &self.models { + brokers.insert(model.command_broker.clone()); + brokers.insert(model.event_broker.clone()); + } + for command in &self.commands { + let broker = self + .command_model(command) + .map(|model| model.command_broker.clone()) + .unwrap_or_else(|| command_broker_for_message(&command.message_name)); + brokers.insert(broker); + } + for event in &self.events { + brokers.insert(event_broker_for_message(&event.message_name)); + } + brokers.into_iter().collect() + } + + fn knative_triggers(&self) -> Vec { + self.commands + .iter() + .map(|command| { + let broker = self + .command_model(command) + .map(|model| model.command_broker.clone()) + .unwrap_or_else(|| command_broker_for_message(&command.message_name)); + KnativeTrigger::new(&command.message_name, &broker, "command") + }) + .chain(self.events.iter().map(|event| { + let broker = event_broker_for_message(&event.message_name); + KnativeTrigger::new(&event.message_name, &broker, "event") + })) + .collect() + } + + fn gitops_deploy_chart_yaml(&self) -> String { + format!( + r#"apiVersion: v2 +name: {chart_name} +description: Deploy chart for {service_name} +type: application +version: 0.1.0 +appVersion: "0.1.0" +"#, + chart_name = k8s_name(&format!("{}-deploy", self.names.package_name)), + service_name = self.names.package_name, + ) + } + + fn gitops_deploy_values_yaml(&self) -> String { + let bus = self + .bus + .map(|bus| format!("bus:\n kind: {}\n", bus.kind())) + .unwrap_or_default(); + format!( + r#"image: + repository: {image_repository} + tag: latest +service: + port: 3000 +{bus} +"#, + image_repository = self.image_repository(), + ) + } + + fn gitops_http_deployment_yaml(&self) -> String { + let name = k8s_name(&self.names.package_name); + let bus_env = self.bus_env_yaml(); + format!( + r#"apiVersion: apps/v1 +kind: Deployment +metadata: + name: {name} + labels: + app.kubernetes.io/name: {name} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: {name} + template: + metadata: + labels: + app.kubernetes.io/name: {name} + spec: + containers: + - name: {name} + image: {image_repository}:latest + ports: + - containerPort: 3000 + env: + - name: BIND_ADDR + value: 0.0.0.0:3000 +{bus_env} +"#, + image_repository = self.image_repository(), + ) + } + + fn gitops_http_service_yaml(&self) -> String { + let name = k8s_name(&self.names.package_name); + format!( + r#"apiVersion: v1 +kind: Service +metadata: + name: {name} + labels: + app.kubernetes.io/name: {name} +spec: + selector: + app.kubernetes.io/name: {name} + ports: + - name: http + port: 80 + targetPort: 3000 +"#, + ) + } + + fn gitops_knative_service_yaml(&self) -> String { + let name = k8s_name(&self.names.package_name); + let bus_env = self.bus_env_yaml(); + format!( + r#"apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: {name} + labels: + app.kubernetes.io/name: {name} +spec: + template: + metadata: + annotations: + autoscaling.knative.dev/min-scale: "0" + spec: + containers: + - image: {image_repository}:latest + ports: + - containerPort: 3000 + env: + - name: BIND_ADDR + value: 0.0.0.0:3000 +{bus_env} +"#, + image_repository = self.image_repository(), + ) + } + + fn gitops_knative_brokers_yaml(&self) -> String { + self.knative_broker_names() + .into_iter() + .map(|broker| { + format!( + r#"apiVersion: eventing.knative.dev/v1 +kind: Broker +metadata: + name: {broker} +"#, + ) + }) + .collect::>() + .join("---\n") + } + + fn gitops_knative_triggers_yaml(&self) -> String { + let service_name = k8s_name(&self.names.package_name); + self.knative_triggers() + .into_iter() + .map(|trigger| { + format!( + r#"apiVersion: eventing.knative.dev/v1 +kind: Trigger +metadata: + name: {name} +spec: + broker: {broker} + filter: + attributes: + type: {event_type} + subscriber: + ref: + apiVersion: serving.knative.dev/v1 + kind: Service + name: {service_name} + uri: /cloudevent/{event_type} +"#, + name = trigger.name, + broker = trigger.broker, + event_type = trigger.event_type, + ) + }) + .collect::>() + .join("---\n") + } + + fn gitops_promote_chart_yaml(&self, promote: GitopsPromoteTarget) -> String { + let suffix = match promote { + GitopsPromoteTarget::Argo => "argo", + GitopsPromoteTarget::Flux => "flux", + }; + format!( + r#"apiVersion: v2 +name: {chart_name} +description: GitOps promotion chart for {service_name} +type: application +version: 0.1.0 +appVersion: "0.1.0" +"#, + chart_name = k8s_name(&format!("{}-{}-promote", self.names.package_name, suffix)), + service_name = self.names.package_name, + ) + } + + fn gitops_promote_values_yaml(&self) -> String { + r#"repoUrl: https://example.invalid/repo.git +targetRevision: HEAD +destinationNamespace: default +"# + .to_string() + } + + fn gitops_argo_application_yaml(&self) -> String { + let name = k8s_name(&self.names.package_name); + format!( + r#"apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: {name} +spec: + project: default + source: + repoURL: https://example.invalid/repo.git + targetRevision: HEAD + path: .gitops/deploy + destination: + server: https://kubernetes.default.svc + namespace: default + syncPolicy: + automated: + prune: true + selfHeal: true +"#, + ) + } + + fn gitops_flux_helmrelease_yaml(&self) -> String { + let name = k8s_name(&self.names.package_name); + format!( + r#"apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: {name}-gitops +spec: + interval: 1m + url: https://example.invalid/repo.git + ref: + branch: main +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: {name} +spec: + interval: 5m + chart: + spec: + chart: .gitops/deploy + sourceRef: + kind: GitRepository + name: {name}-gitops + interval: 1m + targetNamespace: default +"#, + ) + } +} diff --git a/distributed_tooling/src/generate/mod.rs b/distributed_tooling/src/generate/mod.rs index 8d370a0..53e95bf 100644 --- a/distributed_tooling/src/generate/mod.rs +++ b/distributed_tooling/src/generate/mod.rs @@ -7,6 +7,7 @@ //! - [`github`] — GitHub repo parsing (+ workflow templates, follow-up slice). mod github; +mod gitops; mod names; mod service_crate; @@ -38,7 +39,6 @@ pub(crate) struct Scaffold { pub(crate) distributed_dependency_path: String, pub(crate) transport: ServiceTransport, pub(crate) store: StoreTarget, - #[allow(dead_code)] // consumed by GitOps/Knative generation (follow-up slice) pub(crate) bus: Option, pub(crate) include_read_models: bool, pub(crate) gitops: bool, @@ -94,7 +94,6 @@ impl Scaffold { fn generate(self) -> GeneratedProject { let mut files = Vec::new(); - let mut warnings = Vec::new(); let mut post_create_actions = Vec::new(); files.push(file("Cargo.toml", self.cargo_toml())); @@ -128,16 +127,11 @@ impl Scaffold { files.push(file("src/read_models/mod.rs", self.read_models_mod_rs())); } - // GitOps deploy/promote charts and GitHub workflow files are a follow-up - // slice; the spec fields are accepted so the API is stable. Until then, - // surface a warning so the caller knows those artifacts were not emitted. - if self.gitops || self.gitops_promote.is_some() || self.github.is_some() { - warnings.push( - "GitOps and GitHub workflow generation are not yet ported into \ - distributed_tooling; those artifacts were not generated" - .to_string(), - ); - } + // GitOps charts (.gitops/deploy + optional .gitops/promote) and GitHub + // workflow files (+ promotion charts) — staged in the same project. + files.extend(self.gitops_files()); + files.extend(self.github_files()); + if let Some(github) = &self.github { post_create_actions.push(PostCreateAction::EnsureGithubRepository { repo: github.repository.clone(), @@ -146,7 +140,7 @@ impl Scaffold { GeneratedProject { files, - warnings, + warnings: Vec::new(), post_create_actions, } } @@ -250,14 +244,30 @@ mod tests { } #[test] - fn github_yields_a_post_create_action_and_warning() { + fn github_generates_workflows_and_a_post_create_action() { let mut s = spec("orders"); s.github = Some(GithubScaffoldSpec { repository: GithubRepo::parse("hops-ops/orders").unwrap(), - preview_environment_repository: None, - promote_environment_repository: None, + preview_environment_repository: Some(GithubRepo::parse("hops-ops/preview").unwrap()), + promote_environment_repository: Some(GithubRepo::parse("hops-ops/prod").unwrap()), }); let project = generate_service_scaffold(s).unwrap(); + let paths = paths(&project); + for expected in [ + ".github/workflows/version.yaml", + ".github/workflows/release.yaml", + ".github/workflows/preview.yaml", + ".github/workflows/promote.yaml", + ".gitops/preview/helm/Chart.yaml", + ".gitops/promote/helm/templates/application.yaml", + // GitHub presence implies the deploy chart the promotions target. + ".gitops/deploy/Chart.yaml", + ] { + assert!(paths.contains(&expected), "missing {expected} in {paths:?}"); + } + // The image repo is derived from the GitHub repo. + let preview = contents(&project, ".github/workflows/preview.yaml"); + assert!(preview.contains("ghcr.io/hops-ops/orders")); assert_eq!( project.post_create_actions, vec![PostCreateAction::EnsureGithubRepository { @@ -267,7 +277,43 @@ mod tests { }, }] ); - assert!(!project.warnings.is_empty()); + assert!(project.warnings.is_empty()); + } + + #[test] + fn gitops_http_deploy_chart() { + let mut s = spec("orders"); + s.gitops = true; + let project = generate_service_scaffold(s).unwrap(); + let paths = paths(&project); + assert!(paths.contains(&".gitops/deploy/Chart.yaml")); + assert!(paths.contains(&".gitops/deploy/templates/deployment.yaml")); + assert!(paths.contains(&".gitops/deploy/templates/service.yaml")); + assert!(!paths.iter().any(|p| p.contains("knative"))); + } + + #[test] + fn knative_deploy_emits_brokers_and_triggers() { + let mut s = spec("orders"); + s.transport = ServiceTransport::Knative; + s.gitops = true; + s.events = vec!["orders.shipped".to_string()]; + let project = generate_service_scaffold(s).unwrap(); + let paths = paths(&project); + assert!(paths.contains(&".gitops/deploy/templates/knative-service.yaml")); + assert!(paths.contains(&".gitops/deploy/templates/knative-brokers.yaml")); + let triggers = contents(&project, ".gitops/deploy/templates/knative-triggers.yaml"); + assert!(triggers.contains("type: orders.shipped")); + } + + #[test] + fn gitops_promote_flux() { + let mut s = spec("orders"); + s.gitops_promote = Some(crate::GitopsPromoteTarget::Flux); + let project = generate_service_scaffold(s).unwrap(); + let paths = paths(&project); + assert!(paths.contains(&".gitops/promote/Chart.yaml")); + assert!(paths.contains(&".gitops/promote/templates/helmrelease.yaml")); } #[test] diff --git a/distributed_tooling/src/generate/names.rs b/distributed_tooling/src/generate/names.rs index 267e6cf..2ad8e45 100644 --- a/distributed_tooling/src/generate/names.rs +++ b/distributed_tooling/src/generate/names.rs @@ -39,6 +39,8 @@ pub(crate) struct ModelScaffold { pub(crate) type_ident: String, pub(crate) view_ident: String, pub(crate) table_name: String, + pub(crate) command_broker: String, + pub(crate) event_broker: String, } impl ModelScaffold { @@ -59,10 +61,50 @@ impl ModelScaffold { type_ident, view_ident, table_name: format!("{ident}_views"), + command_broker: format!("{name}-commands"), + event_broker: format!("{name}-events"), }) } } +/// A Knative `Trigger` (one per command/event handler). +#[derive(Clone, Debug)] +pub(crate) struct KnativeTrigger { + pub(crate) name: String, + pub(crate) broker: String, + pub(crate) event_type: String, +} + +impl KnativeTrigger { + pub(crate) fn new(event_type: &str, broker: &str, suffix: &str) -> Self { + Self { + name: k8s_name(&format!("{}-{suffix}", event_type.replace('.', "-"))), + broker: broker.to_string(), + event_type: event_type.to_string(), + } + } +} + +/// Broker name for a command message (`-commands`). +pub(crate) fn command_broker_for_message(message_name: &str) -> String { + format!("{}-commands", message_owner(message_name)) +} + +/// Broker name for an event message (`-events`), where `owner` is the +/// first segment of a 3+-segment dotted name, else the first segment. +pub(crate) fn event_broker_for_message(message_name: &str) -> String { + let parts = message_name + .split('.') + .filter(|part| !part.is_empty()) + .collect::>(); + let owner = if parts.len() >= 3 { + parts[0] + } else { + parts.first().copied().unwrap_or("events") + }; + format!("{}-events", k8s_name(owner)) +} + pub(crate) fn model_scaffolds(raw_models: &[String]) -> Result, ScaffoldError> { let mut seen = BTreeSet::new(); let mut models = Vec::new(); diff --git a/distributed_tooling/src/generate/service_crate.rs b/distributed_tooling/src/generate/service_crate.rs index 708e556..d365f3c 100644 --- a/distributed_tooling/src/generate/service_crate.rs +++ b/distributed_tooling/src/generate/service_crate.rs @@ -379,7 +379,7 @@ use serde::{{Deserialize, Serialize}}; ) } - fn command_model(&self, handler: &MessageHandler) -> Option<&ModelScaffold> { + pub(super) fn command_model(&self, handler: &MessageHandler) -> Option<&ModelScaffold> { if self.models.is_empty() { return None; } From a4942c3b4b28408683d8e1322e1fd6299ca561d4 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 4 Jun 2026 23:09:29 -0500 Subject: [PATCH 21/24] refactor: model the three GitHub scaffold flags independently The original hops-cli scaffold exposes --github, --github-preview, and --github-promote as independent flags: --github emits the version/release workflows and the repo-create action; --github-preview emits the preview workflow + .gitops/preview/helm chart; --github-promote emits the promote workflow + .gitops/promote/helm chart. Each can be set without the others (e.g. preview-only), and only --github triggers repo creation. The crate previously nested preview/promote under a required GithubScaffoldSpec repository, which could not represent preview-only and tied the workflows to a service repo. Replace it with three flat Option fields on ServiceScaffoldSpec (github / github_preview / github_promote), mirroring the flags 1:1 and dropping the GithubScaffoldSpec wrapper. The deploy chart is now emitted when any of the five gitops/github signals is set. Adds a regression test for the preview-only path. Prepares the faithful hops-cli rewire onto this crate. Implements [[specs/distributed-service-scaffold-tooling]] [[tasks/distributed-tooling-crate-extraction]] Co-Authored-By: Claude Opus 4.8 (1M context) --- distributed_tooling/src/generate/github.rs | 33 +++++++------- distributed_tooling/src/generate/gitops.rs | 8 +++- distributed_tooling/src/generate/mod.rs | 53 ++++++++++++++++------ distributed_tooling/src/lib.rs | 22 ++++----- 4 files changed, 70 insertions(+), 46 deletions(-) diff --git a/distributed_tooling/src/generate/github.rs b/distributed_tooling/src/generate/github.rs index 4e54898..ef5ab4d 100644 --- a/distributed_tooling/src/generate/github.rs +++ b/distributed_tooling/src/generate/github.rs @@ -32,30 +32,31 @@ pub(crate) fn parse_github_repo(raw: &str) -> Result impl Scaffold { /// The `.github/workflows/*` files and (when preview/promote repos are set) - /// the `.gitops/{preview,promote}/helm` promotion charts. + /// the `.gitops/{preview,promote}/helm` promotion charts. The three GitHub + /// repositories gate independent slices: `github` → version/release workflows; + /// `github_preview` → the preview workflow + chart; `github_promote` → the + /// promote workflow + chart. pub(super) fn github_files(&self) -> Vec { let mut files = Vec::new(); - let Some(github) = &self.github else { - return files; - }; - files.push(file( - ".github/workflows/version.yaml", - github_version_workflow_yaml(), - )); - files.push(file( - ".github/workflows/release.yaml", - github_release_workflow_yaml(), - )); - - if let Some(preview) = &github.preview_environment_repository { + if self.github.is_some() { + files.push(file( + ".github/workflows/version.yaml", + github_version_workflow_yaml(), + )); + files.push(file( + ".github/workflows/release.yaml", + github_release_workflow_yaml(), + )); + } + if let Some(preview) = &self.github_preview { files.push(file( ".github/workflows/preview.yaml", self.github_preview_workflow_yaml(preview), )); files.extend(self.github_promotion_chart(".gitops/preview/helm")); } - if let Some(promote) = &github.promote_environment_repository { + if let Some(promote) = &self.github_promote { files.push(file( ".github/workflows/promote.yaml", self.github_promote_workflow_yaml(promote), @@ -206,7 +207,7 @@ preview: false github_repository = self .github .as_ref() - .map(|g| g.repository.slug()) + .map(|g| g.slug()) .unwrap_or_else(|| "OWNER/REPO".to_string()), ) } diff --git a/distributed_tooling/src/generate/gitops.rs b/distributed_tooling/src/generate/gitops.rs index cc59f41..25e8338 100644 --- a/distributed_tooling/src/generate/gitops.rs +++ b/distributed_tooling/src/generate/gitops.rs @@ -16,7 +16,11 @@ impl Scaffold { /// promotion charts target `.gitops/deploy`. pub(super) fn gitops_files(&self) -> Vec { let mut files = Vec::new(); - let want_deploy = self.gitops || self.gitops_promote.is_some() || self.github.is_some(); + let want_deploy = self.gitops + || self.gitops_promote.is_some() + || self.github.is_some() + || self.github_preview.is_some() + || self.github_promote.is_some(); if want_deploy { files.push(file( ".gitops/deploy/Chart.yaml", @@ -84,7 +88,7 @@ impl Scaffold { pub(super) fn image_repository(&self) -> String { self.github .as_ref() - .map(|g| format!("ghcr.io/{}", g.repository.slug().to_ascii_lowercase())) + .map(|g| format!("ghcr.io/{}", g.slug().to_ascii_lowercase())) .unwrap_or_else(|| format!("ghcr.io/hops-ops/{}", self.names.package_name)) } diff --git a/distributed_tooling/src/generate/mod.rs b/distributed_tooling/src/generate/mod.rs index 53e95bf..e21062c 100644 --- a/distributed_tooling/src/generate/mod.rs +++ b/distributed_tooling/src/generate/mod.rs @@ -4,7 +4,8 @@ //! //! - [`names`] — name/message normalization + validation (the portable rules); //! - [`service_crate`] — the Rust service-crate templates; -//! - [`github`] — GitHub repo parsing (+ workflow templates, follow-up slice). +//! - [`gitops`] — the `.gitops/{deploy,promote}` Helm/Knative charts; +//! - [`github`] — GitHub repo parsing + the release/preview/promote workflows. mod github; mod gitops; @@ -21,8 +22,8 @@ use names::{ }; use crate::{ - BusTarget, GeneratedFile, GeneratedProject, GithubScaffoldSpec, GitopsPromoteTarget, - PostCreateAction, ScaffoldError, ServiceScaffoldSpec, ServiceTransport, StoreTarget, + BusTarget, GeneratedFile, GeneratedProject, GithubRepo, GitopsPromoteTarget, PostCreateAction, + ScaffoldError, ServiceScaffoldSpec, ServiceTransport, StoreTarget, }; /// Generate a Distributed service project from a spec. The public entry point. @@ -43,7 +44,9 @@ pub(crate) struct Scaffold { pub(crate) include_read_models: bool, pub(crate) gitops: bool, pub(crate) gitops_promote: Option, - pub(crate) github: Option, + pub(crate) github: Option, + pub(crate) github_preview: Option, + pub(crate) github_promote: Option, pub(crate) models: Vec, pub(crate) read_models: Vec, pub(crate) commands: Vec, @@ -85,6 +88,8 @@ impl Scaffold { gitops: spec.gitops, gitops_promote: spec.gitops_promote, github: spec.github, + github_preview: spec.github_preview, + github_promote: spec.github_promote, models, read_models, commands, @@ -132,10 +137,9 @@ impl Scaffold { files.extend(self.gitops_files()); files.extend(self.github_files()); - if let Some(github) = &self.github { - post_create_actions.push(PostCreateAction::EnsureGithubRepository { - repo: github.repository.clone(), - }); + if let Some(repo) = &self.github { + post_create_actions + .push(PostCreateAction::EnsureGithubRepository { repo: repo.clone() }); } GeneratedProject { @@ -157,8 +161,8 @@ fn file(path: &str, contents: String) -> GeneratedFile { #[cfg(test)] mod tests { use crate::{ - generate_service_scaffold, GeneratedProject, GithubRepo, GithubScaffoldSpec, - PostCreateAction, ServiceScaffoldSpec, ServiceTransport, StoreTarget, + generate_service_scaffold, GeneratedProject, GithubRepo, PostCreateAction, + ServiceScaffoldSpec, ServiceTransport, StoreTarget, }; fn spec(name: &str) -> ServiceScaffoldSpec { @@ -175,6 +179,8 @@ mod tests { gitops: false, gitops_promote: None, github: None, + github_preview: None, + github_promote: None, } } @@ -246,11 +252,9 @@ mod tests { #[test] fn github_generates_workflows_and_a_post_create_action() { let mut s = spec("orders"); - s.github = Some(GithubScaffoldSpec { - repository: GithubRepo::parse("hops-ops/orders").unwrap(), - preview_environment_repository: Some(GithubRepo::parse("hops-ops/preview").unwrap()), - promote_environment_repository: Some(GithubRepo::parse("hops-ops/prod").unwrap()), - }); + s.github = Some(GithubRepo::parse("hops-ops/orders").unwrap()); + s.github_preview = Some(GithubRepo::parse("hops-ops/preview").unwrap()); + s.github_promote = Some(GithubRepo::parse("hops-ops/prod").unwrap()); let project = generate_service_scaffold(s).unwrap(); let paths = paths(&project); for expected in [ @@ -280,6 +284,25 @@ mod tests { assert!(project.warnings.is_empty()); } + #[test] + fn github_preview_is_independent_of_the_service_repo() { + // Only --github-preview: a preview workflow + chart, the deploy chart it + // targets, but NO version/release workflows and NO repo-create action. + let mut s = spec("orders"); + s.github_preview = Some(GithubRepo::parse("hops-ops/preview").unwrap()); + let project = generate_service_scaffold(s).unwrap(); + let paths = paths(&project); + assert!(paths.contains(&".github/workflows/preview.yaml")); + assert!(paths.contains(&".gitops/preview/helm/Chart.yaml")); + assert!(paths.contains(&".gitops/deploy/Chart.yaml")); + assert!(!paths.contains(&".github/workflows/version.yaml")); + assert!(!paths.contains(&".github/workflows/release.yaml")); + assert!(project.post_create_actions.is_empty()); + // No service repo → the default image repository. + let preview = contents(&project, ".github/workflows/preview.yaml"); + assert!(preview.contains("ghcr.io/hops-ops/orders")); + } + #[test] fn gitops_http_deploy_chart() { let mut s = spec("orders"); diff --git a/distributed_tooling/src/lib.rs b/distributed_tooling/src/lib.rs index 5740c85..223dbd7 100644 --- a/distributed_tooling/src/lib.rs +++ b/distributed_tooling/src/lib.rs @@ -45,8 +45,15 @@ pub struct ServiceScaffoldSpec { pub gitops: bool, /// Generate a GitOps promotion chart for Argo CD or Flux. pub gitops_promote: Option, - /// GitHub repository + release/GitOps workflow scaffolding. - pub github: Option, + /// The service's own GitHub repository: emits the version/release workflows + /// and an `EnsureGithubRepository` post-create action. + pub github: Option, + /// Preview-environment GitOps repository: emits the preview workflow and the + /// `.gitops/preview/helm` promotion chart. Independent of `github`. + pub github_preview: Option, + /// Permanent-environment GitOps repository: emits the promote workflow and the + /// `.gitops/promote/helm` promotion chart. Independent of `github`. + pub github_promote: Option, } /// Runtime transport for the scaffolded service. @@ -103,17 +110,6 @@ pub enum GitopsPromoteTarget { Flux, } -/// GitHub repository + workflow scaffolding inputs. -#[derive(Clone, Debug)] -pub struct GithubScaffoldSpec { - /// The service's own GitHub repository. - pub repository: GithubRepo, - /// Optional preview-environment GitOps repository. - pub preview_environment_repository: Option, - /// Optional permanent-environment GitOps repository. - pub promote_environment_repository: Option, -} - /// An `owner/repo` GitHub identifier. #[derive(Clone, Debug, PartialEq, Eq)] pub struct GithubRepo { From a1b83d3a46095b06cc681ae45362c351a29e81fe Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 4 Jun 2026 23:11:25 -0500 Subject: [PATCH 22/24] feat: expose package_name() for default output-dir derivation The CLI adapter needs the normalized kebab package name to compute the default output directory (./) before generating. Expose the existing ScaffoldNames normalization as a public helper instead of duplicating the casing rule in the CLI. Implements [[tasks/distributed-tooling-crate-extraction]] Co-Authored-By: Claude Opus 4.8 (1M context) --- distributed_tooling/src/generate/mod.rs | 7 +++++++ distributed_tooling/src/lib.rs | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/distributed_tooling/src/generate/mod.rs b/distributed_tooling/src/generate/mod.rs index e21062c..d7dfcd0 100644 --- a/distributed_tooling/src/generate/mod.rs +++ b/distributed_tooling/src/generate/mod.rs @@ -33,6 +33,13 @@ pub fn generate_service_scaffold( Ok(Scaffold::from_spec(spec)?.generate()) } +/// Normalize a free-form service name to its kebab-case package/crate directory +/// name — the same normalization [`generate_service_scaffold`] applies. Callers +/// need this to compute a default output directory before generating. +pub fn package_name(name: &str) -> Result { + Ok(ScaffoldNames::new(name)?.package_name) +} + /// The normalized scaffold: spec values resolved into derived names, models, and /// handlers. Template methods are implemented in the `service_crate` submodule. pub(crate) struct Scaffold { diff --git a/distributed_tooling/src/lib.rs b/distributed_tooling/src/lib.rs index 223dbd7..4732f12 100644 --- a/distributed_tooling/src/lib.rs +++ b/distributed_tooling/src/lib.rs @@ -13,7 +13,7 @@ mod generate; -pub use generate::generate_service_scaffold; +pub use generate::{generate_service_scaffold, package_name}; /// What to scaffold. The pure input to [`generate_service_scaffold`]. /// From 75470332ceb1c6b8983db82033ff7b66c85b64e9 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 5 Jun 2026 19:34:38 -0500 Subject: [PATCH 23/24] =?UTF-8?q?fix:=20address=20PR=20#53=20review=20?= =?UTF-8?q?=E2=80=94=20generated-output=20correctness=20+=20builder=20guar?= =?UTF-8?q?ds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - reject service/model names that yield invalid Rust identifiers, instead of emitting a crate that won't compile - deploy Helm templates honor image.repository/tag rather than hardcoding :latest, so values.yaml/release automation actually drives the image - dedupe Knative trigger names that normalize to the same metadata.name, which otherwise breaks `kubectl apply` - fail fast when dependency builders (with_repo/with_read_model_store) run after handler/bus setup, which silently dropped registrations - reject snapshot frequency 0 (would snapshot on every commit) - correct outbox claim-timing wording: the row is claimed post-commit under a short lease, not within the commit transaction Implements [[tasks/distributed-tooling-crate-extraction]] Co-Authored-By: Claude Opus 4.8 --- README.md | 7 ++-- distributed_tooling/src/generate/gitops.rs | 20 +++++++-- distributed_tooling/src/generate/mod.rs | 49 ++++++++++++++++++++++ distributed_tooling/src/generate/names.rs | 32 ++++++++++++++ src/microsvc/runtime.rs | 6 +-- src/microsvc/service.rs | 14 +++++++ src/snapshot/repository.rs | 12 ++++++ 7 files changed, 130 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index ecc0481..dba1a57 100644 --- a/README.md +++ b/README.md @@ -739,9 +739,10 @@ How a committed row reaches the bus depends on whether a bus is attached to the service: - **Bus attached (`service.with_bus(bus)`)** — `repo.outbox(msg).commit(agg)` - claims the row in the commit transaction and publishes it **immediately** after - commit. A crash before the publish, or a publish failure, leaves the row - claimed under a short lease; when the lease expires the polling worker takes it. + commits the row, then **immediately** after commit claims it under a short + lease and publishes it. A crash before the publish, or a publish failure, + leaves the row claimed under that lease; when the lease expires the polling + worker takes it. - **No bus** — the row is committed `pending` and a worker publishes it. The polling worker is the durable backstop in both cases. It is the same diff --git a/distributed_tooling/src/generate/gitops.rs b/distributed_tooling/src/generate/gitops.rs index 25e8338..53231bb 100644 --- a/distributed_tooling/src/generate/gitops.rs +++ b/distributed_tooling/src/generate/gitops.rs @@ -125,6 +125,11 @@ impl Scaffold { } fn knative_triggers(&self) -> Vec { + // Different message names can normalize to the same `metadata.name` + // (e.g. `orders.Created` and `orders.created`). Deduplicate so the + // generated manifest never declares two Triggers with the same name, + // which would fail `kubectl apply`. + let mut seen = BTreeSet::new(); self.commands .iter() .map(|command| { @@ -138,6 +143,15 @@ impl Scaffold { let broker = event_broker_for_message(&event.message_name); KnativeTrigger::new(&event.message_name, &broker, "event") })) + .map(|mut trigger| { + let base = trigger.name.clone(); + let mut suffix = 2; + while !seen.insert(trigger.name.clone()) { + trigger.name = format!("{base}-{suffix}"); + suffix += 1; + } + trigger + }) .collect() } @@ -194,7 +208,7 @@ spec: spec: containers: - name: {name} - image: {image_repository}:latest + image: {{{{ .Values.image.repository }}}}:{{{{ .Values.image.tag }}}} ports: - containerPort: 3000 env: @@ -202,7 +216,6 @@ spec: value: 0.0.0.0:3000 {bus_env} "#, - image_repository = self.image_repository(), ) } @@ -243,7 +256,7 @@ spec: autoscaling.knative.dev/min-scale: "0" spec: containers: - - image: {image_repository}:latest + - image: {{{{ .Values.image.repository }}}}:{{{{ .Values.image.tag }}}} ports: - containerPort: 3000 env: @@ -251,7 +264,6 @@ spec: value: 0.0.0.0:3000 {bus_env} "#, - image_repository = self.image_repository(), ) } diff --git a/distributed_tooling/src/generate/mod.rs b/distributed_tooling/src/generate/mod.rs index d7dfcd0..d597618 100644 --- a/distributed_tooling/src/generate/mod.rs +++ b/distributed_tooling/src/generate/mod.rs @@ -256,6 +256,55 @@ mod tests { assert!(paths.contains(&"src/read_models/mod.rs")); } + #[test] + fn invalid_rust_identifier_names_are_rejected() { + // A service whose normalized crate identifier would start with a digit + // (or be a keyword) cannot compile — fail at generation, not `cargo build`. + assert!(generate_service_scaffold(spec("3d")).is_err()); + + let mut keyword = spec("orders"); + keyword.models = vec!["enum".to_string()]; + assert!(generate_service_scaffold(keyword).is_err()); + + let mut digit_model = spec("orders"); + digit_model.models = vec!["3d".to_string()]; + assert!(generate_service_scaffold(digit_model).is_err()); + } + + #[test] + fn deploy_templates_consume_helm_image_values() { + // The deploy chart's values.yaml exposes image.repository/tag, so the + // templates must reference them — hardcoding `:latest` makes the knob dead + // and breaks release automation that patches values.yaml. + let mut s = spec("orders"); + s.gitops = true; + let project = generate_service_scaffold(s).unwrap(); + let deployment = contents(&project, ".gitops/deploy/templates/deployment.yaml"); + assert!( + deployment.contains("{{ .Values.image.repository }}:{{ .Values.image.tag }}"), + "deployment.yaml should template the image from values:\n{deployment}" + ); + assert!(!deployment.contains(":latest")); + } + + #[test] + fn knative_triggers_with_colliding_names_are_deduplicated() { + // Two command names that normalize to the same Trigger `metadata.name` + // must still produce two distinctly-named Trigger resources, or + // `kubectl apply` fails on the duplicate. + let mut s = spec("orders"); + s.transport = ServiceTransport::Knative; + s.gitops = true; + s.commands = vec!["orders.created".to_string(), "orders.Created".to_string()]; + let project = generate_service_scaffold(s).unwrap(); + let triggers = contents(&project, ".gitops/deploy/templates/knative-triggers.yaml"); + assert!(triggers.contains("name: orders-created-command")); + assert!( + triggers.contains("name: orders-created-command-2"), + "the colliding trigger should be suffixed:\n{triggers}" + ); + } + #[test] fn github_generates_workflows_and_a_post_create_action() { let mut s = spec("orders"); diff --git a/distributed_tooling/src/generate/names.rs b/distributed_tooling/src/generate/names.rs index 2ad8e45..1ec860c 100644 --- a/distributed_tooling/src/generate/names.rs +++ b/distributed_tooling/src/generate/names.rs @@ -21,6 +21,12 @@ impl ScaffoldNames { )); } let crate_ident = package_name.replace('-', "_"); + if !is_rust_ident(&crate_ident) { + return Err(ScaffoldError::new(format!( + "service name `{input}` yields the invalid Rust crate identifier `{crate_ident}`; \ + start the name with a letter and avoid Rust keywords" + ))); + } let command_name = format!("{crate_ident}.create"); Ok(Self { package_name, @@ -52,6 +58,12 @@ impl ModelScaffold { )); } let ident = name.replace('-', "_"); + if !is_rust_ident(&ident) { + return Err(ScaffoldError::new(format!( + "model name `{raw_name}` yields the invalid Rust identifier `{ident}`; \ + start the name with a letter and avoid Rust keywords" + ))); + } let type_ident = to_pascal_case(&name); let view_ident = format!("{type_ident}View"); Ok(Self { @@ -229,6 +241,26 @@ fn to_rust_ident(value: &str, fallback_prefix: &str) -> String { ident } +/// True if `value` is a usable Rust identifier given this crate's normalization +/// (already lowercased ASCII alphanumerics and `_`): non-empty, not starting +/// with a digit, and not a reserved keyword. Guards against generated crates +/// whose name would not compile (e.g. a service called `3d`). +fn is_rust_ident(value: &str) -> bool { + let Some(first) = value.chars().next() else { + return false; + }; + if !(first.is_ascii_alphabetic() || first == '_') { + return false; + } + if value + .chars() + .any(|char| !(char.is_ascii_alphanumeric() || char == '_')) + { + return false; + } + !is_rust_keyword(value) +} + fn is_rust_keyword(value: &str) -> bool { matches!( value, diff --git a/src/microsvc/runtime.rs b/src/microsvc/runtime.rs index 67de36a..11576c3 100644 --- a/src/microsvc/runtime.rs +++ b/src/microsvc/runtime.rs @@ -34,9 +34,9 @@ where /// /// Two effects, both composing with the rest of the builder: /// - installs an outbox publisher on the repository, so - /// `repo.outbox(msg).commit(agg)` claims the row in the commit transaction - /// and publishes it immediately through this bus (the polling worker stays - /// the crash/retry backstop); + /// `repo.outbox(msg).commit(agg)` publishes immediately after commit + /// through this bus — the row is claimed under a short lease for that + /// publish, and the polling worker stays the crash/retry backstop; /// - captures how to consume, so [`run`](Self::run) listens for the /// registered command names (competing) and subscribes to the event names /// (fan-out). diff --git a/src/microsvc/service.rs b/src/microsvc/service.rs index d7c6d45..6645da3 100644 --- a/src/microsvc/service.rs +++ b/src/microsvc/service.rs @@ -205,6 +205,17 @@ impl Service { } } + /// Fail fast if handlers, specs, or a runner are already registered. The + /// dependency builders (`with_repo`, `with_read_model_store`) reconstruct the + /// service around a new dependency type, which would otherwise silently drop + /// anything registered earlier and leave an empty router with no error. + fn assert_no_registrations(&self, builder: &str) { + assert!( + self.handlers.is_empty() && self.handler_specs.is_empty() && self.runner.is_none(), + "Service::{builder} must be called before registering handlers or attaching a bus" + ); + } + /// Mutable access to the dependencies, used by `with_bus` to install the /// outbox publisher on the repository before the service is shared. pub(crate) fn dependencies_mut(&mut self) -> &mut D { @@ -462,6 +473,7 @@ impl Service<()> { where R: HasRepo + Send + Sync + 'static, { + self.assert_no_registrations("with_repo"); Service::from_dependencies(repo) } @@ -470,6 +482,7 @@ impl Service<()> { where S: HasReadModelStore + Send + Sync + 'static, { + self.assert_no_registrations("with_read_model_store"); Service::from_dependencies(read_model_store) } } @@ -485,6 +498,7 @@ impl Service { where S: HasReadModelStore + Send + Sync + 'static, { + self.assert_no_registrations("with_read_model_store"); Service::from_dependencies(RepoReadModelDependencies::new( self.dependencies, read_model_store, diff --git a/src/snapshot/repository.rs b/src/snapshot/repository.rs index eee1055..dd8f8de 100644 --- a/src/snapshot/repository.rs +++ b/src/snapshot/repository.rs @@ -182,6 +182,11 @@ where /// aggregate is hydrated from a snapshot when one exists. Every other method /// behaves identically with or without snapshots. pub fn with_snapshots(mut self, frequency: u64) -> Self { + assert!( + frequency > 0, + "snapshot frequency must be greater than zero; \ + frequency 0 would snapshot on every commit" + ); self.set_snapshot_policy(SnapshotPolicy::new( frequency, snapshot_record_if_due::, @@ -324,6 +329,13 @@ mod tests { assert_eq!(aggregate.entity.new_events().len(), 1); } + #[test] + #[should_panic(expected = "snapshot frequency must be greater than zero")] + fn with_snapshots_rejects_zero_frequency() { + let _: AggregateRepository<_, TestAggregate> = + AggregateRepository::new(FailingSnapshotRepo::default()).with_snapshots(0); + } + #[test] fn snapshot_due_uses_saturating_version_distance() { assert!(snapshot_due(5, 2, 3)); From d931a6e8de6feff113d1f4cbb0769deb7e62ec18 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 6 Jun 2026 00:20:03 -0500 Subject: [PATCH 24/24] ci: publish distributed_tooling on version tags distributed_tooling was a workspace member but absent from the release pipeline, so it could never become a crates.io dependency. Add a publish-tooling job alongside publish-macros (the crate only depends on serde_json, so it has no internal publish ordering) and gate the release on it. Implements [[tasks/distributed-tooling-crate-extraction]] Co-Authored-By: Claude Opus 4.8 --- .github/workflows/on-v-tag-publish.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/on-v-tag-publish.yaml b/.github/workflows/on-v-tag-publish.yaml index 199001b..4dabfd1 100644 --- a/.github/workflows/on-v-tag-publish.yaml +++ b/.github/workflows/on-v-tag-publish.yaml @@ -17,6 +17,16 @@ jobs: manifest_path: distributed_macros/Cargo.toml cargo_publish_args: "--locked" + # distributed_tooling is a standalone crate (only depends on serde_json), so it + # publishes independently of the macros/core crates. + publish-tooling: + uses: unbounded-tech/workflows-rust/.github/workflows/publish.yaml@feat/cargo-publish + secrets: + crates_io_token: ${{ secrets.CRATES_IO_TOKEN }} + with: + manifest_path: distributed_tooling/Cargo.toml + cargo_publish_args: "--locked" + publish: needs: publish-macros uses: unbounded-tech/workflows-rust/.github/workflows/publish.yaml@feat/cargo-publish @@ -27,7 +37,7 @@ jobs: cargo_publish_args: "--locked" release: - needs: publish + needs: [publish, publish-tooling] permissions: contents: write uses: unbounded-tech/workflow-simple-release/.github/workflows/workflow.yaml@main