Skip to content

otap: real plugin integration + Arrow IPC network transport #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zzylol wants to merge 1 commit into
base: feat/schema-dictionary-record-codec
Choose a base branch
from
Open

otap: real plugin integration + Arrow IPC network transport #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
176 changes: 173 additions & 3 deletions asap-precompute-rs/Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

27 changes: 22 additions & 5 deletions asap-precompute-rs/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,17 @@ thiserror = "1"
# non-default to keep their build cheap.
arrow-array = { version = "53", optional = true }
arrow-schema = { version = "53", optional = true }
# Only pulled in for otap::wire's Arrow IPC serialization of
# SketchStreamBatch across a real transport (see docs/data_model.md's
# "crosses a node or network boundary" framing).
arrow-ipc = { version = "53", optional = true }

# Tokio drives the plugin lifecycle (Stream consumer
# task + Wakeup-driven flush ticker + control-channel poll task +
# graceful drain). Only the `rt`, `sync`, `time`, and `macros`
# sub-features are needed; the full set is pulled in via the `otap`
# feature gate so default-feature consumers don't pay for it.
tokio = { version = "1", features = ["rt", "rt-multi-thread", "sync", "time", "macros"], optional = true }
# graceful drain), plus `otap::wire`'s TCP transport (`net`,
# `io-util`). The full set is pulled in via the `otap` feature gate so
# default-feature consumers don't pay for it.
tokio = { version = "1", features = ["rt", "rt-multi-thread", "sync", "time", "macros", "net", "io-util"], optional = true }
futures = { version = "0.3", optional = true }

[dev-dependencies]
Expand All @@ -32,14 +36,27 @@ serde_json = "1"
# The full tokio runtime is required for #[tokio::test]-driven
# lifecycle tests under `tests/otap_*.rs`.
tokio = { version = "1", features = ["full"] }
# Only used to adapt an `mpsc::UnboundedReceiver` into a `Stream` for
# chaining two `AsapSketchesPlugin`s (producer's EmitReceiver ->
# receiver's start_from_envelopes) in tests and the network-transport
# examples. The library itself is Stream-source-agnostic.
tokio-stream = "0.1"

[features]
default = []
# Enables the OTAP codec + plugin lifecycle at `crate::otap`
# (decode_batch / encode_batch over `arrow::RecordBatch`, plus the
# `AsapSketchesPlugin` Tokio runtime).
otap = ["dep:arrow-array", "dep:arrow-schema", "dep:tokio", "dep:futures"]
otap = ["dep:arrow-array", "dep:arrow-schema", "dep:arrow-ipc", "dep:tokio", "dep:futures"]

[[example]]
name = "sketch_pipeline_demo"
required-features = ["otap"]

[[example]]
name = "sketch_producer_node"
required-features = ["otap"]

[[example]]
name = "sketch_receiver_node"
required-features = ["otap"]
Loading