From 2bc55b1a46cc94ebf479cdbd30fe8bd12810633f Mon Sep 17 00:00:00 2001 From: Bartek Plotka Date: Thu, 12 Nov 2026 07:37:36 +0000 Subject: [PATCH 1/2] workspace: split into prometheus-expfmt, prometheus-relabel, and prometheus-scrape-manager Convert scrape.rs into a Cargo workspace along the pure-CPU sync vs async network/runtime boundary: - crates/expfmt (prometheus-expfmt): wire exposition format negotiation, Layer 0 parsers, ArrowDecoder, and Protobuf DTOs. - crates/relabel (prometheus-relabel): target and metric relabeling plus Stage 1/2 dictionary pushdown primitives. - crates/scrape-manager (prometheus-scrape-manager, publish = false for now): workspace home for upcoming discovery, http_config, and scrape manager modules. Signed-off-by: Bartek Plotka --- .github/workflows/ci.yml | 12 ++-- Cargo.lock | 52 ++++++++++------ Cargo.toml | 61 ++++++++----------- crates/expfmt/Cargo.toml | 41 +++++++++++++ crates/expfmt/README.md | 7 +++ {benches => crates/expfmt/benches}/decode.rs | 14 ++--- .../proto}/io/prometheus/client/metrics.proto | 0 {src => crates/expfmt/src}/arrow.rs | 0 {src => crates/expfmt/src}/arrow/columns.rs | 0 {src => crates/expfmt/src}/arrow/family.rs | 0 {src => crates/expfmt/src}/arrow/intern.rs | 0 {src => crates/expfmt/src}/arrow/options.rs | 0 {src => crates/expfmt/src}/arrow/records.rs | 0 {src => crates/expfmt/src}/arrow/schema.rs | 0 {src => crates/expfmt/src}/arrow/tests.rs | 0 {src => crates/expfmt/src}/dto.rs | 0 .../expfmt/src}/dto/io.prometheus.client.rs | 0 {src => crates/expfmt/src}/format.rs | 0 {src => crates/expfmt/src}/lib.rs | 2 - {src => crates/expfmt/src}/parse.rs | 0 {src => crates/expfmt/src}/parse/float.rs | 0 .../expfmt/src}/parse/go_diff_tests.rs | 2 +- {src => crates/expfmt/src}/parse/omlex.rs | 0 {src => crates/expfmt/src}/parse/omparse.rs | 0 {src => crates/expfmt/src}/parse/promlex.rs | 0 {src => crates/expfmt/src}/parse/promparse.rs | 0 .../expfmt/src}/parse/protobufparse.rs | 0 .../expfmt/tests}/fixtures_smoke.rs | 6 +- crates/relabel/Cargo.toml | 31 ++++++++++ crates/relabel/README.md | 9 +++ .../relabel/benches}/relabel.rs | 8 +-- src/relabel.rs => crates/relabel/src/lib.rs | 0 {tests => crates/relabel/tests}/relabel.rs | 4 +- crates/scrape-manager/Cargo.toml | 24 ++++++++ crates/scrape-manager/README.md | 5 ++ crates/scrape-manager/src/lib.rs | 20 ++++++ tests/otap-compat/Cargo.toml | 6 +- tests/otap-compat/src/lib.rs | 2 +- tests/otap-compat/tests/arrow_otap.rs | 14 ++--- tests/otap-compat/tests/dhat_bounded_mem.rs | 6 +- 40 files changed, 232 insertions(+), 94 deletions(-) create mode 100644 crates/expfmt/Cargo.toml create mode 100644 crates/expfmt/README.md rename {benches => crates/expfmt/benches}/decode.rs (93%) rename {proto => crates/expfmt/proto}/io/prometheus/client/metrics.proto (100%) rename {src => crates/expfmt/src}/arrow.rs (100%) rename {src => crates/expfmt/src}/arrow/columns.rs (100%) rename {src => crates/expfmt/src}/arrow/family.rs (100%) rename {src => crates/expfmt/src}/arrow/intern.rs (100%) rename {src => crates/expfmt/src}/arrow/options.rs (100%) rename {src => crates/expfmt/src}/arrow/records.rs (100%) rename {src => crates/expfmt/src}/arrow/schema.rs (100%) rename {src => crates/expfmt/src}/arrow/tests.rs (100%) rename {src => crates/expfmt/src}/dto.rs (100%) rename {src => crates/expfmt/src}/dto/io.prometheus.client.rs (100%) rename {src => crates/expfmt/src}/format.rs (100%) rename {src => crates/expfmt/src}/lib.rs (95%) rename {src => crates/expfmt/src}/parse.rs (100%) rename {src => crates/expfmt/src}/parse/float.rs (100%) rename {src => crates/expfmt/src}/parse/go_diff_tests.rs (99%) rename {src => crates/expfmt/src}/parse/omlex.rs (100%) rename {src => crates/expfmt/src}/parse/omparse.rs (100%) rename {src => crates/expfmt/src}/parse/promlex.rs (100%) rename {src => crates/expfmt/src}/parse/promparse.rs (100%) rename {src => crates/expfmt/src}/parse/protobufparse.rs (100%) rename {tests => crates/expfmt/tests}/fixtures_smoke.rs (92%) create mode 100644 crates/relabel/Cargo.toml create mode 100644 crates/relabel/README.md rename {benches => crates/relabel/benches}/relabel.rs (98%) rename src/relabel.rs => crates/relabel/src/lib.rs (100%) rename {tests => crates/relabel/tests}/relabel.rs (99%) create mode 100644 crates/scrape-manager/Cargo.toml create mode 100644 crates/scrape-manager/README.md create mode 100644 crates/scrape-manager/src/lib.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fc36a1..9beb494 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,13 +29,13 @@ jobs: - name: Check formatting (OTAP compatibility suite) run: cargo fmt --manifest-path tests/otap-compat/Cargo.toml --all -- --check - name: Run clippy - run: cargo clippy --all-targets --all-features -- -D warnings + run: cargo clippy --workspace --all-targets --all-features -- -D warnings - name: Run clippy (no default features) - run: cargo clippy --all-targets --no-default-features -- -D warnings + run: cargo clippy --workspace --all-targets --no-default-features -- -D warnings - name: Run tests (all features) - run: cargo test --all-features + run: cargo test --workspace --all-features - name: Run tests (no default features) - run: cargo test --no-default-features + run: cargo test --workspace --no-default-features - name: Run clippy (OTAP compatibility suite) env: CARGO_TARGET_DIR: target @@ -53,9 +53,9 @@ jobs: - uses: dtolnay/rust-toolchain@1.88 - uses: Swatinem/rust-cache@v2 - name: Check MSRV (all features) - run: cargo check --all-targets --all-features + run: cargo check --workspace --all-targets --all-features - name: Check MSRV (no default features) - run: cargo check --all-targets --no-default-features + run: cargo check --workspace --all-targets --no-default-features go-fixtures: name: Go testdata sync check diff --git a/Cargo.lock b/Cargo.lock index 5ebb7dc..5482a61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -199,24 +199,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "expfmt" -version = "0.1.0" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-schema", - "benchfmt", - "prost", - "prost-build", - "prost-types", - "protox", - "regex", - "serde", - "serde_json", - "thiserror", -] - [[package]] name = "fastrand" version = "2.5.0" @@ -557,6 +539,40 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prometheus-expfmt" +version = "0.1.0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "benchfmt", + "prost", + "prost-build", + "prost-types", + "protox", + "thiserror", +] + +[[package]] +name = "prometheus-relabel" +version = "0.1.0" +dependencies = [ + "benchfmt", + "regex", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "prometheus-scrape-manager" +version = "0.1.0" +dependencies = [ + "prometheus-expfmt", + "prometheus-relabel", +] + [[package]] name = "prost" version = "0.14.4" diff --git a/Cargo.toml b/Cargo.toml index 0db2232..f1bc98a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,52 +1,39 @@ -[package] -name = "expfmt" +[workspace] +resolver = "3" +members = [ + "crates/expfmt", + "crates/relabel", + "crates/scrape-manager", +] +exclude = ["tests/otap-compat"] + +[workspace.package] version = "0.1.0" edition = "2024" rust-version = "1.88" -description = "Prometheus and OpenMetrics wire exposition format decoders" license = "Apache-2.0" -repository = "https://github.com/prometheus/expfmt-rs" -keywords = ["prometheus", "openmetrics", "metrics", "opentelemetry", "arrow"] -categories = ["parser-implementations", "encoding"] - -[workspace] -exclude = ["tests/otap-compat"] +repository = "https://github.com/prometheus/scrape.rs" -[features] -default = ["arrow", "dto", "relabel"] -arrow = ["dep:arrow-array", "dep:arrow-buffer", "dep:arrow-schema"] -dto = ["dep:prost", "dep:prost-types"] -relabel = ["dep:regex", "dep:serde"] +[workspace.dependencies] +prometheus-expfmt = { version = "0.1.0", path = "crates/expfmt", default-features = false } +prometheus-relabel = { version = "0.1.0", path = "crates/relabel" } -[dependencies] -arrow-array = { version = "59", default-features = false, optional = true } -arrow-buffer = { version = "59", default-features = false, optional = true } -arrow-schema = { version = "59", default-features = false, optional = true } -prost = { version = "0.14", optional = true } -prost-types = { version = "0.14", optional = true } -regex = { version = "1", optional = true } -serde = { version = "1", features = ["derive"], optional = true } -thiserror = "2" - -[dev-dependencies] +arrow-array = { version = "59", default-features = false } +arrow-buffer = { version = "59", default-features = false } +arrow-schema = { version = "59", default-features = false } benchfmt = { git = "https://github.com/bwplotka/benchfmt.rs" } +prost = { version = "0.14" } prost-build = "=0.14.4" +prost-types = { version = "0.14" } protox = "0.10" +regex = { version = "1" } +serde = { version = "1", features = ["derive"] } serde_json = "1" +thiserror = "2" -[[bench]] -name = "decode" -harness = false -required-features = ["arrow"] - -[[bench]] -name = "relabel" -harness = false -required-features = ["relabel"] - -[lints.rust] +[workspace.lints.rust] missing_docs = "warn" unsafe_code = "forbid" -[lints.clippy] +[workspace.lints.clippy] all = "warn" diff --git a/crates/expfmt/Cargo.toml b/crates/expfmt/Cargo.toml new file mode 100644 index 0000000..cec1acc --- /dev/null +++ b/crates/expfmt/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "prometheus-expfmt" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Prometheus and OpenMetrics wire exposition format decoders for Apache Arrow (OTAP) and Protobuf DTOs" +readme = "README.md" +keywords = ["prometheus", "openmetrics", "metrics", "opentelemetry", "arrow"] +categories = ["parser-implementations", "encoding"] +exclude = [ + "benches/**", + "tests/**", +] + +[features] +default = ["arrow", "dto"] +arrow = ["dep:arrow-array", "dep:arrow-buffer", "dep:arrow-schema"] +dto = ["dep:prost", "dep:prost-types"] + +[dependencies] +arrow-array = { workspace = true, optional = true } +arrow-buffer = { workspace = true, optional = true } +arrow-schema = { workspace = true, optional = true } +prost = { workspace = true, optional = true } +prost-types = { workspace = true, optional = true } +thiserror = { workspace = true } + +[dev-dependencies] +benchfmt = { workspace = true } +prost-build = { workspace = true } +protox = { workspace = true } + +[[bench]] +name = "decode" +harness = false +required-features = ["arrow"] + +[lints] +workspace = true diff --git a/crates/expfmt/README.md b/crates/expfmt/README.md new file mode 100644 index 0000000..09e1c67 --- /dev/null +++ b/crates/expfmt/README.md @@ -0,0 +1,7 @@ +# prometheus-expfmt + +Official Rust decoders and content negotiation for Prometheus and OpenMetrics wire exposition formats, part of the [`prometheus/scrape.rs`](https://github.com/prometheus/scrape.rs) workspace: + +- `format`: `Content-Type` and `Accept` negotiation matching [`prometheus/common/expfmt`](https://github.com/prometheus/common/tree/main/expfmt). +- `arrow` *(default feature)*: Zero-copy streaming decoder (`ArrowDecoder`) producing columnar Apache Arrow / OpenTelemetry Arrow (`OtapArrowRecords`) metric tables with bounded memory. +- `dto` *(default feature)*: `io.prometheus.client.MetricFamily` protobuf DTO decoding for CLIs, live-checkers, and test harnesses. diff --git a/benches/decode.rs b/crates/expfmt/benches/decode.rs similarity index 93% rename from benches/decode.rs rename to crates/expfmt/benches/decode.rs index 4a848cf..0d1609a 100644 --- a/benches/decode.rs +++ b/crates/expfmt/benches/decode.rs @@ -11,12 +11,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Throughput and allocation benchmarks for `expfmt::arrow::ArrowDecoder` over the upstream -//! Prometheus `model/textparse` fixture corpus. +//! Throughput and allocation benchmarks for `prometheus_expfmt::arrow::ArrowDecoder` over the +//! upstream Prometheus `model/textparse` fixture corpus. //! //! Recommended CLI invocation: //! ```sh -//! cargo bench --bench decode -- -bench '^BenchmarkDecode' -benchtime 1s -count 6 -cpu 2 -benchmem | tee ${bench}.txt +//! cargo bench -p prometheus-expfmt --bench decode -- -bench '^BenchmarkDecode' -benchtime 1s -count 6 -cpu 2 -benchmem | tee ${bench}.txt //! ``` use std::fs; @@ -27,8 +27,8 @@ use std::pin::{Pin, pin}; use std::task::{Context, Poll, Waker}; use benchfmt::{CountingAlloc, Suite}; -use expfmt::arrow::{ArrowDecoder, DecodeBudget, DecodeOptions, DecodeStep}; -use expfmt::format::Format; +use prometheus_expfmt::arrow::{ArrowDecoder, DecodeBudget, DecodeOptions, DecodeStep}; +use prometheus_expfmt::format::Format; #[global_allocator] static ALLOC: CountingAlloc = CountingAlloc::new(); @@ -103,7 +103,7 @@ fn filter_counter_gauge_families(raw: &str) -> Vec { } fn main() { - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/textparse"); + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../testdata/textparse"); let nometa = fs::read(root.join("alltypes.237mfs.nometa.prom.txt")).unwrap(); let full = fs::read_to_string(root.join("alltypes.237mfs.prom.txt")).unwrap(); let typed_simple = filter_counter_gauge_families(&full); @@ -113,7 +113,7 @@ fn main() { let stream_256_opts = opts.with_max_data_points_per_batch(256); let budget = DecodeBudget::default(); - let mut suite = Suite::from_env("expfmt").with_alloc(&ALLOC); + let mut suite = Suite::from_env("prometheus-expfmt").with_alloc(&ALLOC); for (case, payload) in [ ("alltypes.237mfs.nometa", nometa.as_slice()), diff --git a/proto/io/prometheus/client/metrics.proto b/crates/expfmt/proto/io/prometheus/client/metrics.proto similarity index 100% rename from proto/io/prometheus/client/metrics.proto rename to crates/expfmt/proto/io/prometheus/client/metrics.proto diff --git a/src/arrow.rs b/crates/expfmt/src/arrow.rs similarity index 100% rename from src/arrow.rs rename to crates/expfmt/src/arrow.rs diff --git a/src/arrow/columns.rs b/crates/expfmt/src/arrow/columns.rs similarity index 100% rename from src/arrow/columns.rs rename to crates/expfmt/src/arrow/columns.rs diff --git a/src/arrow/family.rs b/crates/expfmt/src/arrow/family.rs similarity index 100% rename from src/arrow/family.rs rename to crates/expfmt/src/arrow/family.rs diff --git a/src/arrow/intern.rs b/crates/expfmt/src/arrow/intern.rs similarity index 100% rename from src/arrow/intern.rs rename to crates/expfmt/src/arrow/intern.rs diff --git a/src/arrow/options.rs b/crates/expfmt/src/arrow/options.rs similarity index 100% rename from src/arrow/options.rs rename to crates/expfmt/src/arrow/options.rs diff --git a/src/arrow/records.rs b/crates/expfmt/src/arrow/records.rs similarity index 100% rename from src/arrow/records.rs rename to crates/expfmt/src/arrow/records.rs diff --git a/src/arrow/schema.rs b/crates/expfmt/src/arrow/schema.rs similarity index 100% rename from src/arrow/schema.rs rename to crates/expfmt/src/arrow/schema.rs diff --git a/src/arrow/tests.rs b/crates/expfmt/src/arrow/tests.rs similarity index 100% rename from src/arrow/tests.rs rename to crates/expfmt/src/arrow/tests.rs diff --git a/src/dto.rs b/crates/expfmt/src/dto.rs similarity index 100% rename from src/dto.rs rename to crates/expfmt/src/dto.rs diff --git a/src/dto/io.prometheus.client.rs b/crates/expfmt/src/dto/io.prometheus.client.rs similarity index 100% rename from src/dto/io.prometheus.client.rs rename to crates/expfmt/src/dto/io.prometheus.client.rs diff --git a/src/format.rs b/crates/expfmt/src/format.rs similarity index 100% rename from src/format.rs rename to crates/expfmt/src/format.rs diff --git a/src/lib.rs b/crates/expfmt/src/lib.rs similarity index 95% rename from src/lib.rs rename to crates/expfmt/src/lib.rs index 1199f1c..b878c03 100644 --- a/src/lib.rs +++ b/crates/expfmt/src/lib.rs @@ -21,5 +21,3 @@ pub mod format; // Layer 0 parsers remain crate-internal; `arrow` and `dto` expose the public decoding APIs. #[allow(dead_code, unused_imports)] pub(crate) mod parse; -#[cfg(feature = "relabel")] -pub mod relabel; diff --git a/src/parse.rs b/crates/expfmt/src/parse.rs similarity index 100% rename from src/parse.rs rename to crates/expfmt/src/parse.rs diff --git a/src/parse/float.rs b/crates/expfmt/src/parse/float.rs similarity index 100% rename from src/parse/float.rs rename to crates/expfmt/src/parse/float.rs diff --git a/src/parse/go_diff_tests.rs b/crates/expfmt/src/parse/go_diff_tests.rs similarity index 99% rename from src/parse/go_diff_tests.rs rename to crates/expfmt/src/parse/go_diff_tests.rs index 91e77ba..a4e9d1a 100644 --- a/src/parse/go_diff_tests.rs +++ b/crates/expfmt/src/parse/go_diff_tests.rs @@ -195,7 +195,7 @@ fn expected_files(dir: &Path, out: &mut Vec) { /// Compares every expected file under `testdata//` with `render` of its input. fn assert_matches_go(dir: &str, render: impl Fn(&[u8]) -> Vec) { - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata"); + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../testdata"); let mut expected = Vec::new(); expected_files(&root.join(dir), &mut expected); assert!( diff --git a/src/parse/omlex.rs b/crates/expfmt/src/parse/omlex.rs similarity index 100% rename from src/parse/omlex.rs rename to crates/expfmt/src/parse/omlex.rs diff --git a/src/parse/omparse.rs b/crates/expfmt/src/parse/omparse.rs similarity index 100% rename from src/parse/omparse.rs rename to crates/expfmt/src/parse/omparse.rs diff --git a/src/parse/promlex.rs b/crates/expfmt/src/parse/promlex.rs similarity index 100% rename from src/parse/promlex.rs rename to crates/expfmt/src/parse/promlex.rs diff --git a/src/parse/promparse.rs b/crates/expfmt/src/parse/promparse.rs similarity index 100% rename from src/parse/promparse.rs rename to crates/expfmt/src/parse/promparse.rs diff --git a/src/parse/protobufparse.rs b/crates/expfmt/src/parse/protobufparse.rs similarity index 100% rename from src/parse/protobufparse.rs rename to crates/expfmt/src/parse/protobufparse.rs diff --git a/tests/fixtures_smoke.rs b/crates/expfmt/tests/fixtures_smoke.rs similarity index 92% rename from tests/fixtures_smoke.rs rename to crates/expfmt/tests/fixtures_smoke.rs index 0a77e30..6b6cbf2 100644 --- a/tests/fixtures_smoke.rs +++ b/crates/expfmt/tests/fixtures_smoke.rs @@ -19,7 +19,7 @@ use std::path::Path; #[test] fn synced_testdata_corpus_is_present() { - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata"); + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../testdata"); let manifest_path = root.join("manifest.json"); let manifest = fs::read_to_string(&manifest_path).expect("testdata/manifest.json must exist"); assert!(manifest.contains("github.com/prometheus/common")); @@ -48,10 +48,10 @@ fn synced_testdata_corpus_is_present() { #[cfg(feature = "dto")] #[test] fn decode_upstream_expfmt_protobuf_fixtures() { - use expfmt::dto::MetricFamily; + use prometheus_expfmt::dto::MetricFamily; use prost::Message; - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/expfmt"); + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../testdata/expfmt"); for fixture in ["protobuf", "protobuf-multimessage"] { let raw = fs::read(root.join(fixture)).expect("protobuf fixture should be readable"); diff --git a/crates/relabel/Cargo.toml b/crates/relabel/Cargo.toml new file mode 100644 index 0000000..6b9f530 --- /dev/null +++ b/crates/relabel/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "prometheus-relabel" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Prometheus-compatible target and metric label relabeling (relabel_configs and metric_relabel_configs)" +readme = "README.md" +keywords = ["prometheus", "openmetrics", "metrics", "relabel", "observability"] +categories = ["config", "text-processing"] +exclude = [ + "benches/**", + "tests/**", +] + +[dependencies] +regex = { workspace = true } +serde = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +benchfmt = { workspace = true } +serde_json = { workspace = true } + +[[bench]] +name = "relabel" +harness = false + +[lints] +workspace = true diff --git a/crates/relabel/README.md b/crates/relabel/README.md new file mode 100644 index 0000000..e8ff7ae --- /dev/null +++ b/crates/relabel/README.md @@ -0,0 +1,9 @@ +# prometheus-relabel + +Prometheus-compatible target and metric label relabeling (`relabel_configs`, `metric_relabel_configs`, and `honor_labels` key resolution) in Rust, ported from Prometheus [`model/relabel`](https://github.com/prometheus/prometheus/tree/main/model/relabel) and [`scrape/target.go`](https://github.com/prometheus/prometheus/blob/main/scrape/target.go). + +Part of the [`prometheus/scrape.rs`](https://github.com/prometheus/scrape.rs) workspace: + +- Pre-scrape target discovery label population (`populate_discovered_labels`, `populate_labels`) and all 11 Prometheus relabel actions (`replace`, `keep`, `drop`, `hashmod`, `labelmap`, `labeldrop`, `labelkeep`, `lowercase`, `uppercase`, `keepequal`, `dropequal`). +- Stage 1 series `__name__` filter pushdown (`split_metric_name_pushdown`, `keep_metric_name`). +- Stage 2 dictionary-vectorized attribute relabeling and `honor_labels` conflict resolution for columnar metric pipelines. diff --git a/benches/relabel.rs b/crates/relabel/benches/relabel.rs similarity index 98% rename from benches/relabel.rs rename to crates/relabel/benches/relabel.rs index d8c453e..77fb417 100644 --- a/benches/relabel.rs +++ b/crates/relabel/benches/relabel.rs @@ -11,21 +11,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Throughput and allocation benchmarks for `expfmt::relabel` target discovery relabeling, +//! Throughput and allocation benchmarks for `prometheus_relabel` target discovery relabeling, //! per-sample scalar `LabelSet` (`BTreeMap`) metric relabeling, and two-stage //! dictionary-vectorized metric relabeling over `ArrowDecoder`-style interned `(key_id, val_id)` //! series buffers. //! //! Recommended CLI invocation: //! ```sh -//! cargo bench --bench relabel -- -bench '^BenchmarkRelabel' -benchtime 1s -count 6 -cpu 2 -benchmem | tee ${bench}.txt +//! cargo bench -p prometheus-relabel --bench relabel -- -bench '^BenchmarkRelabel' -benchtime 1s -count 6 -cpu 2 -benchmem | tee ${bench}.txt //! ``` use std::collections::BTreeMap; use std::hint::black_box; use benchfmt::{CountingAlloc, Suite}; -use expfmt::relabel::{ +use prometheus_relabel::{ Action, Config, LabelSet, METRIC_NAME_LABEL, Regexp, SingleLabelKeepCache, SingleLabelTransformCache, TargetPopulateConfig, ValidationScheme, eval_key_keep_mask, keep_metric_name, merge_target_and_metric_labels, populate_discovered_labels, populate_labels, @@ -386,7 +386,7 @@ fn main() { let low_card_fixture = build_scrape_fixture(false); let high_card_fixture = build_scrape_fixture(true); - let mut suite = Suite::from_env("expfmt") + let mut suite = Suite::from_env("prometheus-relabel") .with_alloc(&ALLOC) .bench("Relabel/case=kubernetes_target/mode=populate_labels", |b| { let empty_group = BTreeMap::new(); diff --git a/src/relabel.rs b/crates/relabel/src/lib.rs similarity index 100% rename from src/relabel.rs rename to crates/relabel/src/lib.rs diff --git a/tests/relabel.rs b/crates/relabel/tests/relabel.rs similarity index 99% rename from tests/relabel.rs rename to crates/relabel/tests/relabel.rs index c046871..fdeb4d3 100644 --- a/tests/relabel.rs +++ b/crates/relabel/tests/relabel.rs @@ -14,11 +14,9 @@ //! Integration tests for Prometheus target and metric relabeling (`model/relabel` and //! `scrape/target.go`). -#![cfg(feature = "relabel")] - use std::collections::BTreeMap; -use expfmt::relabel::{ +use prometheus_relabel::{ Action, Config, DEFAULT_REPLACEMENT, DictionaryValueTransform, LabelSet, METRIC_NAME_LABEL, Regexp, SingleLabelFilterSummary, SingleLabelKeepCache, SingleLabelTransformCache, TargetPopulateConfig, ValidationScheme, eval_key_keep_mask, eval_single_label_keep_mask, diff --git a/crates/scrape-manager/Cargo.toml b/crates/scrape-manager/Cargo.toml new file mode 100644 index 0000000..c515f63 --- /dev/null +++ b/crates/scrape-manager/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "prometheus-scrape-manager" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Prometheus service discovery, HTTP client configuration, and scrape pool management" +readme = "README.md" +keywords = ["prometheus", "openmetrics", "metrics", "scrape", "service-discovery"] +categories = ["network-programming", "asynchronous"] +# Kept unpublished while initial service discovery, HTTP config, and scrape pool modules land. +publish = false + +[features] +default = ["arrow"] +arrow = ["prometheus-expfmt/arrow"] + +[dependencies] +prometheus-expfmt = { workspace = true } +prometheus-relabel = { workspace = true } + +[lints] +workspace = true diff --git a/crates/scrape-manager/README.md b/crates/scrape-manager/README.md new file mode 100644 index 0000000..17c1927 --- /dev/null +++ b/crates/scrape-manager/README.md @@ -0,0 +1,5 @@ +# prometheus-scrape-manager + +Prometheus service discovery (`discovery`), HTTP/TLS client configuration (`http_config`), and target scrape pool management (`scrape`) in Rust, built on [`prometheus-expfmt`](../expfmt) and [`prometheus-relabel`](../relabel). + +Part of the [`prometheus/scrape.rs`](https://github.com/prometheus/scrape.rs) workspace. diff --git a/crates/scrape-manager/src/lib.rs b/crates/scrape-manager/src/lib.rs new file mode 100644 index 0000000..5378518 --- /dev/null +++ b/crates/scrape-manager/src/lib.rs @@ -0,0 +1,20 @@ +// Copyright 2026 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Prometheus service discovery, HTTP scrape client configuration, and target scrape management. +//! +//! Re-exports [`prometheus_expfmt`] and [`prometheus_relabel`] and hosts the async runtime I/O +//! layers (`http_config`, `discovery`, and `scrape`) for [`prometheus/scrape.rs`](https://github.com/prometheus/scrape.rs). + +pub use prometheus_expfmt as expfmt; +pub use prometheus_relabel as relabel; diff --git a/tests/otap-compat/Cargo.toml b/tests/otap-compat/Cargo.toml index 32221a3..7df6332 100644 --- a/tests/otap-compat/Cargo.toml +++ b/tests/otap-compat/Cargo.toml @@ -4,10 +4,10 @@ version = "0.0.0" edition = "2024" publish = false -# Standalone workspace isolating `otel-arrow-dfe-pdata` from the root `expfmt` crate. +# Standalone workspace isolating `otel-arrow-dfe-pdata` from the root `prometheus-expfmt` crate. # Upstream `otel-arrow-dfe-pdata` has no feature flag to disable `datafusion`, `tonic`, `tokio`, # `nix`, and `zstd`, so resolving it pulls ~380 transitive crates (~5.3k lines in `Cargo.lock`) -# and requires MSRV 1.94 versus `expfmt`'s MSRV 1.88. +# and requires MSRV 1.94 versus `prometheus-expfmt`'s MSRV 1.88. # # Per Rust's lockfile guidance (https://blog.rust-lang.org/2023/08/29/committing-lockfiles/), # root `Cargo.lock` stays committed to pin MSRV 1.88 and byte-exact `prost-build` checks, while @@ -17,8 +17,8 @@ publish = false [dev-dependencies] dhat = "0.3" -expfmt = { path = "../..", default-features = false, features = ["arrow"] } otel-arrow-dfe-pdata = { version = "0.62", features = ["testing"] } +prometheus-expfmt = { path = "../../crates/expfmt", default-features = false, features = ["arrow"] } tokio = { version = "1.48", default-features = false, features = ["macros", "rt", "sync", "time"] } [lints.rust] diff --git a/tests/otap-compat/src/lib.rs b/tests/otap-compat/src/lib.rs index b5549fa..9ac7717 100644 --- a/tests/otap-compat/src/lib.rs +++ b/tests/otap-compat/src/lib.rs @@ -11,4 +11,4 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Integration test harness validating `expfmt::arrow` output against `otel-arrow-dfe-pdata`. +//! Integration test harness validating `prometheus_expfmt::arrow` output against `otel-arrow-dfe-pdata`. diff --git a/tests/otap-compat/tests/arrow_otap.rs b/tests/otap-compat/tests/arrow_otap.rs index 441a770..33f0f29 100644 --- a/tests/otap-compat/tests/arrow_otap.rs +++ b/tests/otap-compat/tests/arrow_otap.rs @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Integration tests verifying `expfmt::arrow::ArrowDecoder` against `otel-arrow-dfe-pdata` +//! Integration tests verifying `prometheus_expfmt::arrow::ArrowDecoder` against `otel-arrow-dfe-pdata` //! schema checks, transport-optimized ID encoding and decoding, and OTLP semantic equivalence. use std::fs; @@ -19,12 +19,6 @@ use std::path::Path; use std::rc::Rc; use std::time::Duration; -use expfmt::arrow::{ - ArrowDecoder, CounterSuffixPolicy, DecodeBudget, DecodeCursor, DecodeError, DecodeOptions, - DecodeStep, MetricPayloadType, OtapMetricRecords, PROMETHEUS_TYPE_METADATA_KEY, - UnitSuffixPolicy, UntypedPolicy, -}; -use expfmt::format::Format; use otel_arrow_dfe_pdata::otap::{Metrics, OtapArrowRecords, OtapBatchStore}; use otel_arrow_dfe_pdata::proto::OtlpProtoMessage; use otel_arrow_dfe_pdata::proto::opentelemetry::arrow::v1::ArrowPayloadType; @@ -39,6 +33,12 @@ use otel_arrow_dfe_pdata::schema::{ }; use otel_arrow_dfe_pdata::testing::equiv::assert_equivalent; use otel_arrow_dfe_pdata::testing::round_trip::{otap_to_otlp, to_metrics_data}; +use prometheus_expfmt::arrow::{ + ArrowDecoder, CounterSuffixPolicy, DecodeBudget, DecodeCursor, DecodeError, DecodeOptions, + DecodeStep, MetricPayloadType, OtapMetricRecords, PROMETHEUS_TYPE_METADATA_KEY, + UnitSuffixPolicy, UntypedPolicy, +}; +use prometheus_expfmt::format::Format; fn assert_matches_otap_schema(records: &OtapMetricRecords) { for (payload_type, batch) in records.iter() { diff --git a/tests/otap-compat/tests/dhat_bounded_mem.rs b/tests/otap-compat/tests/dhat_bounded_mem.rs index b0f8501..3ca7d15 100644 --- a/tests/otap-compat/tests/dhat_bounded_mem.rs +++ b/tests/otap-compat/tests/dhat_bounded_mem.rs @@ -19,10 +19,12 @@ use std::fs; use std::path::Path; use std::rc::Rc; -use expfmt::arrow::{ArrowDecoder, DecodeBudget, DecodeOptions, DecodeStep, OtapMetricRecords}; -use expfmt::format::Format; use otel_arrow_dfe_pdata::otap::{Metrics, OtapArrowRecords, OtapBatchStore}; use otel_arrow_dfe_pdata::proto::opentelemetry::arrow::v1::ArrowPayloadType; +use prometheus_expfmt::arrow::{ + ArrowDecoder, DecodeBudget, DecodeOptions, DecodeStep, OtapMetricRecords, +}; +use prometheus_expfmt::format::Format; #[global_allocator] static ALLOC: dhat::Alloc = dhat::Alloc; From 29d6bb0ddd0d41c0d49273085bac1bb20f1ff42b Mon Sep 17 00:00:00 2001 From: Bartek Plotka Date: Thu, 12 Nov 2026 07:40:45 +0000 Subject: [PATCH 2/2] ci, docs: add Apache-2.0 LICENSE, crates.io release workflow, and workspace docs Prepare workspace crates for crates.io publication and automated GitHub Release publishing: - Add Apache-2.0 LICENSE at the repository root and symlink into each crate directory so packaged .crate tarballs include the license text. - Add .github/workflows/release.yml using rust-lang/crates-io-auth-action (OIDC Trusted Publishing) and cargo publish --workspace --locked on published GitHub Releases (plus workflow_dispatch --dry-run). - Update README.md, AGENTS.md, and specs/SPEC.md for the three-crate workspace layout. Signed-off-by: Bartek Plotka --- .github/workflows/release.yml | 77 +++++++++++++ AGENTS.md | 14 +-- LICENSE | 201 ++++++++++++++++++++++++++++++++++ README.md | 25 ++--- crates/expfmt/LICENSE | 1 + crates/relabel/LICENSE | 1 + crates/scrape-manager/LICENSE | 1 + specs/SPEC.md | 2 +- 8 files changed, 301 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 LICENSE create mode 120000 crates/expfmt/LICENSE create mode 120000 crates/relabel/LICENSE create mode 120000 crates/scrape-manager/LICENSE diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..503a138 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,77 @@ +name: Release + +on: + release: + types: [published] + workflow_dispatch: + inputs: + dry_run: + description: "Run cargo publish --dry-run without uploading to crates.io" + required: true + type: boolean + default: true + +permissions: + contents: read + id-token: write + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + +jobs: + publish: + name: Publish to crates.io + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - uses: dtolnay/rust-toolchain@stable + + - name: Verify release tag matches workspace crate versions + if: github.event_name == 'release' + env: + TAG_NAME: ${{ github.event.release.tag_name }} + run: | + expected="${TAG_NAME#v}" + if [ "$expected" = "$TAG_NAME" ]; then + echo "Release tag '${TAG_NAME}' must start with 'v' (for example, v0.1.0)" + exit 1 + fi + actual=$(cargo metadata --no-deps --format-version 1 \ + | jq -r '.packages[] | select(.publish == null or (.publish | length > 0)) | "\(.name)@\(.version)"') + echo "Publishable crates:" + echo "$actual" + mismatched=$(cargo metadata --no-deps --format-version 1 \ + | jq -r --arg expected "$expected" \ + '.packages[] | select(.publish == null or (.publish | length > 0)) | select(.version != $expected) | "\(.name)@\(.version)"') + if [ -n "$mismatched" ]; then + echo "Release tag ${TAG_NAME} (version ${expected}) does not match crate versions:" + echo "$mismatched" + exit 1 + fi + + - name: Run workspace tests and doc checks + env: + RUSTDOCFLAGS: "--deny warnings" + run: | + cargo test --workspace --locked --all-features + cargo test --workspace --locked --no-default-features + cargo doc --workspace --locked --no-deps --all-features + + - name: Dry-run workspace publish + if: github.event_name == 'workflow_dispatch' && inputs.dry_run + run: cargo publish --workspace --locked --dry-run + + - name: Authenticate with crates.io (OIDC Trusted Publishing) + if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && !inputs.dry_run) + id: crates-io-auth + uses: rust-lang/crates-io-auth-action@3268eb413133e2960fa4ae0e963ca76380c54cd9 # v1.0.6 + + - name: Publish workspace crates to crates.io + if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && !inputs.dry_run) + env: + CARGO_REGISTRY_TOKEN: ${{ steps.crates-io-auth.outputs.token }} + run: cargo publish --workspace --locked diff --git a/AGENTS.md b/AGENTS.md index 34445d0..90bc47a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,13 +8,13 @@ Run these before you push. CI runs the same commands, see [.github/workflows/ci. ```sh cargo fmt --all -- --check -cargo clippy --all-targets --all-features -- -D warnings -cargo clippy --all-targets --no-default-features -- -D warnings -cargo test --all-features -cargo test --no-default-features +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo clippy --workspace --all-targets --no-default-features -- -D warnings +cargo test --workspace --all-features +cargo test --workspace --no-default-features ``` -If you touch `src/arrow`, also run the OTAP compatibility suite. It has its own manifest: +If you touch `crates/expfmt/src/arrow`, also run the OTAP compatibility suite. It has its own manifest: ```sh cargo test --manifest-path tests/otap-compat/Cargo.toml @@ -24,7 +24,7 @@ Do not use Rust language or std features newer than `rust-version` in `Cargo.tom ## Spec -[specs/SPEC.md](specs/SPEC.md) is the specification contract for `scrape.rs` (`expfmt`, `relabel`, and scrape pipeline integration). If you change behavior the spec describes, update the spec in the same PR. +[specs/SPEC.md](specs/SPEC.md) is the specification contract for `scrape.rs` (`prometheus-expfmt`, `prometheus-relabel`, and `prometheus-scrape-manager`). If you change behavior the spec describes, update the spec in the same PR. ## Test data @@ -46,7 +46,7 @@ If Rust must differ from Go on purpose, explain each difference in the PR body w - Write all benchmarks with [`benchfmt.rs`](https://github.com/bwplotka/benchfmt.rs) using `benchfmt::Suite` and `benchfmt::CountingAlloc` in `harness = false` `[[bench]]` targets so every benchmark emits Go [Benchmark Data Format](https://go.googlesource.com/proposal/+/master/design/14313-benchmark-format.md) lines with `ns/op`, `MB/s`, `B/op`, and `allocs/op` for [`benchstat`](https://pkg.go.dev/golang.org/x/perf/cmd/benchstat). Do not use Criterion or custom benchmark output formats. - Name benchmarks with `/key=value` sub-benchmark dimensions such as `Decode/case=alltypes.237mfs/mode=decode`, call `b.report_allocs()` and `while b.keep_running()`, and include a `Recommended CLI invocation:` doc comment at the top of each benchmark file. -- Performance improvements and hot-path changes require running benchmarks before and after the change, for example `cargo bench --bench decode -- -bench '^BenchmarkDecode' -benchtime 1s -count 6 -cpu 2 -benchmem | tee ${bench}.txt`, and comparing with `benchstat before.txt after.txt`. +- Performance improvements and hot-path changes require running benchmarks before and after the change, for example `cargo bench -p prometheus-expfmt --bench decode -- -bench '^BenchmarkDecode' -benchtime 1s -count 6 -cpu 2 -benchmem | tee ${bench}.txt`, and comparing with `benchstat before.txt after.txt`. - Always output benchmark results in raw `benchstat` format in the PR body and review comments, and always include links to the exact `before` and `after` commits that the benchmark refers to. Rerun benchmarks and update the commit links whenever addressing review comments that touch benchmarked code or benchmarks. - If a subset of benchmark results shows a regression, address it or explain in the PR body why that case is not important. - Reuse allocations in hot paths where possible, such as `Vec::clear`, scratch buffers, and interner arenas. When reusing buffers exposed through APIs or traits, document lifetime and copy requirements on the trait or type definition. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index be7d196..2968c7c 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ > **Status: Experimental (Work in Progress)** -Official Rust libraries for Prometheus and OpenMetrics metric collection pipelines: wire exposition format decoding (`expfmt`), target and metric relabeling, standard service discoveries, and scrape management. +Official Rust libraries for Prometheus and OpenMetrics metric collection pipelines: wire exposition format decoding (`prometheus-expfmt`), target and metric relabeling (`prometheus-relabel`), and service discovery and scrape management (`prometheus-scrape-manager`). ## Overview @@ -22,23 +22,22 @@ Service discoveries The project serves two main use cases: 1. Columnar scraping for OpenTelemetry Arrow. [`open-telemetry/otel-arrow#3277`](https://github.com/open-telemetry/otel-arrow/issues/3277) defines an OTAP-native `prometheus_scrape` receiver in Rust. `scrape.rs` streams scrape bytes through early `__name__` family filtering, metric family assembly, and dictionary-vectorized relabeling directly into Apache Arrow (`OtapArrowRecords`) tables on thread-per-core `!Send` runtimes with bounded memory. -2. Standalone Rust Prometheus tooling. Validation harnesses such as OpenTelemetry Weaver live-checking `/metrics` endpoints against semantic conventions, CLIs, and custom collectors can use individual components (such as `expfmt` DTO decoding or `relabel`) without pulling in the full scrape runtime. +2. Standalone Rust Prometheus tooling. Validation harnesses such as OpenTelemetry Weaver live-checking `/metrics` endpoints against semantic conventions, CLIs, and custom collectors can depend on individual pure-CPU crates (`prometheus-expfmt` or `prometheus-relabel`) without pulling in async HTTP or service discovery dependencies. -## Components and scope +## Workspace crates -| Component | Status | Specification | Scope | -| :--- | :--- | :--- | :--- | -| `expfmt` | In progress | [specs/SPEC.md](./specs/SPEC.md) | `Content-Type` and `Accept` negotiation, zero-copy streaming parsers for Prometheus Text 0.0.4, OpenMetrics 1.0, OpenMetrics 2.0, and Prometheus Protobuf, primary Apache Arrow / OTAP decoder (`ArrowDecoder`), and [`MetricFamily`](https://github.com/prometheus/prometheus/blob/main/prompb/io/prometheus/client/metrics.proto) protobuf DTO decoder. | -| `relabel` | In progress | [specs/SPEC.md](./specs/SPEC.md#target-and-metric-relabeling-architecture) | Prometheus-compatible target and metric relabeling (`replace`, `keep`, `drop`, `hashmod`, `labelmap`, `labeldrop`, `labelkeep`, `lowercase`, `uppercase`, `keepequal`, `dropequal`), Stage 1 `__name__` filter pushdown, and Stage 2 dictionary-vectorized relabeling and `honor_labels` resolution. | -| `discovery` | Planned | TBD | Standard Prometheus service discovery mechanisms (static targets, file-based SD, HTTP SD, DNS SD, Kubernetes SD) emitting target groups and `__meta_*` labels. | -| `scrape` | Planned | TBD | Bounded HTTP scrape scheduling, target pool lifecycle, compression and content negotiation, stale marker handling, target state reporting, and self-telemetry. | +| Crate | Path | Status | Specification | Scope | +| :--- | :--- | :--- | :--- | :--- | +| `prometheus-expfmt` | [`crates/expfmt`](./crates/expfmt) | In progress | [specs/SPEC.md](./specs/SPEC.md) | `Content-Type` and `Accept` negotiation, zero-copy streaming parsers for Prometheus Text 0.0.4, OpenMetrics 1.0, OpenMetrics 2.0, and Prometheus Protobuf, primary Apache Arrow / OTAP decoder (`ArrowDecoder`), and [`MetricFamily`](https://github.com/prometheus/prometheus/blob/main/prompb/io/prometheus/client/metrics.proto) protobuf DTO decoder. | +| `prometheus-relabel` | [`crates/relabel`](./crates/relabel) | In progress | [specs/SPEC.md](./specs/SPEC.md#target-and-metric-relabeling-architecture) | Prometheus-compatible target and metric relabeling (`replace`, `keep`, `drop`, `hashmod`, `labelmap`, `labeldrop`, `labelkeep`, `lowercase`, `uppercase`, `keepequal`, `dropequal`), Stage 1 `__name__` filter pushdown, and Stage 2 dictionary-vectorized relabeling and `honor_labels` resolution. | +| `prometheus-scrape-manager` | [`crates/scrape-manager`](./crates/scrape-manager) | Planned | TBD | Prometheus HTTP/TLS client configuration (`http_config`), service discovery mechanisms (`discovery`: static, file, HTTP, DNS, Kubernetes), and bounded target scrape pool scheduling (`scrape`). | ## Development and testing -Run the unit and fixture test suite: +Run the workspace unit and fixture test suite: ```sh -cargo test --all-features +cargo test --workspace --all-features ``` Run the [`otel-arrow-dfe-pdata`](https://github.com/open-telemetry/otel-arrow/tree/main/rust/otap-dataflow/crates/pdata) compatibility test suite: @@ -56,6 +55,6 @@ cd tests/go-harness && go test ./... Run the decoding and relabeling benchmarks in Go `benchfmt` format for [`benchstat`](https://pkg.go.dev/golang.org/x/perf/cmd/benchstat): ```sh -cargo bench --bench decode -- -bench '^BenchmarkDecode' -benchtime 1s -count 6 -cpu 2 -benchmem | tee decode.txt -cargo bench --bench relabel -- -bench '^BenchmarkRelabel' -benchtime 1s -count 6 -cpu 2 -benchmem | tee relabel.txt +cargo bench -p prometheus-expfmt --bench decode -- -bench '^BenchmarkDecode' -benchtime 1s -count 6 -cpu 2 -benchmem | tee decode.txt +cargo bench -p prometheus-relabel --bench relabel -- -bench '^BenchmarkRelabel' -benchtime 1s -count 6 -cpu 2 -benchmem | tee relabel.txt ``` diff --git a/crates/expfmt/LICENSE b/crates/expfmt/LICENSE new file mode 120000 index 0000000..30cff74 --- /dev/null +++ b/crates/expfmt/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/crates/relabel/LICENSE b/crates/relabel/LICENSE new file mode 120000 index 0000000..30cff74 --- /dev/null +++ b/crates/relabel/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/crates/scrape-manager/LICENSE b/crates/scrape-manager/LICENSE new file mode 120000 index 0000000..30cff74 --- /dev/null +++ b/crates/scrape-manager/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/specs/SPEC.md b/specs/SPEC.md index 839466d..6becd52 100644 --- a/specs/SPEC.md +++ b/specs/SPEC.md @@ -1,6 +1,6 @@ # `scrape.rs` specification and implementation plan -`scrape.rs` (`expfmt` crate) provides official Rust decoders for Prometheus and OpenMetrics wire exposition formats along with integrated Prometheus target and metric relabeling (`relabel`). +`scrape.rs` provides official Rust decoders for Prometheus and OpenMetrics wire exposition formats (`prometheus-expfmt`), Prometheus target and metric relabeling (`prometheus-relabel`), and service discovery and scrape management (`prometheus-scrape-manager`). ## Background and goals