From c3ec0e582b7442c04467fbb5b598f45f719daf8e Mon Sep 17 00:00:00 2001
From: Gagan Trivedi <gagandeeptrivedi47@gmail.com>
Date: Tue, 27 Oct 2026 13:55:35 +0530
Subject: [PATCH 01/11] feat(ingest): reject events older than 10 days

Bounds how many daily ClickHouse partitions one insert can touch, so a
batch with scattered old timestamps cannot fail or overload the insert.

Refs #3
---
 README.md            |   2 +-
 src/routes/ingest.rs | 135 ++++++++++++++++++++++++++++---------------
 src/routes/mod.rs    |   9 +--
 src/test_helpers.rs  |   5 ++
 4 files changed, 100 insertions(+), 51 deletions(-)

diff --git a/README.md b/README.md
index b9a9f54..e8ac513 100644
--- a/README.md
+++ b/README.md
@@ -42,7 +42,7 @@ An empty `events` array adds a top-level `error` field: `{ "accepted": 0, "rejec
 }
 ```
 
-`event` and `timestamp` (epoch millis) are required. `$flag_exposure` is a reserved system event and requires `feature_name` (omitting it rejects that event); `identifier`, `value`, `traits`, and `metadata` are optional.
+`event` and `timestamp` (epoch millis) are required; `timestamp` must be after 2020-01-01, no more than 24h in the future and no more than 10 days in the past (otherwise that event is rejected). `$flag_exposure` is a reserved system event and requires `feature_name` (omitting it rejects that event); `identifier`, `value`, `traits`, and `metadata` are optional.
 
 Field limits — an over-limit field rejects that event: `event` ≤ 256 chars; `feature_name` and `identifier` ≤ 2000 chars, trait keys ≤ 200 chars, trait string values ≤ 2000 chars (all matching Flagsmith's model limits); serialized `value` ≤ 128 KiB (fits Flagsmith's default 20,000-char flag value limit), `traits` ≤ 32 KiB, `metadata` ≤ 8 KiB. `traits` must be a JSON object.
 
diff --git a/src/routes/ingest.rs b/src/routes/ingest.rs
index 4a039e0..e8681af 100644
--- a/src/routes/ingest.rs
+++ b/src/routes/ingest.rs
@@ -39,8 +39,13 @@ fn parse_sdk_user_agent(header: &str) -> (String, String) {
 
 /// 2020-01-01T00:00:00Z in epoch-millis; mirrors ClickHouse's `timestamp_sane` constraint.
 const MIN_TIMESTAMP_MS: i64 = 1_577_836_800_000;
-/// Future clock-skew allowance.
-const MAX_FUTURE_SKEW_MS: i64 = 24 * 60 * 60 * 1000;
+const HOUR_MS: i64 = 60 * 60 * 1000;
+const DAY_MS: i64 = 24 * HOUR_MS;
+/// Future clock-skew allowance, in hours after now.
+const MAX_FUTURE_SKEW_HOURS: i64 = 24;
+/// Oldest accepted event, in days before now. Bounds how many daily
+/// ClickHouse partitions one insert can touch.
+const MAX_EVENT_AGE_DAYS: i64 = 10;
 
 /// Event names live in our own namespace and a LowCardinality ClickHouse column.
 const MAX_EVENT_NAME_CHARS: usize = 256;
@@ -72,9 +77,26 @@ fn json_size(value: &serde_json::Value) -> usize {
         .unwrap_or(usize::MAX)
 }
 
+fn validate_event_timestamps(timestamp: i64, now: i64) -> Result<(), String> {
+    if timestamp <= MIN_TIMESTAMP_MS {
+        return Err("timestamp must be an epoch-millis value after 2020-01-01".to_string());
+    }
+    if timestamp > now + MAX_FUTURE_SKEW_HOURS * HOUR_MS {
+        return Err(format!(
+            "timestamp must not be more than {MAX_FUTURE_SKEW_HOURS}h in the future"
+        ));
+    }
+    if timestamp < now - MAX_EVENT_AGE_DAYS * DAY_MS {
+        return Err(format!(
+            "timestamp must not be more than {MAX_EVENT_AGE_DAYS} days in the past"
+        ));
+    }
+    Ok(())
+}
+
 /// Validates one event against the ingest schema rules. Error messages name
 /// the offending field; the caller prefixes the batch index.
-fn validate_event(event: &Event, max_timestamp: i64) -> Result<(), String> {
+fn validate_event(event: &Event, now: i64) -> Result<(), String> {
     if event.event.trim().is_empty() {
         return Err("event must be non-empty".to_string());
     }
@@ -110,12 +132,7 @@ fn validate_event(event: &Event, max_timestamp: i64) -> Result<(), String> {
             ));
         }
     }
-    if event.timestamp <= MIN_TIMESTAMP_MS {
-        return Err("timestamp must be an epoch-millis value after 2020-01-01".to_string());
-    }
-    if event.timestamp > max_timestamp {
-        return Err("timestamp must not be more than 24h in the future".to_string());
-    }
+    validate_event_timestamps(event.timestamp, now)?;
     if json_size(&event.value) > MAX_VALUE_BYTES {
         return Err(format!(
             "value must serialize to at most {MAX_VALUE_BYTES} bytes"
@@ -161,11 +178,11 @@ fn validate_event(event: &Event, max_timestamp: i64) -> Result<(), String> {
 /// event is validated independently: one invalid event never blocks its
 /// neighbours.
 fn partition_events(events: &[Event]) -> (Vec<&Event>, Vec<RejectedEvent>) {
-    let max_timestamp = Utc::now().timestamp_millis() + MAX_FUTURE_SKEW_MS;
+    let now = Utc::now().timestamp_millis();
     let mut valid = Vec::with_capacity(events.len());
     let mut rejected = Vec::new();
     for (i, event) in events.iter().enumerate() {
-        match validate_event(event, max_timestamp) {
+        match validate_event(event, now) {
             Ok(()) => valid.push(event),
             Err(error) => rejected.push(RejectedEvent {
                 index: i,
@@ -288,7 +305,7 @@ mod tests {
 
     use crate::sink::{KafkaAuth, KafkaSink, KafkaSinkConfig};
     use crate::test_helpers::{
-        assert_topic_empty, mock_kafka, read_records, FLAGSMITH_WAREHOUSE_TOPIC,
+        assert_topic_empty, mock_kafka, now_ms, read_records, FLAGSMITH_WAREHOUSE_TOPIC,
     };
 
     fn parse(json: &str) -> EventBatch {
@@ -348,9 +365,10 @@ mod tests {
 
     #[test]
     fn test_validate_accepts_exposure_with_feature_name() {
-        assert!(accepts(
-            r#"{"events":[{"event":"$flag_exposure","feature_name":"dark_mode","value":"control","timestamp":1700000000000}]}"#,
-        ));
+        let ts = now_ms();
+        assert!(accepts(&format!(
+            r#"{{"events":[{{"event":"$flag_exposure","feature_name":"dark_mode","value":"control","timestamp":{ts}}}]}}"#,
+        )));
     }
 
     #[test]
@@ -422,10 +440,31 @@ mod tests {
     }
 
     #[test]
-    fn test_validate_accepts_custom_event() {
-        assert!(accepts(
-            r#"{"events":[{"event":"purchase","value":42,"timestamp":1700000000000}]}"#
+    fn test_validate_accepts_timestamp_within_max_age() {
+        let ts = now_ms() - 10 * DAY_MS + 60_000;
+        assert!(accepts(&format!(
+            r#"{{"events":[{{"event":"purchase","timestamp":{ts}}}]}}"#
+        )));
+    }
+
+    #[test]
+    fn test_validate_rejects_timestamp_older_than_max_age() {
+        let ts = now_ms() - 11 * DAY_MS;
+        let (_, rejected) = partition(&format!(
+            r#"{{"events":[{{"event":"purchase","timestamp":{ts}}}]}}"#
         ));
+        assert_eq!(
+            rejected[0].error,
+            "events[0].timestamp must not be more than 10 days in the past"
+        );
+    }
+
+    #[test]
+    fn test_validate_accepts_custom_event() {
+        let ts = now_ms();
+        assert!(accepts(&format!(
+            r#"{{"events":[{{"event":"purchase","value":42,"timestamp":{ts}}}]}}"#
+        )));
     }
 
     #[test]
@@ -433,13 +472,14 @@ mod tests {
         // One invalid event never blocks its neighbours; the rejection names
         // the offending index and carries the same message the old whole-batch
         // 400 used.
-        let (accepted, rejected) = partition(
-            r#"{"events":[
-                {"event":"purchase","value":42,"timestamp":1700000000000},
-                {"event":"purchase","timestamp":1},
-                {"event":"$flag_exposure","feature_name":"dark_mode","timestamp":1700000000000}
-            ]}"#,
-        );
+        let ts = now_ms();
+        let (accepted, rejected) = partition(&format!(
+            r#"{{"events":[
+                {{"event":"purchase","value":42,"timestamp":{ts}}},
+                {{"event":"purchase","timestamp":1}},
+                {{"event":"$flag_exposure","feature_name":"dark_mode","timestamp":{ts}}}
+            ]}}"#,
+        ));
         assert_eq!(accepted, 2);
         assert_eq!(rejected.len(), 1);
         assert_eq!(rejected[0].index, 1);
@@ -557,6 +597,11 @@ mod tests {
         app_with_destination(sink, None)
     }
 
+    fn purchase_json() -> String {
+        let ts = now_ms();
+        format!(r#"{{"events":[{{"event":"purchase","value":42,"timestamp":{ts}}}]}}"#)
+    }
+
     fn post_events(json: &str) -> Request<Body> {
         Request::builder()
             .method("POST")
@@ -574,14 +619,15 @@ mod tests {
     #[tokio::test]
     async fn test_handler_mixed_batch_returns_202_and_forwards_valid_only() {
         let kafka = mock_kafka(&[FLAGSMITH_WAREHOUSE_TOPIC]);
+        let ts = now_ms();
         let response = app(&kafka.sink)
-            .oneshot(post_events(
-                r#"{"events":[
-                    {"event":"purchase","value":42,"timestamp":1700000000000},
-                    {"event":"purchase","timestamp":1},
-                    {"event":"$flag_exposure","feature_name":"dark_mode","timestamp":1700000000000}
-                ]}"#,
-            ))
+            .oneshot(post_events(&format!(
+                r#"{{"events":[
+                    {{"event":"purchase","value":42,"timestamp":{ts}}},
+                    {{"event":"purchase","timestamp":1}},
+                    {{"event":"$flag_exposure","feature_name":"dark_mode","timestamp":{ts}}}
+                ]}}"#,
+            )))
             .await
             .unwrap();
 
@@ -601,10 +647,11 @@ mod tests {
         // The destination String columns (and the ClickPipes field mapping)
         // take flat scalar fields, so dict fields go on the wire JSON-encoded.
         let kafka = mock_kafka(&[FLAGSMITH_WAREHOUSE_TOPIC]);
+        let ts = now_ms();
         let response = app(&kafka.sink)
-            .oneshot(post_events(
-                r#"{"events":[{"event":"purchase","traits":{"logins":42,"plan":"pro"},"metadata":{"source":"csv"},"timestamp":1700000000000}]}"#,
-            ))
+            .oneshot(post_events(&format!(
+                r#"{{"events":[{{"event":"purchase","traits":{{"logins":42,"plan":"pro"}},"metadata":{{"source":"csv"}},"timestamp":{ts}}}]}}"#,
+            )))
             .await
             .unwrap();
 
@@ -659,9 +706,7 @@ mod tests {
         let sink = unreachable_sink();
         let started = std::time::Instant::now();
         let response = app(&sink)
-            .oneshot(post_events(
-                r#"{"events":[{"event":"purchase","value":42,"timestamp":1700000000000}]}"#,
-            ))
+            .oneshot(post_events(&purchase_json()))
             .await
             .unwrap();
 
@@ -688,9 +733,7 @@ mod tests {
             .expect("topic error");
         let started = std::time::Instant::now();
         let response = app(&kafka.sink)
-            .oneshot(post_events(
-                r#"{"events":[{"event":"purchase","value":42,"timestamp":1700000000000}]}"#,
-            ))
+            .oneshot(post_events(&purchase_json()))
             .await
             .unwrap();
 
@@ -707,9 +750,7 @@ mod tests {
     async fn test_handler_forwards_to_configured_destination() {
         let kafka = mock_kafka(&[FLAGSMITH_WAREHOUSE_TOPIC, "events-dest-x"]);
         let response = app_with_destination(&kafka.sink, Some("events-dest-x"))
-            .oneshot(post_events(
-                r#"{"events":[{"event":"purchase","value":42,"timestamp":1700000000000}]}"#,
-            ))
+            .oneshot(post_events(&purchase_json()))
             .await
             .unwrap();
 
@@ -722,8 +763,9 @@ mod tests {
     #[tokio::test]
     async fn test_handler_delivers_large_batches() {
         let kafka = mock_kafka(&[FLAGSMITH_WAREHOUSE_TOPIC]);
+        let ts = now_ms();
         let events: Vec<String> = (0..501)
-            .map(|i| format!(r#"{{"event":"purchase","value":{i},"timestamp":1700000000000}}"#))
+            .map(|i| format!(r#"{{"event":"purchase","value":{i},"timestamp":{ts}}}"#))
             .collect();
         let body = format!(r#"{{"events":[{}]}}"#, events.join(","));
         let response = app(&kafka.sink).oneshot(post_events(&body)).await.unwrap();
@@ -737,7 +779,8 @@ mod tests {
     }
 
     fn event_json(fields: &str) -> String {
-        format!(r#"{{"events":[{{{fields},"timestamp":1700000000000}}]}}"#)
+        let ts = now_ms();
+        format!(r#"{{"events":[{{{fields},"timestamp":{ts}}}]}}"#)
     }
 
     #[test]
diff --git a/src/routes/mod.rs b/src/routes/mod.rs
index 7e12d2d..d2c0db3 100644
--- a/src/routes/mod.rs
+++ b/src/routes/mod.rs
@@ -44,7 +44,7 @@ mod tests {
     use super::*;
     use crate::environment_keys::EnvironmentKeys;
     use crate::test_helpers::{
-        assert_topic_empty, create_environment_keys, insert_environment_key, mock_kafka,
+        assert_topic_empty, create_environment_keys, insert_environment_key, mock_kafka, now_ms,
         read_records, FLAGSMITH_WAREHOUSE_TOPIC,
     };
 
@@ -63,9 +63,10 @@ mod tests {
             .uri("/v1/events")
             .header("content-type", "application/json")
             .header("X-Environment-Key", key)
-            .body(Body::from(
-                r#"{"events":[{"event":"purchase","value":42,"timestamp":1700000000000}]}"#,
-            ))
+            .body(Body::from(format!(
+                r#"{{"events":[{{"event":"purchase","value":42,"timestamp":{}}}]}}"#,
+                now_ms()
+            )))
             .unwrap()
     }
 
diff --git a/src/test_helpers.rs b/src/test_helpers.rs
index 9545c21..4f089f5 100644
--- a/src/test_helpers.rs
+++ b/src/test_helpers.rs
@@ -14,6 +14,11 @@ pub(crate) const FLAGSMITH_WAREHOUSE_TOPIC: &str = "events-default";
 
 pub(crate) type MockKafkaCluster = MockCluster<'static, DefaultProducerContext>;
 
+/// Event timestamps must be recent to pass validation.
+pub(crate) fn now_ms() -> i64 {
+    Utc::now().timestamp_millis()
+}
+
 /// The cluster is torn down on drop, so it must outlive every request in the
 /// test.
 pub(crate) struct MockKafka {

From 70fd3b302c4d53bad3bc05a4641b27a797edf8e2 Mon Sep 17 00:00:00 2001
From: Gagan Trivedi <gagandeeptrivedi47@gmail.com>
Date: Tue, 27 Oct 2026 14:47:57 +0530
Subject: [PATCH 02/11] feat(sink): write events to ClickHouse in batches

SINK=clickhouse buffers accepted events in memory and writes them with
one plain INSERT per flush, every SINK_FLUSH_INTERVAL_MS or earlier at
SINK_BATCH_MAX_ROWS. Requests return 202 once buffered; a full buffer
(SINK_BUFFER_MAX_BYTES) returns 503. Failed writes are retried, then
dropped. The batching is generic so other sinks can reuse it.

Refs #3
---
 Cargo.lock             | 325 ++++++++++++++++++++++++++++++++++++++--
 Cargo.toml             |   1 +
 README.md              |  22 ++-
 src/auth/middleware.rs |   8 +-
 src/config.rs          |  94 +++++++++++-
 src/main.rs            |  19 ++-
 src/routes/mod.rs      |   2 +-
 src/sink/batching.rs   | 328 +++++++++++++++++++++++++++++++++++++++++
 src/sink/clickhouse.rs | 197 +++++++++++++++++++++++++
 src/sink/mod.rs        |  13 ++
 10 files changed, 981 insertions(+), 28 deletions(-)
 create mode 100644 src/sink/batching.rs
 create mode 100644 src/sink/clickhouse.rs

diff --git a/Cargo.lock b/Cargo.lock
index a8ddebf..1d08c2d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -100,6 +100,29 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
 
+[[package]]
+name = "aws-lc-rs"
+version = "1.18.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13"
+dependencies = [
+ "aws-lc-sys",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-lc-sys"
+version = "0.46.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc"
+dependencies = [
+ "cc",
+ "cmake",
+ "dunce",
+ "fs_extra",
+ "pkg-config",
+]
+
 [[package]]
 name = "axum"
 version = "0.7.9"
@@ -185,6 +208,21 @@ dependencies = [
  "generic-array",
 ]
 
+[[package]]
+name = "bnum"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "119771309b95163ec7aaf79810da82f7cd0599c19722d48b9c03894dca833966"
+
+[[package]]
+name = "bstr"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.19.0"
@@ -202,6 +240,9 @@ name = "bytes"
 version = "1.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
+dependencies = [
+ "serde",
+]
 
 [[package]]
 name = "cc"
@@ -221,6 +262,17 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
 
+[[package]]
+name = "chacha20"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
+dependencies = [
+ "cfg-if",
+ "cpufeatures 0.3.1",
+ "rand_core 0.10.1",
+]
+
 [[package]]
 name = "chrono"
 version = "0.4.39"
@@ -236,6 +288,66 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
+[[package]]
+name = "cityhash-rs"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93a719913643003b84bd13022b4b7e703c09342cd03b679c4641c7d2e50dc34d"
+
+[[package]]
+name = "clickhouse"
+version = "0.15.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6ad2112cf4ad543ceed12e48b779e82e8f3a8172374cac98d59e59feda35c69"
+dependencies = [
+ "bnum",
+ "bstr",
+ "bytes",
+ "cityhash-rs",
+ "clickhouse-macros",
+ "clickhouse-types",
+ "futures-channel",
+ "futures-util",
+ "half",
+ "hashbrown 0.17.1",
+ "http-body-util",
+ "hyper",
+ "hyper-rustls",
+ "hyper-util",
+ "lz4_flex",
+ "rand 0.9.5",
+ "rustls",
+ "serde",
+ "serde_json",
+ "thiserror 2.0.21",
+ "tokio",
+ "tracing",
+ "twox-hash",
+ "url",
+]
+
+[[package]]
+name = "clickhouse-macros"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff6669899e23cb87b43daf7996f0ea3b9c07d0fb933d745bb7b815b052515ae3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "serde_derive_internals",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "clickhouse-types"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7ddab58fabf3bdde7b6fa401f72b9abc7566ff015264c89aaaa71cf1d173d06"
+dependencies = [
+ "bytes",
+ "thiserror 2.0.21",
+]
+
 [[package]]
 name = "cmake"
 version = "0.1.54"
@@ -292,6 +404,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "cpufeatures"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "crc"
 version = "3.4.0"
@@ -349,6 +470,12 @@ version = "0.8.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
 
+[[package]]
+name = "crunchy"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
+
 [[package]]
 name = "crypto-common"
 version = "0.1.7"
@@ -399,6 +526,12 @@ version = "0.15.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
 
+[[package]]
+name = "dunce"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
+
 [[package]]
 name = "either"
 version = "1.15.0"
@@ -460,6 +593,7 @@ dependencies = [
  "axum",
  "bytes",
  "chrono",
+ "clickhouse",
  "futures",
  "moka",
  "rdkafka",
@@ -501,6 +635,12 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
 
+[[package]]
+name = "foldhash"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
+
 [[package]]
 name = "form_urlencoded"
 version = "1.2.2"
@@ -510,6 +650,12 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "fs_extra"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
+
 [[package]]
 name = "futures"
 version = "0.3.31"
@@ -639,10 +785,33 @@ checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
 dependencies = [
  "cfg-if",
  "libc",
- "r-efi",
+ "r-efi 5.3.0",
  "wasip2",
 ]
 
+[[package]]
+name = "getrandom"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi 6.0.0",
+ "rand_core 0.10.1",
+]
+
+[[package]]
+name = "half"
+version = "2.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
+dependencies = [
+ "cfg-if",
+ "crunchy",
+ "zerocopy",
+]
+
 [[package]]
 name = "hashbrown"
 version = "0.15.5"
@@ -651,7 +820,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
 dependencies = [
  "allocator-api2",
  "equivalent",
- "foldhash",
+ "foldhash 0.1.5",
 ]
 
 [[package]]
@@ -659,6 +828,11 @@ name = "hashbrown"
 version = "0.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash 0.2.0",
+]
 
 [[package]]
 name = "hashlink"
@@ -772,6 +946,23 @@ dependencies = [
  "pin-utils",
  "smallvec",
  "tokio",
+ "want",
+]
+
+[[package]]
+name = "hyper-rustls"
+version = "0.27.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53"
+dependencies = [
+ "http",
+ "hyper",
+ "hyper-util",
+ "rustls",
+ "tokio",
+ "tokio-rustls",
+ "tower-service",
+ "webpki-roots 1.0.9",
 ]
 
 [[package]]
@@ -781,13 +972,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f"
 dependencies = [
  "bytes",
+ "futures-channel",
  "futures-core",
+ "futures-util",
  "http",
  "http-body",
  "hyper",
+ "libc",
  "pin-project-lite",
+ "socket2",
  "tokio",
  "tower-service",
+ "tracing",
 ]
 
 [[package]]
@@ -1028,6 +1224,12 @@ version = "0.4.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
 
+[[package]]
+name = "lz4_flex"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226"
+
 [[package]]
 name = "matchers"
 version = "0.2.0"
@@ -1126,7 +1328,7 @@ dependencies = [
  "num-integer",
  "num-iter",
  "num-traits",
- "rand",
+ "rand 0.8.6",
  "smallvec",
  "zeroize",
 ]
@@ -1356,6 +1558,12 @@ version = "5.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
 
+[[package]]
+name = "r-efi"
+version = "6.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
+
 [[package]]
 name = "rand"
 version = "0.8.6"
@@ -1363,8 +1571,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a"
 dependencies = [
  "libc",
- "rand_chacha",
- "rand_core",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
+dependencies = [
+ "rand_chacha 0.9.0",
+ "rand_core 0.9.5",
+]
+
+[[package]]
+name = "rand"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af"
+dependencies = [
+ "chacha20",
+ "getrandom 0.4.3",
+ "rand_core 0.10.1",
 ]
 
 [[package]]
@@ -1374,7 +1603,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
 dependencies = [
  "ppv-lite86",
- "rand_core",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.9.5",
 ]
 
 [[package]]
@@ -1386,6 +1625,21 @@ dependencies = [
  "getrandom 0.2.16",
 ]
 
+[[package]]
+name = "rand_core"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
+dependencies = [
+ "getrandom 0.3.4",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
+
 [[package]]
 name = "rdkafka"
 version = "0.39.0"
@@ -1481,7 +1735,7 @@ dependencies = [
  "num-traits",
  "pkcs1",
  "pkcs8",
- "rand_core",
+ "rand_core 0.6.4",
  "signature",
  "spki",
  "subtle",
@@ -1494,6 +1748,7 @@ version = "0.23.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f"
 dependencies = [
+ "aws-lc-rs",
  "once_cell",
  "ring",
  "rustls-pki-types",
@@ -1517,6 +1772,7 @@ version = "0.103.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52"
 dependencies = [
+ "aws-lc-rs",
  "ring",
  "rustls-pki-types",
  "untrusted",
@@ -1570,6 +1826,17 @@ dependencies = [
  "syn 2.0.119",
 ]
 
+[[package]]
+name = "serde_derive_internals"
+version = "0.29.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
 [[package]]
 name = "serde_json"
 version = "1.0.151"
@@ -1613,7 +1880,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8"
 dependencies = [
  "cfg-if",
- "cpufeatures",
+ "cpufeatures 0.2.17",
  "digest",
 ]
 
@@ -1624,7 +1891,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
 dependencies = [
  "cfg-if",
- "cpufeatures",
+ "cpufeatures 0.2.17",
  "digest",
 ]
 
@@ -1659,7 +1926,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
 dependencies = [
  "digest",
- "rand_core",
+ "rand_core 0.6.4",
 ]
 
 [[package]]
@@ -1830,7 +2097,7 @@ dependencies = [
  "memchr",
  "once_cell",
  "percent-encoding",
- "rand",
+ "rand 0.8.6",
  "rsa",
  "serde",
  "sha1",
@@ -1869,7 +2136,7 @@ dependencies = [
  "md-5",
  "memchr",
  "once_cell",
- "rand",
+ "rand 0.8.6",
  "serde",
  "serde_json",
  "sha2",
@@ -2067,6 +2334,16 @@ dependencies = [
  "syn 3.0.3",
 ]
 
+[[package]]
+name = "tokio-rustls"
+version = "0.26.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db"
+dependencies = [
+ "rustls",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-stream"
 version = "0.1.18"
@@ -2244,6 +2521,21 @@ dependencies = [
  "tracing-serde",
 ]
 
+[[package]]
+name = "try-lock"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
+
+[[package]]
+name = "twox-hash"
+version = "2.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2"
+dependencies = [
+ "rand 0.10.3",
+]
+
 [[package]]
 name = "typenum"
 version = "1.20.1"
@@ -2330,6 +2622,15 @@ version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
 
+[[package]]
+name = "want"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b"
+dependencies = [
+ "try-lock",
+]
+
 [[package]]
 name = "wasi"
 version = "0.11.1+wasi-snapshot-preview1"
diff --git a/Cargo.toml b/Cargo.toml
index 20117d8..e54e7bd 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -15,6 +15,7 @@ sqlx = { version = "0.8", default-features = false, features = ["postgres", "run
 
 rdkafka = { version = "0.39", features = ["cmake-build", "ssl-vendored", "tokio", "zstd"] }
 futures = "0.3"
+clickhouse = { version = "0.15", features = ["lz4", "rustls-tls"] }
 
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
diff --git a/README.md b/README.md
index e8ac513..19cc5a5 100644
--- a/README.md
+++ b/README.md
@@ -52,17 +52,25 @@ On the wire to Kafka, `traits` and `metadata` are produced as JSON-encoded strin
 
 | Env var | Required | Default |
 |---|---|---|
-| `SINK` | no | `kafka` (the only sink so far) |
-| `KAFKA_BOOTSTRAP_SERVERS` | yes | — |
-| `KAFKA_TOPIC` | yes | — |
-| `KAFKA_EXTERNAL_TOPIC` | yes | — |
+| `SINK` | no | `kafka` (or `clickhouse`) |
+| `KAFKA_BOOTSTRAP_SERVERS` | with `kafka` | — |
+| `KAFKA_TOPIC` | with `kafka` | — |
+| `KAFKA_EXTERNAL_TOPIC` | with `kafka` | — |
 | `KAFKA_AUTH` | no | `scram` (`none` for plaintext local brokers) |
 | `KAFKA_USERNAME` | with `scram` | — |
 | `KAFKA_PASSWORD` | with `scram` | — |
+| `CLICKHOUSE_URL` | with `clickhouse` | — |
+| `CLICKHOUSE_USER` | no | `default` |
+| `CLICKHOUSE_PASSWORD` | no | empty |
+| `CLICKHOUSE_DATABASE` | no | `flagsmith_exp` |
+| `CLICKHOUSE_TABLE` | no | `events` |
+| `SINK_FLUSH_INTERVAL_MS` | no | `1000` |
+| `SINK_BATCH_MAX_ROWS` | no | `50000` |
+| `SINK_BUFFER_MAX_BYTES` | no | `67108864` (64 MiB) |
 | `DATABASE_URL` | yes | — |
 | `LISTEN_ADDR` | no | `0.0.0.0:8080` |
 
-Events from environments that use an external warehouse go to `KAFKA_EXTERNAL_TOPIC`; all others go to `KAFKA_TOPIC`. The server looks presented keys up in the `experimentation_environment_keys` view in Flagsmith's Postgres, at `DATABASE_URL`.
+With `SINK=kafka`, events from environments that use an external warehouse go to `KAFKA_EXTERNAL_TOPIC`; all others go to `KAFKA_TOPIC`. With `SINK=clickhouse`, every event goes to the one ClickHouse table. The server looks presented keys up in the `experimentation_environment_keys` view in Flagsmith's Postgres, at `DATABASE_URL`.
 
 ### Database role
 
@@ -79,6 +87,10 @@ The view reads its tables with its owner's privileges, so the role needs no acce
 
 With `KAFKA_AUTH=scram`, the producer authenticates with SASL/SCRAM-SHA-512 over TLS; MSK serves it on port 9096. The task definition injects both credentials from the `AmazonMSK_*` secret in Secrets Manager.
 
+### ClickHouse sink
+
+With `SINK=clickhouse`, `202` means the events are buffered in memory, not yet stored. A background task writes the buffer every `SINK_FLUSH_INTERVAL_MS`, or sooner once `SINK_BATCH_MAX_ROWS` rows are waiting, as one `INSERT` per batch. A failed batch is retried 3 times and then dropped with an error log. Once the buffer reaches `SINK_BUFFER_MAX_BYTES`, requests get `503` until a flush frees space. A normal shutdown (SIGTERM) flushes the buffer, but events still buffered are lost if the process crashes.
+
 ## Run locally
 
 ```bash
diff --git a/src/auth/middleware.rs b/src/auth/middleware.rs
index 507796a..1b4128e 100644
--- a/src/auth/middleware.rs
+++ b/src/auth/middleware.rs
@@ -22,7 +22,8 @@ pub struct EnvironmentContext {
 #[derive(Clone)]
 pub struct ContextState {
     pub environment_keys: EnvironmentKeys,
-    pub external_topic: Arc<str>,
+    /// Kafka only; other sinks write every environment to one place.
+    pub external_topic: Option<Arc<str>>,
 }
 
 pub async fn extract_environment_context(
@@ -60,7 +61,8 @@ pub async fn extract_environment_context(
 
     let destination = environment_key
         .uses_external_warehouse
-        .then(|| state.external_topic.clone());
+        .then(|| state.external_topic.clone())
+        .flatten();
 
     request.extensions_mut().insert(EnvironmentContext {
         client_api_key: environment_key.client_api_key,
@@ -87,7 +89,7 @@ mod tests {
     fn test_state(pool: PgPool) -> ContextState {
         ContextState {
             environment_keys: EnvironmentKeys::new(pool, Duration::from_secs(300)),
-            external_topic: Arc::from("external-topic"),
+            external_topic: Some(Arc::from("external-topic")),
         }
     }
 
diff --git a/src/config.rs b/src/config.rs
index 4e0e38f..2046f55 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -1,6 +1,8 @@
 use std::env;
+use std::str::FromStr;
+use std::time::Duration;
 
-use crate::sink::KafkaAuth;
+use crate::sink::{BatchConfig, ClickHouseSinkConfig, KafkaAuth};
 
 #[derive(Debug, Clone)]
 pub struct Config {
@@ -12,6 +14,10 @@ pub struct Config {
 #[derive(Debug, Clone)]
 pub enum SinkConfig {
     Kafka(KafkaConfig),
+    ClickHouse {
+        clickhouse: ClickHouseSinkConfig,
+        batch: BatchConfig,
+    },
 }
 
 #[derive(Debug, Clone)]
@@ -26,6 +32,10 @@ impl Config {
     pub fn from_env() -> Result<Self, ConfigError> {
         let sink = match parse_sink(env::var("SINK").ok().as_deref())? {
             SinkKind::Kafka => SinkConfig::Kafka(KafkaConfig::from_env()?),
+            SinkKind::ClickHouse => SinkConfig::ClickHouse {
+                clickhouse: clickhouse_from_env()?,
+                batch: batch_from_env()?,
+            },
         };
 
         Ok(Config {
@@ -60,16 +70,63 @@ fn required(name: &'static str) -> Result<String, ConfigError> {
 #[derive(Debug, PartialEq)]
 enum SinkKind {
     Kafka,
+    ClickHouse,
 }
 
 /// Which sink `SINK` selects; unset means Kafka.
 fn parse_sink(value: Option<&str>) -> Result<SinkKind, ConfigError> {
     match value.unwrap_or("kafka") {
         "kafka" => Ok(SinkKind::Kafka),
+        "clickhouse" => Ok(SinkKind::ClickHouse),
         other => Err(ConfigError::InvalidSink(other.to_string())),
     }
 }
 
+fn clickhouse_from_env() -> Result<ClickHouseSinkConfig, ConfigError> {
+    Ok(ClickHouseSinkConfig {
+        url: required("CLICKHOUSE_URL")?,
+        user: env::var("CLICKHOUSE_USER").unwrap_or_else(|_| "default".to_string()),
+        password: env::var("CLICKHOUSE_PASSWORD").unwrap_or_default(),
+        database: env::var("CLICKHOUSE_DATABASE").unwrap_or_else(|_| "flagsmith_exp".to_string()),
+        table: env::var("CLICKHOUSE_TABLE").unwrap_or_else(|_| "events".to_string()),
+    })
+}
+
+/// Shared by every batching sink.
+fn batch_from_env() -> Result<BatchConfig, ConfigError> {
+    Ok(BatchConfig {
+        flush_interval: Duration::from_millis(parse_positive(
+            "SINK_FLUSH_INTERVAL_MS",
+            env::var("SINK_FLUSH_INTERVAL_MS").ok(),
+            1000,
+        )?),
+        max_rows: parse_positive(
+            "SINK_BATCH_MAX_ROWS",
+            env::var("SINK_BATCH_MAX_ROWS").ok(),
+            50_000,
+        )?,
+        max_bytes: parse_positive(
+            "SINK_BUFFER_MAX_BYTES",
+            env::var("SINK_BUFFER_MAX_BYTES").ok(),
+            64 * 1024 * 1024,
+        )?,
+    })
+}
+
+fn parse_positive<T: FromStr + Default + PartialOrd>(
+    name: &'static str,
+    value: Option<String>,
+    default: T,
+) -> Result<T, ConfigError> {
+    match value {
+        None => Ok(default),
+        Some(v) => match v.parse() {
+            Ok(n) if n > T::default() => Ok(n),
+            _ => Err(ConfigError::InvalidNumber(name, v)),
+        },
+    }
+}
+
 fn parse_auth(
     mode: &str,
     username: Option<String>,
@@ -91,8 +148,10 @@ pub enum ConfigError {
     MissingRequired(&'static str),
     #[error("Invalid value for {0}: {1:?} (expected scram or none)")]
     Invalid(&'static str, String),
-    #[error("Invalid value for SINK: {0:?} (expected kafka)")]
+    #[error("Invalid value for SINK: {0:?} (expected kafka or clickhouse)")]
     InvalidSink(String),
+    #[error("Invalid value for {0}: {1:?} (expected a positive integer)")]
+    InvalidNumber(&'static str, String),
 }
 
 #[cfg(test)]
@@ -140,13 +199,40 @@ mod tests {
     fn parse_sink_defaults_to_kafka() {
         assert_eq!(parse_sink(None).unwrap(), SinkKind::Kafka);
         assert_eq!(parse_sink(Some("kafka")).unwrap(), SinkKind::Kafka);
+        assert_eq!(
+            parse_sink(Some("clickhouse")).unwrap(),
+            SinkKind::ClickHouse
+        );
     }
 
     #[test]
     fn parse_sink_rejects_unknown_value() {
         assert!(matches!(
-            parse_sink(Some("clickhouse")),
-            Err(ConfigError::InvalidSink(v)) if v == "clickhouse"
+            parse_sink(Some("postgres")),
+            Err(ConfigError::InvalidSink(v)) if v == "postgres"
         ));
     }
+
+    #[test]
+    fn parse_positive_defaults_when_unset() {
+        assert_eq!(parse_positive("N", None, 1000u64).unwrap(), 1000);
+    }
+
+    #[test]
+    fn parse_positive_reads_a_positive_integer() {
+        assert_eq!(
+            parse_positive("N", Some("50".to_string()), 1000usize).unwrap(),
+            50
+        );
+    }
+
+    #[test]
+    fn parse_positive_rejects_zero_and_non_numbers() {
+        for value in ["0", "-1", "ten", ""] {
+            assert!(matches!(
+                parse_positive("N", Some(value.to_string()), 1000u64),
+                Err(ConfigError::InvalidNumber("N", v)) if v == value
+            ));
+        }
+    }
 }
diff --git a/src/main.rs b/src/main.rs
index d5185e6..199f48c 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -18,7 +18,7 @@ use tracing::info;
 use auth::ContextState;
 use config::{Config, SinkConfig};
 use environment_keys::EnvironmentKeys;
-use sink::{KafkaSink, KafkaSinkConfig, Sink};
+use sink::{BatchingSink, ClickHouseWriter, KafkaSink, KafkaSinkConfig, Sink};
 
 const ENVIRONMENT_KEY_REFRESH_AFTER: Duration = Duration::from_secs(300);
 
@@ -47,7 +47,20 @@ async fn main() -> anyhow::Result<()> {
                 kafka.topic.clone(),
                 kafka.auth.clone(),
             ))?;
-            (Sink::Kafka(sink), kafka.external_topic.as_str())
+            (Sink::Kafka(sink), Some(kafka.external_topic.as_str()))
+        }
+        SinkConfig::ClickHouse { clickhouse, batch } => {
+            info!(
+                listen_addr = %config.listen_addr,
+                clickhouse = ?clickhouse,
+                batch = ?batch,
+                "Starting flagsmith-analytics server"
+            );
+            let writer = ClickHouseWriter::new(clickhouse.clone());
+            (
+                Sink::ClickHouse(BatchingSink::new(writer, batch.clone())),
+                None,
+            )
         }
     };
 
@@ -57,7 +70,7 @@ async fn main() -> anyhow::Result<()> {
         .connect_lazy(&config.database_url)?;
     let context_state = ContextState {
         environment_keys: EnvironmentKeys::new(pool, ENVIRONMENT_KEY_REFRESH_AFTER),
-        external_topic: Arc::from(external_topic),
+        external_topic: external_topic.map(Arc::from),
     };
 
     let app = routes::app(sink.clone(), context_state);
diff --git a/src/routes/mod.rs b/src/routes/mod.rs
index d2c0db3..667f5c3 100644
--- a/src/routes/mod.rs
+++ b/src/routes/mod.rs
@@ -53,7 +53,7 @@ mod tests {
     fn context_state(pool: PgPool) -> ContextState {
         ContextState {
             environment_keys: EnvironmentKeys::new(pool, Duration::from_secs(300)),
-            external_topic: Arc::from(EXTERNAL_WAREHOUSE_TOPIC),
+            external_topic: Some(Arc::from(EXTERNAL_WAREHOUSE_TOPIC)),
         }
     }
 
diff --git a/src/sink/batching.rs b/src/sink/batching.rs
new file mode 100644
index 0000000..f57e1e0
--- /dev/null
+++ b/src/sink/batching.rs
@@ -0,0 +1,328 @@
+use std::future::Future;
+use std::hash::{BuildHasher, RandomState};
+use std::mem;
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+
+use tokio::sync::Notify;
+use tokio::task::JoinHandle;
+use tokio::time::{Instant, MissedTickBehavior};
+use tracing::{error, warn};
+
+use super::SinkError;
+use crate::models::EventRecord;
+
+/// Retries after the first failed write, before the batch is dropped.
+const MAX_RETRIES: u32 = 3;
+/// Doubles per retry: about 200 ms, 400 ms, 800 ms before jitter.
+const RETRY_BASE_DELAY: Duration = Duration::from_millis(200);
+
+/// Writes one batch of owned rows to a store.
+pub trait BatchWriter: Send + Sync + 'static {
+    type Row: Send + Sync + 'static;
+
+    fn row(record: &EventRecord<'_>) -> Self::Row;
+
+    /// Approximate in-memory size, counted against the buffer limit.
+    fn row_size(row: &Self::Row) -> usize;
+
+    fn write(&self, rows: &[Self::Row]) -> impl Future<Output = Result<(), SinkError>> + Send;
+}
+
+#[derive(Debug, Clone)]
+pub struct BatchConfig {
+    pub flush_interval: Duration,
+    /// Flush early once this many rows are buffered; also the most rows per write.
+    pub max_rows: usize,
+    /// Bound on buffered plus in-flight rows; `put` fails beyond it.
+    pub max_bytes: usize,
+}
+
+struct Buffer<R> {
+    rows: Vec<R>,
+    /// Size of `rows` plus the batch being flushed.
+    bytes: usize,
+}
+
+struct Shared<W: BatchWriter> {
+    writer: W,
+    config: BatchConfig,
+    buffer: Mutex<Buffer<W::Row>>,
+    flush_now: Notify,
+    stop: Notify,
+}
+
+/// Buffers rows in memory and writes them in batches from one background
+/// task. `put` returns once the rows are buffered, so rows still buffered are
+/// lost if the process dies.
+pub struct BatchingSink<W: BatchWriter> {
+    shared: Arc<Shared<W>>,
+    flusher: Arc<Mutex<Option<JoinHandle<()>>>>,
+}
+
+impl<W: BatchWriter> Clone for BatchingSink<W> {
+    fn clone(&self) -> Self {
+        Self {
+            shared: self.shared.clone(),
+            flusher: self.flusher.clone(),
+        }
+    }
+}
+
+impl<W: BatchWriter> BatchingSink<W> {
+    /// Starts the flusher task; needs a Tokio runtime.
+    pub fn new(writer: W, config: BatchConfig) -> Self {
+        let shared = Arc::new(Shared {
+            writer,
+            config,
+            buffer: Mutex::new(Buffer {
+                rows: Vec::new(),
+                bytes: 0,
+            }),
+            flush_now: Notify::new(),
+            stop: Notify::new(),
+        });
+        let flusher = tokio::spawn(run(shared.clone()));
+        Self {
+            shared,
+            flusher: Arc::new(Mutex::new(Some(flusher))),
+        }
+    }
+
+    /// Buffer every record, or none of them if that would pass the byte limit.
+    pub fn put(&self, records: &[EventRecord<'_>]) -> Result<(), SinkError> {
+        let rows: Vec<W::Row> = records.iter().map(W::row).collect();
+        let bytes: usize = rows.iter().map(W::row_size).sum();
+        let config = &self.shared.config;
+        let mut buffer = self.shared.buffer.lock().expect("buffer lock");
+        if buffer.bytes + bytes > config.max_bytes {
+            return Err(SinkError::BufferFull);
+        }
+        buffer.bytes += bytes;
+        buffer.rows.extend(rows);
+        if buffer.rows.len() >= config.max_rows {
+            self.shared.flush_now.notify_one();
+        }
+        Ok(())
+    }
+
+    /// Stop the flusher and write what is buffered, bounded by `timeout`.
+    pub async fn shutdown(self, timeout: Duration) {
+        self.shared.stop.notify_one();
+        let Some(flusher) = self.flusher.lock().expect("flusher lock").take() else {
+            return;
+        };
+        match tokio::time::timeout(timeout, flusher).await {
+            Ok(Ok(())) => {}
+            Ok(Err(e)) => warn!(error = %e, "Sink flush task failed"),
+            Err(_) => warn!("Sink final flush incomplete"),
+        }
+    }
+}
+
+async fn run<W: BatchWriter>(shared: Arc<Shared<W>>) {
+    let period = shared.config.flush_interval;
+    let mut interval = tokio::time::interval_at(Instant::now() + period, period);
+    interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
+    loop {
+        let stopping = tokio::select! {
+            _ = interval.tick() => false,
+            _ = shared.flush_now.notified() => false,
+            _ = shared.stop.notified() => true,
+        };
+        flush(&shared).await;
+        if stopping {
+            return;
+        }
+    }
+}
+
+async fn flush<W: BatchWriter>(shared: &Shared<W>) {
+    let rows = mem::take(&mut shared.buffer.lock().expect("buffer lock").rows);
+    if rows.is_empty() {
+        return;
+    }
+    for chunk in rows.chunks(shared.config.max_rows) {
+        write_with_retries(&shared.writer, chunk).await;
+    }
+    let flushed: usize = rows.iter().map(W::row_size).sum();
+    shared.buffer.lock().expect("buffer lock").bytes -= flushed;
+}
+
+async fn write_with_retries<W: BatchWriter>(writer: &W, rows: &[W::Row]) {
+    let mut retry = 0;
+    loop {
+        match writer.write(rows).await {
+            Ok(()) => return,
+            Err(e) if retry < MAX_RETRIES => {
+                warn!(rows = rows.len(), retry, error = %e, "Sink batch write failed, retrying");
+                tokio::time::sleep(backoff(retry)).await;
+                retry += 1;
+            }
+            Err(e) => {
+                error!(rows = rows.len(), error = %e, "Sink batch dropped after retries");
+                return;
+            }
+        }
+    }
+}
+
+/// Exponential backoff with ±50% jitter, so tasks that failed together do not
+/// retry together.
+fn backoff(retry: u32) -> Duration {
+    let jitter_permille = RandomState::new().hash_one(retry) % 1000;
+    (RETRY_BASE_DELAY * 2u32.pow(retry)).mul_f64(0.5 + jitter_permille as f64 / 1000.0)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    use super::*;
+
+    const ROW_SIZE: usize = 100;
+    static NULL: serde_json::Value = serde_json::Value::Null;
+
+    /// Records every write attempt; fails all of them when `fail` is set.
+    #[derive(Clone, Default)]
+    struct FakeWriter {
+        fail: bool,
+        attempts: Arc<AtomicUsize>,
+        written: Arc<Mutex<Vec<Vec<i64>>>>,
+    }
+
+    impl BatchWriter for FakeWriter {
+        type Row = i64;
+
+        fn row(record: &EventRecord<'_>) -> i64 {
+            record.timestamp
+        }
+
+        fn row_size(_row: &i64) -> usize {
+            ROW_SIZE
+        }
+
+        async fn write(&self, rows: &[i64]) -> Result<(), SinkError> {
+            self.attempts.fetch_add(1, Ordering::SeqCst);
+            if self.fail {
+                return Err(SinkError::BufferFull);
+            }
+            self.written.lock().unwrap().push(rows.to_vec());
+            Ok(())
+        }
+    }
+
+    fn records(timestamps: &[i64]) -> Vec<EventRecord<'static>> {
+        timestamps
+            .iter()
+            .map(|&timestamp| EventRecord {
+                environment_key: "env",
+                collected_at: timestamp,
+                sdk_language: "rust",
+                sdk_version: "1.0.0",
+                event: "purchase",
+                feature_name: None,
+                identifier: None,
+                value: &NULL,
+                traits: None,
+                metadata: None,
+                timestamp,
+            })
+            .collect()
+    }
+
+    fn config(flush_interval: Duration, max_rows: usize, max_bytes: usize) -> BatchConfig {
+        BatchConfig {
+            flush_interval,
+            max_rows,
+            max_bytes,
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn flushes_on_the_interval() {
+        let writer = FakeWriter::default();
+        let sink = BatchingSink::new(writer.clone(), config(Duration::from_secs(1), 100, 10_000));
+
+        sink.put(&records(&[1, 2])).unwrap();
+        tokio::time::sleep(Duration::from_millis(500)).await;
+        assert!(writer.written.lock().unwrap().is_empty());
+
+        tokio::time::sleep(Duration::from_millis(600)).await;
+        assert_eq!(*writer.written.lock().unwrap(), vec![vec![1, 2]]);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn flushes_early_on_max_rows() {
+        let writer = FakeWriter::default();
+        let sink = BatchingSink::new(writer.clone(), config(Duration::from_secs(3600), 2, 10_000));
+
+        sink.put(&records(&[1, 2])).unwrap();
+        tokio::time::sleep(Duration::from_millis(10)).await;
+
+        assert_eq!(*writer.written.lock().unwrap(), vec![vec![1, 2]]);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn rejects_a_put_that_would_overflow_the_buffer() {
+        let writer = FakeWriter::default();
+        let sink = BatchingSink::new(
+            writer.clone(),
+            config(Duration::from_secs(3600), 100, 2 * ROW_SIZE),
+        );
+
+        sink.put(&records(&[1])).unwrap();
+        assert!(matches!(
+            sink.put(&records(&[2, 3])),
+            Err(SinkError::BufferFull)
+        ));
+        sink.put(&records(&[4])).unwrap();
+
+        sink.shutdown(Duration::from_secs(1)).await;
+        assert_eq!(*writer.written.lock().unwrap(), vec![vec![1, 4]]);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn drops_a_batch_after_the_retries_and_frees_its_space() {
+        let writer = FakeWriter {
+            fail: true,
+            ..FakeWriter::default()
+        };
+        let sink = BatchingSink::new(
+            writer.clone(),
+            config(Duration::from_secs(1), 100, ROW_SIZE),
+        );
+
+        sink.put(&records(&[1])).unwrap();
+        tokio::time::sleep(Duration::from_secs(10)).await;
+
+        assert_eq!(
+            writer.attempts.load(Ordering::SeqCst),
+            1 + MAX_RETRIES as usize
+        );
+        sink.put(&records(&[2])).unwrap();
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn shutdown_writes_what_is_buffered() {
+        let writer = FakeWriter::default();
+        let sink = BatchingSink::new(
+            writer.clone(),
+            config(Duration::from_secs(3600), 100, 10_000),
+        );
+
+        sink.put(&records(&[1, 2, 3])).unwrap();
+        sink.shutdown(Duration::from_secs(1)).await;
+
+        assert_eq!(*writer.written.lock().unwrap(), vec![vec![1, 2, 3]]);
+    }
+
+    #[test]
+    fn backoff_doubles_within_the_jitter_band() {
+        for retry in 0..MAX_RETRIES {
+            let base = RETRY_BASE_DELAY * 2u32.pow(retry);
+            let delay = backoff(retry);
+            assert!(delay >= base / 2 && delay < base * 3 / 2, "{delay:?}");
+        }
+    }
+}
diff --git a/src/sink/clickhouse.rs b/src/sink/clickhouse.rs
new file mode 100644
index 0000000..8a6afa3
--- /dev/null
+++ b/src/sink/clickhouse.rs
@@ -0,0 +1,197 @@
+use std::fmt;
+use std::mem;
+use std::time::Duration;
+
+use clickhouse::{Client, Compression, Row};
+use serde::Serialize;
+
+use super::batching::BatchWriter;
+use super::SinkError;
+use crate::models::EventRecord;
+
+/// Bounds sending and committing one batch of up to `SINK_BATCH_MAX_ROWS` rows.
+const INSERT_TIMEOUT: Duration = Duration::from_secs(30);
+
+#[derive(Clone)]
+pub struct ClickHouseSinkConfig {
+    pub url: String,
+    pub user: String,
+    pub password: String,
+    pub database: String,
+    pub table: String,
+}
+
+/// Keeps the password out of logs.
+impl fmt::Debug for ClickHouseSinkConfig {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("ClickHouseSinkConfig")
+            .field("url", &self.url)
+            .field("user", &self.user)
+            .field("database", &self.database)
+            .field("table", &self.table)
+            .finish_non_exhaustive()
+    }
+}
+
+/// One row of the `events` table. Field names are the column names. Every
+/// column is non-Nullable, so absent optionals are written as "", as
+/// ClickPipes does.
+#[derive(Debug, PartialEq, Row, Serialize)]
+pub struct ClickHouseRow {
+    environment_key: String,
+    event: String,
+    feature_name: String,
+    /// DateTime64(3), sent as epoch millis.
+    timestamp: i64,
+    collected_at: i64,
+    identifier: String,
+    value: String,
+    traits: String,
+    metadata: String,
+    sdk_language: String,
+    sdk_version: String,
+}
+
+/// `value` is a String column: a JSON string goes in raw, null as "", and
+/// anything else as its JSON text.
+fn value_column(value: &serde_json::Value) -> String {
+    match value {
+        serde_json::Value::String(s) => s.clone(),
+        serde_json::Value::Null => String::new(),
+        other => other.to_string(),
+    }
+}
+
+pub struct ClickHouseWriter {
+    client: Client,
+    table: String,
+}
+
+impl ClickHouseWriter {
+    /// Connects lazily, on the first insert.
+    pub fn new(config: ClickHouseSinkConfig) -> Self {
+        let client = Client::default()
+            .with_url(config.url)
+            .with_user(config.user)
+            .with_password(config.password)
+            .with_database(config.database)
+            .with_compression(Compression::Lz4);
+        Self {
+            client,
+            table: config.table,
+        }
+    }
+}
+
+impl BatchWriter for ClickHouseWriter {
+    type Row = ClickHouseRow;
+
+    fn row(record: &EventRecord<'_>) -> ClickHouseRow {
+        ClickHouseRow {
+            environment_key: record.environment_key.to_string(),
+            event: record.event.to_string(),
+            feature_name: record.feature_name.unwrap_or_default().to_string(),
+            timestamp: record.timestamp,
+            collected_at: record.collected_at,
+            identifier: record.identifier.unwrap_or_default().to_string(),
+            value: value_column(record.value),
+            traits: record.traits.map(ToString::to_string).unwrap_or_default(),
+            metadata: record.metadata.map(ToString::to_string).unwrap_or_default(),
+            sdk_language: record.sdk_language.to_string(),
+            sdk_version: record.sdk_version.to_string(),
+        }
+    }
+
+    fn row_size(row: &ClickHouseRow) -> usize {
+        mem::size_of::<ClickHouseRow>()
+            + row.environment_key.len()
+            + row.event.len()
+            + row.feature_name.len()
+            + row.identifier.len()
+            + row.value.len()
+            + row.traits.len()
+            + row.metadata.len()
+            + row.sdk_language.len()
+            + row.sdk_version.len()
+    }
+
+    /// One plain `INSERT` per batch; succeeds once ClickHouse has committed it.
+    async fn write(&self, rows: &[ClickHouseRow]) -> Result<(), SinkError> {
+        let mut insert = self
+            .client
+            .insert::<ClickHouseRow>(&self.table)
+            .await?
+            .with_timeouts(Some(INSERT_TIMEOUT), Some(INSERT_TIMEOUT));
+        for row in rows {
+            insert.write(row).await?;
+        }
+        insert.end().await?;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use serde_json::json;
+
+    use super::*;
+
+    #[test]
+    fn row_fills_every_column_and_blanks_absent_optionals() {
+        let value = json!(42);
+        let traits = json!({"plan": "pro"});
+        let record = EventRecord {
+            environment_key: "env",
+            collected_at: 1_700_000_000_001,
+            sdk_language: "python",
+            sdk_version: "1.0.0",
+            event: "purchase",
+            feature_name: None,
+            identifier: None,
+            value: &value,
+            traits: Some(&traits),
+            metadata: None,
+            timestamp: 1_700_000_000_000,
+        };
+
+        assert_eq!(
+            ClickHouseWriter::row(&record),
+            ClickHouseRow {
+                environment_key: "env".to_string(),
+                event: "purchase".to_string(),
+                feature_name: String::new(),
+                timestamp: 1_700_000_000_000,
+                collected_at: 1_700_000_000_001,
+                identifier: String::new(),
+                value: "42".to_string(),
+                traits: r#"{"plan":"pro"}"#.to_string(),
+                metadata: String::new(),
+                sdk_language: "python".to_string(),
+                sdk_version: "1.0.0".to_string(),
+            }
+        );
+    }
+
+    #[test]
+    fn value_column_writes_strings_raw_and_null_as_empty() {
+        assert_eq!(value_column(&json!("control")), "control");
+        assert_eq!(value_column(&json!(null)), "");
+        assert_eq!(value_column(&json!(42)), "42");
+        assert_eq!(value_column(&json!(true)), "true");
+        assert_eq!(value_column(&json!({"a": [1, 2]})), r#"{"a":[1,2]}"#);
+    }
+
+    #[test]
+    fn debug_output_omits_the_password() {
+        let config = ClickHouseSinkConfig {
+            url: "https://clickhouse:8443".to_string(),
+            user: "ingest".to_string(),
+            password: "hunter2".to_string(),
+            database: "flagsmith_exp".to_string(),
+            table: "events".to_string(),
+        };
+        let shown = format!("{config:?}");
+        assert!(shown.contains("ingest"));
+        assert!(!shown.contains("hunter2"));
+    }
+}
diff --git a/src/sink/mod.rs b/src/sink/mod.rs
index 4369129..277aed4 100644
--- a/src/sink/mod.rs
+++ b/src/sink/mod.rs
@@ -1,7 +1,11 @@
+mod batching;
+mod clickhouse;
 mod kafka;
 
 use std::time::Duration;
 
+pub use batching::{BatchConfig, BatchingSink};
+pub use clickhouse::{ClickHouseSinkConfig, ClickHouseWriter};
 pub use kafka::{KafkaAuth, KafkaSink, KafkaSinkConfig};
 
 use crate::models::EventRecord;
@@ -10,6 +14,7 @@ use crate::models::EventRecord;
 #[derive(Clone)]
 pub enum Sink {
     Kafka(KafkaSink),
+    ClickHouse(BatchingSink<ClickHouseWriter>),
 }
 
 impl Sink {
@@ -22,6 +27,8 @@ impl Sink {
     ) -> Result<(), SinkError> {
         match self {
             Self::Kafka(sink) => Ok(sink.put(destination, records).await?),
+            // A single table: there is no per-environment destination.
+            Self::ClickHouse(sink) => sink.put(records),
         }
     }
 
@@ -29,6 +36,7 @@ impl Sink {
     pub fn fatal_error(&self) -> Option<String> {
         match self {
             Self::Kafka(sink) => sink.fatal_error(),
+            Self::ClickHouse(_) => None,
         }
     }
 
@@ -36,6 +44,7 @@ impl Sink {
     pub async fn shutdown(self, timeout: Duration) {
         match self {
             Self::Kafka(sink) => sink.shutdown(timeout).await,
+            Self::ClickHouse(sink) => sink.shutdown(timeout).await,
         }
     }
 }
@@ -44,4 +53,8 @@ impl Sink {
 pub enum SinkError {
     #[error(transparent)]
     Kafka(#[from] kafka::DeliveryError),
+    #[error("ClickHouse insert failed: {0}")]
+    ClickHouse(#[from] ::clickhouse::error::Error),
+    #[error("Sink buffer is full")]
+    BufferFull,
 }

From ac2af7abc8def624e76cfff7bdeb8c449bb58df9 Mon Sep 17 00:00:00 2001
From: Gagan Trivedi <gagandeeptrivedi47@gmail.com>
Date: Tue, 27 Oct 2026 14:57:30 +0530
Subject: [PATCH 03/11] test(sink): ClickHouse integration tests

Tests the ClickHouse sink against a real server: flushed rows land in
the events table and read back intact, and shutdown writes buffered
rows. CI runs a clickhouse-server service for them.

Refs #3
---
 .github/workflows/ci.yml |  13 +++++
 README.md                |   2 +-
 src/routes/ingest.rs     |  37 +++++++++++++-
 src/sink/clickhouse.rs   | 100 +++++++++++++++++++++++++++++++++++++
 src/test_helpers.rs      | 105 ++++++++++++++++++++++++++++++++++++++-
 5 files changed, 254 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index eab9134..8d21937 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -31,8 +31,21 @@ jobs:
           --health-interval 5s
           --health-timeout 5s
           --health-retries 10
+      clickhouse:
+        image: clickhouse/clickhouse-server:25.8
+        env:
+          # Lets the default user connect without a password.
+          CLICKHOUSE_SKIP_USER_SETUP: 1
+        ports:
+          - 8123:8123
+        options: >-
+          --health-cmd "clickhouse-client --query 'SELECT 1'"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
     env:
       DATABASE_URL: postgres://postgres:password@localhost:5432/postgres
+      CLICKHOUSE_URL: http://localhost:8123
     steps:
       - uses: actions/checkout@v6
 
diff --git a/README.md b/README.md
index 19cc5a5..e6bd7b6 100644
--- a/README.md
+++ b/README.md
@@ -102,7 +102,7 @@ curl localhost:8080/health
 
 The Kafka producer and Postgres pool connect lazily, so `/health` responds even without either available.
 
-`cargo test` needs a Postgres it can create databases in, at `DATABASE_URL`: each database test gets a fresh one.
+`cargo test` needs a Postgres it can create databases in, at `DATABASE_URL`, and a ClickHouse it can create databases in, at `CLICKHOUSE_URL` (with `CLICKHOUSE_USER` and `CLICKHOUSE_PASSWORD` if needed): each database test gets a fresh one.
 
 ## CI / CD
 
diff --git a/src/routes/ingest.rs b/src/routes/ingest.rs
index e8681af..fbe0459 100644
--- a/src/routes/ingest.rs
+++ b/src/routes/ingest.rs
@@ -303,7 +303,10 @@ mod tests {
     use rdkafka::types::RDKafkaRespErr;
     use tower::ServiceExt;
 
-    use crate::sink::{KafkaAuth, KafkaSink, KafkaSinkConfig};
+    use crate::sink::{
+        BatchConfig, BatchingSink, ClickHouseSinkConfig, ClickHouseWriter, KafkaAuth, KafkaSink,
+        KafkaSinkConfig,
+    };
     use crate::test_helpers::{
         assert_topic_empty, mock_kafka, now_ms, read_records, FLAGSMITH_WAREHOUSE_TOPIC,
     };
@@ -721,6 +724,38 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_handler_full_clickhouse_buffer_returns_503() {
+        // Room for one buffered event; the unreachable server and the hour-long
+        // interval keep it buffered.
+        let sink = Sink::ClickHouse(BatchingSink::new(
+            ClickHouseWriter::new(ClickHouseSinkConfig {
+                url: "http://127.0.0.1:1".to_string(),
+                user: "default".to_string(),
+                password: String::new(),
+                database: "flagsmith_exp".to_string(),
+                table: "events".to_string(),
+            }),
+            BatchConfig {
+                flush_interval: Duration::from_secs(3600),
+                max_rows: 1000,
+                max_bytes: 400,
+            },
+        ));
+
+        let first = app(&sink)
+            .oneshot(post_events(&purchase_json()))
+            .await
+            .unwrap();
+        let second = app(&sink)
+            .oneshot(post_events(&purchase_json()))
+            .await
+            .unwrap();
+
+        assert_eq!(first.status(), StatusCode::ACCEPTED);
+        assert_eq!(second.status(), StatusCode::SERVICE_UNAVAILABLE);
+    }
+
     #[tokio::test]
     async fn test_handler_broker_rejection_returns_503_quickly() {
         let kafka = mock_kafka(&[FLAGSMITH_WAREHOUSE_TOPIC]);
diff --git a/src/sink/clickhouse.rs b/src/sink/clickhouse.rs
index 8a6afa3..ea86955 100644
--- a/src/sink/clickhouse.rs
+++ b/src/sink/clickhouse.rs
@@ -37,6 +37,7 @@ impl fmt::Debug for ClickHouseSinkConfig {
 /// column is non-Nullable, so absent optionals are written as "", as
 /// ClickPipes does.
 #[derive(Debug, PartialEq, Row, Serialize)]
+#[cfg_attr(test, derive(serde::Deserialize))]
 pub struct ClickHouseRow {
     environment_key: String,
     event: String,
@@ -135,6 +136,8 @@ mod tests {
     use serde_json::json;
 
     use super::*;
+    use crate::sink::{BatchConfig, BatchingSink};
+    use crate::test_helpers::{now_ms, ClickHouseTestDatabase};
 
     #[test]
     fn row_fills_every_column_and_blanks_absent_optionals() {
@@ -194,4 +197,101 @@ mod tests {
         assert!(shown.contains("ingest"));
         assert!(!shown.contains("hunter2"));
     }
+
+    fn batch_config(flush_interval: Duration) -> BatchConfig {
+        BatchConfig {
+            flush_interval,
+            max_rows: 1000,
+            max_bytes: 1024 * 1024,
+        }
+    }
+
+    #[tokio::test]
+    async fn flushed_rows_land_in_the_events_table() {
+        let db = ClickHouseTestDatabase::create().await;
+        let sink = BatchingSink::new(
+            ClickHouseWriter::new(db.config.clone()),
+            batch_config(Duration::from_millis(100)),
+        );
+        let ts = now_ms();
+        let control = json!("control");
+        let null = json!(null);
+        let traits = json!({"plan": "pro"});
+        let metadata = json!({"source": "csv"});
+        let records = [
+            EventRecord {
+                environment_key: "env",
+                collected_at: ts + 1,
+                sdk_language: "python",
+                sdk_version: "1.0.0",
+                event: "$flag_exposure",
+                feature_name: Some("dark_mode"),
+                identifier: Some("user_1"),
+                value: &control,
+                traits: Some(&traits),
+                metadata: Some(&metadata),
+                timestamp: ts,
+            },
+            EventRecord {
+                environment_key: "env",
+                collected_at: ts + 1,
+                sdk_language: "python",
+                sdk_version: "1.0.0",
+                event: "purchase",
+                feature_name: None,
+                identifier: None,
+                value: &null,
+                traits: None,
+                metadata: None,
+                timestamp: ts + 2,
+            },
+        ];
+
+        sink.put(&records).unwrap();
+        db.wait_for_rows(2).await;
+
+        let stored: Vec<ClickHouseRow> = db
+            .client
+            .query("SELECT ?fields FROM events ORDER BY timestamp")
+            .fetch_all()
+            .await
+            .unwrap();
+        let expected: Vec<ClickHouseRow> = records.iter().map(ClickHouseWriter::row).collect();
+        assert_eq!(stored, expected);
+        assert_eq!(stored[0].value, "control");
+        assert_eq!(stored[1].value, "");
+
+        sink.shutdown(Duration::from_secs(5)).await;
+        db.drop().await;
+    }
+
+    #[tokio::test]
+    async fn shutdown_writes_buffered_rows() {
+        let db = ClickHouseTestDatabase::create().await;
+        let sink = BatchingSink::new(
+            ClickHouseWriter::new(db.config.clone()),
+            batch_config(Duration::from_secs(3600)),
+        );
+        let value = json!(42);
+        let record = EventRecord {
+            environment_key: "env",
+            collected_at: now_ms(),
+            sdk_language: "rust",
+            sdk_version: "1.0.0",
+            event: "purchase",
+            feature_name: None,
+            identifier: None,
+            value: &value,
+            traits: None,
+            metadata: None,
+            timestamp: now_ms(),
+        };
+
+        sink.put(&[record.clone(), record.clone(), record]).unwrap();
+        assert_eq!(db.count().await, 0);
+        sink.shutdown(Duration::from_secs(5)).await;
+
+        assert_eq!(db.count().await, 3);
+        db.drop().await;
+    }
 }
diff --git a/src/test_helpers.rs b/src/test_helpers.rs
index 4f089f5..6bfe1c0 100644
--- a/src/test_helpers.rs
+++ b/src/test_helpers.rs
@@ -1,6 +1,9 @@
+use std::env;
+use std::sync::atomic::{AtomicUsize, Ordering};
 use std::time::Duration;
 
 use chrono::{DateTime, Utc};
+use clickhouse::Client;
 use rdkafka::config::ClientConfig;
 use rdkafka::consumer::{Consumer, StreamConsumer};
 use rdkafka::mocking::MockCluster;
@@ -8,7 +11,7 @@ use rdkafka::producer::DefaultProducerContext;
 use rdkafka::Message;
 use sqlx::PgPool;
 
-use crate::sink::{KafkaAuth, KafkaSink, KafkaSinkConfig, Sink};
+use crate::sink::{ClickHouseSinkConfig, KafkaAuth, KafkaSink, KafkaSinkConfig, Sink};
 
 pub(crate) const FLAGSMITH_WAREHOUSE_TOPIC: &str = "events-default";
 
@@ -112,3 +115,103 @@ pub(crate) async fn insert_environment_key(
     .await
     .unwrap();
 }
+
+/// The production `events` table, from Flagsmith's
+/// experimentation/clickhouse/events.sql.
+const EVENTS_TABLE: &str = "
+    CREATE TABLE {database}.events
+    (
+        environment_key      LowCardinality(String),
+        event                LowCardinality(String),
+        feature_name         LowCardinality(String),
+        timestamp            DateTime64(3),
+        collected_at         DateTime64(3),
+        identifier           String,
+        value                String                          CODEC(ZSTD(3)),
+        traits               String                          CODEC(ZSTD(3)),
+        metadata             String                          CODEC(ZSTD(3)),
+        sdk_language         LowCardinality(String),
+        sdk_version          LowCardinality(String),
+        INDEX idx_identity identifier TYPE bloom_filter GRANULARITY 4,
+        CONSTRAINT environment_key_not_empty CHECK environment_key != '',
+        CONSTRAINT event_not_empty           CHECK event != '',
+        CONSTRAINT timestamp_sane            CHECK timestamp > toDateTime64('2020-01-01 00:00:00', 3)
+    )
+    ENGINE = MergeTree
+    PARTITION BY toYYYYMMDD(timestamp)
+    ORDER BY (environment_key, event, feature_name, timestamp, identifier)
+";
+
+static NEXT_DATABASE: AtomicUsize = AtomicUsize::new(0);
+
+/// A fresh database holding the `events` table, on the ClickHouse at
+/// `CLICKHOUSE_URL` (with `CLICKHOUSE_USER` and `CLICKHOUSE_PASSWORD`, as the
+/// server reads them). Call `drop` at the end of the test.
+pub(crate) struct ClickHouseTestDatabase {
+    pub(crate) config: ClickHouseSinkConfig,
+    pub(crate) client: Client,
+}
+
+impl ClickHouseTestDatabase {
+    pub(crate) async fn create() -> Self {
+        let url = env::var("CLICKHOUSE_URL").expect("CLICKHOUSE_URL must be set");
+        let user = env::var("CLICKHOUSE_USER").unwrap_or_else(|_| "default".to_string());
+        let password = env::var("CLICKHOUSE_PASSWORD").unwrap_or_default();
+        let database = format!(
+            "test_{}_{}",
+            Utc::now().timestamp_millis(),
+            NEXT_DATABASE.fetch_add(1, Ordering::SeqCst)
+        );
+        let server = Client::default()
+            .with_url(&url)
+            .with_user(&user)
+            .with_password(&password);
+        server
+            .query(&format!("CREATE DATABASE {database}"))
+            .execute()
+            .await
+            .expect("create database");
+        server
+            .query(&EVENTS_TABLE.replace("{database}", &database))
+            .execute()
+            .await
+            .expect("create events table");
+        Self {
+            client: server.with_database(&database),
+            config: ClickHouseSinkConfig {
+                url,
+                user,
+                password,
+                database,
+                table: "events".to_string(),
+            },
+        }
+    }
+
+    pub(crate) async fn count(&self) -> u64 {
+        self.client
+            .query("SELECT count() FROM events")
+            .fetch_one()
+            .await
+            .expect("count events")
+    }
+
+    /// Polls until `n` rows are stored; flushes run in the background.
+    pub(crate) async fn wait_for_rows(&self, n: u64) {
+        for _ in 0..100 {
+            if self.count().await >= n {
+                return;
+            }
+            tokio::time::sleep(Duration::from_millis(100)).await;
+        }
+        panic!("expected {n} rows within 10s, found {}", self.count().await);
+    }
+
+    pub(crate) async fn drop(self) {
+        self.client
+            .query(&format!("DROP DATABASE {}", self.config.database))
+            .execute()
+            .await
+            .expect("drop database");
+    }
+}

From d01d4dcae7f47dff59f4a1350aedd8cc5cbde417 Mon Sep 17 00:00:00 2001
From: Gagan Trivedi <gagandeeptrivedi47@gmail.com>
Date: Wed, 28 Oct 2026 14:50:53 +0530
Subject: [PATCH 04/11] refactor(sink): buffer owned EventRecords

EventRecord owns its fields, so BatchingSink buffers records directly and
sizes them with one approx_size. BatchWriter shrinks to write(), and the
ClickHouse writer builds borrowed rows at flush time. Kafka's wire bytes
are unchanged.

Refs #3
---
 src/models/event.rs    |  62 ++++++++--
 src/routes/ingest.rs   |  55 ++++-----
 src/sink/batching.rs   | 107 ++++++++---------
 src/sink/clickhouse.rs | 255 +++++++++++++++++++++--------------------
 src/sink/kafka.rs      |   2 +-
 src/sink/mod.rs        |   4 +-
 6 files changed, 260 insertions(+), 225 deletions(-)

diff --git a/src/models/event.rs b/src/models/event.rs
index 2f44b77..d2e5834 100644
--- a/src/models/event.rs
+++ b/src/models/event.rs
@@ -1,3 +1,5 @@
+use std::mem;
+
 use serde::{Deserialize, Serialize, Serializer};
 
 /// Reserved system event for flag-exposure tracking. `feature_name` is required
@@ -60,35 +62,73 @@ pub struct IngestResponse {
 /// only maps flat scalar fields — a nested object field is left out of the
 /// pipe's field mapping and the column silently defaults to ''.
 #[derive(Debug, Clone, Serialize)]
-pub struct EventRecord<'a> {
-    pub environment_key: &'a str,
+pub struct EventRecord {
+    pub environment_key: String,
     pub collected_at: i64,
-    pub sdk_language: &'a str,
-    pub sdk_version: &'a str,
+    pub sdk_language: String,
+    pub sdk_version: String,
 
-    pub event: &'a str,
+    pub event: String,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub feature_name: Option<&'a str>,
+    pub feature_name: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub identifier: Option<&'a str>,
+    pub identifier: Option<String>,
     #[serde(skip_serializing_if = "serde_json::Value::is_null")]
-    pub value: &'a serde_json::Value,
+    pub value: serde_json::Value,
     #[serde(
         skip_serializing_if = "Option::is_none",
         serialize_with = "json_string"
     )]
-    pub traits: Option<&'a serde_json::Value>,
+    pub traits: Option<serde_json::Value>,
     #[serde(
         skip_serializing_if = "Option::is_none",
         serialize_with = "json_string"
     )]
-    pub metadata: Option<&'a serde_json::Value>,
+    pub metadata: Option<serde_json::Value>,
     pub timestamp: i64,
 }
 
+impl EventRecord {
+    /// Approximate memory held, counted against a sink's buffer limit.
+    pub fn approx_size(&self) -> usize {
+        mem::size_of::<Self>()
+            + self.environment_key.len()
+            + self.sdk_language.len()
+            + self.sdk_version.len()
+            + self.event.len()
+            + self.feature_name.as_ref().map_or(0, String::len)
+            + self.identifier.as_ref().map_or(0, String::len)
+            + value_size(&self.value)
+            + self.traits.as_ref().map_or(0, value_size)
+            + self.metadata.as_ref().map_or(0, value_size)
+    }
+}
+
+/// Rough per-entry overhead of a JSON object's map.
+const MAP_ENTRY_OVERHEAD: usize = 32;
+
+/// Heap held by a JSON value; its inline size is counted by its parent.
+fn value_size(value: &serde_json::Value) -> usize {
+    use serde_json::Value;
+    match value {
+        Value::String(s) => s.len(),
+        Value::Array(items) => items
+            .iter()
+            .map(|item| mem::size_of::<Value>() + value_size(item))
+            .sum(),
+        Value::Object(map) => map
+            .iter()
+            .map(|(key, item)| {
+                MAP_ENTRY_OVERHEAD + key.len() + mem::size_of::<Value>() + value_size(item)
+            })
+            .sum(),
+        Value::Null | Value::Bool(_) | Value::Number(_) => 0,
+    }
+}
+
 /// Serializes a JSON value as its encoded text, in a string.
 fn json_string<S: Serializer>(
-    value: &Option<&serde_json::Value>,
+    value: &Option<serde_json::Value>,
     serializer: S,
 ) -> Result<S::Ok, S::Error> {
     match value {
diff --git a/src/routes/ingest.rs b/src/routes/ingest.rs
index fbe0459..d769925 100644
--- a/src/routes/ingest.rs
+++ b/src/routes/ingest.rs
@@ -177,12 +177,12 @@ fn validate_event(event: &Event, now: i64) -> Result<(), String> {
 /// Splits a batch into forwardable events and per-event rejections. Each
 /// event is validated independently: one invalid event never blocks its
 /// neighbours.
-fn partition_events(events: &[Event]) -> (Vec<&Event>, Vec<RejectedEvent>) {
+fn partition_events(events: Vec<Event>) -> (Vec<Event>, Vec<RejectedEvent>) {
     let now = Utc::now().timestamp_millis();
     let mut valid = Vec::with_capacity(events.len());
     let mut rejected = Vec::new();
-    for (i, event) in events.iter().enumerate() {
-        match validate_event(event, now) {
+    for (i, event) in events.into_iter().enumerate() {
+        match validate_event(&event, now) {
             Ok(()) => valid.push(event),
             Err(error) => rejected.push(RejectedEvent {
                 index: i,
@@ -216,12 +216,13 @@ pub async fn ingest_batch(
         return (StatusCode::BAD_REQUEST, Json(body)).into_response();
     }
 
-    let (valid, rejected) = partition_events(&batch.events);
+    let batch_size = batch.events.len();
+    let (valid, rejected) = partition_events(batch.events);
     if !rejected.is_empty() {
         warn!(
             client_api_key = %mask_key(&ctx.client_api_key),
             rejected_count = rejected.len(),
-            batch_size = batch.events.len(),
+            batch_size,
             first_error = %rejected[0].error,
             "Rejected invalid events from batch"
         );
@@ -244,24 +245,24 @@ pub async fn ingest_batch(
 
     // TODO: move this to a method.
     let records: Vec<EventRecord> = valid
-        .iter()
+        .into_iter()
         .map(|event| EventRecord {
-            environment_key: &ctx.client_api_key,
+            environment_key: ctx.client_api_key.to_string(),
             collected_at,
-            sdk_language: &sdk_language,
-            sdk_version: &sdk_version,
-            event: &event.event,
-            feature_name: event.feature_name.as_deref(),
-            identifier: event.identifier.as_deref(),
-            value: &event.value,
-            traits: event.traits.as_ref(),
-            metadata: event.metadata.as_ref(),
+            sdk_language: sdk_language.clone(),
+            sdk_version: sdk_version.clone(),
+            event: event.event,
+            feature_name: event.feature_name,
+            identifier: event.identifier,
+            value: event.value,
+            traits: event.traits,
+            metadata: event.metadata,
             timestamp: event.timestamp,
         })
         .collect();
 
     let record_count = records.len();
-    match sink.put(ctx.destination.as_deref(), &records).await {
+    match sink.put(ctx.destination.as_deref(), records).await {
         Ok(()) => {
             debug!(
                 client_api_key = %mask_key(&ctx.client_api_key),
@@ -318,7 +319,7 @@ mod tests {
     /// Partition the batch and return (accepted count, rejections).
     fn partition(json: &str) -> (usize, Vec<RejectedEvent>) {
         let batch = parse(json);
-        let (valid, rejected) = partition_events(&batch.events);
+        let (valid, rejected) = partition_events(batch.events);
         (valid.len(), rejected)
     }
 
@@ -545,18 +546,18 @@ mod tests {
         let batch = parse(
             r#"{"events":[{"event":"$flag_exposure","feature_name":"dark_mode","identifier":"user_1","value":"control","timestamp":1700000000000}]}"#,
         );
-        let event = &batch.events[0];
+        let event = batch.events.into_iter().next().unwrap();
         let record = EventRecord {
-            environment_key: "env_key",
+            environment_key: "env_key".to_string(),
             collected_at: 1700000000001,
-            sdk_language: "python",
-            sdk_version: "1.0.0",
-            event: &event.event,
-            feature_name: event.feature_name.as_deref(),
-            identifier: event.identifier.as_deref(),
-            value: &event.value,
-            traits: event.traits.as_ref(),
-            metadata: event.metadata.as_ref(),
+            sdk_language: "python".to_string(),
+            sdk_version: "1.0.0".to_string(),
+            event: event.event,
+            feature_name: event.feature_name,
+            identifier: event.identifier,
+            value: event.value,
+            traits: event.traits,
+            metadata: event.metadata,
             timestamp: event.timestamp,
         };
 
diff --git a/src/sink/batching.rs b/src/sink/batching.rs
index f57e1e0..0d63c39 100644
--- a/src/sink/batching.rs
+++ b/src/sink/batching.rs
@@ -17,16 +17,9 @@ const MAX_RETRIES: u32 = 3;
 /// Doubles per retry: about 200 ms, 400 ms, 800 ms before jitter.
 const RETRY_BASE_DELAY: Duration = Duration::from_millis(200);
 
-/// Writes one batch of owned rows to a store.
+/// Writes one batch of records to a store.
 pub trait BatchWriter: Send + Sync + 'static {
-    type Row: Send + Sync + 'static;
-
-    fn row(record: &EventRecord<'_>) -> Self::Row;
-
-    /// Approximate in-memory size, counted against the buffer limit.
-    fn row_size(row: &Self::Row) -> usize;
-
-    fn write(&self, rows: &[Self::Row]) -> impl Future<Output = Result<(), SinkError>> + Send;
+    fn write(&self, records: &[EventRecord]) -> impl Future<Output = Result<(), SinkError>> + Send;
 }
 
 #[derive(Debug, Clone)]
@@ -38,23 +31,23 @@ pub struct BatchConfig {
     pub max_bytes: usize,
 }
 
-struct Buffer<R> {
-    rows: Vec<R>,
-    /// Size of `rows` plus the batch being flushed.
+struct Buffer {
+    records: Vec<EventRecord>,
+    /// Size of `records` plus the batch being flushed.
     bytes: usize,
 }
 
 struct Shared<W: BatchWriter> {
     writer: W,
     config: BatchConfig,
-    buffer: Mutex<Buffer<W::Row>>,
+    buffer: Mutex<Buffer>,
     flush_now: Notify,
     stop: Notify,
 }
 
-/// Buffers rows in memory and writes them in batches from one background
-/// task. `put` returns once the rows are buffered, so rows still buffered are
-/// lost if the process dies.
+/// Buffers records in memory and writes them in batches from one background
+/// task. `put` returns once the records are buffered, so records still
+/// buffered are lost if the process dies.
 pub struct BatchingSink<W: BatchWriter> {
     shared: Arc<Shared<W>>,
     flusher: Arc<Mutex<Option<JoinHandle<()>>>>,
@@ -76,7 +69,7 @@ impl<W: BatchWriter> BatchingSink<W> {
             writer,
             config,
             buffer: Mutex::new(Buffer {
-                rows: Vec::new(),
+                records: Vec::new(),
                 bytes: 0,
             }),
             flush_now: Notify::new(),
@@ -90,17 +83,16 @@ impl<W: BatchWriter> BatchingSink<W> {
     }
 
     /// Buffer every record, or none of them if that would pass the byte limit.
-    pub fn put(&self, records: &[EventRecord<'_>]) -> Result<(), SinkError> {
-        let rows: Vec<W::Row> = records.iter().map(W::row).collect();
-        let bytes: usize = rows.iter().map(W::row_size).sum();
+    pub fn put(&self, records: Vec<EventRecord>) -> Result<(), SinkError> {
+        let bytes: usize = records.iter().map(EventRecord::approx_size).sum();
         let config = &self.shared.config;
         let mut buffer = self.shared.buffer.lock().expect("buffer lock");
         if buffer.bytes + bytes > config.max_bytes {
             return Err(SinkError::BufferFull);
         }
         buffer.bytes += bytes;
-        buffer.rows.extend(rows);
-        if buffer.rows.len() >= config.max_rows {
+        buffer.records.extend(records);
+        if buffer.records.len() >= config.max_rows {
             self.shared.flush_now.notify_one();
         }
         Ok(())
@@ -138,29 +130,29 @@ async fn run<W: BatchWriter>(shared: Arc<Shared<W>>) {
 }
 
 async fn flush<W: BatchWriter>(shared: &Shared<W>) {
-    let rows = mem::take(&mut shared.buffer.lock().expect("buffer lock").rows);
-    if rows.is_empty() {
+    let records = mem::take(&mut shared.buffer.lock().expect("buffer lock").records);
+    if records.is_empty() {
         return;
     }
-    for chunk in rows.chunks(shared.config.max_rows) {
+    for chunk in records.chunks(shared.config.max_rows) {
         write_with_retries(&shared.writer, chunk).await;
     }
-    let flushed: usize = rows.iter().map(W::row_size).sum();
+    let flushed: usize = records.iter().map(EventRecord::approx_size).sum();
     shared.buffer.lock().expect("buffer lock").bytes -= flushed;
 }
 
-async fn write_with_retries<W: BatchWriter>(writer: &W, rows: &[W::Row]) {
+async fn write_with_retries<W: BatchWriter>(writer: &W, records: &[EventRecord]) {
     let mut retry = 0;
     loop {
-        match writer.write(rows).await {
+        match writer.write(records).await {
             Ok(()) => return,
             Err(e) if retry < MAX_RETRIES => {
-                warn!(rows = rows.len(), retry, error = %e, "Sink batch write failed, retrying");
+                warn!(rows = records.len(), retry, error = %e, "Sink batch write failed, retrying");
                 tokio::time::sleep(backoff(retry)).await;
                 retry += 1;
             }
             Err(e) => {
-                error!(rows = rows.len(), error = %e, "Sink batch dropped after retries");
+                error!(rows = records.len(), error = %e, "Sink batch dropped after retries");
                 return;
             }
         }
@@ -180,9 +172,6 @@ mod tests {
 
     use super::*;
 
-    const ROW_SIZE: usize = 100;
-    static NULL: serde_json::Value = serde_json::Value::Null;
-
     /// Records every write attempt; fails all of them when `fail` is set.
     #[derive(Clone, Default)]
     struct FakeWriter {
@@ -192,38 +181,29 @@ mod tests {
     }
 
     impl BatchWriter for FakeWriter {
-        type Row = i64;
-
-        fn row(record: &EventRecord<'_>) -> i64 {
-            record.timestamp
-        }
-
-        fn row_size(_row: &i64) -> usize {
-            ROW_SIZE
-        }
-
-        async fn write(&self, rows: &[i64]) -> Result<(), SinkError> {
+        async fn write(&self, records: &[EventRecord]) -> Result<(), SinkError> {
             self.attempts.fetch_add(1, Ordering::SeqCst);
             if self.fail {
                 return Err(SinkError::BufferFull);
             }
-            self.written.lock().unwrap().push(rows.to_vec());
+            let timestamps = records.iter().map(|record| record.timestamp).collect();
+            self.written.lock().unwrap().push(timestamps);
             Ok(())
         }
     }
 
-    fn records(timestamps: &[i64]) -> Vec<EventRecord<'static>> {
+    fn records(timestamps: &[i64]) -> Vec<EventRecord> {
         timestamps
             .iter()
             .map(|&timestamp| EventRecord {
-                environment_key: "env",
+                environment_key: "env".to_string(),
                 collected_at: timestamp,
-                sdk_language: "rust",
-                sdk_version: "1.0.0",
-                event: "purchase",
+                sdk_language: "rust".to_string(),
+                sdk_version: "1.0.0".to_string(),
+                event: "purchase".to_string(),
                 feature_name: None,
                 identifier: None,
-                value: &NULL,
+                value: serde_json::Value::Null,
                 traits: None,
                 metadata: None,
                 timestamp,
@@ -231,6 +211,11 @@ mod tests {
             .collect()
     }
 
+    /// Every test record has the same size.
+    fn record_size() -> usize {
+        records(&[0])[0].approx_size()
+    }
+
     fn config(flush_interval: Duration, max_rows: usize, max_bytes: usize) -> BatchConfig {
         BatchConfig {
             flush_interval,
@@ -244,7 +229,7 @@ mod tests {
         let writer = FakeWriter::default();
         let sink = BatchingSink::new(writer.clone(), config(Duration::from_secs(1), 100, 10_000));
 
-        sink.put(&records(&[1, 2])).unwrap();
+        sink.put(records(&[1, 2])).unwrap();
         tokio::time::sleep(Duration::from_millis(500)).await;
         assert!(writer.written.lock().unwrap().is_empty());
 
@@ -257,7 +242,7 @@ mod tests {
         let writer = FakeWriter::default();
         let sink = BatchingSink::new(writer.clone(), config(Duration::from_secs(3600), 2, 10_000));
 
-        sink.put(&records(&[1, 2])).unwrap();
+        sink.put(records(&[1, 2])).unwrap();
         tokio::time::sleep(Duration::from_millis(10)).await;
 
         assert_eq!(*writer.written.lock().unwrap(), vec![vec![1, 2]]);
@@ -268,15 +253,15 @@ mod tests {
         let writer = FakeWriter::default();
         let sink = BatchingSink::new(
             writer.clone(),
-            config(Duration::from_secs(3600), 100, 2 * ROW_SIZE),
+            config(Duration::from_secs(3600), 100, 2 * record_size()),
         );
 
-        sink.put(&records(&[1])).unwrap();
+        sink.put(records(&[1])).unwrap();
         assert!(matches!(
-            sink.put(&records(&[2, 3])),
+            sink.put(records(&[2, 3])),
             Err(SinkError::BufferFull)
         ));
-        sink.put(&records(&[4])).unwrap();
+        sink.put(records(&[4])).unwrap();
 
         sink.shutdown(Duration::from_secs(1)).await;
         assert_eq!(*writer.written.lock().unwrap(), vec![vec![1, 4]]);
@@ -290,17 +275,17 @@ mod tests {
         };
         let sink = BatchingSink::new(
             writer.clone(),
-            config(Duration::from_secs(1), 100, ROW_SIZE),
+            config(Duration::from_secs(1), 100, record_size()),
         );
 
-        sink.put(&records(&[1])).unwrap();
+        sink.put(records(&[1])).unwrap();
         tokio::time::sleep(Duration::from_secs(10)).await;
 
         assert_eq!(
             writer.attempts.load(Ordering::SeqCst),
             1 + MAX_RETRIES as usize
         );
-        sink.put(&records(&[2])).unwrap();
+        sink.put(records(&[2])).unwrap();
     }
 
     #[tokio::test(start_paused = true)]
@@ -311,7 +296,7 @@ mod tests {
             config(Duration::from_secs(3600), 100, 10_000),
         );
 
-        sink.put(&records(&[1, 2, 3])).unwrap();
+        sink.put(records(&[1, 2, 3])).unwrap();
         sink.shutdown(Duration::from_secs(1)).await;
 
         assert_eq!(*writer.written.lock().unwrap(), vec![vec![1, 2, 3]]);
diff --git a/src/sink/clickhouse.rs b/src/sink/clickhouse.rs
index ea86955..c41ab0d 100644
--- a/src/sink/clickhouse.rs
+++ b/src/sink/clickhouse.rs
@@ -1,5 +1,5 @@
+use std::borrow::Cow;
 use std::fmt;
-use std::mem;
 use std::time::Duration;
 
 use clickhouse::{Client, Compression, Row};
@@ -33,36 +33,58 @@ impl fmt::Debug for ClickHouseSinkConfig {
     }
 }
 
-/// One row of the `events` table. Field names are the column names. Every
-/// column is non-Nullable, so absent optionals are written as "", as
-/// ClickPipes does.
+/// One row of the `events` table, borrowed from its record. Field names are
+/// the column names. Every column is non-Nullable, so absent optionals are
+/// written as "", as ClickPipes does.
 #[derive(Debug, PartialEq, Row, Serialize)]
-#[cfg_attr(test, derive(serde::Deserialize))]
-pub struct ClickHouseRow {
-    environment_key: String,
-    event: String,
-    feature_name: String,
+struct ClickHouseRow<'a> {
+    environment_key: &'a str,
+    event: &'a str,
+    feature_name: &'a str,
     /// DateTime64(3), sent as epoch millis.
     timestamp: i64,
     collected_at: i64,
-    identifier: String,
-    value: String,
+    identifier: &'a str,
+    value: Cow<'a, str>,
     traits: String,
     metadata: String,
-    sdk_language: String,
-    sdk_version: String,
+    sdk_language: &'a str,
+    sdk_version: &'a str,
+}
+
+impl<'a> From<&'a EventRecord> for ClickHouseRow<'a> {
+    fn from(record: &'a EventRecord) -> Self {
+        ClickHouseRow {
+            environment_key: &record.environment_key,
+            event: &record.event,
+            feature_name: record.feature_name.as_deref().unwrap_or_default(),
+            timestamp: record.timestamp,
+            collected_at: record.collected_at,
+            identifier: record.identifier.as_deref().unwrap_or_default(),
+            value: value_column(&record.value),
+            traits: json_column(record.traits.as_ref()),
+            metadata: json_column(record.metadata.as_ref()),
+            sdk_language: &record.sdk_language,
+            sdk_version: &record.sdk_version,
+        }
+    }
 }
 
 /// `value` is a String column: a JSON string goes in raw, null as "", and
 /// anything else as its JSON text.
-fn value_column(value: &serde_json::Value) -> String {
+fn value_column(value: &serde_json::Value) -> Cow<'_, str> {
     match value {
-        serde_json::Value::String(s) => s.clone(),
-        serde_json::Value::Null => String::new(),
-        other => other.to_string(),
+        serde_json::Value::String(s) => Cow::Borrowed(s),
+        serde_json::Value::Null => Cow::Borrowed(""),
+        other => Cow::Owned(other.to_string()),
     }
 }
 
+/// JSON text, or "" when absent.
+fn json_column(value: Option<&serde_json::Value>) -> String {
+    value.map(ToString::to_string).unwrap_or_default()
+}
+
 pub struct ClickHouseWriter {
     client: Client,
     table: String,
@@ -85,46 +107,16 @@ impl ClickHouseWriter {
 }
 
 impl BatchWriter for ClickHouseWriter {
-    type Row = ClickHouseRow;
-
-    fn row(record: &EventRecord<'_>) -> ClickHouseRow {
-        ClickHouseRow {
-            environment_key: record.environment_key.to_string(),
-            event: record.event.to_string(),
-            feature_name: record.feature_name.unwrap_or_default().to_string(),
-            timestamp: record.timestamp,
-            collected_at: record.collected_at,
-            identifier: record.identifier.unwrap_or_default().to_string(),
-            value: value_column(record.value),
-            traits: record.traits.map(ToString::to_string).unwrap_or_default(),
-            metadata: record.metadata.map(ToString::to_string).unwrap_or_default(),
-            sdk_language: record.sdk_language.to_string(),
-            sdk_version: record.sdk_version.to_string(),
-        }
-    }
-
-    fn row_size(row: &ClickHouseRow) -> usize {
-        mem::size_of::<ClickHouseRow>()
-            + row.environment_key.len()
-            + row.event.len()
-            + row.feature_name.len()
-            + row.identifier.len()
-            + row.value.len()
-            + row.traits.len()
-            + row.metadata.len()
-            + row.sdk_language.len()
-            + row.sdk_version.len()
-    }
-
     /// One plain `INSERT` per batch; succeeds once ClickHouse has committed it.
-    async fn write(&self, rows: &[ClickHouseRow]) -> Result<(), SinkError> {
+    /// Rows are built one at a time, so the batch is never copied whole.
+    async fn write(&self, records: &[EventRecord]) -> Result<(), SinkError> {
         let mut insert = self
             .client
-            .insert::<ClickHouseRow>(&self.table)
+            .insert::<ClickHouseRow<'_>>(&self.table)
             .await?
             .with_timeouts(Some(INSERT_TIMEOUT), Some(INSERT_TIMEOUT));
-        for row in rows {
-            insert.write(row).await?;
+        for record in records {
+            insert.write(&ClickHouseRow::from(record)).await?;
         }
         insert.end().await?;
         Ok(())
@@ -133,44 +125,51 @@ impl BatchWriter for ClickHouseWriter {
 
 #[cfg(test)]
 mod tests {
+    use serde::Deserialize;
     use serde_json::json;
 
     use super::*;
     use crate::sink::{BatchConfig, BatchingSink};
     use crate::test_helpers::{now_ms, ClickHouseTestDatabase};
 
-    #[test]
-    fn row_fills_every_column_and_blanks_absent_optionals() {
-        let value = json!(42);
-        let traits = json!({"plan": "pro"});
-        let record = EventRecord {
-            environment_key: "env",
-            collected_at: 1_700_000_000_001,
-            sdk_language: "python",
-            sdk_version: "1.0.0",
-            event: "purchase",
+    fn record(event: &str, timestamp: i64) -> EventRecord {
+        EventRecord {
+            environment_key: "env".to_string(),
+            collected_at: timestamp + 1,
+            sdk_language: "python".to_string(),
+            sdk_version: "1.0.0".to_string(),
+            event: event.to_string(),
             feature_name: None,
             identifier: None,
-            value: &value,
-            traits: Some(&traits),
+            value: serde_json::Value::Null,
+            traits: None,
             metadata: None,
-            timestamp: 1_700_000_000_000,
+            timestamp,
+        }
+    }
+
+    #[test]
+    fn row_fills_every_column_and_blanks_absent_optionals() {
+        let record = EventRecord {
+            value: json!(42),
+            traits: Some(json!({"plan": "pro"})),
+            ..record("purchase", 1_700_000_000_000)
         };
 
         assert_eq!(
-            ClickHouseWriter::row(&record),
+            ClickHouseRow::from(&record),
             ClickHouseRow {
-                environment_key: "env".to_string(),
-                event: "purchase".to_string(),
-                feature_name: String::new(),
+                environment_key: "env",
+                event: "purchase",
+                feature_name: "",
                 timestamp: 1_700_000_000_000,
                 collected_at: 1_700_000_000_001,
-                identifier: String::new(),
-                value: "42".to_string(),
+                identifier: "",
+                value: Cow::Borrowed("42"),
                 traits: r#"{"plan":"pro"}"#.to_string(),
                 metadata: String::new(),
-                sdk_language: "python".to_string(),
-                sdk_version: "1.0.0".to_string(),
+                sdk_language: "python",
+                sdk_version: "1.0.0",
             }
         );
     }
@@ -206,6 +205,22 @@ mod tests {
         }
     }
 
+    /// A row as read back from the table.
+    #[derive(Debug, PartialEq, Row, Deserialize)]
+    struct StoredRow {
+        environment_key: String,
+        event: String,
+        feature_name: String,
+        timestamp: i64,
+        collected_at: i64,
+        identifier: String,
+        value: String,
+        traits: String,
+        metadata: String,
+        sdk_language: String,
+        sdk_version: String,
+    }
+
     #[tokio::test]
     async fn flushed_rows_land_in_the_events_table() {
         let db = ClickHouseTestDatabase::create().await;
@@ -214,52 +229,58 @@ mod tests {
             batch_config(Duration::from_millis(100)),
         );
         let ts = now_ms();
-        let control = json!("control");
-        let null = json!(null);
-        let traits = json!({"plan": "pro"});
-        let metadata = json!({"source": "csv"});
-        let records = [
-            EventRecord {
-                environment_key: "env",
-                collected_at: ts + 1,
-                sdk_language: "python",
-                sdk_version: "1.0.0",
-                event: "$flag_exposure",
-                feature_name: Some("dark_mode"),
-                identifier: Some("user_1"),
-                value: &control,
-                traits: Some(&traits),
-                metadata: Some(&metadata),
-                timestamp: ts,
-            },
+        let records = vec![
             EventRecord {
-                environment_key: "env",
-                collected_at: ts + 1,
-                sdk_language: "python",
-                sdk_version: "1.0.0",
-                event: "purchase",
-                feature_name: None,
-                identifier: None,
-                value: &null,
-                traits: None,
-                metadata: None,
-                timestamp: ts + 2,
+                feature_name: Some("dark_mode".to_string()),
+                identifier: Some("user_1".to_string()),
+                value: json!("control"),
+                traits: Some(json!({"plan": "pro"})),
+                metadata: Some(json!({"source": "csv"})),
+                ..record("$flag_exposure", ts)
             },
+            record("purchase", ts + 2),
         ];
 
-        sink.put(&records).unwrap();
+        sink.put(records).unwrap();
         db.wait_for_rows(2).await;
 
-        let stored: Vec<ClickHouseRow> = db
+        let stored: Vec<StoredRow> = db
             .client
             .query("SELECT ?fields FROM events ORDER BY timestamp")
             .fetch_all()
             .await
             .unwrap();
-        let expected: Vec<ClickHouseRow> = records.iter().map(ClickHouseWriter::row).collect();
-        assert_eq!(stored, expected);
-        assert_eq!(stored[0].value, "control");
-        assert_eq!(stored[1].value, "");
+        assert_eq!(
+            stored,
+            vec![
+                StoredRow {
+                    environment_key: "env".to_string(),
+                    event: "$flag_exposure".to_string(),
+                    feature_name: "dark_mode".to_string(),
+                    timestamp: ts,
+                    collected_at: ts + 1,
+                    identifier: "user_1".to_string(),
+                    value: "control".to_string(),
+                    traits: r#"{"plan":"pro"}"#.to_string(),
+                    metadata: r#"{"source":"csv"}"#.to_string(),
+                    sdk_language: "python".to_string(),
+                    sdk_version: "1.0.0".to_string(),
+                },
+                StoredRow {
+                    environment_key: "env".to_string(),
+                    event: "purchase".to_string(),
+                    feature_name: String::new(),
+                    timestamp: ts + 2,
+                    collected_at: ts + 3,
+                    identifier: String::new(),
+                    value: String::new(),
+                    traits: String::new(),
+                    metadata: String::new(),
+                    sdk_language: "python".to_string(),
+                    sdk_version: "1.0.0".to_string(),
+                },
+            ]
+        );
 
         sink.shutdown(Duration::from_secs(5)).await;
         db.drop().await;
@@ -272,22 +293,10 @@ mod tests {
             ClickHouseWriter::new(db.config.clone()),
             batch_config(Duration::from_secs(3600)),
         );
-        let value = json!(42);
-        let record = EventRecord {
-            environment_key: "env",
-            collected_at: now_ms(),
-            sdk_language: "rust",
-            sdk_version: "1.0.0",
-            event: "purchase",
-            feature_name: None,
-            identifier: None,
-            value: &value,
-            traits: None,
-            metadata: None,
-            timestamp: now_ms(),
-        };
+        let record = record("purchase", now_ms());
 
-        sink.put(&[record.clone(), record.clone(), record]).unwrap();
+        sink.put(vec![record.clone(), record.clone(), record])
+            .unwrap();
         assert_eq!(db.count().await, 0);
         sink.shutdown(Duration::from_secs(5)).await;
 
diff --git a/src/sink/kafka.rs b/src/sink/kafka.rs
index d41fb13..6a79477 100644
--- a/src/sink/kafka.rs
+++ b/src/sink/kafka.rs
@@ -148,7 +148,7 @@ impl KafkaSink {
     pub async fn put(
         &self,
         destination: Option<&str>,
-        records: &[EventRecord<'_>],
+        records: &[EventRecord],
     ) -> Result<(), DeliveryError> {
         let topic = destination.unwrap_or(&self.default_topic);
         let total = records.len();
diff --git a/src/sink/mod.rs b/src/sink/mod.rs
index 277aed4..3ac45ad 100644
--- a/src/sink/mod.rs
+++ b/src/sink/mod.rs
@@ -23,10 +23,10 @@ impl Sink {
     pub async fn put(
         &self,
         destination: Option<&str>,
-        records: &[EventRecord<'_>],
+        records: Vec<EventRecord>,
     ) -> Result<(), SinkError> {
         match self {
-            Self::Kafka(sink) => Ok(sink.put(destination, records).await?),
+            Self::Kafka(sink) => Ok(sink.put(destination, &records).await?),
             // A single table: there is no per-environment destination.
             Self::ClickHouse(sink) => sink.put(records),
         }

From c1ef814f41ef64ad42554e840822aaefb7786e84 Mon Sep 17 00:00:00 2001
From: Gagan Trivedi <gagandeeptrivedi47@gmail.com>
Date: Wed, 28 Oct 2026 14:54:40 +0530
Subject: [PATCH 05/11] refactor(sink): rename ClickHouseSinkConfig to
 ClickHouseConfig

It configures the writer, not a sink, and reads oddly next to
config::SinkConfig.

Refs #3
---
 src/config.rs          |  8 ++++----
 src/routes/ingest.rs   |  4 ++--
 src/sink/clickhouse.rs | 10 +++++-----
 src/sink/mod.rs        |  2 +-
 src/test_helpers.rs    |  6 +++---
 5 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/config.rs b/src/config.rs
index 2046f55..5ccaaa1 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -2,7 +2,7 @@ use std::env;
 use std::str::FromStr;
 use std::time::Duration;
 
-use crate::sink::{BatchConfig, ClickHouseSinkConfig, KafkaAuth};
+use crate::sink::{BatchConfig, ClickHouseConfig, KafkaAuth};
 
 #[derive(Debug, Clone)]
 pub struct Config {
@@ -15,7 +15,7 @@ pub struct Config {
 pub enum SinkConfig {
     Kafka(KafkaConfig),
     ClickHouse {
-        clickhouse: ClickHouseSinkConfig,
+        clickhouse: ClickHouseConfig,
         batch: BatchConfig,
     },
 }
@@ -82,8 +82,8 @@ fn parse_sink(value: Option<&str>) -> Result<SinkKind, ConfigError> {
     }
 }
 
-fn clickhouse_from_env() -> Result<ClickHouseSinkConfig, ConfigError> {
-    Ok(ClickHouseSinkConfig {
+fn clickhouse_from_env() -> Result<ClickHouseConfig, ConfigError> {
+    Ok(ClickHouseConfig {
         url: required("CLICKHOUSE_URL")?,
         user: env::var("CLICKHOUSE_USER").unwrap_or_else(|_| "default".to_string()),
         password: env::var("CLICKHOUSE_PASSWORD").unwrap_or_default(),
diff --git a/src/routes/ingest.rs b/src/routes/ingest.rs
index d769925..baf5013 100644
--- a/src/routes/ingest.rs
+++ b/src/routes/ingest.rs
@@ -305,7 +305,7 @@ mod tests {
     use tower::ServiceExt;
 
     use crate::sink::{
-        BatchConfig, BatchingSink, ClickHouseSinkConfig, ClickHouseWriter, KafkaAuth, KafkaSink,
+        BatchConfig, BatchingSink, ClickHouseConfig, ClickHouseWriter, KafkaAuth, KafkaSink,
         KafkaSinkConfig,
     };
     use crate::test_helpers::{
@@ -730,7 +730,7 @@ mod tests {
         // Room for one buffered event; the unreachable server and the hour-long
         // interval keep it buffered.
         let sink = Sink::ClickHouse(BatchingSink::new(
-            ClickHouseWriter::new(ClickHouseSinkConfig {
+            ClickHouseWriter::new(ClickHouseConfig {
                 url: "http://127.0.0.1:1".to_string(),
                 user: "default".to_string(),
                 password: String::new(),
diff --git a/src/sink/clickhouse.rs b/src/sink/clickhouse.rs
index c41ab0d..19ed2ba 100644
--- a/src/sink/clickhouse.rs
+++ b/src/sink/clickhouse.rs
@@ -13,7 +13,7 @@ use crate::models::EventRecord;
 const INSERT_TIMEOUT: Duration = Duration::from_secs(30);
 
 #[derive(Clone)]
-pub struct ClickHouseSinkConfig {
+pub struct ClickHouseConfig {
     pub url: String,
     pub user: String,
     pub password: String,
@@ -22,9 +22,9 @@ pub struct ClickHouseSinkConfig {
 }
 
 /// Keeps the password out of logs.
-impl fmt::Debug for ClickHouseSinkConfig {
+impl fmt::Debug for ClickHouseConfig {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("ClickHouseSinkConfig")
+        f.debug_struct("ClickHouseConfig")
             .field("url", &self.url)
             .field("user", &self.user)
             .field("database", &self.database)
@@ -92,7 +92,7 @@ pub struct ClickHouseWriter {
 
 impl ClickHouseWriter {
     /// Connects lazily, on the first insert.
-    pub fn new(config: ClickHouseSinkConfig) -> Self {
+    pub fn new(config: ClickHouseConfig) -> Self {
         let client = Client::default()
             .with_url(config.url)
             .with_user(config.user)
@@ -185,7 +185,7 @@ mod tests {
 
     #[test]
     fn debug_output_omits_the_password() {
-        let config = ClickHouseSinkConfig {
+        let config = ClickHouseConfig {
             url: "https://clickhouse:8443".to_string(),
             user: "ingest".to_string(),
             password: "hunter2".to_string(),
diff --git a/src/sink/mod.rs b/src/sink/mod.rs
index 3ac45ad..8ffd5b2 100644
--- a/src/sink/mod.rs
+++ b/src/sink/mod.rs
@@ -5,7 +5,7 @@ mod kafka;
 use std::time::Duration;
 
 pub use batching::{BatchConfig, BatchingSink};
-pub use clickhouse::{ClickHouseSinkConfig, ClickHouseWriter};
+pub use clickhouse::{ClickHouseConfig, ClickHouseWriter};
 pub use kafka::{KafkaAuth, KafkaSink, KafkaSinkConfig};
 
 use crate::models::EventRecord;
diff --git a/src/test_helpers.rs b/src/test_helpers.rs
index 6bfe1c0..767bc92 100644
--- a/src/test_helpers.rs
+++ b/src/test_helpers.rs
@@ -11,7 +11,7 @@ use rdkafka::producer::DefaultProducerContext;
 use rdkafka::Message;
 use sqlx::PgPool;
 
-use crate::sink::{ClickHouseSinkConfig, KafkaAuth, KafkaSink, KafkaSinkConfig, Sink};
+use crate::sink::{ClickHouseConfig, KafkaAuth, KafkaSink, KafkaSinkConfig, Sink};
 
 pub(crate) const FLAGSMITH_WAREHOUSE_TOPIC: &str = "events-default";
 
@@ -148,7 +148,7 @@ static NEXT_DATABASE: AtomicUsize = AtomicUsize::new(0);
 /// `CLICKHOUSE_URL` (with `CLICKHOUSE_USER` and `CLICKHOUSE_PASSWORD`, as the
 /// server reads them). Call `drop` at the end of the test.
 pub(crate) struct ClickHouseTestDatabase {
-    pub(crate) config: ClickHouseSinkConfig,
+    pub(crate) config: ClickHouseConfig,
     pub(crate) client: Client,
 }
 
@@ -178,7 +178,7 @@ impl ClickHouseTestDatabase {
             .expect("create events table");
         Self {
             client: server.with_database(&database),
-            config: ClickHouseSinkConfig {
+            config: ClickHouseConfig {
                 url,
                 user,
                 password,

From b87770e3ad1ef872c41e3f1637869274fa342e24 Mon Sep 17 00:00:00 2001
From: Gagan Trivedi <gagandeeptrivedi47@gmail.com>
Date: Wed, 28 Oct 2026 15:10:52 +0530
Subject: [PATCH 06/11] refactor(sink): flush the ClickHouse buffer by event
 count

Byte estimates of parsed JSON were guesswork. The buffer is now
unbounded: put never waits or fails, and the flusher takes the whole
buffer every SINK_FLUSH_INTERVAL_MS, or once SINK_FLUSH_MAX_EVENTS
(5000) events are waiting, as one INSERT. This drops
SINK_BATCH_MAX_ROWS, SINK_BUFFER_MAX_BYTES and SinkError::BufferFull.

Refs #3
---
 README.md              |   5 +-
 src/config.rs          |  13 ++---
 src/models/event.rs    |  40 --------------
 src/routes/ingest.rs   |  37 +------------
 src/sink/batching.rs   | 121 ++++++++++++++++-------------------------
 src/sink/clickhouse.rs |  10 ++--
 src/sink/mod.rs        |   7 ++-
 7 files changed, 63 insertions(+), 170 deletions(-)

diff --git a/README.md b/README.md
index e6bd7b6..81c3e0e 100644
--- a/README.md
+++ b/README.md
@@ -65,8 +65,7 @@ On the wire to Kafka, `traits` and `metadata` are produced as JSON-encoded strin
 | `CLICKHOUSE_DATABASE` | no | `flagsmith_exp` |
 | `CLICKHOUSE_TABLE` | no | `events` |
 | `SINK_FLUSH_INTERVAL_MS` | no | `1000` |
-| `SINK_BATCH_MAX_ROWS` | no | `50000` |
-| `SINK_BUFFER_MAX_BYTES` | no | `67108864` (64 MiB) |
+| `SINK_FLUSH_MAX_EVENTS` | no | `5000` |
 | `DATABASE_URL` | yes | — |
 | `LISTEN_ADDR` | no | `0.0.0.0:8080` |
 
@@ -89,7 +88,7 @@ With `KAFKA_AUTH=scram`, the producer authenticates with SASL/SCRAM-SHA-512 over
 
 ### ClickHouse sink
 
-With `SINK=clickhouse`, `202` means the events are buffered in memory, not yet stored. A background task writes the buffer every `SINK_FLUSH_INTERVAL_MS`, or sooner once `SINK_BATCH_MAX_ROWS` rows are waiting, as one `INSERT` per batch. A failed batch is retried 3 times and then dropped with an error log. Once the buffer reaches `SINK_BUFFER_MAX_BYTES`, requests get `503` until a flush frees space. A normal shutdown (SIGTERM) flushes the buffer, but events still buffered are lost if the process crashes.
+With `SINK=clickhouse`, `202` means the events are buffered in memory, not yet stored. A background task takes the whole buffer every `SINK_FLUSH_INTERVAL_MS`, or sooner once `SINK_FLUSH_MAX_EVENTS` events are waiting, and writes it as one `INSERT`. A failed batch is retried 3 times and then dropped with an error log. The buffer is unbounded: requests never wait or get `503`, so while a write is slow or failing (up to about 2 minutes with retries) it grows by the ingest rate times the write time, and the next flush takes it all. A normal shutdown (SIGTERM) flushes the buffer, but events still buffered are lost if the process crashes.
 
 ## Run locally
 
diff --git a/src/config.rs b/src/config.rs
index 5ccaaa1..e883a64 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -100,15 +100,10 @@ fn batch_from_env() -> Result<BatchConfig, ConfigError> {
             env::var("SINK_FLUSH_INTERVAL_MS").ok(),
             1000,
         )?),
-        max_rows: parse_positive(
-            "SINK_BATCH_MAX_ROWS",
-            env::var("SINK_BATCH_MAX_ROWS").ok(),
-            50_000,
-        )?,
-        max_bytes: parse_positive(
-            "SINK_BUFFER_MAX_BYTES",
-            env::var("SINK_BUFFER_MAX_BYTES").ok(),
-            64 * 1024 * 1024,
+        flush_max_events: parse_positive(
+            "SINK_FLUSH_MAX_EVENTS",
+            env::var("SINK_FLUSH_MAX_EVENTS").ok(),
+            5000,
         )?,
     })
 }
diff --git a/src/models/event.rs b/src/models/event.rs
index d2e5834..194a2db 100644
--- a/src/models/event.rs
+++ b/src/models/event.rs
@@ -1,5 +1,3 @@
-use std::mem;
-
 use serde::{Deserialize, Serialize, Serializer};
 
 /// Reserved system event for flag-exposure tracking. `feature_name` is required
@@ -88,44 +86,6 @@ pub struct EventRecord {
     pub timestamp: i64,
 }
 
-impl EventRecord {
-    /// Approximate memory held, counted against a sink's buffer limit.
-    pub fn approx_size(&self) -> usize {
-        mem::size_of::<Self>()
-            + self.environment_key.len()
-            + self.sdk_language.len()
-            + self.sdk_version.len()
-            + self.event.len()
-            + self.feature_name.as_ref().map_or(0, String::len)
-            + self.identifier.as_ref().map_or(0, String::len)
-            + value_size(&self.value)
-            + self.traits.as_ref().map_or(0, value_size)
-            + self.metadata.as_ref().map_or(0, value_size)
-    }
-}
-
-/// Rough per-entry overhead of a JSON object's map.
-const MAP_ENTRY_OVERHEAD: usize = 32;
-
-/// Heap held by a JSON value; its inline size is counted by its parent.
-fn value_size(value: &serde_json::Value) -> usize {
-    use serde_json::Value;
-    match value {
-        Value::String(s) => s.len(),
-        Value::Array(items) => items
-            .iter()
-            .map(|item| mem::size_of::<Value>() + value_size(item))
-            .sum(),
-        Value::Object(map) => map
-            .iter()
-            .map(|(key, item)| {
-                MAP_ENTRY_OVERHEAD + key.len() + mem::size_of::<Value>() + value_size(item)
-            })
-            .sum(),
-        Value::Null | Value::Bool(_) | Value::Number(_) => 0,
-    }
-}
-
 /// Serializes a JSON value as its encoded text, in a string.
 fn json_string<S: Serializer>(
     value: &Option<serde_json::Value>,
diff --git a/src/routes/ingest.rs b/src/routes/ingest.rs
index baf5013..c23dee8 100644
--- a/src/routes/ingest.rs
+++ b/src/routes/ingest.rs
@@ -304,10 +304,7 @@ mod tests {
     use rdkafka::types::RDKafkaRespErr;
     use tower::ServiceExt;
 
-    use crate::sink::{
-        BatchConfig, BatchingSink, ClickHouseConfig, ClickHouseWriter, KafkaAuth, KafkaSink,
-        KafkaSinkConfig,
-    };
+    use crate::sink::{KafkaAuth, KafkaSink, KafkaSinkConfig};
     use crate::test_helpers::{
         assert_topic_empty, mock_kafka, now_ms, read_records, FLAGSMITH_WAREHOUSE_TOPIC,
     };
@@ -725,38 +722,6 @@ mod tests {
         );
     }
 
-    #[tokio::test]
-    async fn test_handler_full_clickhouse_buffer_returns_503() {
-        // Room for one buffered event; the unreachable server and the hour-long
-        // interval keep it buffered.
-        let sink = Sink::ClickHouse(BatchingSink::new(
-            ClickHouseWriter::new(ClickHouseConfig {
-                url: "http://127.0.0.1:1".to_string(),
-                user: "default".to_string(),
-                password: String::new(),
-                database: "flagsmith_exp".to_string(),
-                table: "events".to_string(),
-            }),
-            BatchConfig {
-                flush_interval: Duration::from_secs(3600),
-                max_rows: 1000,
-                max_bytes: 400,
-            },
-        ));
-
-        let first = app(&sink)
-            .oneshot(post_events(&purchase_json()))
-            .await
-            .unwrap();
-        let second = app(&sink)
-            .oneshot(post_events(&purchase_json()))
-            .await
-            .unwrap();
-
-        assert_eq!(first.status(), StatusCode::ACCEPTED);
-        assert_eq!(second.status(), StatusCode::SERVICE_UNAVAILABLE);
-    }
-
     #[tokio::test]
     async fn test_handler_broker_rejection_returns_503_quickly() {
         let kafka = mock_kafka(&[FLAGSMITH_WAREHOUSE_TOPIC]);
diff --git a/src/sink/batching.rs b/src/sink/batching.rs
index 0d63c39..70d0b87 100644
--- a/src/sink/batching.rs
+++ b/src/sink/batching.rs
@@ -25,22 +25,15 @@ pub trait BatchWriter: Send + Sync + 'static {
 #[derive(Debug, Clone)]
 pub struct BatchConfig {
     pub flush_interval: Duration,
-    /// Flush early once this many rows are buffered; also the most rows per write.
-    pub max_rows: usize,
-    /// Bound on buffered plus in-flight rows; `put` fails beyond it.
-    pub max_bytes: usize,
-}
-
-struct Buffer {
-    records: Vec<EventRecord>,
-    /// Size of `records` plus the batch being flushed.
-    bytes: usize,
+    /// Flush early once this many events are buffered. Not a cap: the buffer
+    /// is unbounded and keeps growing while a write is slow or failing.
+    pub flush_max_events: usize,
 }
 
 struct Shared<W: BatchWriter> {
     writer: W,
     config: BatchConfig,
-    buffer: Mutex<Buffer>,
+    buffer: Mutex<Vec<EventRecord>>,
     flush_now: Notify,
     stop: Notify,
 }
@@ -68,10 +61,7 @@ impl<W: BatchWriter> BatchingSink<W> {
         let shared = Arc::new(Shared {
             writer,
             config,
-            buffer: Mutex::new(Buffer {
-                records: Vec::new(),
-                bytes: 0,
-            }),
+            buffer: Mutex::new(Vec::new()),
             flush_now: Notify::new(),
             stop: Notify::new(),
         });
@@ -82,20 +72,13 @@ impl<W: BatchWriter> BatchingSink<W> {
         }
     }
 
-    /// Buffer every record, or none of them if that would pass the byte limit.
-    pub fn put(&self, records: Vec<EventRecord>) -> Result<(), SinkError> {
-        let bytes: usize = records.iter().map(EventRecord::approx_size).sum();
-        let config = &self.shared.config;
+    /// Buffer the records; never blocks or fails.
+    pub fn put(&self, records: Vec<EventRecord>) {
         let mut buffer = self.shared.buffer.lock().expect("buffer lock");
-        if buffer.bytes + bytes > config.max_bytes {
-            return Err(SinkError::BufferFull);
-        }
-        buffer.bytes += bytes;
-        buffer.records.extend(records);
-        if buffer.records.len() >= config.max_rows {
+        buffer.extend(records);
+        if buffer.len() >= self.shared.config.flush_max_events {
             self.shared.flush_now.notify_one();
         }
-        Ok(())
     }
 
     /// Stop the flusher and write what is buffered, bounded by `timeout`.
@@ -130,15 +113,11 @@ async fn run<W: BatchWriter>(shared: Arc<Shared<W>>) {
 }
 
 async fn flush<W: BatchWriter>(shared: &Shared<W>) {
-    let records = mem::take(&mut shared.buffer.lock().expect("buffer lock").records);
+    let records = mem::take(&mut *shared.buffer.lock().expect("buffer lock"));
     if records.is_empty() {
         return;
     }
-    for chunk in records.chunks(shared.config.max_rows) {
-        write_with_retries(&shared.writer, chunk).await;
-    }
-    let flushed: usize = records.iter().map(EventRecord::approx_size).sum();
-    shared.buffer.lock().expect("buffer lock").bytes -= flushed;
+    write_with_retries(&shared.writer, &records).await;
 }
 
 async fn write_with_retries<W: BatchWriter>(writer: &W, records: &[EventRecord]) {
@@ -172,10 +151,12 @@ mod tests {
 
     use super::*;
 
-    /// Records every write attempt; fails all of them when `fail` is set.
+    /// Records every write attempt; each takes `delay`, and all fail when
+    /// `fail` is set.
     #[derive(Clone, Default)]
     struct FakeWriter {
         fail: bool,
+        delay: Duration,
         attempts: Arc<AtomicUsize>,
         written: Arc<Mutex<Vec<Vec<i64>>>>,
     }
@@ -183,8 +164,9 @@ mod tests {
     impl BatchWriter for FakeWriter {
         async fn write(&self, records: &[EventRecord]) -> Result<(), SinkError> {
             self.attempts.fetch_add(1, Ordering::SeqCst);
+            tokio::time::sleep(self.delay).await;
             if self.fail {
-                return Err(SinkError::BufferFull);
+                return Err(clickhouse::error::Error::Custom("failed".to_string()).into());
             }
             let timestamps = records.iter().map(|record| record.timestamp).collect();
             self.written.lock().unwrap().push(timestamps);
@@ -211,25 +193,19 @@ mod tests {
             .collect()
     }
 
-    /// Every test record has the same size.
-    fn record_size() -> usize {
-        records(&[0])[0].approx_size()
-    }
-
-    fn config(flush_interval: Duration, max_rows: usize, max_bytes: usize) -> BatchConfig {
+    fn config(flush_interval: Duration, flush_max_events: usize) -> BatchConfig {
         BatchConfig {
             flush_interval,
-            max_rows,
-            max_bytes,
+            flush_max_events,
         }
     }
 
     #[tokio::test(start_paused = true)]
     async fn flushes_on_the_interval() {
         let writer = FakeWriter::default();
-        let sink = BatchingSink::new(writer.clone(), config(Duration::from_secs(1), 100, 10_000));
+        let sink = BatchingSink::new(writer.clone(), config(Duration::from_secs(1), 100));
 
-        sink.put(records(&[1, 2])).unwrap();
+        sink.put(records(&[1, 2]));
         tokio::time::sleep(Duration::from_millis(500)).await;
         assert!(writer.written.lock().unwrap().is_empty());
 
@@ -238,65 +214,64 @@ mod tests {
     }
 
     #[tokio::test(start_paused = true)]
-    async fn flushes_early_on_max_rows() {
+    async fn flushes_early_at_flush_max_events() {
         let writer = FakeWriter::default();
-        let sink = BatchingSink::new(writer.clone(), config(Duration::from_secs(3600), 2, 10_000));
+        let sink = BatchingSink::new(writer.clone(), config(Duration::from_secs(3600), 2));
 
-        sink.put(records(&[1, 2])).unwrap();
+        sink.put(records(&[1]));
         tokio::time::sleep(Duration::from_millis(10)).await;
+        assert!(writer.written.lock().unwrap().is_empty());
 
+        sink.put(records(&[2]));
+        tokio::time::sleep(Duration::from_millis(10)).await;
         assert_eq!(*writer.written.lock().unwrap(), vec![vec![1, 2]]);
     }
 
     #[tokio::test(start_paused = true)]
-    async fn rejects_a_put_that_would_overflow_the_buffer() {
-        let writer = FakeWriter::default();
-        let sink = BatchingSink::new(
-            writer.clone(),
-            config(Duration::from_secs(3600), 100, 2 * record_size()),
-        );
+    async fn keeps_buffering_while_a_write_is_slow() {
+        let writer = FakeWriter {
+            delay: Duration::from_secs(1),
+            ..FakeWriter::default()
+        };
+        let sink = BatchingSink::new(writer.clone(), config(Duration::from_secs(3600), 2));
 
-        sink.put(records(&[1])).unwrap();
-        assert!(matches!(
-            sink.put(records(&[2, 3])),
-            Err(SinkError::BufferFull)
-        ));
-        sink.put(records(&[4])).unwrap();
+        sink.put(records(&[1, 2]));
+        tokio::time::sleep(Duration::from_millis(1)).await;
+        // [1, 2] is being written; the buffer grows past the threshold.
+        sink.put(records(&[3, 4]));
+        sink.put(records(&[5]));
 
-        sink.shutdown(Duration::from_secs(1)).await;
-        assert_eq!(*writer.written.lock().unwrap(), vec![vec![1, 4]]);
+        sink.shutdown(Duration::from_secs(5)).await;
+        assert_eq!(
+            *writer.written.lock().unwrap(),
+            vec![vec![1, 2], vec![3, 4, 5]]
+        );
     }
 
     #[tokio::test(start_paused = true)]
-    async fn drops_a_batch_after_the_retries_and_frees_its_space() {
+    async fn drops_a_batch_after_the_retries() {
         let writer = FakeWriter {
             fail: true,
             ..FakeWriter::default()
         };
-        let sink = BatchingSink::new(
-            writer.clone(),
-            config(Duration::from_secs(1), 100, record_size()),
-        );
+        let sink = BatchingSink::new(writer.clone(), config(Duration::from_secs(1), 1));
 
-        sink.put(records(&[1])).unwrap();
+        sink.put(records(&[1]));
         tokio::time::sleep(Duration::from_secs(10)).await;
 
         assert_eq!(
             writer.attempts.load(Ordering::SeqCst),
             1 + MAX_RETRIES as usize
         );
-        sink.put(records(&[2])).unwrap();
+        sink.put(records(&[2]));
     }
 
     #[tokio::test(start_paused = true)]
     async fn shutdown_writes_what_is_buffered() {
         let writer = FakeWriter::default();
-        let sink = BatchingSink::new(
-            writer.clone(),
-            config(Duration::from_secs(3600), 100, 10_000),
-        );
+        let sink = BatchingSink::new(writer.clone(), config(Duration::from_secs(3600), 100));
 
-        sink.put(records(&[1, 2, 3])).unwrap();
+        sink.put(records(&[1, 2, 3]));
         sink.shutdown(Duration::from_secs(1)).await;
 
         assert_eq!(*writer.written.lock().unwrap(), vec![vec![1, 2, 3]]);
diff --git a/src/sink/clickhouse.rs b/src/sink/clickhouse.rs
index 19ed2ba..5c58d72 100644
--- a/src/sink/clickhouse.rs
+++ b/src/sink/clickhouse.rs
@@ -9,7 +9,7 @@ use super::batching::BatchWriter;
 use super::SinkError;
 use crate::models::EventRecord;
 
-/// Bounds sending and committing one batch of up to `SINK_BATCH_MAX_ROWS` rows.
+/// Bounds sending and committing one batch of up to `SINK_FLUSH_MAX_EVENTS` rows.
 const INSERT_TIMEOUT: Duration = Duration::from_secs(30);
 
 #[derive(Clone)]
@@ -200,8 +200,7 @@ mod tests {
     fn batch_config(flush_interval: Duration) -> BatchConfig {
         BatchConfig {
             flush_interval,
-            max_rows: 1000,
-            max_bytes: 1024 * 1024,
+            flush_max_events: 1000,
         }
     }
 
@@ -241,7 +240,7 @@ mod tests {
             record("purchase", ts + 2),
         ];
 
-        sink.put(records).unwrap();
+        sink.put(records);
         db.wait_for_rows(2).await;
 
         let stored: Vec<StoredRow> = db
@@ -295,8 +294,7 @@ mod tests {
         );
         let record = record("purchase", now_ms());
 
-        sink.put(vec![record.clone(), record.clone(), record])
-            .unwrap();
+        sink.put(vec![record.clone(), record.clone(), record]);
         assert_eq!(db.count().await, 0);
         sink.shutdown(Duration::from_secs(5)).await;
 
diff --git a/src/sink/mod.rs b/src/sink/mod.rs
index 8ffd5b2..f22d2e2 100644
--- a/src/sink/mod.rs
+++ b/src/sink/mod.rs
@@ -28,7 +28,10 @@ impl Sink {
         match self {
             Self::Kafka(sink) => Ok(sink.put(destination, &records).await?),
             // A single table: there is no per-environment destination.
-            Self::ClickHouse(sink) => sink.put(records),
+            Self::ClickHouse(sink) => {
+                sink.put(records);
+                Ok(())
+            }
         }
     }
 
@@ -55,6 +58,4 @@ pub enum SinkError {
     Kafka(#[from] kafka::DeliveryError),
     #[error("ClickHouse insert failed: {0}")]
     ClickHouse(#[from] ::clickhouse::error::Error),
-    #[error("Sink buffer is full")]
-    BufferFull,
 }

From ba59c6b7393d3369c4ca9927f4c380e4556362c3 Mon Sep 17 00:00:00 2001
From: Gagan Trivedi <gagandeeptrivedi47@gmail.com>
Date: Wed, 28 Oct 2026 16:05:41 +0530
Subject: [PATCH 07/11] refactor(sink): trim comments and tidy test names

Drop comments that restate the code, inline json_column, rename value_column to value_to_text, and prefix the sink and config tests with test_.

Refs #3
---
 src/config.rs          |  6 ++---
 src/routes/ingest.rs   |  2 +-
 src/sink/batching.rs   | 16 +++++--------
 src/sink/clickhouse.rs | 53 +++++++++++++++++++++---------------------
 src/test_helpers.rs    |  2 +-
 5 files changed, 37 insertions(+), 42 deletions(-)

diff --git a/src/config.rs b/src/config.rs
index e883a64..3236201 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -209,12 +209,12 @@ mod tests {
     }
 
     #[test]
-    fn parse_positive_defaults_when_unset() {
+    fn test_parse_positive_defaults_when_unset() {
         assert_eq!(parse_positive("N", None, 1000u64).unwrap(), 1000);
     }
 
     #[test]
-    fn parse_positive_reads_a_positive_integer() {
+    fn test_parse_positive_reads_a_positive_integer() {
         assert_eq!(
             parse_positive("N", Some("50".to_string()), 1000usize).unwrap(),
             50
@@ -222,7 +222,7 @@ mod tests {
     }
 
     #[test]
-    fn parse_positive_rejects_zero_and_non_numbers() {
+    fn test_parse_positive_rejects_zero_and_non_numbers() {
         for value in ["0", "-1", "ten", ""] {
             assert!(matches!(
                 parse_positive("N", Some(value.to_string()), 1000u64),
diff --git a/src/routes/ingest.rs b/src/routes/ingest.rs
index c23dee8..beb9079 100644
--- a/src/routes/ingest.rs
+++ b/src/routes/ingest.rs
@@ -539,7 +539,7 @@ mod tests {
     }
 
     #[test]
-    fn test_event_record_omits_nulls_and_renames_fields() {
+    fn test_event_record_omits_absent_fields() {
         let batch = parse(
             r#"{"events":[{"event":"$flag_exposure","feature_name":"dark_mode","identifier":"user_1","value":"control","timestamp":1700000000000}]}"#,
         );
diff --git a/src/sink/batching.rs b/src/sink/batching.rs
index 70d0b87..6848a8d 100644
--- a/src/sink/batching.rs
+++ b/src/sink/batching.rs
@@ -38,9 +38,6 @@ struct Shared<W: BatchWriter> {
     stop: Notify,
 }
 
-/// Buffers records in memory and writes them in batches from one background
-/// task. `put` returns once the records are buffered, so records still
-/// buffered are lost if the process dies.
 pub struct BatchingSink<W: BatchWriter> {
     shared: Arc<Shared<W>>,
     flusher: Arc<Mutex<Option<JoinHandle<()>>>>,
@@ -56,7 +53,6 @@ impl<W: BatchWriter> Clone for BatchingSink<W> {
 }
 
 impl<W: BatchWriter> BatchingSink<W> {
-    /// Starts the flusher task; needs a Tokio runtime.
     pub fn new(writer: W, config: BatchConfig) -> Self {
         let shared = Arc::new(Shared {
             writer,
@@ -201,7 +197,7 @@ mod tests {
     }
 
     #[tokio::test(start_paused = true)]
-    async fn flushes_on_the_interval() {
+    async fn test_flushes_on_the_interval() {
         let writer = FakeWriter::default();
         let sink = BatchingSink::new(writer.clone(), config(Duration::from_secs(1), 100));
 
@@ -214,7 +210,7 @@ mod tests {
     }
 
     #[tokio::test(start_paused = true)]
-    async fn flushes_early_at_flush_max_events() {
+    async fn test_flushes_early_at_flush_max_events() {
         let writer = FakeWriter::default();
         let sink = BatchingSink::new(writer.clone(), config(Duration::from_secs(3600), 2));
 
@@ -228,7 +224,7 @@ mod tests {
     }
 
     #[tokio::test(start_paused = true)]
-    async fn keeps_buffering_while_a_write_is_slow() {
+    async fn test_keeps_buffering_while_a_write_is_slow() {
         let writer = FakeWriter {
             delay: Duration::from_secs(1),
             ..FakeWriter::default()
@@ -249,7 +245,7 @@ mod tests {
     }
 
     #[tokio::test(start_paused = true)]
-    async fn drops_a_batch_after_the_retries() {
+    async fn test_drops_a_batch_after_the_retries() {
         let writer = FakeWriter {
             fail: true,
             ..FakeWriter::default()
@@ -267,7 +263,7 @@ mod tests {
     }
 
     #[tokio::test(start_paused = true)]
-    async fn shutdown_writes_what_is_buffered() {
+    async fn test_shutdown_writes_what_is_buffered() {
         let writer = FakeWriter::default();
         let sink = BatchingSink::new(writer.clone(), config(Duration::from_secs(3600), 100));
 
@@ -278,7 +274,7 @@ mod tests {
     }
 
     #[test]
-    fn backoff_doubles_within_the_jitter_band() {
+    fn test_backoff_doubles_within_the_jitter_band() {
         for retry in 0..MAX_RETRIES {
             let base = RETRY_BASE_DELAY * 2u32.pow(retry);
             let delay = backoff(retry);
diff --git a/src/sink/clickhouse.rs b/src/sink/clickhouse.rs
index 5c58d72..0443234 100644
--- a/src/sink/clickhouse.rs
+++ b/src/sink/clickhouse.rs
@@ -33,9 +33,8 @@ impl fmt::Debug for ClickHouseConfig {
     }
 }
 
-/// One row of the `events` table, borrowed from its record. Field names are
-/// the column names. Every column is non-Nullable, so absent optionals are
-/// written as "", as ClickPipes does.
+/// Field names must match the `events` columns. Every column is
+/// non-Nullable, so absent optionals are written as "", as ClickPipes does.
 #[derive(Debug, PartialEq, Row, Serialize)]
 struct ClickHouseRow<'a> {
     environment_key: &'a str,
@@ -61,18 +60,24 @@ impl<'a> From<&'a EventRecord> for ClickHouseRow<'a> {
             timestamp: record.timestamp,
             collected_at: record.collected_at,
             identifier: record.identifier.as_deref().unwrap_or_default(),
-            value: value_column(&record.value),
-            traits: json_column(record.traits.as_ref()),
-            metadata: json_column(record.metadata.as_ref()),
+            value: value_to_text(&record.value),
+            traits: record
+                .traits
+                .as_ref()
+                .map(ToString::to_string)
+                .unwrap_or_default(),
+            metadata: record
+                .metadata
+                .as_ref()
+                .map(ToString::to_string)
+                .unwrap_or_default(),
             sdk_language: &record.sdk_language,
             sdk_version: &record.sdk_version,
         }
     }
 }
 
-/// `value` is a String column: a JSON string goes in raw, null as "", and
-/// anything else as its JSON text.
-fn value_column(value: &serde_json::Value) -> Cow<'_, str> {
+fn value_to_text(value: &serde_json::Value) -> Cow<'_, str> {
     match value {
         serde_json::Value::String(s) => Cow::Borrowed(s),
         serde_json::Value::Null => Cow::Borrowed(""),
@@ -80,11 +85,6 @@ fn value_column(value: &serde_json::Value) -> Cow<'_, str> {
     }
 }
 
-/// JSON text, or "" when absent.
-fn json_column(value: Option<&serde_json::Value>) -> String {
-    value.map(ToString::to_string).unwrap_or_default()
-}
-
 pub struct ClickHouseWriter {
     client: Client,
     table: String,
@@ -108,7 +108,6 @@ impl ClickHouseWriter {
 
 impl BatchWriter for ClickHouseWriter {
     /// One plain `INSERT` per batch; succeeds once ClickHouse has committed it.
-    /// Rows are built one at a time, so the batch is never copied whole.
     async fn write(&self, records: &[EventRecord]) -> Result<(), SinkError> {
         let mut insert = self
             .client
@@ -149,7 +148,7 @@ mod tests {
     }
 
     #[test]
-    fn row_fills_every_column_and_blanks_absent_optionals() {
+    fn test_absent_optionals_become_empty_strings() {
         let record = EventRecord {
             value: json!(42),
             traits: Some(json!({"plan": "pro"})),
@@ -175,16 +174,16 @@ mod tests {
     }
 
     #[test]
-    fn value_column_writes_strings_raw_and_null_as_empty() {
-        assert_eq!(value_column(&json!("control")), "control");
-        assert_eq!(value_column(&json!(null)), "");
-        assert_eq!(value_column(&json!(42)), "42");
-        assert_eq!(value_column(&json!(true)), "true");
-        assert_eq!(value_column(&json!({"a": [1, 2]})), r#"{"a":[1,2]}"#);
+    fn test_value_to_text_leaves_strings_unquoted() {
+        assert_eq!(value_to_text(&json!("control")), "control");
+        assert_eq!(value_to_text(&json!(null)), "");
+        assert_eq!(value_to_text(&json!(42)), "42");
+        assert_eq!(value_to_text(&json!(true)), "true");
+        assert_eq!(value_to_text(&json!({"a": [1, 2]})), r#"{"a":[1,2]}"#);
     }
 
     #[test]
-    fn debug_output_omits_the_password() {
+    fn test_debug_output_omits_the_password() {
         let config = ClickHouseConfig {
             url: "https://clickhouse:8443".to_string(),
             user: "ingest".to_string(),
@@ -221,7 +220,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn flushed_rows_land_in_the_events_table() {
+    async fn test_flushed_rows_land_in_the_events_table() {
         let db = ClickHouseTestDatabase::create().await;
         let sink = BatchingSink::new(
             ClickHouseWriter::new(db.config.clone()),
@@ -282,11 +281,11 @@ mod tests {
         );
 
         sink.shutdown(Duration::from_secs(5)).await;
-        db.drop().await;
+        db.drop_db().await;
     }
 
     #[tokio::test]
-    async fn shutdown_writes_buffered_rows() {
+    async fn test_shutdown_writes_buffered_rows() {
         let db = ClickHouseTestDatabase::create().await;
         let sink = BatchingSink::new(
             ClickHouseWriter::new(db.config.clone()),
@@ -299,6 +298,6 @@ mod tests {
         sink.shutdown(Duration::from_secs(5)).await;
 
         assert_eq!(db.count().await, 3);
-        db.drop().await;
+        db.drop_db().await;
     }
 }
diff --git a/src/test_helpers.rs b/src/test_helpers.rs
index 767bc92..ed98364 100644
--- a/src/test_helpers.rs
+++ b/src/test_helpers.rs
@@ -207,7 +207,7 @@ impl ClickHouseTestDatabase {
         panic!("expected {n} rows within 10s, found {}", self.count().await);
     }
 
-    pub(crate) async fn drop(self) {
+    pub(crate) async fn drop_db(self) {
         self.client
             .query(&format!("DROP DATABASE {}", self.config.database))
             .execute()

From 4175491e4ae10f9c06ad8f0972635c00944fb4fd Mon Sep 17 00:00:00 2001
From: Gagan Trivedi <gagandeeptrivedi47@gmail.com>
Date: Wed, 28 Oct 2026 16:14:42 +0530
Subject: [PATCH 08/11] refactor(config): read positive env vars in
 positive_env_var

Rename parse_positive to positive_env_var and have it read the variable itself, so callers name it once.

Refs #3
---
 src/config.rs | 40 ++++++++++++++++++++--------------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/src/config.rs b/src/config.rs
index 3236201..63a2aac 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -95,25 +95,16 @@ fn clickhouse_from_env() -> Result<ClickHouseConfig, ConfigError> {
 /// Shared by every batching sink.
 fn batch_from_env() -> Result<BatchConfig, ConfigError> {
     Ok(BatchConfig {
-        flush_interval: Duration::from_millis(parse_positive(
-            "SINK_FLUSH_INTERVAL_MS",
-            env::var("SINK_FLUSH_INTERVAL_MS").ok(),
-            1000,
-        )?),
-        flush_max_events: parse_positive(
-            "SINK_FLUSH_MAX_EVENTS",
-            env::var("SINK_FLUSH_MAX_EVENTS").ok(),
-            5000,
-        )?,
+        flush_interval: Duration::from_millis(positive_env_var("SINK_FLUSH_INTERVAL_MS", 1000)?),
+        flush_max_events: positive_env_var("SINK_FLUSH_MAX_EVENTS", 5000)?,
     })
 }
 
-fn parse_positive<T: FromStr + Default + PartialOrd>(
+fn positive_env_var<T: FromStr + Default + PartialOrd>(
     name: &'static str,
-    value: Option<String>,
     default: T,
 ) -> Result<T, ConfigError> {
-    match value {
+    match env::var(name).ok() {
         None => Ok(default),
         Some(v) => match v.parse() {
             Ok(n) if n > T::default() => Ok(n),
@@ -208,25 +199,34 @@ mod tests {
         ));
     }
 
+    // Each test uses its own variable, since the environment is shared
+    // across test threads.
+
     #[test]
-    fn test_parse_positive_defaults_when_unset() {
-        assert_eq!(parse_positive("N", None, 1000u64).unwrap(), 1000);
+    fn test_positive_env_var_defaults_when_unset() {
+        assert_eq!(
+            positive_env_var("TEST_POSITIVE_ENV_VAR_UNSET", 1000u64).unwrap(),
+            1000
+        );
     }
 
     #[test]
-    fn test_parse_positive_reads_a_positive_integer() {
+    fn test_positive_env_var_reads_a_positive_integer() {
+        env::set_var("TEST_POSITIVE_ENV_VAR_SET", "50");
         assert_eq!(
-            parse_positive("N", Some("50".to_string()), 1000usize).unwrap(),
+            positive_env_var("TEST_POSITIVE_ENV_VAR_SET", 1000usize).unwrap(),
             50
         );
     }
 
     #[test]
-    fn test_parse_positive_rejects_zero_and_non_numbers() {
+    fn test_positive_env_var_rejects_zero_and_non_numbers() {
+        const NAME: &str = "TEST_POSITIVE_ENV_VAR_INVALID";
         for value in ["0", "-1", "ten", ""] {
+            env::set_var(NAME, value);
             assert!(matches!(
-                parse_positive("N", Some(value.to_string()), 1000u64),
-                Err(ConfigError::InvalidNumber("N", v)) if v == value
+                positive_env_var(NAME, 1000u64),
+                Err(ConfigError::InvalidNumber(NAME, v)) if v == value
             ));
         }
     }

From 2b6fc18c2a7b9d01327719b63f0bc14637b3c3a3 Mon Sep 17 00:00:00 2001
From: Gagan Trivedi <gagandeeptrivedi47@gmail.com>
Date: Wed, 28 Oct 2026 16:19:42 +0530
Subject: [PATCH 09/11] refactor(config): name env helpers after the config
 they build

Refs #3
---
 src/config.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/config.rs b/src/config.rs
index 63a2aac..c3c3f10 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -33,8 +33,8 @@ impl Config {
         let sink = match parse_sink(env::var("SINK").ok().as_deref())? {
             SinkKind::Kafka => SinkConfig::Kafka(KafkaConfig::from_env()?),
             SinkKind::ClickHouse => SinkConfig::ClickHouse {
-                clickhouse: clickhouse_from_env()?,
-                batch: batch_from_env()?,
+                clickhouse: clickhouse_config_from_env()?,
+                batch: batch_config_from_env()?,
             },
         };
 
@@ -82,7 +82,7 @@ fn parse_sink(value: Option<&str>) -> Result<SinkKind, ConfigError> {
     }
 }
 
-fn clickhouse_from_env() -> Result<ClickHouseConfig, ConfigError> {
+fn clickhouse_config_from_env() -> Result<ClickHouseConfig, ConfigError> {
     Ok(ClickHouseConfig {
         url: required("CLICKHOUSE_URL")?,
         user: env::var("CLICKHOUSE_USER").unwrap_or_else(|_| "default".to_string()),
@@ -93,7 +93,7 @@ fn clickhouse_from_env() -> Result<ClickHouseConfig, ConfigError> {
 }
 
 /// Shared by every batching sink.
-fn batch_from_env() -> Result<BatchConfig, ConfigError> {
+fn batch_config_from_env() -> Result<BatchConfig, ConfigError> {
     Ok(BatchConfig {
         flush_interval: Duration::from_millis(positive_env_var("SINK_FLUSH_INTERVAL_MS", 1000)?),
         flush_max_events: positive_env_var("SINK_FLUSH_MAX_EVENTS", 5000)?,

From 715df4d88958b028888618691848fdab476d0fd8 Mon Sep 17 00:00:00 2001
From: Gagan Trivedi <gagandeeptrivedi47@gmail.com>
Date: Fri, 30 Oct 2026 11:25:12 +0530
Subject: [PATCH 10/11] feat(sink): retry ClickHouse writes for about 17
 seconds

Wait 1, 2, 4, 5, 5 s between attempts before dropping a batch, so
writes survive a ClickHouse restart instead of giving up after about
1.4 s. The delays are a fixed list; jitter is dropped, since each
process has a single flusher.

Refs #3
---
 README.md            |  2 +-
 src/sink/batching.rs | 49 +++++++++++++++-----------------------------
 2 files changed, 18 insertions(+), 33 deletions(-)

diff --git a/README.md b/README.md
index 81c3e0e..42fc6b1 100644
--- a/README.md
+++ b/README.md
@@ -88,7 +88,7 @@ With `KAFKA_AUTH=scram`, the producer authenticates with SASL/SCRAM-SHA-512 over
 
 ### ClickHouse sink
 
-With `SINK=clickhouse`, `202` means the events are buffered in memory, not yet stored. A background task takes the whole buffer every `SINK_FLUSH_INTERVAL_MS`, or sooner once `SINK_FLUSH_MAX_EVENTS` events are waiting, and writes it as one `INSERT`. A failed batch is retried 3 times and then dropped with an error log. The buffer is unbounded: requests never wait or get `503`, so while a write is slow or failing (up to about 2 minutes with retries) it grows by the ingest rate times the write time, and the next flush takes it all. A normal shutdown (SIGTERM) flushes the buffer, but events still buffered are lost if the process crashes.
+With `SINK=clickhouse`, `202` means the events are buffered in memory, not yet stored. A background task takes the whole buffer every `SINK_FLUSH_INTERVAL_MS`, or sooner once `SINK_FLUSH_MAX_EVENTS` events are waiting, and writes it as one `INSERT`. A failed batch is retried 5 times over about 17 seconds, enough to ride out a ClickHouse restart, and then dropped with an error log. The buffer is unbounded: requests never wait or get `503`, so while a write is slow or failing (up to about 3 minutes with retries) it grows by the ingest rate times the write time, and the next flush takes it all. A normal shutdown (SIGTERM) flushes the buffer, but events still buffered are lost if the process crashes.
 
 ## Run locally
 
diff --git a/src/sink/batching.rs b/src/sink/batching.rs
index 6848a8d..76b755e 100644
--- a/src/sink/batching.rs
+++ b/src/sink/batching.rs
@@ -1,5 +1,4 @@
 use std::future::Future;
-use std::hash::{BuildHasher, RandomState};
 use std::mem;
 use std::sync::{Arc, Mutex};
 use std::time::Duration;
@@ -12,10 +11,15 @@ use tracing::{error, warn};
 use super::SinkError;
 use crate::models::EventRecord;
 
-/// Retries after the first failed write, before the batch is dropped.
-const MAX_RETRIES: u32 = 3;
-/// Doubles per retry: about 200 ms, 400 ms, 800 ms before jitter.
-const RETRY_BASE_DELAY: Duration = Duration::from_millis(200);
+/// Waits between write attempts before the batch is dropped: about 17 s in
+/// all, to ride out a ClickHouse restart.
+const RETRY_DELAYS: [Duration; 5] = [
+    Duration::from_secs(1),
+    Duration::from_secs(2),
+    Duration::from_secs(4),
+    Duration::from_secs(5),
+    Duration::from_secs(5),
+];
 
 /// Writes one batch of records to a store.
 pub trait BatchWriter: Send + Sync + 'static {
@@ -117,28 +121,18 @@ async fn flush<W: BatchWriter>(shared: &Shared<W>) {
 }
 
 async fn write_with_retries<W: BatchWriter>(writer: &W, records: &[EventRecord]) {
-    let mut retry = 0;
-    loop {
+    for (retry, delay) in RETRY_DELAYS.into_iter().enumerate() {
         match writer.write(records).await {
             Ok(()) => return,
-            Err(e) if retry < MAX_RETRIES => {
-                warn!(rows = records.len(), retry, error = %e, "Sink batch write failed, retrying");
-                tokio::time::sleep(backoff(retry)).await;
-                retry += 1;
-            }
             Err(e) => {
-                error!(rows = records.len(), error = %e, "Sink batch dropped after retries");
-                return;
+                warn!(rows = records.len(), retry, error = %e, "Sink batch write failed, retrying");
+                tokio::time::sleep(delay).await;
             }
         }
     }
-}
-
-/// Exponential backoff with ±50% jitter, so tasks that failed together do not
-/// retry together.
-fn backoff(retry: u32) -> Duration {
-    let jitter_permille = RandomState::new().hash_one(retry) % 1000;
-    (RETRY_BASE_DELAY * 2u32.pow(retry)).mul_f64(0.5 + jitter_permille as f64 / 1000.0)
+    if let Err(e) = writer.write(records).await {
+        error!(rows = records.len(), error = %e, "Sink batch dropped after retries");
+    }
 }
 
 #[cfg(test)]
@@ -253,11 +247,11 @@ mod tests {
         let sink = BatchingSink::new(writer.clone(), config(Duration::from_secs(1), 1));
 
         sink.put(records(&[1]));
-        tokio::time::sleep(Duration::from_secs(10)).await;
+        tokio::time::sleep(Duration::from_secs(60)).await;
 
         assert_eq!(
             writer.attempts.load(Ordering::SeqCst),
-            1 + MAX_RETRIES as usize
+            1 + RETRY_DELAYS.len()
         );
         sink.put(records(&[2]));
     }
@@ -272,13 +266,4 @@ mod tests {
 
         assert_eq!(*writer.written.lock().unwrap(), vec![vec![1, 2, 3]]);
     }
-
-    #[test]
-    fn test_backoff_doubles_within_the_jitter_band() {
-        for retry in 0..MAX_RETRIES {
-            let base = RETRY_BASE_DELAY * 2u32.pow(retry);
-            let delay = backoff(retry);
-            assert!(delay >= base / 2 && delay < base * 3 / 2, "{delay:?}");
-        }
-    }
 }

From fd7feba5115a27592533fbfc3cd3f2d11fc5a546 Mon Sep 17 00:00:00 2001
From: Gagan Trivedi <gagandeeptrivedi47@gmail.com>
Date: Fri, 30 Oct 2026 13:01:22 +0530
Subject: [PATCH 11/11] feat(sink): handle ClickHouse errors by kind and check
 the table at startup

Classify write errors as transient, bad data or config. Transient errors
are retried; bad data and config errors drop the batch at once. A config
error also fails /health until restart. At startup, check that ClickHouse
is reachable and the table matches the row, so bad credentials or a
missing table fail before serving.

Refs #3
---
 README.md              |   4 +-
 src/main.rs            |   5 ++
 src/routes/health.rs   |   2 +-
 src/sink/batching.rs   | 110 ++++++++++++++++++++++------
 src/sink/clickhouse.rs | 161 ++++++++++++++++++++++++++++++++++++++++-
 src/sink/mod.rs        |  24 +++++-
 6 files changed, 279 insertions(+), 27 deletions(-)

diff --git a/README.md b/README.md
index 42fc6b1..3f225e4 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@ Rust + axum, deployed as an ARM64 container on ECS Fargate.
 
 | Method | Path | Description |
 |---|---|---|
-| GET | `/health` | Liveness probe → `{"status":"healthy"}`. Touches no downstream; returns `503` once the Kafka producer has hit a fatal error, so ECS replaces the task. |
+| GET | `/health` | Liveness probe → `{"status":"healthy"}`. Touches no downstream; returns `503` once the sink has hit a fatal error (the Kafka producer's, or a ClickHouse config error), so ECS replaces the task. |
 | POST | `/v1/events` | Ingest an event batch. Requires the `X-Environment-Key` header. |
 
 `POST /v1/events` validates each event independently — invalid events are rejected individually and never block valid ones in the same batch.
@@ -88,7 +88,7 @@ With `KAFKA_AUTH=scram`, the producer authenticates with SASL/SCRAM-SHA-512 over
 
 ### ClickHouse sink
 
-With `SINK=clickhouse`, `202` means the events are buffered in memory, not yet stored. A background task takes the whole buffer every `SINK_FLUSH_INTERVAL_MS`, or sooner once `SINK_FLUSH_MAX_EVENTS` events are waiting, and writes it as one `INSERT`. A failed batch is retried 5 times over about 17 seconds, enough to ride out a ClickHouse restart, and then dropped with an error log. The buffer is unbounded: requests never wait or get `503`, so while a write is slow or failing (up to about 3 minutes with retries) it grows by the ingest rate times the write time, and the next flush takes it all. A normal shutdown (SIGTERM) flushes the buffer, but events still buffered are lost if the process crashes.
+With `SINK=clickhouse`, `202` means the events are buffered in memory, not yet stored. A background task takes the whole buffer every `SINK_FLUSH_INTERVAL_MS`, or sooner once `SINK_FLUSH_MAX_EVENTS` events are waiting, and writes it as one `INSERT`. A batch that fails with a transient error (network, timeout, overload, or anything unrecognised) is retried 5 times over about 17 seconds, enough to ride out a ClickHouse restart, and then dropped with an error log; these never affect `/health`. Rows the table rejects (parse, type or constraint errors) drop the batch at once, with an error log, and leave `/health` alone. A config error (bad credentials or grants, or a missing database, table or column) drops the batch at once and fails `/health` until restart. At startup the server checks that ClickHouse is reachable, accepts the credentials, and has a table whose columns match, and exits with an error if not. The buffer is unbounded: requests never wait or get `503`, so while a write is slow or failing (up to about 3 minutes with retries) it grows by the ingest rate times the write time, and the next flush takes it all. A normal shutdown (SIGTERM) flushes the buffer, but events still buffered are lost if the process crashes.
 
 ## Run locally
 
diff --git a/src/main.rs b/src/main.rs
index 199f48c..bea09fc 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -11,6 +11,7 @@ mod test_helpers;
 use std::sync::Arc;
 use std::time::Duration;
 
+use anyhow::Context;
 use sqlx::postgres::PgPoolOptions;
 use tokio::signal;
 use tracing::info;
@@ -57,6 +58,10 @@ async fn main() -> anyhow::Result<()> {
                 "Starting flagsmith-analytics server"
             );
             let writer = ClickHouseWriter::new(clickhouse.clone());
+            writer
+                .check()
+                .await
+                .context("ClickHouse startup check failed")?;
             (
                 Sink::ClickHouse(BatchingSink::new(writer, batch.clone())),
                 None,
diff --git a/src/routes/health.rs b/src/routes/health.rs
index d696d36..ae5783e 100644
--- a/src/routes/health.rs
+++ b/src/routes/health.rs
@@ -10,7 +10,7 @@ pub struct HealthResponse {
     error: Option<String>,
 }
 
-/// Liveness plus one local check: a fatal producer error is permanent, so
+/// Liveness plus one local check: a fatal sink error is permanent, so
 /// report it and let ECS replace the task. No downstream call, so this cannot hang.
 pub async fn health_check(State(sink): State<Sink>) -> (StatusCode, Json<HealthResponse>) {
     match sink.fatal_error() {
diff --git a/src/sink/batching.rs b/src/sink/batching.rs
index 76b755e..e2667dc 100644
--- a/src/sink/batching.rs
+++ b/src/sink/batching.rs
@@ -1,6 +1,6 @@
 use std::future::Future;
 use std::mem;
-use std::sync::{Arc, Mutex};
+use std::sync::{Arc, Mutex, OnceLock};
 use std::time::Duration;
 
 use tokio::sync::Notify;
@@ -8,11 +8,11 @@ use tokio::task::JoinHandle;
 use tokio::time::{Instant, MissedTickBehavior};
 use tracing::{error, warn};
 
-use super::SinkError;
+use super::{ErrorKind, SinkError};
 use crate::models::EventRecord;
 
-/// Waits between write attempts before the batch is dropped: about 17 s in
-/// all, to ride out a ClickHouse restart.
+/// Waits between attempts to write a batch that failed with a transient error,
+/// before it is dropped: about 17 s in all, to ride out a ClickHouse restart.
 const RETRY_DELAYS: [Duration; 5] = [
     Duration::from_secs(1),
     Duration::from_secs(2),
@@ -40,6 +40,8 @@ struct Shared<W: BatchWriter> {
     buffer: Mutex<Vec<EventRecord>>,
     flush_now: Notify,
     stop: Notify,
+    /// The first config error; stays set until restart.
+    fatal_error: OnceLock<String>,
 }
 
 pub struct BatchingSink<W: BatchWriter> {
@@ -64,6 +66,7 @@ impl<W: BatchWriter> BatchingSink<W> {
             buffer: Mutex::new(Vec::new()),
             flush_now: Notify::new(),
             stop: Notify::new(),
+            fatal_error: OnceLock::new(),
         });
         let flusher = tokio::spawn(run(shared.clone()));
         Self {
@@ -81,6 +84,12 @@ impl<W: BatchWriter> BatchingSink<W> {
         }
     }
 
+    /// A config error, such as bad credentials or a missing table, that
+    /// stops every write until the server is fixed.
+    pub fn fatal_error(&self) -> Option<String> {
+        self.shared.fatal_error.get().cloned()
+    }
+
     /// Stop the flusher and write what is buffered, bounded by `timeout`.
     pub async fn shutdown(self, timeout: Duration) {
         self.shared.stop.notify_one();
@@ -117,21 +126,36 @@ async fn flush<W: BatchWriter>(shared: &Shared<W>) {
     if records.is_empty() {
         return;
     }
-    write_with_retries(&shared.writer, &records).await;
+    write_with_retries(shared, &records).await;
 }
 
-async fn write_with_retries<W: BatchWriter>(writer: &W, records: &[EventRecord]) {
-    for (retry, delay) in RETRY_DELAYS.into_iter().enumerate() {
-        match writer.write(records).await {
-            Ok(()) => return,
-            Err(e) => {
-                warn!(rows = records.len(), retry, error = %e, "Sink batch write failed, retrying");
-                tokio::time::sleep(delay).await;
+/// Retries transient errors per `RETRY_DELAYS`. Drops the batch at once on
+/// bad data or a config error, and records a config error to fail the health
+/// check.
+async fn write_with_retries<W: BatchWriter>(shared: &Shared<W>, records: &[EventRecord]) {
+    let rows = records.len();
+    for retry in 0.. {
+        let Err(e) = shared.writer.write(records).await else {
+            return;
+        };
+        match e.kind() {
+            ErrorKind::Transient => {}
+            ErrorKind::BadData => {
+                error!(rows, error = %e, "Sink batch dropped: rows rejected");
+                return;
+            }
+            ErrorKind::Config => {
+                error!(rows, error = %e, "Sink batch dropped: sink misconfigured");
+                let _ = shared.fatal_error.set(e.to_string());
+                return;
             }
         }
-    }
-    if let Err(e) = writer.write(records).await {
-        error!(rows = records.len(), error = %e, "Sink batch dropped after retries");
+        let Some(&delay) = RETRY_DELAYS.get(retry) else {
+            error!(rows, error = %e, "Sink batch dropped after retries");
+            return;
+        };
+        warn!(rows, retry, error = %e, "Sink batch write failed, retrying");
+        tokio::time::sleep(delay).await;
     }
 }
 
@@ -141,11 +165,12 @@ mod tests {
 
     use super::*;
 
-    /// Records every write attempt; each takes `delay`, and all fail when
-    /// `fail` is set.
+    /// Records every write attempt; each takes `delay`, and the first
+    /// `failures` fail with `error`.
     #[derive(Clone, Default)]
     struct FakeWriter {
-        fail: bool,
+        failures: usize,
+        error: Option<fn() -> clickhouse::error::Error>,
         delay: Duration,
         attempts: Arc<AtomicUsize>,
         written: Arc<Mutex<Vec<Vec<i64>>>>,
@@ -153,10 +178,11 @@ mod tests {
 
     impl BatchWriter for FakeWriter {
         async fn write(&self, records: &[EventRecord]) -> Result<(), SinkError> {
-            self.attempts.fetch_add(1, Ordering::SeqCst);
+            let attempt = self.attempts.fetch_add(1, Ordering::SeqCst);
             tokio::time::sleep(self.delay).await;
-            if self.fail {
-                return Err(clickhouse::error::Error::Custom("failed".to_string()).into());
+            if attempt < self.failures {
+                let error = self.error.expect("an error to fail with");
+                return Err(error().into());
             }
             let timestamps = records.iter().map(|record| record.timestamp).collect();
             self.written.lock().unwrap().push(timestamps);
@@ -241,7 +267,8 @@ mod tests {
     #[tokio::test(start_paused = true)]
     async fn test_drops_a_batch_after_the_retries() {
         let writer = FakeWriter {
-            fail: true,
+            failures: usize::MAX,
+            error: Some(|| clickhouse::error::Error::TimedOut),
             ..FakeWriter::default()
         };
         let sink = BatchingSink::new(writer.clone(), config(Duration::from_secs(1), 1));
@@ -253,7 +280,46 @@ mod tests {
             writer.attempts.load(Ordering::SeqCst),
             1 + RETRY_DELAYS.len()
         );
+        assert_eq!(sink.fatal_error(), None);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn test_drops_bad_data_without_retries_and_stays_healthy() {
+        let writer = FakeWriter {
+            failures: 1,
+            error: Some(|| clickhouse::error::Error::Custom("bad row".to_string())),
+            ..FakeWriter::default()
+        };
+        let sink = BatchingSink::new(writer.clone(), config(Duration::from_secs(3600), 1));
+
+        sink.put(records(&[1]));
+        tokio::time::sleep(Duration::from_secs(60)).await;
+        assert_eq!(writer.attempts.load(Ordering::SeqCst), 1);
+        assert!(writer.written.lock().unwrap().is_empty());
+        assert_eq!(sink.fatal_error(), None);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn test_drops_a_batch_without_retries_on_a_config_error() {
+        let writer = FakeWriter {
+            failures: 1,
+            error: Some(|| clickhouse::error::Error::SchemaMismatch("no column".to_string())),
+            ..FakeWriter::default()
+        };
+        let sink = BatchingSink::new(writer.clone(), config(Duration::from_secs(3600), 1));
+
+        sink.put(records(&[1]));
+        tokio::time::sleep(Duration::from_secs(60)).await;
+        assert_eq!(writer.attempts.load(Ordering::SeqCst), 1);
+        assert!(writer.written.lock().unwrap().is_empty());
+        let fatal_error = sink.fatal_error();
+        assert!(fatal_error.is_some());
+
+        // Stays set after a later write succeeds.
         sink.put(records(&[2]));
+        tokio::time::sleep(Duration::from_millis(10)).await;
+        assert_eq!(*writer.written.lock().unwrap(), vec![vec![2]]);
+        assert_eq!(sink.fatal_error(), fatal_error);
     }
 
     #[tokio::test(start_paused = true)]
diff --git a/src/sink/clickhouse.rs b/src/sink/clickhouse.rs
index 0443234..972d067 100644
--- a/src/sink/clickhouse.rs
+++ b/src/sink/clickhouse.rs
@@ -2,16 +2,38 @@ use std::borrow::Cow;
 use std::fmt;
 use std::time::Duration;
 
+use clickhouse::error::Error;
 use clickhouse::{Client, Compression, Row};
 use serde::Serialize;
 
 use super::batching::BatchWriter;
-use super::SinkError;
+use super::{ErrorKind, SinkError};
 use crate::models::EventRecord;
 
 /// Bounds sending and committing one batch of up to `SINK_FLUSH_MAX_EVENTS` rows.
 const INSERT_TIMEOUT: Duration = Duration::from_secs(30);
 
+/// Server error codes for rows the table rejects.
+const BAD_DATA_ERROR_CODES: [u32; 5] = [
+    6,   // CANNOT_PARSE_TEXT
+    27,  // CANNOT_PARSE_INPUT_ASSERTION_FAILED
+    53,  // TYPE_MISMATCH
+    117, // INCORRECT_DATA
+    469, // VIOLATED_CONSTRAINT
+];
+
+/// Server error codes for bad credentials or grants, or a missing database,
+/// table or column.
+const CONFIG_ERROR_CODES: [u32; 7] = [
+    16,  // NO_SUCH_COLUMN_IN_TABLE
+    60,  // UNKNOWN_TABLE
+    81,  // UNKNOWN_DATABASE
+    192, // UNKNOWN_USER
+    194, // REQUIRED_PASSWORD
+    497, // ACCESS_DENIED
+    516, // AUTHENTICATION_FAILED
+];
+
 #[derive(Clone)]
 pub struct ClickHouseConfig {
     pub url: String,
@@ -85,6 +107,29 @@ fn value_to_text(value: &serde_json::Value) -> Cow<'_, str> {
     }
 }
 
+/// Anything unrecognised, such as a network error, timeout or overloaded
+/// server, counts as transient.
+pub(super) fn error_kind(error: &Error) -> ErrorKind {
+    match error {
+        // A row that fails to serialize.
+        Error::Custom(_) => ErrorKind::BadData,
+        // `ClickHouseRow` doesn't match the table.
+        Error::SchemaMismatch(_) => ErrorKind::Config,
+        Error::BadResponse(reason) => match error_code(reason) {
+            Some(code) if BAD_DATA_ERROR_CODES.contains(&code) => ErrorKind::BadData,
+            Some(code) if CONFIG_ERROR_CODES.contains(&code) => ErrorKind::Config,
+            _ => ErrorKind::Transient,
+        },
+        _ => ErrorKind::Transient,
+    }
+}
+
+/// The code of a server exception, e.g. 60 in "Code: 60. DB::Exception: ...".
+fn error_code(reason: &str) -> Option<u32> {
+    let rest = reason.strip_prefix("Code: ")?;
+    rest[..rest.find('.')?].parse().ok()
+}
+
 pub struct ClickHouseWriter {
     client: Client,
     table: String,
@@ -104,6 +149,16 @@ impl ClickHouseWriter {
             table: config.table,
         }
     }
+
+    /// Fails if ClickHouse is unreachable, rejects the credentials, or the
+    /// table's columns don't match `ClickHouseRow`. Column types are only
+    /// checked on the first write.
+    pub async fn check(&self) -> Result<(), SinkError> {
+        // Opening an insert fetches the table's schema and matches it against
+        // the row; dropping it unused sends nothing.
+        let _ = self.client.insert::<ClickHouseRow<'_>>(&self.table).await?;
+        Ok(())
+    }
 }
 
 impl BatchWriter for ClickHouseWriter {
@@ -196,6 +251,110 @@ mod tests {
         assert!(!shown.contains("hunter2"));
     }
 
+    #[test]
+    fn test_error_code_reads_the_server_exception_code() {
+        assert_eq!(
+            error_code("Code: 60. DB::Exception: Table x.events does not exist."),
+            Some(60)
+        );
+        assert_eq!(error_code("Code: abc. DB::Exception"), None);
+        assert_eq!(error_code("502 Bad Gateway"), None);
+    }
+
+    #[test]
+    fn test_error_kind_separates_bad_data_and_config_from_outages() {
+        let bad_response = |reason: &str| Error::BadResponse(reason.to_string());
+        let cases = [
+            (
+                bad_response("Code: 469. DB::Exception: Constraint violated"),
+                ErrorKind::BadData,
+            ),
+            (Error::Custom("bad row".to_string()), ErrorKind::BadData),
+            (
+                bad_response("Code: 516. DB::Exception: Authentication failed"),
+                ErrorKind::Config,
+            ),
+            (
+                bad_response("Code: 81. DB::Exception: Database x does not exist"),
+                ErrorKind::Config,
+            ),
+            (Error::SchemaMismatch("x".to_string()), ErrorKind::Config),
+            (
+                bad_response("Code: 202. DB::Exception: Too many simultaneous queries"),
+                ErrorKind::Transient,
+            ),
+            (
+                bad_response("503 Service Unavailable"),
+                ErrorKind::Transient,
+            ),
+            (Error::TimedOut, ErrorKind::Transient),
+            (Error::Network("refused".into()), ErrorKind::Transient),
+        ];
+        for (error, kind) in cases {
+            assert_eq!(error_kind(&error), kind, "{error}");
+        }
+    }
+
+    #[tokio::test]
+    async fn test_check_passes_for_the_events_table() {
+        let db = ClickHouseTestDatabase::create().await;
+        ClickHouseWriter::new(db.config.clone())
+            .check()
+            .await
+            .unwrap();
+        db.drop_db().await;
+    }
+
+    #[tokio::test]
+    async fn test_check_fails_with_a_config_error_on_bad_config() {
+        let db = ClickHouseTestDatabase::create().await;
+        db.client
+            .query("ALTER TABLE events RENAME COLUMN sdk_version TO version")
+            .execute()
+            .await
+            .unwrap();
+        let configs = [
+            ClickHouseConfig {
+                password: "wrong".to_string(),
+                ..db.config.clone()
+            },
+            ClickHouseConfig {
+                database: "missing".to_string(),
+                ..db.config.clone()
+            },
+            ClickHouseConfig {
+                table: "missing".to_string(),
+                ..db.config.clone()
+            },
+            // The renamed column.
+            db.config.clone(),
+        ];
+
+        for config in configs {
+            let error = ClickHouseWriter::new(config.clone())
+                .check()
+                .await
+                .unwrap_err();
+            assert_eq!(error.kind(), ErrorKind::Config, "{config:?}: {error}");
+        }
+        db.drop_db().await;
+    }
+
+    #[tokio::test]
+    async fn test_check_fails_with_a_transient_error_when_unreachable() {
+        let writer = ClickHouseWriter::new(ClickHouseConfig {
+            url: "http://127.0.0.1:1".to_string(),
+            user: "default".to_string(),
+            password: String::new(),
+            database: "default".to_string(),
+            table: "events".to_string(),
+        });
+        assert_eq!(
+            writer.check().await.unwrap_err().kind(),
+            ErrorKind::Transient
+        );
+    }
+
     fn batch_config(flush_interval: Duration) -> BatchConfig {
         BatchConfig {
             flush_interval,
diff --git a/src/sink/mod.rs b/src/sink/mod.rs
index f22d2e2..c37f1c3 100644
--- a/src/sink/mod.rs
+++ b/src/sink/mod.rs
@@ -39,7 +39,7 @@ impl Sink {
     pub fn fatal_error(&self) -> Option<String> {
         match self {
             Self::Kafka(sink) => sink.fatal_error(),
-            Self::ClickHouse(_) => None,
+            Self::ClickHouse(sink) => sink.fatal_error(),
         }
     }
 
@@ -59,3 +59,25 @@ pub enum SinkError {
     #[error("ClickHouse insert failed: {0}")]
     ClickHouse(#[from] ::clickhouse::error::Error),
 }
+
+/// What a failed write means for the batch and the server.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum ErrorKind {
+    /// May pass on retry: network, timeout, overload, or unrecognised.
+    Transient,
+    /// The store rejects the rows; retrying cannot help.
+    BadData,
+    /// The server is misconfigured, e.g. bad credentials or a missing
+    /// table; nothing will be written until it is fixed.
+    Config,
+}
+
+impl SinkError {
+    /// Only batching sinks retry; the Kafka producer retries internally.
+    pub fn kind(&self) -> ErrorKind {
+        match self {
+            Self::Kafka(_) => ErrorKind::Transient,
+            Self::ClickHouse(e) => clickhouse::error_kind(e),
+        }
+    }
+}