feat(master): support etcd-backed HA scheduler

.github/workflows/rust-test.yml

@@ -48,5 +48,30 @@ jobs:
         run: cargo test -p lance-context-core -p lance-context-master --no-run
       - name: Run unit tests
         run: cargo test -p lance-context-core -p lance-context-master --lib
+      - name: Start etcd for HA scheduler test
+        run: |
+          ETCD_VERSION=3.7.0
+          curl -fsSL -o /tmp/etcd.tar.gz \
+            "https://github.com/etcd-io/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz"
+          mkdir -p /tmp/etcd
+          tar -xzf /tmp/etcd.tar.gz -C /tmp/etcd --strip-components=1
+          nohup /tmp/etcd/etcd \
+            --data-dir /tmp/etcd-data \
+            --listen-client-urls http://127.0.0.1:2379 \
+            --advertise-client-urls http://127.0.0.1:2379 \
+            >/tmp/etcd.log 2>&1 &
+          for _ in $(seq 1 30); do
+            curl -fsS http://127.0.0.1:2379/health && exit 0
+            sleep 1
+          done
+          cat /tmp/etcd.log
+          exit 1
+      - name: Run etcd scheduler integration test
+        env:
+          ETCD_TEST_ENDPOINTS: http://127.0.0.1:2379
+        run: |
+          cargo test -p lance-context-master \
+            task_store::tests::etcd_coordinates_dedupe_claims_and_target_locks \
+            -- --ignored --exact
       - name: Run doc tests
         run: cargo test -p lance-context-core --doc

README.md

@@ -138,7 +138,7 @@ crates/lance-context-client  # HTTP client for the server
 crates/lance-context-master  # Control plane, scheduler, and admin UI
 crates/lance-context         # Re-export crate for downstream clients
 python/                      # Python bindings (PyO3) + tests
-deploy/kubernetes/           # Kubernetes master + RocksDB PVC example
+deploy/kubernetes/           # Kubernetes master examples: RocksDB/PVC or etcd HA
 examples/                    # Runnable example projects
 ```
 

crates/lance-context-master/Cargo.toml

@@ -21,6 +21,7 @@ arrow-schema = "58"
 axum = { version = "0.8", features = ["json"] }
 chrono = { version = "0.4", default-features = false, features = ["clock"] }
 clap = { version = "4", features = ["derive", "env"] }
+etcd-client = { version = "0.19", features = ["tls"] }
 futures = "0.3"
 lance = "7.0.0"
 metrics = "0.24"

crates/lance-context-master/src/config.rs

@@ -1,6 +1,15 @@
 //! Command-line / environment configuration for the master control-plane.
 
-use clap::Parser;
+use clap::{Parser, ValueEnum};
+
+/// Durable scheduler state backend.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
+pub enum TaskStoreBackend {
+    /// Single-master RocksDB on a dedicated local PVC.
+    Rocksdb,
+    /// Shared etcd state with lease-based task claims for HA masters.
+    Etcd,
+}
 
 /// Control-plane (master) process for lance-context rollout stores.
 #[derive(Debug, Clone, Parser)]
@@ -56,16 +65,61 @@ pub struct MasterConfig {
     #[arg(long, env = "TASK_CONCURRENCY", default_value_t = 4)]
     pub task_concurrency: usize,
 
+    /// Scheduler state backend. `rocksdb` requires exactly one master and a
+    /// dedicated PVC; `etcd` supports multiple stateless master replicas.
+    #[arg(
+        long,
+        env = "TASK_STORE_BACKEND",
+        value_enum,
+        default_value_t = TaskStoreBackend::Rocksdb
+    )]
+    pub task_store_backend: TaskStoreBackend,
+
     /// Local RocksDB directory for durable scheduler task state. This must be
     /// backed by persistent local storage in production; it must not be an
-    /// object-store URI or a volume shared by multiple master processes.
+    /// object-store URI or a volume shared by multiple master processes. Used
+    /// only when `TASK_STORE_BACKEND=rocksdb`.
     #[arg(
         long,
         env = "TASK_DB_PATH",
         default_value = "./master-data/tasks.rocksdb"
     )]
     pub task_db_path: String,
 
+    /// Comma-separated etcd v3 endpoints. Required when
+    /// `TASK_STORE_BACKEND=etcd`.
+    #[arg(long, env = "ETCD_ENDPOINTS", value_delimiter = ',')]
+    pub etcd_endpoints: Vec<String>,
+
+    /// Namespace for all lance-context master keys in etcd.
+    #[arg(long, env = "ETCD_PREFIX", default_value = "/lance-context/master")]
+    pub etcd_prefix: String,
+
+    /// Optional etcd username. `ETCD_PASSWORD` must also be set.
+    #[arg(long, env = "ETCD_USERNAME")]
+    pub etcd_username: Option<String>,
+
+    /// Optional etcd password. `ETCD_USERNAME` must also be set.
+    #[arg(long, env = "ETCD_PASSWORD")]
+    pub etcd_password: Option<String>,
+
+    /// Optional PEM CA certificate path for etcd TLS.
+    #[arg(long, env = "ETCD_CA_CERT")]
+    pub etcd_ca_cert: Option<String>,
+
+    /// Optional PEM client certificate path for etcd mutual TLS.
+    #[arg(long, env = "ETCD_CLIENT_CERT")]
+    pub etcd_client_cert: Option<String>,
+
+    /// Optional PEM client private-key path for etcd mutual TLS.
+    #[arg(long, env = "ETCD_CLIENT_KEY")]
+    pub etcd_client_key: Option<String>,
+
+    /// TTL for etcd task claims and distributed locks. The master renews leases
+    /// while work is running; orphaned tasks are requeued after expiry.
+    #[arg(long, env = "ETCD_LEASE_TTL_SECS", default_value_t = 30)]
+    pub etcd_lease_ttl_secs: i64,
+
     /// Maximum number of terminal scheduler tasks retained in RocksDB and the
     /// queue UI. Queued and running tasks are never pruned, and at least one
     /// terminal task is retained for status polling.

crates/lance-context-master/src/main.rs

@@ -48,7 +48,7 @@ async fn main() {
     // Background stats scanner (detached; stops when the last Arc drops).
     let _scanner = scanner::spawn_scanner(&state);
 
-    // Single serial compaction worker (+ optional periodic auto-sweep).
+    // Durable scheduler poller (+ optional coordinated auto-sweep).
     let _scheduler = scheduler::spawn_scheduler(&state);
 
     let mut app = Router::new().nest("/api/v1", routes::api_router());