feat(master): persist scheduler tasks in RocksDB

.github/workflows/release.yml

@@ -56,7 +56,7 @@ jobs:
     - name: Install system dependencies
       run: |
         sudo apt update
-        sudo apt install -y protobuf-compiler libssl-dev
+        sudo apt install -y clang libclang-dev protobuf-compiler libssl-dev
 
     - uses: actions-rust-lang/setup-rust-toolchain@v1
       with:

.github/workflows/rust-publish.yml

@@ -36,7 +36,7 @@ jobs:
       - name: Install dependencies
         run: |
           sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
+          sudo apt install -y clang libclang-dev protobuf-compiler libssl-dev
 
       - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
@@ -46,10 +46,15 @@ jobs:
 
       - uses: rui314/setup-mold@v1
 
+      - name: Validate Rust crate packages
+        if: github.event_name == 'pull_request'
+        run: cargo package --workspace --exclude lance-context-python --allow-dirty --no-verify --all-features --locked
+
       - name: Publish Rust crates
+        if: github.event_name != 'pull_request'
         uses: katyo/publish-crates@v2
         with:
           registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
           args: "--all-features"
           path: .
-          dry-run: ${{ github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'dry_run') }}
+          dry-run: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'dry_run' }}

.github/workflows/rust-test.yml

@@ -43,10 +43,10 @@ jobs:
       - name: Install dependencies
         run: |
           sudo apt update
-          sudo apt install -y protobuf-compiler
+          sudo apt install -y clang libclang-dev protobuf-compiler
       - name: Build tests
-        run: cargo test -p lance-context-core --no-run
+        run: cargo test -p lance-context-core -p lance-context-master --no-run
       - name: Run unit tests
-        run: cargo test -p lance-context-core --lib
+        run: cargo test -p lance-context-core -p lance-context-master --lib
       - name: Run doc tests
         run: cargo test -p lance-context-core --doc

.github/workflows/style.yml

@@ -33,7 +33,7 @@ jobs:
 
   clippy:
     runs-on: ubuntu-24.04
-    timeout-minutes: 15
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
       - uses: Swatinem/rust-cache@v2
@@ -45,7 +45,7 @@ jobs:
       - name: Install dependencies
         run: |
           sudo apt update
-          sudo apt install -y protobuf-compiler
+          sudo apt install -y clang libclang-dev protobuf-compiler
       - name: Clippy
         run: cargo clippy --workspace --all-targets -- -D warnings
 

.gitignore

@@ -74,6 +74,7 @@ docs/_build/
 # PyBuilder
 .pybuilder/
 target/
+/master-data/
 
 # Jupyter Notebook
 .ipynb_checkpoints

README.md

@@ -128,8 +128,10 @@ crates/lance-context-core    # Rust engine: Context + Rollout stores (no Python
 crates/lance-context-api     # Shared request/response types (DTOs)
 crates/lance-context-server  # HTTP server for remote access
 crates/lance-context-client  # HTTP client for the server
+crates/lance-context-master  # Control plane, scheduler, and admin UI
 crates/lance-context         # Re-export crate for downstream clients
 python/                      # Python bindings (PyO3) + tests
+deploy/kubernetes/           # Kubernetes master + RocksDB PVC example
 examples/                    # Runnable example projects
 ```
 

crates/lance-context-master/Cargo.toml

@@ -25,6 +25,7 @@ futures = "0.3"
 lance = "7.0.0"
 metrics = "0.24"
 reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
+rocksdb = { version = "0.24", default-features = false }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 tokio = { version = "1", features = ["rt-multi-thread", "macros", "signal", "sync"] }

crates/lance-context-master/src/config.rs

@@ -56,6 +56,22 @@ pub struct MasterConfig {
     #[arg(long, env = "TASK_CONCURRENCY", default_value_t = 4)]
     pub task_concurrency: usize,
 
+    /// Local RocksDB directory for durable scheduler task state. This must be
+    /// backed by persistent local storage in production; it must not be an
+    /// object-store URI or a volume shared by multiple master processes.
+    #[arg(
+        long,
+        env = "TASK_DB_PATH",
+        default_value = "./master-data/tasks.rocksdb"
+    )]
+    pub task_db_path: String,
+
+    /// Maximum number of terminal scheduler tasks retained in RocksDB and the
+    /// queue UI. Queued and running tasks are never pruned, and at least one
+    /// terminal task is retained for status polling.
+    #[arg(long, env = "TASK_HISTORY_LIMIT", default_value_t = 1_000)]
+    pub task_history_limit: usize,
+
     /// Directory of built UI assets to serve. When unset, only the JSON API is
     /// exposed.
     #[arg(long, env = "UI_DIR")]

crates/lance-context-master/src/lib.rs

@@ -8,3 +8,4 @@ pub mod scanner;
 pub mod scheduler;
 pub mod state;
 pub mod stats_store;
+pub mod task_store;

crates/lance-context-master/src/main.rs

@@ -22,6 +22,10 @@ async fn main() {
     let config = MasterConfig::parse();
     let addr = format!("{}:{}", config.host, config.port);
 
+    // Install the recorder before state recovery and scheduler startup so
+    // recovered queue depth is represented immediately.
+    let metrics_handle = lance_context_metrics::install_recorder();
+
     if let Err(e) = create_local_dir_if_needed(&config.data_dir) {
         tracing::error!(
             "Failed to create local data directory '{}': {}",
@@ -47,9 +51,6 @@ async fn main() {
     // Single serial compaction worker (+ optional periodic auto-sweep).
     let _scheduler = scheduler::spawn_scheduler(&state);
 
-    // Install the Prometheus recorder once, before any metrics are emitted.
-    let metrics_handle = lance_context_metrics::install_recorder();
-
     let mut app = Router::new().nest("/api/v1", routes::api_router());
 
     // Serve the built UI (SPA) as a fallback when a `--ui-dir` is configured.

crates/lance-context-master/src/routes.rs

@@ -277,7 +277,9 @@ pub async fn compact_experiment(
             name
         )));
     }
-    let task = scheduler::enqueue(&state, TaskKind::Compact, &name).await;
+    let task = scheduler::enqueue(&state, TaskKind::Compact, &name)
+        .await
+        .map_err(MasterError::from_lance)?;
     Ok((StatusCode::ACCEPTED, Json(task_to_compact_status(&task))))
 }
 
@@ -312,7 +314,9 @@ pub async fn enqueue_task(
             req.target
         )));
     }
-    let task = scheduler::enqueue(&state, req.kind, &req.target).await;
+    let task = scheduler::enqueue(&state, req.kind, &req.target)
+        .await
+        .map_err(MasterError::from_lance)?;
     Ok((StatusCode::ACCEPTED, Json(task)))
 }
 
@@ -418,6 +422,12 @@ mod tests {
             target_rows_per_fragment: 1_048_576,
             worker_endpoints: vec![],
             task_concurrency: 4,
+            task_db_path: dir
+                .path()
+                .join("master-tasks")
+                .to_string_lossy()
+                .to_string(),
+            task_history_limit: 1_000,
             ui_dir: None,
         }
     }