feat: add V8 code cache with in-memory LRU

Replace the old snapshot-based caching with V8 code cache:
- On first execution, transpile and run the worker code
- In the background, create a code cache (compiled bytecode)
- On subsequent requests, load from in-memory LRU (5000 entries)

Code cache bundles embed source + bytecode in a simple wire format:
[magic 0xC0DECA5E][source_len][source][cache_bytes]

Author: max-lt

Cargo.lock

@@ -37,6 +37,7 @@ tokio = "1.49.0"
 tokio-util = { version = "0.7", features = ["rt"] }
 flume = "0.12.0"
 once_cell = "1.19"
+# filetime = "0.2"  # was used by disk-based snapshot cache
 env_logger = "0.11.6"
 lru = "0.16.3"
 
@@ -62,7 +63,7 @@ openworkers-core = { git = "https://github.com/openworkers/openworkers-core", ta
 openworkers-transform = { git = "https://github.com/openworkers/openworkers-transform", tag = "v0.1.0" }
 
 # Runtime backend (v8 only for now, others require older version of core)
-openworkers-runtime-v8 = { git = "https://github.com/openworkers/openworkers-runtime-v8", tag = "v0.13.2", optional = true, features = ["ptrcomp"] }
+openworkers-runtime-v8 = { path = "../openworkers-runtime-v8", optional = true, features = ["ptrcomp"] }
 
 # WASM runtime (optional)
 # openworkers-runtime-wasm = { path = "../openworkers-runtime-wasm", optional = true }

Cargo.toml

@@ -11,6 +11,8 @@ pub mod nats;
 pub mod ops;
 pub mod runtime;
 pub mod services;
+#[cfg(feature = "v8")]
+pub mod snapshot_cache;
 pub mod store;
 pub mod task_executor;
 pub mod telemetry;

src/lib.rs

@@ -0,0 +1,45 @@
+//! In-memory LRU cache for worker snapshots.
+//!
+//! Stores V8 snapshot blobs in memory, keyed by `(worker_id, version)`.
+//! This avoids disk I/O on every request and keeps snapshot bytes readily
+//! available for V8 isolate creation.
+
+use std::num::NonZeroUsize;
+use std::sync::Mutex;
+
+use lru::LruCache;
+use once_cell::sync::Lazy;
+
+/// Default max number of cached snapshots.
+const DEFAULT_MAX_ENTRIES: usize = 500;
+
+/// Cache key: (worker_id, version)
+type CacheKey = (String, i32);
+
+/// Global in-memory LRU cache for worker snapshots.
+static SNAPSHOT_CACHE: Lazy<Mutex<LruCache<CacheKey, Vec<u8>>>> = Lazy::new(|| {
+    let max_entries = std::env::var("SNAPSHOT_CACHE_MAX")
+        .ok()
+        .and_then(|v| v.parse().ok())
+        .unwrap_or(DEFAULT_MAX_ENTRIES);
+
+    tracing::info!("Snapshot cache: in-memory, max_entries={}", max_entries);
+
+    Mutex::new(LruCache::new(
+        NonZeroUsize::new(max_entries).unwrap_or(NonZeroUsize::new(DEFAULT_MAX_ENTRIES).unwrap()),
+    ))
+});
+
+/// Try to read a cached worker snapshot from memory.
+pub fn get(worker_id: &str, version: i32) -> Option<Vec<u8>> {
+    let key = (worker_id.to_string(), version);
+    let mut cache = SNAPSHOT_CACHE.lock().unwrap();
+    cache.get(&key).cloned()
+}
+
+/// Store a worker snapshot in memory.
+pub fn put(worker_id: &str, version: i32, snapshot: &[u8]) {
+    let key = (worker_id.to_string(), version);
+    let mut cache = SNAPSHOT_CACHE.lock().unwrap();
+    cache.put(key, snapshot.to_vec());
+}

src/snapshot_cache.rs

@@ -169,6 +169,11 @@ pub async fn execute_task_await_v8_pooled(
     let components = prepare_task_components(&config)
         .ok_or_else(|| TerminationReason::Other("Failed to prepare script".to_string()))?;
 
+    // Capture JS code for background code cache creation (before script is moved)
+    let js_code_for_snapshot = components.script.code.as_js().map(|s| s.to_string());
+    let worker_id_for_snapshot = config.worker_data.id.clone();
+    let version_for_snapshot = config.worker_data.version;
+
     // Use user_id (tenant) for isolate pool isolation instead of worker_id
     // This prevents a single tenant from monopolizing resources via multiple workers
     let owner_id = config.worker_data.user_id.clone();
@@ -223,6 +228,37 @@ pub async fn execute_task_await_v8_pooled(
                 // CRITICAL: Flush logs before returning
                 components.log_handler.flush();
 
+                // After successful JS execution, create code cache in background
+                if result.is_ok()
+                    && let Some(js_code) = js_code_for_snapshot
+                {
+                    let worker_id = worker_id_for_snapshot;
+                    let version = version_for_snapshot;
+
+                    tokio::task::spawn_blocking(move || {
+                        match openworkers_runtime_v8::create_code_cache(&js_code) {
+                            Ok(cache) => {
+                                let packed =
+                                    openworkers_runtime_v8::pack_code_cache(&js_code, &cache);
+                                crate::snapshot_cache::put(&worker_id, version, &packed);
+                                tracing::debug!(
+                                    "Created code cache: worker={}, version={}, size={}",
+                                    crate::utils::short_id(&worker_id),
+                                    version,
+                                    packed.len()
+                                );
+                            }
+                            Err(e) => {
+                                tracing::warn!(
+                                    "Failed to create code cache for worker={}: {}",
+                                    crate::utils::short_id(&worker_id),
+                                    e
+                                );
+                            }
+                        }
+                    });
+                }
+
                 result
             }
             .instrument(span)

src/task_executor.rs

@@ -5,27 +5,11 @@
 //! - wasm only: direct type alias to WasmWorker (zero overhead)
 //! - both: enum with runtime dispatch based on CodeType
 
-use std::num::NonZeroUsize;
-use std::sync::{Arc, Mutex};
-
-use lru::LruCache;
-use once_cell::sync::Lazy;
+use std::sync::Arc;
 
 use crate::ops::RunnerOperations;
 use crate::store::{CodeType, WorkerWithBindings, bindings_to_infos};
 
-// =============================================================================
-// Transpiled code cache
-// =============================================================================
-
-/// Cache key: (worker_id, version)
-type CacheKey = (String, i32);
-
-/// LRU cache for transpiled JavaScript code
-/// Capacity: 1000 workers (transpiled code is relatively small)
-static TRANSPILE_CACHE: Lazy<Mutex<LruCache<CacheKey, String>>> =
-    Lazy::new(|| Mutex::new(LruCache::new(NonZeroUsize::new(1000).unwrap())));
-
 #[cfg(all(feature = "v8", feature = "wasm"))]
 use openworkers_core::Event;
 
@@ -113,35 +97,33 @@ pub async fn create_worker(
 // Script preparation (shared across all configurations)
 // =============================================================================
 
-/// Parse worker code based on code type (with caching for JS/TS)
+/// Parse worker code based on code type.
+///
+/// For JS/TS workers: checks the code cache (fast path) and returns cached
+/// bytecode. On cache miss, transpiles the source and returns JS — a code
+/// cache entry is created in the background after first successful execution.
 fn parse_code(data: &WorkerWithBindings) -> Result<WorkerCode, TerminationReason> {
     match data.code_type {
         CodeType::Javascript | CodeType::Typescript => {
             #[cfg(feature = "v8")]
             {
-                let cache_key = (data.id.clone(), data.version);
-
-                // Try to get from cache first
-                {
-                    let mut cache = TRANSPILE_CACHE.lock().unwrap();
-
-                    if let Some(cached_code) = cache.get(&cache_key) {
-                        tracing::debug!(
-                            "transpile cache HIT: worker={}, version={}",
-                            crate::utils::short_id(&data.id),
-                            data.version
-                        );
-                        return Ok(WorkerCode::js(cached_code.clone()));
-                    }
+                // Fast path: check code cache
+                if let Some(snapshot) = crate::snapshot_cache::get(&data.id, data.version) {
+                    tracing::debug!(
+                        "code cache HIT: worker={}, version={}",
+                        crate::utils::short_id(&data.id),
+                        data.version
+                    );
+                    return Ok(WorkerCode::snapshot(snapshot));
                 }
 
                 tracing::debug!(
-                    "transpile cache MISS: worker={}, version={}",
+                    "code cache MISS: worker={}, version={}",
                     crate::utils::short_id(&data.id),
                     data.version
                 );
 
-                // Transpile and cache
+                // Slow path: transpile code (code cache created after first execution)
                 let language = match data.code_type {
                     CodeType::Javascript => openworkers_transform::CodeLanguage::JavaScript,
                     CodeType::Typescript => openworkers_transform::CodeLanguage::TypeScript,
@@ -156,11 +138,6 @@ fn parse_code(data: &WorkerWithBindings) -> Result<WorkerCode, TerminationReason
                         ))
                     })?;
 
-                {
-                    let mut cache = TRANSPILE_CACHE.lock().unwrap();
-                    cache.put(cache_key, transpiled.clone());
-                }
-
                 Ok(WorkerCode::js(transpiled))
             }