[pull] canary from vercel:canary

Cargo.toml

@@ -358,6 +358,7 @@ tracing = "0.1.44"
 tracing-subscriber = "0.3.16"
 triomphe = { git = "https://github.com/sokra/triomphe", branch = "sokra/unstable" }
 xxhash-rust = { version = "0.8.12", features = ["xxh3"] }
+zstd = "0.13.2"
 unsize = "1.1.0"
 unty = "0.0.4"
 url = "2.2.2"

lerna.json

@@ -15,5 +15,5 @@
       "registry": "https://registry.npmjs.org/"
     }
   },
-  "version": "16.4.0-canary.14"
+  "version": "16.4.0-canary.15"
 }

packages/create-next-app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-next-app",
-  "version": "16.4.0-canary.14",
+  "version": "16.4.0-canary.15",
   "keywords": [
     "react",
     "next",

packages/devlow-bench/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@vercel/devlow-bench",
   "private": true,
-  "version": "16.4.0-canary.14",
+  "version": "16.4.0-canary.15",
   "description": "Benchmarking tool for the developer workflow",
   "repository": {
     "type": "git",

packages/eslint-config-next/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eslint-config-next",
-  "version": "16.4.0-canary.14",
+  "version": "16.4.0-canary.15",
   "description": "ESLint configuration used by Next.js.",
   "license": "MIT",
   "repository": {
@@ -12,7 +12,7 @@
     "dist"
   ],
   "dependencies": {
-    "@next/eslint-plugin-next": "16.4.0-canary.14",
+    "@next/eslint-plugin-next": "16.4.0-canary.15",
     "eslint-import-resolver-node": "^0.3.6",
     "eslint-import-resolver-typescript": "^3.5.2",
     "eslint-plugin-import": "^2.32.0",

packages/eslint-plugin-internal/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@next/eslint-plugin-internal",
   "private": true,
-  "version": "16.4.0-canary.14",
+  "version": "16.4.0-canary.15",
   "description": "ESLint plugin for working on Next.js.",
   "exports": {
     ".": "./src/eslint-plugin-internal.js"

packages/eslint-plugin-next/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@next/eslint-plugin-next",
-  "version": "16.4.0-canary.14",
+  "version": "16.4.0-canary.15",
   "description": "ESLint plugin for Next.js.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

packages/font/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@next/font",
   "private": true,
-  "version": "16.4.0-canary.14",
+  "version": "16.4.0-canary.15",
   "repository": {
     "url": "vercel/next.js",
     "directory": "packages/font"

packages/next-bundle-analyzer/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@next/bundle-analyzer",
-  "version": "16.4.0-canary.14",
+  "version": "16.4.0-canary.15",
   "main": "index.js",
   "types": "index.d.ts",
   "license": "MIT",

packages/next-codemod/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@next/codemod",
-  "version": "16.4.0-canary.14",
+  "version": "16.4.0-canary.15",
   "license": "MIT",
   "repository": {
     "type": "git",

packages/next-env/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@next/env",
-  "version": "16.4.0-canary.14",
+  "version": "16.4.0-canary.15",
   "keywords": [
     "react",
     "next",

packages/next-mdx/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@next/mdx",
-  "version": "16.4.0-canary.14",
+  "version": "16.4.0-canary.15",
   "main": "index.js",
   "license": "MIT",
   "repository": {

packages/next-playwright/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@next/playwright",
-  "version": "16.4.0-canary.14",
+  "version": "16.4.0-canary.15",
   "repository": {
     "url": "vercel/next.js",
     "directory": "packages/next-playwright"

packages/next-plugin-storybook/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@next/plugin-storybook",
-  "version": "16.4.0-canary.14",
+  "version": "16.4.0-canary.15",
   "repository": {
     "url": "vercel/next.js",
     "directory": "packages/next-plugin-storybook"

packages/next-polyfill-module/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@next/polyfill-module",
-  "version": "16.4.0-canary.14",
+  "version": "16.4.0-canary.15",
   "description": "A standard library polyfill for ES Modules supporting browsers (Edge 16+, Firefox 60+, Chrome 61+, Safari 10.1+)",
   "main": "dist/polyfill-module.js",
   "license": "MIT",

packages/next-polyfill-nomodule/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@next/polyfill-nomodule",
-  "version": "16.4.0-canary.14",
+  "version": "16.4.0-canary.15",
   "description": "A polyfill for non-dead, nomodule browsers.",
   "main": "dist/polyfill-nomodule.js",
   "license": "MIT",

packages/next-routing/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@next/routing",
-  "version": "16.4.0-canary.14",
+  "version": "16.4.0-canary.15",
   "keywords": [
     "react",
     "next",

packages/next-rspack/package.json

@@ -1,6 +1,6 @@
 {
   "name": "next-rspack",
-  "version": "16.4.0-canary.14",
+  "version": "16.4.0-canary.15",
   "repository": {
     "url": "vercel/next.js",
     "directory": "packages/next-rspack"

packages/next-swc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@next/swc",
-  "version": "16.4.0-canary.14",
+  "version": "16.4.0-canary.15",
   "private": true,
   "files": [
     "native/"

packages/next/package.json

@@ -1,6 +1,6 @@
 {
   "name": "next",
-  "version": "16.4.0-canary.14",
+  "version": "16.4.0-canary.15",
   "description": "The React Framework",
   "main": "./dist/server/next.js",
   "license": "MIT",
@@ -100,7 +100,7 @@
     ]
   },
   "dependencies": {
-    "@next/env": "16.4.0-canary.14",
+    "@next/env": "16.4.0-canary.15",
     "@swc/helpers": "0.5.23",
     "baseline-browser-mapping": "^2.9.19",
     "caniuse-lite": "^1.0.30001579",
@@ -164,11 +164,11 @@
     "@modelcontextprotocol/sdk": "1.18.1",
     "@mswjs/interceptors": "0.42.0",
     "@napi-rs/triples": "1.2.0",
-    "@next/font": "16.4.0-canary.14",
-    "@next/polyfill-module": "16.4.0-canary.14",
-    "@next/polyfill-nomodule": "16.4.0-canary.14",
-    "@next/react-refresh-utils": "16.4.0-canary.14",
-    "@next/swc": "16.4.0-canary.14",
+    "@next/font": "16.4.0-canary.15",
+    "@next/polyfill-module": "16.4.0-canary.15",
+    "@next/polyfill-nomodule": "16.4.0-canary.15",
+    "@next/react-refresh-utils": "16.4.0-canary.15",
+    "@next/swc": "16.4.0-canary.15",
     "@opentelemetry/api": "1.6.0",
     "@playwright/test": "1.61.0",
     "@rspack/core": "1.6.7",

packages/react-refresh-utils/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@next/react-refresh-utils",
-  "version": "16.4.0-canary.14",
+  "version": "16.4.0-canary.15",
   "description": "An experimental package providing utilities for React Refresh.",
   "repository": {
     "url": "vercel/next.js",

packages/third-parties/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@next/third-parties",
-  "version": "16.4.0-canary.14",
+  "version": "16.4.0-canary.15",
   "repository": {
     "url": "vercel/next.js",
     "directory": "packages/third-parties"
@@ -26,7 +26,7 @@
     "third-party-capital": "1.0.20"
   },
   "devDependencies": {
-    "next": "16.4.0-canary.14",
+    "next": "16.4.0-canary.15",
     "outdent": "0.8.0",
     "prettier": "2.5.1",
     "typescript": "6.0.2"

turbopack/crates/turbo-persistence/Cargo.toml

@@ -38,6 +38,7 @@ smallvec = { workspace = true }
 thread_local = { workspace = true }
 tracing = { workspace = true }
 xxhash-rust = { workspace = true }
+zstd = { workspace = true }
 
 [dev-dependencies]
 criterion = { workspace = true }

turbopack/crates/turbo-persistence/README.md

@@ -56,6 +56,7 @@ A meta file can contain metadata about multiple SST files. The metadata is store
 - Header
   - 4 bytes magic number (0xFE4ADA4A)
   - 4 bytes key family
+  - 1 byte compression algorithm, which must match the configuration used to open the database
   - 4 bytes count of obsolete SST files
   - foreach obsolete SST file
     - 4 bytes sequence number of the obsolete SST file
@@ -88,9 +89,11 @@ The SST file contains only data without any header.
 
 #### Block Compression
 
-Blocks can be stored compressed (LZ4) or uncompressed. The 4-byte header distinguishes them:
+Blocks can be stored compressed or uncompressed. The compression algorithm is specified in the meta file.
 
-- **Header > 0**: Block is LZ4 compressed. Header value is the uncompressed length.
+The 4-byte header distinguishes compressed from uncompressed storage:
+
+- **Header > 0**: Block is compressed with the family's configured algorithm. Header value is the uncompressed length.
 - **Header = 0**: Block is stored uncompressed. Actual length is derived from block offsets.
 
 #### Block Checksum
@@ -231,7 +234,7 @@ The plain value compressed with dynamic compression. Each blob file has an 8-byt
 
 - 4 bytes: uncompressed length (u32 big-endian)
 - 4 bytes: CRC32 checksum of the compressed data (u32 big-endian)
-- remaining bytes: LZ4-compressed value data
+- remaining bytes: value data compressed with the blob's key-family configuration
 
 The checksum is verified on the compressed data **before** decompression when the blob is read.
 

turbopack/crates/turbo-persistence/benches/mod.rs

@@ -10,9 +10,9 @@ use quick_cache::sync::GuardResult;
 use rand::{RngExt, SeedableRng, rngs::SmallRng, seq::SliceRandom};
 use tempfile::TempDir;
 use turbo_persistence::{
-    ArcBytes, BlockCache, CompactConfig, DbConfig as TpDbConfig, Entry, EntryValue, FamilyConfig,
-    FamilyKind, MetaEntryFlags, SerialScheduler, StaticSortedFile, StaticSortedFileMetaData,
-    TurboPersistence, hash_key, write_static_stored_file,
+    ArcBytes, BlockCache, CompactConfig, Compression, DbConfig as TpDbConfig, Entry, EntryValue,
+    FamilyConfig, FamilyKind, MetaEntryFlags, SerialScheduler, StaticSortedFile,
+    StaticSortedFileMetaData, TurboPersistence, hash_key, write_static_stored_file,
 };
 use turbo_tasks_malloc::TurboMalloc;
 
@@ -622,6 +622,7 @@ fn prefill_multi_value_database(
         family_configs: [FamilyConfig {
             name: "test",
             kind: FamilyKind::MultiValue,
+            compression: Compression::Lz4,
         }],
     };
     let db =
@@ -696,6 +697,7 @@ fn open_multi_value_db(path: &Path) -> TurboPersistence<SerialScheduler, 1> {
         family_configs: [FamilyConfig {
             name: "test",
             kind: FamilyKind::MultiValue,
+            compression: Compression::Lz4,
         }],
     };
     TurboPersistence::<SerialScheduler, 1>::open_with_config(path.to_path_buf(), db_config).unwrap()
@@ -964,6 +966,7 @@ fn bench_write_multi_value(c: &mut Criterion) {
                             family_configs: [FamilyConfig {
                                 name: "test",
                                 kind: FamilyKind::MultiValue,
+                                compression: Compression::Lz4,
                             }],
                         };
                         let db = TurboPersistence::<SerialScheduler, 1>::open_with_config(
@@ -1201,15 +1204,20 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
             // Create temp directory and write SST file
             let tempdir = tempfile::tempdir().unwrap();
             let sst_path = tempdir.path().join("00000001.sst");
-            let (meta, _file) =
-                write_static_stored_file(&entries, &sst_path, MetaEntryFlags::FRESH).unwrap();
+            let (meta, _file) = write_static_stored_file(
+                &entries,
+                &sst_path,
+                MetaEntryFlags::FRESH,
+                Compression::Lz4,
+            )
+            .unwrap();
 
             // Open the SST file
             let sst_meta = StaticSortedFileMetaData {
                 sequence_number: 1,
                 block_count: meta.block_count,
             };
-            let sst = StaticSortedFile::open(tempdir.path(), sst_meta).unwrap();
+            let sst = StaticSortedFile::open(tempdir.path(), sst_meta, Compression::Lz4).unwrap();
 
             // Create block caches
             let key_block_cache: BlockCache = BlockCache::with(