Support M6/32 GiB Flint, oQ3e selection and prefill throughput

.github/workflows/release.yml

@@ -17,10 +17,14 @@ jobs:
       - name: Build and package
         run: |
           version=${GITHUB_REF_NAME#v}
-          uv python install 3.12.12
-          python_root="$(uv python dir)/cpython-3.12.12-macos-aarch64-none"
-          "$python_root/bin/python3.12" packaging/build-mlx-runtime.py --python-root "$python_root" --output .build/bundle-mlx
+          if ! xcrun -sdk macosx metal --version > /dev/null 2>&1; then
+            xcodebuild -downloadComponent MetalToolchain
+          fi
+          uv python install 3.14.10
+          python_root="$(uv python dir)/cpython-3.14.10-macos-aarch64-none"
+          "$python_root/bin/python3.14" packaging/build-mlx-runtime.py --python-root "$python_root" --output .build/bundle-mlx
           python3 packaging/package.py --runtime-dir .build/bundle-mlx --manifest packaging/flint-int4-mlx.json \
+            --low-memory-manifest packaging/flint-mixed-3-4bit-mlx.json \
             --version "$version" --output dist/rec-darwin-arm64
           test "$(dist/rec-darwin-arm64 --version)" = "rec $version"
       - name: Publish

.github/workflows/test.yml

@@ -41,4 +41,9 @@ jobs:
         with:
           node-version: '22'
       - *cache-clients
+      - name: Ensure Metal compiler
+        run: |
+          if ! xcrun -sdk macosx metal --version > /dev/null 2>&1; then
+            xcodebuild -downloadComponent MetalToolchain
+          fi
       - run: make test-engine

Makefile

@@ -20,16 +20,16 @@ test-cli:
 	python3 tests/installer_test.py
 	python3 tests/package_test.py
 
-PYTHON_ROOT = $(shell uv python dir)/cpython-3.12.12-macos-aarch64-none
+PYTHON_ROOT = $(shell uv python dir)/cpython-3.14.10-macos-aarch64-none
 
 test-engine:
-	cd engine && uv venv --allow-existing --python 3.12 && uv pip install '.[validation]'
+	cd engine && uv venv --allow-existing --python 3.14.10 && uv pip install '.[validation]'
 	cd engine && .venv/bin/pytest -v -rs
-	uv python install 3.12.12
+	uv python install 3.14.10
 	rm -rf .build/bundle-mlx .build/smoke
-	$(PYTHON_ROOT)/bin/python3.12 packaging/build-mlx-runtime.py --python-root $(PYTHON_ROOT) --output .build/bundle-mlx
+	$(PYTHON_ROOT)/bin/python3.14 packaging/build-mlx-runtime.py --python-root $(PYTHON_ROOT) --output .build/bundle-mlx
 	python3 tests/build_smoke_runtime.py --runtime .build/bundle-mlx --output .build/smoke
 	python3 packaging/package.py --runtime-dir .build/smoke/runtime --manifest .build/smoke/smoke-mlx.json --output .build/rec-smoke
 	python3 tests/smoke_test.py .build/rec-smoke
 	go build -o .build/rec ./cli
-	python3 tests/protocol_test.py .build/bundle-mlx/python/bin/python3.12 .build/rec
+	python3 tests/protocol_test.py .build/bundle-mlx/python/bin/python3.14 .build/rec

README.md

@@ -26,11 +26,16 @@ excluded.</sub></p>
 
 ## Requirements
 
-- Apple Silicon Mac with an M4 Pro, M4 Max, M5 Pro, or M5 Max
-- At least 48 GB unified memory
+- Apple Silicon Mac with an M4 Pro, M4 Max, M5 Pro, M5 Max, or M6
+- At least 32 GB unified memory
 - About 30 GB free disk space
 - [Node.js 22](https://nodejs.org/en/download) and npm for pi
 
+On Macs with 32 GB (or less than 48 GB), packaged builds select the community
+[Flint mixed 3/4-bit checkpoint](https://huggingface.co/mlx-community/flint-mixed-3-4bit)
+(17.8 GB), with a 262,144-token default context. Macs with 48 GB or more use the
+official INT4 checkpoint. The mixed checkpoint is an unofficial community conversion.
+Use `rec doctor` to see the selected revision and context.
 
 ## Install
 
@@ -84,6 +89,15 @@ rec serve
 Keep this terminal open. When `flint is ready.` appears, connect a client from
 another terminal.
 
+After each prompt finishes processing, the server logs prefill throughput:
+
+```text
+Prefill: prompt=74051 common_prefix=62433 reused=62433 new=11618 200 t/s
+```
+
+The rate is newly processed prompt tokens divided by prefill time. Fully cached
+prompts report `n/a t/s`. This measures prefill, rather than generated-token speed.
+
 To use your own install of pi or OpenCode, run:
 
 ```bash
@@ -141,3 +155,38 @@ Run `rec COMMAND --help` for a command's options.
 - Community guidelines: [Code of conduct](CODE_OF_CONDUCT.md).
 - Security vulnerabilities: [Private reporting instructions](SECURITY.md).
 - License: [Apache 2.0](LICENSE).
+
+## Runtime build
+
+The bundled runtime uses CPython 3.14.10, nanobind 3.1.0, and MLX built from
+GitHub commit `fc344c986c4169d99f76b5d95ae6b4a58a359a63` (HEAD when upgraded).
+
+### Choose the Flint quantization
+
+This development binary includes two small-memory checkpoints:
+
+```sh
+rec serve --quant 3/4
+rec serve --quant oq3e
+```
+
+Stop the existing server before switching (`rec stop`). `--quant` also works
+with `rec pull` and `rec doctor`; `auto` preserves RAM-based selection. Explicit
+`--manifest` and explicit `--quant` are mutually exclusive.
+
+`3/4` selects `mlx-community/flint-mixed-3-4bit` with the existing optimized
+Rec kernels. `oq3e` selects pinned `mlx-community/flint-oQ3e` weights, preserving
+their per-module 3/6/8-bit affine quantization through native MLX operations.
+The oQ3e path reuses a matching prompt prefix using isolated native MLX KV
+snapshots and BF16 KV. It is not the resident fused decode path, so performance
+differs. Both small-memory profiles default to 262,144 total tokens. Override
+with `--context TOKENS`; the architectural maximum does not guarantee available RAM.
+
+Prefix reuse is automatic for oQ3e. A snapshot immediately before the previous
+prompt's final token can be reused when it matches the beginning of the next
+request. Different prefixes recompute safely; this is a single in-memory prefix,
+not a disk cache or arbitrary branching cache. Decode runs on separate cache
+objects, including sliding-window offsets, so it cannot mutate the saved prefix.
+The log and API usage report reused/cached prompt tokens. On one 252-token M6
+smoke prompt, reuse was 251 tokens: 0.029 s prefill versus 0.255 s uncached after
+warm-up. Deterministic cached and uncached output token IDs matched exactly.

cli/cli.go

@@ -90,6 +90,7 @@ func run(args []string) error {
 	}
 	flags := flag.NewFlagSet(command, flag.ContinueOnError)
 	manifest := flags.String("manifest", "", "")
+	quant := flags.String("quant", "auto", "")
 	runtimeDir := flags.String("runtime-dir", "", "")
 	var contextTokens int
 	port := 11500
@@ -108,6 +109,9 @@ func run(args []string) error {
 		return fmt.Errorf("port must be between 1 and 65535")
 	}
 	m, e := loadManifest(*manifest)
+	if e == nil {
+		m, e = selectQuant(m, *quant, *manifest)
+	}
 	if e == nil && command == "serve" {
 		flags.Visit(func(f *flag.Flag) {
 			if f.Name == "context" {

cli/engine.go

@@ -15,10 +15,10 @@ func launchCommand(p Paths, m Manifest, runtimeDir, model string, port int) (str
 }
 
 func checkAppleChip(name string) error {
-	for _, chip := range []string{"Apple M4 Pro", "Apple M4 Max", "Apple M5 Pro", "Apple M5 Max"} {
+	for _, chip := range []string{"Apple M4 Pro", "Apple M4 Max", "Apple M5 Pro", "Apple M5 Max", "Apple M6"} {
 		if name == chip {
 			return nil
 		}
 	}
-	return fmt.Errorf("Flint requires an Apple M4 Pro, M4 Max, M5 Pro or M5 Max; detected %s", name)
+	return fmt.Errorf("Flint requires an Apple M4 Pro, M4 Max, M5 Pro, M5 Max or M6; detected %s", name)
 }

cli/help.go

@@ -15,6 +15,7 @@ type commandHelp struct {
 }
 
 var developmentOptions = [][2]string{
+	{"--quant PROFILE", "Checkpoint: auto, 3/4, or oq3e (native MLX)"},
 	{"--manifest FILE", "Use this manifest instead of the bundled one"},
 	{"--runtime-dir DIR", "Use this runtime instead of the bundled one; needs --manifest"},
 }

cli/manifest.go

@@ -149,12 +149,36 @@ func loadManifest(path string) (Manifest, error) {
 		return Manifest{}, err
 	}
 	m, err := decodeManifest(data)
+	if err == nil && path == "" {
+		lowData, lowErr := assets.ReadFile("assets/manifest-low-memory.json")
+		if lowErr == nil {
+			low, decodeErr := decodeManifest(lowData)
+			if decodeErr != nil {
+				return Manifest{}, decodeErr
+			}
+			ram, memoryErr := physicalMemory()
+			if memoryErr != nil {
+				return Manifest{}, memoryErr
+			}
+			m = manifestForMemory(m, low, ram)
+		} else if !os.IsNotExist(lowErr) {
+			return Manifest{}, lowErr
+		}
+	}
 	if err == nil && (m.OS != runtime.GOOS || m.Arch != runtime.GOARCH) {
 		err = fmt.Errorf("bundle targets %s/%s, this machine is %s/%s", m.OS, m.Arch, runtime.GOOS, runtime.GOARCH)
 	}
 	return m, err
 }
 
+// Select the smaller checkpoint below 48 GiB. Explicit --manifest bypasses this.
+func manifestForMemory(primary, low Manifest, ram int64) Manifest {
+	if ram < 48*1024*1024*1024 {
+		return low
+	}
+	return primary
+}
+
 func digest(b []byte) string        { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) }
 func modelDigest(m Manifest) string { b, _ := json.Marshal(m.Files); return digest(b) }
 
@@ -191,3 +215,27 @@ func verifyFileInto(path string, f File, progress io.Writer) error {
 	}
 	return nil
 }
+
+// Explicit profiles share the packaged engine but retain separate model hashes.
+func selectQuant(current Manifest, quant, manifest string) (Manifest, error) {
+	if quant == "auto" {
+		return current, nil
+	}
+	if manifest != "" {
+		return Manifest{}, fmt.Errorf("--quant cannot be combined with --manifest")
+	}
+	name := ""
+	switch quant {
+	case "3/4", "mixed-3-4":
+		name = "manifest-low-memory.json"
+	case "oq3e":
+		name = "manifest-oq3e.json"
+	default:
+		return Manifest{}, fmt.Errorf("unknown quant %q; choose auto, 3/4, or oq3e", quant)
+	}
+	data, err := assets.ReadFile("assets/" + name)
+	if err != nil {
+		return Manifest{}, fmt.Errorf("quant %s is not included in this binary; rebuild the package", quant)
+	}
+	return decodeManifest(data)
+}

cli/memory_test.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+)
+
+func TestManifestForMemory(t *testing.T) {
+	primary := Manifest{Revision: "int4", Context: 262144, Entry: "flint-int4.json"}
+	low := Manifest{Revision: "mixed", Context: 131072, Entry: "config.json"}
+	for _, gib := range []int64{16, 24, 32, 36, 47, 48, 64, 128} {
+		want := primary
+		if gib < 48 {
+			want = low
+		}
+		got := manifestForMemory(primary, low, gib<<30)
+		if got.Revision != want.Revision || got.Context != want.Context || got.Entry != want.Entry {
+			t.Fatalf("%d GiB: got %+v, want %+v", gib, got, want)
+		}
+	}
+}
+
+func TestEmbeddedLowMemoryManifest(t *testing.T) {
+	data, err := assets.ReadFile("assets/manifest-low-memory.json")
+	if os.IsNotExist(err) {
+		t.Skip("no alternate profile packaged")
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	low, err := decodeManifest(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+	data, err = assets.ReadFile("assets/manifest.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	primary, err := decodeManifest(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Run("explicit profile", func(t *testing.T) {
+		if runtime.GOOS != primary.OS || runtime.GOARCH != primary.Arch {
+			t.Skip("explicit profiles require a matching host target")
+		}
+		// An explicit manifest must retain the requested profile on a 32 GiB Mac.
+		path := filepath.Join(t.TempDir(), "manifest.json")
+		if err := os.WriteFile(path, data, 0600); err != nil {
+			t.Fatal(err)
+		}
+		explicit, err := loadManifest(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if explicit.Revision != primary.Revision {
+			t.Fatal("explicit profile was replaced")
+		}
+	})
+	if low.RuntimeSHA != primary.RuntimeSHA || low.MinRAM != 32<<30 || low.Context != 262144 || low.Entry != "config.json" {
+		t.Fatalf("invalid low-memory profile: %+v", low)
+	}
+}
+
+func TestQuantSelectionErrors(t *testing.T) {
+	for _, quant := range []string{"bad", "4bit"} {
+		if _, err := selectQuant(Manifest{}, quant, ""); err == nil {
+			t.Fatalf("accepted %q", quant)
+		}
+	}
+	if _, err := selectQuant(Manifest{}, "oq3e", "custom.json"); err == nil {
+		t.Fatal("accepted conflicting manifest")
+	}
+	m, err := selectQuant(Manifest{Revision: "test"}, "auto", "custom.json")
+	if err != nil || m.Revision != "test" {
+		t.Fatal("auto changed explicit manifest")
+	}
+}

cli/platform.go

@@ -21,11 +21,7 @@ func preflight(m Manifest) error {
 	if runtime.GOOS != "darwin" || runtime.GOARCH != "arm64" {
 		return fmt.Errorf("rec requires an Apple Silicon Mac")
 	}
-	s, err := command("/usr/sbin/sysctl", "-n", "hw.memsize")
-	if err != nil {
-		return err
-	}
-	ram, err := strconv.ParseInt(s, 10, 64)
+	ram, err := physicalMemory()
 	if err != nil {
 		return err
 	}
@@ -41,3 +37,11 @@ func preflight(m Manifest) error {
 	}
 	return checkAppleChip(chip)
 }
+
+func physicalMemory() (int64, error) {
+	s, err := command("/usr/sbin/sysctl", "-n", "hw.memsize")
+	if err != nil {
+		return 0, err
+	}
+	return strconv.ParseInt(s, 10, 64)
+}

cli/rec_test.go

@@ -356,7 +356,7 @@ func TestManifestSchemaAndPaths(t *testing.T) {
 }
 
 func TestAppleChipAllowlist(t *testing.T) {
-	for chip, ok := range map[string]bool{"Apple M4 Pro": true, "Apple M5 Pro": true, "Apple M5 Max": true, "Apple M4 Max": true, "Apple M4": false, "Apple M1 Pro": false} {
+	for chip, ok := range map[string]bool{"Apple M4 Pro": true, "Apple M5 Pro": true, "Apple M5 Max": true, "Apple M4 Max": true, "Apple M6": true, "Apple M6 Pro": false, "Apple M4": false, "Apple M1 Pro": false} {
 		if (checkAppleChip(chip) == nil) != ok {
 			t.Fatalf("%s: want supported=%t", chip, ok)
 		}

engine/M6_TUNING.md

@@ -0,0 +1,24 @@
+# M6 schedule experiments
+
+M6 uses a dedicated device profile: 36 attention workers, 32 expert workers, 32 output-head workers, 2048-token prefill, and generic mixed experts. No tested candidate showed a consistent improvement sufficient to replace it.
+
+Apple documents 12 GPU cores versus 10 on base M5, up to 170 GB/s versus 153 GB/s bandwidth, and updated Dynamic Caching: https://www.apple.com/newsroom/2026/08/apple-introduces-m6-and-m5-ultra-for-a-big-leap-in-performance-and-ai-compute/
+
+This motivated occupancy and chunk-size experiments. These scalar Metal kernels do not automatically use the Neural Engine or GPU Neural Accelerators.
+
+Local M6 32 GiB, mixed 3/4-bit Flint, MLX HEAD, temperature 1/top-p .95/top-k 20, 128 generated tokens after 16-token warmup, cold prompt cache. One measured run per candidate; baseline repeated at the end. This is exploratory tuning, not statistically established speedup evidence.
+
+| Configuration (attention/expert/head; chunk; specialized) | 34 tokens | 4095 tokens | 8192 tokens |
+|---|---:|---:|---:|
+| 36/32/32; 2048; off | 85.1 | 75.4 | 70.6 |
+| 48/32/32; 2048; off | 84.2 | 74.4 | 68.0 |
+| 36/32/16; 2048; off | 80.6 | 74.9 | 67.6 |
+| 36/32/32; 1024; off | 81.9 | 75.5 | 69.9 |
+| 32/32/32; 2048; off | 84.2 | 73.8 | 67.1 |
+| 40/32/32; 2048; off | 83.3 | 73.0 | 67.3 |
+| 32/32/32; 2048; on | 75.2 | 75.0 | 69.8 |
+| Baseline repeat | 84.6 | 73.9 | 67.9 |
+
+Throughput is tokens/second. Baseline drift at 8192 tokens (70.6 to 67.9) means small differences are inconclusive. A 24-expert-worker trial was rejected by the existing 32/64-worker constraint.
+
+The earlier 4096-token/M5-specialization experiment passed 61 scalar/packet, 9 EOS and 3 cancellation cases but retained 72 independent native-reference failures. This tuning does not resolve that limitation. Candidate schedules above received generation/performance checks; they are not promoted as numerically validated alternatives.

engine/hatch_build.py

@@ -1,5 +1,6 @@
 import subprocess
 import sys
+import sysconfig
 from pathlib import Path
 
 from hatchling.builders.hooks.plugin.interface import BuildHookInterface
@@ -12,4 +13,6 @@ def initialize(self, version, build_data):
         subprocess.run([sys.executable, str(Path(self.root) / "native/build.py")], check=True)
         build_data["pure_python"] = False
         build_data["infer_tag"] = True
-        build_data["artifacts"] = ["src/rec_mlx/_grammar_native*.so"]
+        build_data["artifacts"] = [
+            "src/rec_mlx/_grammar_native" + sysconfig.get_config_var("EXT_SUFFIX")
+        ]