From 67c9756af47bbce3c5612123085edf06d3a32de7 Mon Sep 17 00:00:00 2001 From: wcole Date: Tue, 27 Oct 2026 17:05:01 -0400 Subject: [PATCH 1/3] Add host arg to cli and engine --- README.md | 80 ++++++++++++++++++++++++++++++ cli/cli.go | 12 ++++- cli/context_test.go | 58 +++++++++++++++++----- cli/engine.go | 4 +- cli/help.go | 3 +- cli/launch.go | 6 ++- cli/launch_test.go | 23 +++++++++ cli/pi_test.go | 2 + cli/serve.go | 82 ++++++++++++++++++++++++------- cli/serve_test.go | 41 ++++++++++++++-- cli/testdata/fake-rec-mlx | 17 ++++++- engine/src/rec_mlx/cli.py | 10 ++-- engine/src/rec_mlx/server.py | 12 +++++ engine/tests/test_cli_defaults.py | 28 +++++++++++ engine/tests/test_serving.py | 28 +++++++++++ tests/data/smoke-mlx-server.py | 17 ++++++- 16 files changed, 378 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 0e08a2a..92ab096 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,16 @@ rec serve Keep this terminal open. When `flint is ready.` appears, connect a client from another terminal. +To expose the server on your LAN, bind to all interfaces: + +```bash +rec serve --host 0.0.0.0 +``` + +From another machine, connect to `http://:11500/v1` with the +`rec auth token` key. Traffic is unencrypted, so only use this on trusted +networks. + To use your own install of pi or OpenCode, run: ```bash @@ -114,6 +124,76 @@ The Anthropic-compatible API is available at `http://127.0.0.1:11500` and send the token from `rec auth token` as a bearer token or as `x-api-key`. The model is `flint`. +
+Connect pi or OpenCode on another machine + +On the Mac running rec, collect three values: + +- **Base URL**: `http://:11500/v1`. +- **API key**: the output of `rec auth token`. +- **Context**: shown by `rec status` (262,144 tokens by default). Use it as both + the context and output limit below. rec fits each reply to the space left. + +On the other machine, make the key available to the harness: + +```bash +export REC_API_KEY='' +``` + +**pi**: add the provider to `~/.pi/agent/models.json`. If the file exists, merge +`rec` into its `providers`. + +```json +{ + "providers": { + "rec": { + "baseUrl": "http://:11500/v1", + "api": "openai-completions", + "apiKey": "REC_API_KEY", + "models": [{ + "id": "flint", "name": "Flint", "reasoning": false, "input": ["text"], + "contextWindow": 262144, "maxTokens": 262144, + "compat": { + "supportsStore": false, "supportsDeveloperRole": false, + "supportsReasoningEffort": false, "supportsStrictMode": false, + "maxTokensField": "max_tokens" + } + }] + } + } +} +``` + +Select it with `/model`, or start pi with `pi --provider rec --model flint`. + +**OpenCode 2.x**: add the provider to `~/.config/opencode/opencode.json`. If the +file exists, merge `rec` into its `providers`. + +```json +{ + "model": "rec/flint", + "providers": { + "rec": { + "package": "@opencode/ai/providers/openai-compatible", + "name": "Recursive", + "settings": { "baseURL": "http://:11500/v1", "apiKey": "{env:REC_API_KEY}" }, + "models": { + "flint": { + "name": "Flint", + "limit": { "context": 262144, "output": 65536 }, + "body": { "max_completion_tokens": 262144 }, + "capabilities": { "tools": true, "input": ["text"], "output": ["text"] } + } + } + } + } +} +``` + +`model` makes Flint the default; remove it to keep yours. + +
+ ## Commands diff --git a/cli/cli.go b/cli/cli.go index 49e42a6..0b5945b 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -93,9 +93,11 @@ func run(args []string) error { runtimeDir := flags.String("runtime-dir", "", "") var contextTokens int port := 11500 + var host string if command == "serve" { flags.IntVar(&contextTokens, "context", 0, "") flags.IntVar(&port, "port", port, "") + flags.StringVar(&host, "host", "127.0.0.1", "") } flags.SetOutput(io.Discard) if e := flags.Parse(rest); e != nil { @@ -107,6 +109,14 @@ func run(args []string) error { if port < 1 || port > 65535 { return fmt.Errorf("port must be between 1 and 65535") } + if command == "serve" { + // Canonicalize so the engine, Status.Listen and sameInstance agree on one spelling. + addr, err := parseHost(host) + if err != nil { + return err + } + host = addr.String() + } m, e := loadManifest(*manifest) if e == nil && command == "serve" { flags.Visit(func(f *flag.Flag) { @@ -139,7 +149,7 @@ func run(args []string) error { } return e } - return serve(ctx, p, m, *runtimeDir, port, os.Stderr, background) + return serve(ctx, p, m, *runtimeDir, host, port, os.Stderr, background) } func doctor(p Paths, m Manifest, manifestErr error) error { diff --git a/cli/context_test.go b/cli/context_test.go index 1d7a4b8..2357cc9 100644 --- a/cli/context_test.go +++ b/cli/context_test.go @@ -52,41 +52,75 @@ func TestReleaseManifestContext(t *testing.T) { func TestExistingInstanceUsesEffectiveProfile(t *testing.T) { m := Manifest{Model: "flint", Revision: "test", RuntimeSHA: "runtime", Context: 4096} s := Status{Ready: true, Model: m.Model, Revision: m.Revision, Runtime: m.RuntimeSHA, - Context: m.Context, Instance: instanceKey(m), Endpoint: "http://127.0.0.1:11500/v1"} - if !sameInstance(s, m, 11500) { + Context: m.Context, Instance: instanceKey(m), Listen: "127.0.0.1:11500", Endpoint: "http://127.0.0.1:11500/v1"} + if !sameInstance(s, m, "127.0.0.1", 11500) { t.Fatal("identical instance rejected") } explicit, _ := withContext(m, 4096) - if !sameInstance(s, explicit, 11500) { + if !sameInstance(s, explicit, "127.0.0.1", 11500) { t.Fatal("explicit default rejected") } changed := m changed.Context = 16384 - if sameInstance(s, changed, 11500) { + if sameInstance(s, changed, "127.0.0.1", 11500) { t.Fatal("reused a smaller context") } changed = m changed.MinRAM++ - if sameInstance(s, changed, 11500) { + if sameInstance(s, changed, "127.0.0.1", 11500) { t.Fatal("reused different serving settings") } s.Instance = "" - if sameInstance(s, m, 11500) { + if sameInstance(s, m, "127.0.0.1", 11500) { t.Fatal("legacy server without profile must restart") } } -func TestContextReachesEngine(t *testing.T) { +func TestSameInstanceRequiresSameHost(t *testing.T) { + m := Manifest{Model: "flint", Revision: "test", RuntimeSHA: "runtime", Context: 4096} + wildcard := Status{Ready: true, Model: m.Model, Revision: m.Revision, Runtime: m.RuntimeSHA, + Context: m.Context, Instance: instanceKey(m), Listen: "0.0.0.0:11500", Endpoint: "http://127.0.0.1:11500/v1"} + if sameInstance(wildcard, m, "127.0.0.1", 11500) { + t.Fatal("reused a server bound to a different host") + } + if !sameInstance(wildcard, m, "0.0.0.0", 11500) { + t.Fatal("a server bound to all interfaces was rejected") + } + specific := Status{Ready: true, Model: m.Model, Revision: m.Revision, Runtime: m.RuntimeSHA, + Context: m.Context, Instance: instanceKey(m), Listen: "192.168.1.5:11500", Endpoint: "http://192.168.1.5:11500/v1"} + if sameInstance(specific, m, "127.0.0.1", 11500) { + t.Fatal("reused a server bound to another interface") + } + if !sameInstance(specific, m, "192.168.1.5", 11500) { + t.Fatal("an instance on a specific interface was rejected") + } + legacy := Status{Ready: true, Model: m.Model, Revision: m.Revision, Runtime: m.RuntimeSHA, + Context: m.Context, Instance: instanceKey(m), Endpoint: "http://127.0.0.1:11500/v1"} + if sameInstance(legacy, m, "127.0.0.1", 11500) { + t.Fatal("legacy server without listen address must restart") + } +} + +func TestHostAndContextReachesEngine(t *testing.T) { m := releaseManifest(t) - _, args, _ := launchCommand(Paths{}, m, "/runtime", "/model/flint-int4.json", 11500) - found := false + _, args, _ := launchCommand(Paths{}, m, "/runtime", "/model/flint-int4.json", "127.0.0.1", 11500) + hostArg := "" + for i, arg := range args { + if arg == "--host" && i+1 < len(args) { + hostArg = args[i+1] + } + } + if hostArg != "127.0.0.1" { + t.Fatalf("the requested host did not reach the engine: %q", hostArg) + } + contextLimit := "" for i, arg := range args { if arg == "--context-limit" && i+1 < len(args) { - found = args[i+1] == "262144" + contextLimit = args[i+1] } } - if !found { - t.Fatal("default 256K allocation did not reach the engine") + if contextLimit != "262144" { + t.Fatalf("default 256K allocation did not reach the engine: %q", contextLimit) } if _, err := withContext(m, 524288); err == nil { t.Fatal("512K exceeds the engine limit") diff --git a/cli/engine.go b/cli/engine.go index aaff247..161262f 100644 --- a/cli/engine.go +++ b/cli/engine.go @@ -6,10 +6,10 @@ import ( "strconv" ) -func launchCommand(p Paths, m Manifest, runtimeDir, model string, port int) (string, []string, []string) { +func launchCommand(p Paths, m Manifest, runtimeDir, model, host string, port int) (string, []string, []string) { return filepath.Join(runtimeDir, runtimeServer), []string{ "serve", m.Model, "--model-dir", filepath.Dir(model), "--manifest", model, - "--port", strconv.Itoa(port), "--context-limit", strconv.Itoa(m.Context), + "--host", host, "--port", strconv.Itoa(port), "--context-limit", strconv.Itoa(m.Context), "--api-key-file", filepath.Join(p.Config, "api-key"), }, []string{"HF_HUB_OFFLINE=1", "TOKENIZERS_PARALLELISM=false"} } diff --git a/cli/help.go b/cli/help.go index 2bc4474..eb5cf02 100644 --- a/cli/help.go +++ b/cli/help.go @@ -26,7 +26,8 @@ var commands = []commandHelp{ {"exec", "pi|opencode ARGS", "Run a harness command, such as installing a plugin", nil}, {"serve", "[MODEL]", "Start the model server in this terminal", append([][2]string{ {"--context TOKENS", "Total input and output tokens (default: set by the release)"}, - {"--port PORT", "Local port (default: 11500)"}, + {"--port PORT", "Port (default: 11500)"}, + {"--host ADDRESS", "IP address to listen on (default: 127.0.0.1; 0.0.0.0 for all interfaces)"}, }, developmentOptions...)}, {"status", "", "Show whether the server is running", nil}, {"stop", "", "Stop the server", nil}, diff --git a/cli/launch.go b/cli/launch.go index 57b6e59..65527d1 100644 --- a/cli/launch.go +++ b/cli/launch.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "io" + "net/netip" "net/url" "os" "os/exec" @@ -309,7 +310,10 @@ func serverPhase(log string) string { func validateClientStatus(s Status) error { endpoint, err := url.Parse(s.Endpoint) - if err != nil || endpoint.Scheme != "http" || endpoint.Hostname() != "127.0.0.1" || endpoint.Port() == "" || endpoint.Path != "/v1" || endpoint.User != nil || endpoint.RawQuery != "" || endpoint.Fragment != "" { + if err != nil || endpoint.Scheme != "http" || endpoint.Port() == "" || endpoint.Path != "/v1" || endpoint.User != nil || endpoint.RawQuery != "" || endpoint.Fragment != "" { + return fmt.Errorf("invalid local Rec endpoint") + } + if _, err := netip.ParseAddr(endpoint.Hostname()); err != nil { return fmt.Errorf("invalid local Rec endpoint") } if !s.Ready || s.Model != "flint" || s.Context < 2048 { diff --git a/cli/launch_test.go b/cli/launch_test.go index 4c094c2..2323506 100644 --- a/cli/launch_test.go +++ b/cli/launch_test.go @@ -123,3 +123,26 @@ func TestHarnessShellCommand(t *testing.T) { t.Fatalf("shell command does not reproduce the environment: %v %q", err, out) } } + +func TestValidateClientStatusEndpoint(t *testing.T) { + for _, tc := range []struct { + name string + endpoint string + ok bool + }{ + {"loopback", "http://127.0.0.1:11500/v1", true}, + {"lan", "http://192.168.1.5:11500/v1", true}, + {"ipv6 loopback", "http://[::1]:11500/v1", true}, + {"https", "https://127.0.0.1:11500/v1", false}, + {"hostname", "http://localhost:11500/v1", false}, + {"missing port", "http://127.0.0.1/v1", false}, + } { + t.Run(tc.name, func(t *testing.T) { + s := Status{Ready: true, Model: "flint", Context: 8192, Endpoint: tc.endpoint} + err := validateClientStatus(s) + if (err == nil) != tc.ok { + t.Fatalf("validateClientStatus(%q) error = %v, want ok = %v", tc.endpoint, err, tc.ok) + } + }) + } +} diff --git a/cli/pi_test.go b/cli/pi_test.go index 27e038d..e072a20 100644 --- a/cli/pi_test.go +++ b/cli/pi_test.go @@ -53,6 +53,8 @@ func TestClientsRejectInvalidStatus(t *testing.T) { {Ready: true, Model: "other", Context: 8192, Endpoint: "http://127.0.0.1:11500/v1"}, {Ready: false, Model: "flint", Context: 8192, Endpoint: "http://127.0.0.1:11500/v1"}, {Ready: true, Model: "flint", Context: 1024, Endpoint: "http://127.0.0.1:11500/v1"}, + {Ready: true, Model: "flint", Context: 8192, Endpoint: "http://localhost:11500/v1"}, + {Ready: true, Model: "flint", Context: 8192, Endpoint: "http://127.0.0.1/v1"}, } { p := Paths{Data: t.TempDir()} if _, _, err := piConfig(p, s); err == nil { diff --git a/cli/serve.go b/cli/serve.go index c4def68..6c4ae4c 100644 --- a/cli/serve.go +++ b/cli/serve.go @@ -10,10 +10,12 @@ import ( "io" "net" "net/http" + "net/netip" "os" "os/exec" "os/signal" "path/filepath" + "strconv" "strings" "sync" "syscall" @@ -28,6 +30,7 @@ type Status struct { Revision string `json:"revision"` Runtime string `json:"runtime_sha256"` Endpoint string `json:"endpoint"` + Listen string `json:"listen"` Context int `json:"context"` MaxContext int `json:"max_context"` Instance string `json:"instance_sha256"` @@ -39,10 +42,39 @@ func instanceKey(m Manifest) string { return digest(b) } -func sameInstance(s Status, m Manifest, port int) bool { +func sameInstance(s Status, m Manifest, host string, port int) bool { return s.Ready && s.Model == m.Model && s.Runtime == m.RuntimeSHA && s.Revision == m.Revision && s.Context == m.Context && - s.Instance == instanceKey(m) && s.Endpoint == fmt.Sprintf("http://127.0.0.1:%d/v1", port) + s.Instance == instanceKey(m) && s.Listen == net.JoinHostPort(host, strconv.Itoa(port)) && + s.Endpoint == clientEndpoint(host, port) +} + +// parseHost validates the --host flag: IP literals only, such as 127.0.0.1 or 0.0.0.0. +// Zoned addresses (fe80::1%en0) are rejected because they cannot appear in a URL +// unescaped. IPv4-mapped IPv6 is unmapped so Go and the engine pick one family. +func parseHost(host string) (netip.Addr, error) { + addr, err := netip.ParseAddr(host) + if err != nil || addr.Zone() != "" { + return netip.Addr{}, fmt.Errorf("--host must be an IP address, such as 127.0.0.1 or 0.0.0.0") + } + return addr.Unmap(), nil +} + +// clientEndpoint maps the requested host to the address clients should connect +// to; an unspecified bind address means the loopback of the same family. +func clientEndpoint(host string, port int) string { + addr, err := parseHost(host) + if err != nil { + return fmt.Sprintf("http://%s:%d/v1", host, port) + } + if addr.IsUnspecified() { + if addr.Is4() { + addr = netip.AddrFrom4([4]byte{127, 0, 0, 1}) + } else { + addr = netip.IPv6Loopback() + } + } + return "http://" + net.JoinHostPort(addr.String(), strconv.Itoa(port)) + "/v1" } func control(p Paths, action string) (Status, error) { @@ -210,28 +242,28 @@ func probe(ctx context.Context, base, key, model string, phase ...func(string)) return nil } -func serve(parent context.Context, p Paths, m Manifest, override string, port int, out io.Writer, background ...bool) error { +func serve(parent context.Context, p Paths, m Manifest, override, host string, port int, out io.Writer, background ...bool) error { l, e := lock(filepath.Join(p.State, "instance.lock")) if e != nil { if s, err := control(p, "status"); err == nil { - if sameInstance(s, m, port) { + if sameInstance(s, m, host, port) { newScreen(out).succeeded(m.Model+" is already running", s.Endpoint) return nil } - return fmt.Errorf("another instance is starting or uses a different profile (running context %d, requested %d); use rec status or rec stop", s.Context, m.Context) + return fmt.Errorf("another instance is starting or uses a different profile (running on %s with context %d, requested %s with context %d); use rec status or rec stop", s.Listen, s.Context, net.JoinHostPort(host, strconv.Itoa(port)), m.Context) } return e } defer unlock(l) ctx, cancel := context.WithCancel(parent) defer cancel() - address := fmt.Sprintf("127.0.0.1:%d", port) + address := net.JoinHostPort(host, strconv.Itoa(port)) check, e := net.Listen("tcp", address) if e != nil { - return fmt.Errorf("port %d is unavailable; use --port with a free port", port) + return fmt.Errorf("%s is unavailable; use --host/--port with a free local address", address) } check.Close() - status := Status{Engine: m.Engine, Version: Version, Model: m.Model, Revision: m.Revision, Runtime: m.RuntimeSHA, Endpoint: "http://" + address + "/v1", Context: m.Context, MaxContext: m.contextLimit(), Instance: instanceKey(m)} + status := Status{Engine: m.Engine, Version: Version, Model: m.Model, Revision: m.Revision, Runtime: m.RuntimeSHA, Endpoint: clientEndpoint(host, port), Listen: address, Context: m.Context, MaxContext: m.contextLimit(), Instance: instanceKey(m)} var mu sync.Mutex socket := filepath.Join(p.State, "control.sock") os.Remove(socket) @@ -284,7 +316,7 @@ func serve(parent context.Context, p Paths, m Manifest, override string, port in if e != nil { return e } - executable, args, extra := launchCommand(p, m, runtimeDir, model, port) + executable, args, extra := launchCommand(p, m, runtimeDir, model, host, port) self, e := os.Executable() if e != nil { return e @@ -324,7 +356,9 @@ func serve(parent context.Context, p Paths, m Manifest, override string, port in probeCtx, stopProbe := context.WithTimeout(ctx, 10*time.Minute) defer stopProbe() ready := make(chan error, 1) - go func() { ready <- probe(probeCtx, "http://"+address, key, m.Model, loading.phase) }() + go func() { + ready <- probe(probeCtx, strings.TrimSuffix(status.Endpoint, "/v1"), key, m.Model, loading.phase) + }() select { case e = <-done: loading.fail() @@ -350,7 +384,7 @@ func serve(parent context.Context, p Paths, m Manifest, override string, port in mu.Lock() status.Ready = true mu.Unlock() - printReady(ui, m, status.Endpoint, len(background) > 0 && background[0]) + printReady(ui, m, status, len(background) > 0 && background[0]) select { case <-ctx.Done(): return nil @@ -360,22 +394,32 @@ func serve(parent context.Context, p Paths, m Manifest, override string, port in } } -func printReady(ui *screen, m Manifest, endpoint string, background bool) { - var rows strings.Builder - for i, row := range [][2]string{ - {"Base URL", endpoint}, +func printReady(ui *screen, m Manifest, s Status, background bool) { + host, _, _ := net.SplitHostPort(s.Listen) + listen, _ := netip.ParseAddr(host) + rows := [][2]string{{"Base URL", s.Endpoint}} + // Only a wildcard bind differs from the Base URL host. + if listen.IsUnspecified() { + rows = append(rows, [2]string{"Listen", s.Listen + " (all interfaces)"}) + } + rows = append(rows, [][2]string{ {"Model", m.Model}, {"Checkpoint", m.Revision}, {"Context", m.contextSummary()}, {"API key", accentStyle.Render("rec auth token")}, - } { + }...) + var panel strings.Builder + for i, row := range rows { if i > 0 { - rows.WriteString("\n") + panel.WriteString("\n") } - rows.WriteString(faintStyle.Render(fmt.Sprintf("%-10s", row[0])) + " " + row[1]) + panel.WriteString(faintStyle.Render(fmt.Sprintf("%-10s", row[0])) + " " + row[1]) } ui.println("") - ui.println(panelStyle.Render(successStyle.Render("✓ ") + boldStyle.Render(m.Model+" is ready.") + "\n\n" + rows.String())) + ui.println(panelStyle.Render(successStyle.Render("✓ ") + boldStyle.Render(m.Model+" is ready.") + "\n\n" + panel.String())) + if !listen.IsLoopback() { + ui.step(warningStyle.Render("!"), "Reachable from the network over unencrypted HTTP. Use only on trusted networks.", "") + } if background { ui.println(faintStyle.Render("Running in the background. Use `rec stop` to stop.")) } else { diff --git a/cli/serve_test.go b/cli/serve_test.go index 050d204..6396acb 100644 --- a/cli/serve_test.go +++ b/cli/serve_test.go @@ -48,6 +48,38 @@ func (b *lockedBuffer) String() string { return b.b.String() } +func TestParseHostAndClientEndpoint(t *testing.T) { + for _, tc := range []struct { + host string + endpoint string + }{ + {"127.0.0.1", "http://127.0.0.1:11500/v1"}, + {"0.0.0.0", "http://127.0.0.1:11500/v1"}, + {"::", "http://[::1]:11500/v1"}, + {"192.168.1.5", "http://192.168.1.5:11500/v1"}, + {"fe80::1", "http://[fe80::1]:11500/v1"}, + } { + if got := clientEndpoint(tc.host, 11500); got != tc.endpoint { + t.Fatalf("clientEndpoint(%q, 11500) = %q, want %q", tc.host, got, tc.endpoint) + } + } + for host, canonical := range map[string]string{"0:0::0": "::", "::ffff:127.0.0.1": "127.0.0.1"} { + if addr, err := parseHost(host); err != nil || addr.String() != canonical { + t.Fatalf("parseHost(%q) = %v, %v; want %s", host, addr, err, canonical) + } + } + for _, bad := range []string{"localhost", "", "example.com", "fe80::1%en0"} { + if _, err := parseHost(bad); err == nil { + t.Fatalf("parseHost(%q) accepted a non-IP address", bad) + } + } + for _, good := range []string{"127.0.0.1", "0.0.0.0", "::", "192.168.1.5", "fe80::1"} { + if _, err := parseHost(good); err != nil { + t.Fatalf("parseHost(%q) rejected a valid IP: %v", good, err) + } + } +} + func TestServeLifecycle(t *testing.T) { if runtime.GOOS != "darwin" || runtime.GOARCH != "arm64" { t.Skip("serving requires Apple Silicon") @@ -95,7 +127,7 @@ func TestServeLifecycle(t *testing.T) { var out lockedBuffer done := make(chan error, 1) - go func() { done <- serve(context.Background(), p, m, runtimeDir, port, &out) }() + go func() { done <- serve(context.Background(), p, m, runtimeDir, "127.0.0.1", port, &out) }() deadline := time.Now().Add(2 * time.Minute) var s Status for !s.Ready { @@ -113,14 +145,17 @@ func TestServeLifecycle(t *testing.T) { if !strings.Contains(out.String(), "test is ready.") || s.Endpoint != fmt.Sprintf("http://127.0.0.1:%d/v1", port) || s.Engine != "mlx" { t.Fatalf("wrong readiness report %+v:\n%s", s, out.String()) } + if s.Listen != fmt.Sprintf("127.0.0.1:%d", port) { + t.Fatalf("wrong listen address: %q", s.Listen) + } var again bytes.Buffer - if err := serve(context.Background(), p, m, runtimeDir, port, &again); err != nil || !strings.Contains(again.String(), "test is already running") { + if err := serve(context.Background(), p, m, runtimeDir, "127.0.0.1", port, &again); err != nil || !strings.Contains(again.String(), "test is already running") { t.Fatalf("identical run did not reuse the server: %v %s", err, again.String()) } changed := m changed.Context, changed.MaxContext = 4096, 4096 - if err := serve(context.Background(), p, changed, runtimeDir, port, io.Discard); err == nil { + if err := serve(context.Background(), p, changed, runtimeDir, "127.0.0.1", port, io.Discard); err == nil { t.Fatal("a different context reused the running server") } diff --git a/cli/testdata/fake-rec-mlx b/cli/testdata/fake-rec-mlx index 8a56828..19c1fe1 100755 --- a/cli/testdata/fake-rec-mlx +++ b/cli/testdata/fake-rec-mlx @@ -3,6 +3,7 @@ import argparse import http.server import json +import socket from pathlib import Path parser = argparse.ArgumentParser() @@ -12,6 +13,7 @@ for flag in ("--model-dir", "--manifest", "--api-key-file"): parser.add_argument(flag, type=Path, required=True) parser.add_argument("--port", type=int, required=True) parser.add_argument("--context-limit", type=int, required=True) +parser.add_argument("--host", required=True) args = parser.parse_args() assert (args.model_dir / args.manifest.name).is_file() key = args.api_key_file.read_text().strip() @@ -46,4 +48,17 @@ class Handler(http.server.BaseHTTPRequestHandler): self.reply({"choices": [{"index": 0, "message": message, "finish_reason": "stop"}]}) -http.server.ThreadingHTTPServer(("127.0.0.1", args.port), Handler).serve_forever() +class Server(http.server.ThreadingHTTPServer): + def __init__(self, address, handler, **kwargs): + self._bind_host = address[0] + self.address_family = socket.AF_INET6 if ":" in address[0] else socket.AF_INET + super().__init__(address, handler, **kwargs) + + def server_bind(self): + if self.address_family == socket.AF_INET6 and self._bind_host == "::": + self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + super().server_bind() + + +server = Server((args.host, args.port), Handler) +server.serve_forever() diff --git a/engine/src/rec_mlx/cli.py b/engine/src/rec_mlx/cli.py index 7b04258..068f336 100644 --- a/engine/src/rec_mlx/cli.py +++ b/engine/src/rec_mlx/cli.py @@ -33,7 +33,10 @@ def main(): help="total input and output tokens (default: %(default)s)", ) parser.add_argument( - "--port", type=int, default=8080, help="loopback port for serve (default: %(default)s)" + "--port", type=int, default=8080, help="port for serve (default: %(default)s)" + ) + parser.add_argument( + "--host", default="127.0.0.1", help="address for serve to listen on" ) parser.add_argument( "--grammar", @@ -123,13 +126,14 @@ def main(): server = make_server( engine, - ("127.0.0.1", args.port), + (args.host, args.port), args.metrics, defaults=settings, api_key=api_key, grammar_mode=args.grammar, ) - print(f"Listening on http://127.0.0.1:{server.server_port}/v1", flush=True) + host = f"[{args.host}]" if ":" in args.host and not args.host.startswith("[") else args.host + print(f"Listening on http://{host}:{server.server_port}/v1", flush=True) def stop(signum, frame): server.generator.cancel() diff --git a/engine/src/rec_mlx/server.py b/engine/src/rec_mlx/server.py index 4b81ab0..2ceafb2 100644 --- a/engine/src/rec_mlx/server.py +++ b/engine/src/rec_mlx/server.py @@ -23,8 +23,20 @@ class RequestError(ValueError): class Server(ThreadingHTTPServer): + """HTTP server that binds either IPv4 or IPv6, and IPv4-mapped IPv6 when asked.""" # The default backlog of 5 resets a burst of connections before the accept loop takes them. request_queue_size = 32 + def __init__(self, address, handler, **kwargs): + self._bind_host = address[0] + self.address_family = socket.AF_INET6 if ":" in address[0] else socket.AF_INET + super().__init__(address, handler, **kwargs) + + def server_bind(self): + if self.address_family == socket.AF_INET6 and self._bind_host == "::": + # Accept IPv4 clients too, matching stdlib http.server's DualStackServer. + # The option must be set before bind; afterwards it fails with EINVAL. + self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + super().server_bind() def make_server( diff --git a/engine/tests/test_cli_defaults.py b/engine/tests/test_cli_defaults.py index 4fb343e..45f92fe 100644 --- a/engine/tests/test_cli_defaults.py +++ b/engine/tests/test_cli_defaults.py @@ -49,6 +49,7 @@ def check_defaults(*args, _fast_sync=fast_sync, _engine=engine, **kwargs): construct.side_effect = check_defaults main() + self.assertEqual(make.call_args.args[1], ("127.0.0.1", 8080)) self.assertEqual(make.call_args.kwargs["grammar_mode"], grammar) self.assertEqual(construct.call_args.kwargs["enforce_tool_schemas"], enforce) ready = json.loads(stdout.getvalue().splitlines()[0]) @@ -56,3 +57,30 @@ def check_defaults(*args, _fast_sync=fast_sync, _engine=engine, **kwargs): self.assertEqual(ready["fast_sync"], fast_sync) self.assertEqual(ready["enforce_tool_schemas"], enforce) self.assertEqual(ready["chat_template"], "flint") + + def test_serve_host_wildcard_binds_all_interfaces(self): + engine = Mock(identity={"model_id": "flint"}, startup={"chat_template": "flint"}) + server = Mock(server_port=8080) + with ( + patch( + "sys.argv", + [ + "rec-mlx", + "serve", + "flint", + "--model-dir", + ".", + "--manifest", + "model.json", + "--host", + "0.0.0.0", + ], + ), + patch.dict(os.environ, {"MLX_METAL_FAST_SYNCH": "1"}), + patch("rec_mlx.cli.Engine", return_value=engine), + patch("rec_mlx.server.make_server", return_value=server) as make, + patch("rec_mlx.cli.signal.signal"), + redirect_stdout(io.StringIO()), + ): + main() + self.assertEqual(make.call_args.args[1], ("0.0.0.0", 8080)) diff --git a/engine/tests/test_serving.py b/engine/tests/test_serving.py index f3d9cbf..b0a4166 100644 --- a/engine/tests/test_serving.py +++ b/engine/tests/test_serving.py @@ -488,6 +488,34 @@ def post(path, headers): server.server_close() thread.join() + def test_make_server_binds_ipv6_and_dual_stack(self): + if not socket.has_ipv6: + self.skipTest("IPv6 loopback is unavailable") + # "::" must also accept IPv4 clients; IPV6_V6ONLY has to be cleared before bind. + for bind, clients in (("::1", ("::1",)), ("::", ("::1", "127.0.0.1"))): + with self.subTest(bind=bind): + server = make_server(self.engine, (bind, 0), Path(self.temp.name) / "metrics") + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + self.assertEqual(server.address_family, socket.AF_INET6) + port = server.server_address[1] + for client in clients: + conn = http.client.HTTPConnection(client, port, timeout=5) + try: + conn.request("GET", "/health") + response = conn.getresponse() + self.assertEqual(response.status, 200) + self.assertEqual(json.loads(response.read())["status"], "ready") + finally: + conn.close() + finally: + server.shutdown() + server.generator.cancel() + server.generator.join() + server.server_close() + thread.join() + if __name__ == "__main__": unittest.main() diff --git a/tests/data/smoke-mlx-server.py b/tests/data/smoke-mlx-server.py index 830c10e..1480c42 100644 --- a/tests/data/smoke-mlx-server.py +++ b/tests/data/smoke-mlx-server.py @@ -9,6 +9,7 @@ import argparse import http.server import json +import socket import time from pathlib import Path @@ -20,6 +21,7 @@ parser.add_argument("model") parser.add_argument("--model-dir", type=Path, required=True) parser.add_argument("--manifest", type=Path, required=True) +parser.add_argument("--host", required=True) parser.add_argument("--port", type=int, required=True) parser.add_argument("--context-limit", type=int, required=True) parser.add_argument("--api-key-file", type=Path, required=True) @@ -126,5 +128,16 @@ def do_POST(self): ) -print(f"Listening on http://127.0.0.1:{args.port}/v1", flush=True) -http.server.HTTPServer(("127.0.0.1", args.port), Handler).serve_forever() +class Server(http.server.HTTPServer): + # Bind like the Flint server: IPv6 literals need AF_INET6, and "::" also takes IPv4. + address_family = socket.AF_INET6 if ":" in args.host else socket.AF_INET + + def server_bind(self): + if args.host == "::": + self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + super().server_bind() + + +host = f"[{args.host}]" if ":" in args.host else args.host +print(f"Listening on http://{host}:{args.port}/v1", flush=True) +Server((args.host, args.port), Handler).serve_forever() From b499337b0d68f5834e22e184044c81a6bbd92ff0 Mon Sep 17 00:00:00 2001 From: wcole Date: Wed, 4 Nov 2026 23:22:43 -0500 Subject: [PATCH 2/3] Require key for mlx-serve when `--host` is specified --- engine/src/rec_mlx/cli.py | 26 ++++- engine/src/rec_mlx/server.py | 2 + engine/tests/test_cli_defaults.py | 163 ++++++++++++++++++------------ 3 files changed, 126 insertions(+), 65 deletions(-) diff --git a/engine/src/rec_mlx/cli.py b/engine/src/rec_mlx/cli.py index 068f336..90e172c 100644 --- a/engine/src/rec_mlx/cli.py +++ b/engine/src/rec_mlx/cli.py @@ -1,6 +1,7 @@ """Run Flint as a terminal chat or a local OpenAI-compatible server.""" import argparse +import ipaddress import json import logging import os @@ -36,7 +37,10 @@ def main(): "--port", type=int, default=8080, help="port for serve (default: %(default)s)" ) parser.add_argument( - "--host", default="127.0.0.1", help="address for serve to listen on" + "--host", + default="127.0.0.1", + help="IP address for serve to listen on; non-loopback requires --api-key-file " + "(default: %(default)s)", ) parser.add_argument( "--grammar", @@ -95,11 +99,20 @@ def main(): ) except ValueError as error: parser.error(str(error)) + # Check valid key and host before Engine load api_key = None if args.api_key_file is not None: api_key = args.api_key_file.read_text().strip() if not api_key or any(c.isspace() for c in api_key): parser.error("API-key file must contain one nonempty token") + if args.command == "serve": + try: + address = parse_host(args.host) + except ValueError as error: + parser.error(str(error)) + if not address.is_loopback and api_key is None: + parser.error("--api-key-file is required when --host is not a loopback address") + args.host = str(address) engine = Engine( args.model_dir, args.manifest, @@ -153,6 +166,17 @@ def stop(signum, frame): chat(engine, settings, args.metrics, grammar_mode=args.grammar) +def parse_host(host): + """Validate --host like the rec CLI: IP literals only, no zone, IPv4-mapped IPv6 unmapped.""" + try: + address = ipaddress.ip_address(host) + except ValueError: + address = None + if address is None or getattr(address, "scope_id", None): + raise ValueError("--host must be an IP address, such as 127.0.0.1 or 0.0.0.0") + return getattr(address, "ipv4_mapped", None) or address + + def chat(engine, settings, metrics_path, *, grammar_mode="flint"): constraint = engine.constraints.tools() if grammar_mode == "flint" else None messages = [] diff --git a/engine/src/rec_mlx/server.py b/engine/src/rec_mlx/server.py index 2ceafb2..7cb4127 100644 --- a/engine/src/rec_mlx/server.py +++ b/engine/src/rec_mlx/server.py @@ -24,8 +24,10 @@ class RequestError(ValueError): class Server(ThreadingHTTPServer): """HTTP server that binds either IPv4 or IPv6, and IPv4-mapped IPv6 when asked.""" + # The default backlog of 5 resets a burst of connections before the accept loop takes them. request_queue_size = 32 + def __init__(self, address, handler, **kwargs): self._bind_host = address[0] self.address_family = socket.AF_INET6 if ":" in address[0] else socket.AF_INET diff --git a/engine/tests/test_cli_defaults.py b/engine/tests/test_cli_defaults.py index 45f92fe..929e445 100644 --- a/engine/tests/test_cli_defaults.py +++ b/engine/tests/test_cli_defaults.py @@ -1,14 +1,51 @@ import io import json import os +import tempfile import unittest -from contextlib import redirect_stdout +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +from types import SimpleNamespace from unittest.mock import Mock, patch from rec_mlx.cli import main class DefaultsTests(unittest.TestCase): + def serve(self, *options): + """Run `rec-mlx serve` with a mocked engine and server and record what happened.""" + engine = Mock(identity={"model_id": "flint"}, startup={"chat_template": "flint"}) + seen = {} + + def construct_engine(*args, **kwargs): + seen["fast_synch"] = os.environ["MLX_METAL_FAST_SYNCH"] + return engine + + argv = ["rec-mlx", "serve", "flint", "--model-dir", ".", "--manifest", "model.json"] + code = None + with ( + patch("sys.argv", [*argv, *options]), + # A stale value proves main() sets the variable before the engine loads. + patch.dict(os.environ, {"MLX_METAL_FAST_SYNCH": "stale"}), + patch("rec_mlx.cli.Engine", side_effect=construct_engine) as construct, + patch("rec_mlx.server.make_server", return_value=Mock(server_port=8080)) as make, + patch("rec_mlx.cli.signal.signal"), + redirect_stdout(io.StringIO()) as stdout, + redirect_stderr(io.StringIO()) as stderr, + ): + try: + main() + except SystemExit as exit: + code = exit.code + return SimpleNamespace( + construct=construct, + make=make, + exit=code, + fast_synch=seen.get("fast_synch"), + stdout=stdout.getvalue(), + stderr=stderr.getvalue(), + ) + def test_flint_grammar_and_fast_sync_defaults_and_opt_outs(self): for options, fast_sync, grammar, enforce in ( ([], True, "flint", False), @@ -17,70 +54,68 @@ def test_flint_grammar_and_fast_sync_defaults_and_opt_outs(self): (["--enforce-tool-schemas"], True, "flint", True), ): with self.subTest(options=options): - engine = Mock(identity={"model_id": "flint"}, startup={"chat_template": "flint"}) - server = Mock(server_port=8080) - with ( - patch( - "sys.argv", - [ - "rec-mlx", - "serve", - "flint", - "--model-dir", - ".", - "--manifest", - "model.json", - *options, - ], - ), - patch.dict(os.environ, {"MLX_METAL_FAST_SYNCH": "0" if fast_sync else "1"}), - patch("rec_mlx.cli.Engine", return_value=engine) as construct, - patch("rec_mlx.server.make_server", return_value=server) as make, - patch("rec_mlx.cli.signal.signal"), - redirect_stdout(io.StringIO()) as stdout, - ): + run = self.serve(*options) + self.assertIsNone(run.exit) + self.assertEqual(run.fast_synch, "1" if fast_sync else "0") + self.assertNotIn("chat_template", run.construct.call_args.kwargs) + self.assertEqual(run.construct.call_args.kwargs["enforce_tool_schemas"], enforce) + self.assertEqual(run.make.call_args.args[1], ("127.0.0.1", 8080)) + self.assertEqual(run.make.call_args.kwargs["grammar_mode"], grammar) + ready = json.loads(run.stdout.splitlines()[0]) + self.assertEqual(ready["grammar"], grammar) + self.assertEqual(ready["fast_sync"], fast_sync) + self.assertEqual(ready["enforce_tool_schemas"], enforce) + self.assertEqual(ready["chat_template"], "flint") - def check_defaults(*args, _fast_sync=fast_sync, _engine=engine, **kwargs): - self.assertEqual( - os.environ["MLX_METAL_FAST_SYNCH"], "1" if _fast_sync else "0" - ) - self.assertNotIn("chat_template", kwargs) - return _engine + def test_non_loopback_host_without_api_key_is_rejected_before_loading(self): + for host in ("0.0.0.0", "::", "192.168.1.5"): + with self.subTest(host=host): + run = self.serve("--host", host) + self.assertEqual(run.exit, 2) + run.construct.assert_not_called() + run.make.assert_not_called() + self.assertIn("--api-key-file is required", run.stderr) - construct.side_effect = check_defaults - main() - self.assertEqual(make.call_args.args[1], ("127.0.0.1", 8080)) - self.assertEqual(make.call_args.kwargs["grammar_mode"], grammar) - self.assertEqual(construct.call_args.kwargs["enforce_tool_schemas"], enforce) - ready = json.loads(stdout.getvalue().splitlines()[0]) - self.assertEqual(ready["grammar"], grammar) - self.assertEqual(ready["fast_sync"], fast_sync) - self.assertEqual(ready["enforce_tool_schemas"], enforce) - self.assertEqual(ready["chat_template"], "flint") + def test_non_loopback_host_with_invalid_api_key_file_is_rejected_before_loading(self): + for contents in ("", "\n", " \t\n", "two tokens\n"): + with self.subTest(contents=contents), tempfile.TemporaryDirectory() as directory: + key = Path(directory) / "api-key" + key.write_text(contents) + run = self.serve("--host", "0.0.0.0", "--api-key-file", str(key)) + self.assertEqual(run.exit, 2) + run.construct.assert_not_called() + run.make.assert_not_called() + self.assertIn("API-key file must contain one nonempty token", run.stderr) - def test_serve_host_wildcard_binds_all_interfaces(self): - engine = Mock(identity={"model_id": "flint"}, startup={"chat_template": "flint"}) - server = Mock(server_port=8080) - with ( - patch( - "sys.argv", - [ - "rec-mlx", - "serve", - "flint", - "--model-dir", - ".", - "--manifest", - "model.json", - "--host", - "0.0.0.0", - ], - ), - patch.dict(os.environ, {"MLX_METAL_FAST_SYNCH": "1"}), - patch("rec_mlx.cli.Engine", return_value=engine), - patch("rec_mlx.server.make_server", return_value=server) as make, - patch("rec_mlx.cli.signal.signal"), - redirect_stdout(io.StringIO()), + def test_host_must_be_an_ip_literal(self): + for host in ("localhost", "example.com", "", "fe80::1%en0"): + with self.subTest(host=host): + run = self.serve("--host", host) + self.assertEqual(run.exit, 2) + run.construct.assert_not_called() + run.make.assert_not_called() + self.assertIn("--host must be an IP address", run.stderr) + + def test_wildcard_bind_with_api_key(self): + with tempfile.TemporaryDirectory() as directory: + key = Path(directory) / "api-key" + key.write_text("secret\n") + run = self.serve("--host", "0.0.0.0", "--api-key-file", str(key)) + self.assertIsNone(run.exit) + run.construct.assert_called_once() + self.assertEqual(run.make.call_args.args[1], ("0.0.0.0", 8080)) + self.assertEqual(run.make.call_args.kwargs["api_key"], "secret") + + def test_loopback_hosts_need_no_api_key(self): + for host, bind in ( + ("127.0.0.1", "127.0.0.1"), + ("127.0.0.2", "127.0.0.2"), + ("::1", "::1"), + ("::ffff:127.0.0.1", "127.0.0.1"), ): - main() - self.assertEqual(make.call_args.args[1], ("0.0.0.0", 8080)) + with self.subTest(host=host): + run = self.serve("--host", host) + self.assertIsNone(run.exit) + run.construct.assert_called_once() + self.assertEqual(run.make.call_args.args[1], (bind, 8080)) + self.assertIsNone(run.make.call_args.kwargs["api_key"]) From 3c6390b1f26c787d00f9cc0fda29be4f258ff8f5 Mon Sep 17 00:00:00 2001 From: wcole Date: Wed, 4 Nov 2026 23:48:08 -0500 Subject: [PATCH 3/3] Add IP to `rec status` when served with `--host` --- cli/cli.go | 3 +++ cli/serve.go | 14 ++++++++++++++ cli/serve_test.go | 19 +++++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/cli/cli.go b/cli/cli.go index 0b5945b..3fc53c9 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -203,5 +203,8 @@ func printStatus(ui *screen, action string, s Status) { } if action == "status" { ui.println(faintStyle.Render(fmt.Sprintf(" Context %s tokens · rec %s", formatTokens(s.Context), s.Version))) + if listen := networkListen(s); listen != "" { + ui.println(faintStyle.Render(" Listen " + listen + " · reachable from the network")) + } } } diff --git a/cli/serve.go b/cli/serve.go index 6c4ae4c..330e0a8 100644 --- a/cli/serve.go +++ b/cli/serve.go @@ -394,6 +394,20 @@ func serve(parent context.Context, p Paths, m Manifest, override, host string, p } } +// networkListen describes the listen address of a server reachable from other +// machines, or returns "" for loopback and servers that predate Status.Listen. +func networkListen(s Status) string { + host, _, _ := net.SplitHostPort(s.Listen) + addr, err := netip.ParseAddr(host) + if err != nil || addr.IsLoopback() { + return "" + } + if addr.IsUnspecified() { + return s.Listen + " (all interfaces)" + } + return s.Listen +} + func printReady(ui *screen, m Manifest, s Status, background bool) { host, _, _ := net.SplitHostPort(s.Listen) listen, _ := netip.ParseAddr(host) diff --git a/cli/serve_test.go b/cli/serve_test.go index 6396acb..8ccafd3 100644 --- a/cli/serve_test.go +++ b/cli/serve_test.go @@ -80,6 +80,25 @@ func TestParseHostAndClientEndpoint(t *testing.T) { } } +func TestStatusShowsNetworkListenAddress(t *testing.T) { + for _, tc := range []struct{ listen, endpoint, want string }{ + {"127.0.0.1:11500", "http://127.0.0.1:11500/v1", ""}, + {"", "http://127.0.0.1:11500/v1", ""}, + {"0.0.0.0:11500", "http://127.0.0.1:11500/v1", "Listen 0.0.0.0:11500 (all interfaces) · reachable from the network"}, + {"[::]:11500", "http://[::1]:11500/v1", "Listen [::]:11500 (all interfaces) · reachable from the network"}, + {"192.168.1.5:11500", "http://192.168.1.5:11500/v1", "Listen 192.168.1.5:11500 · reachable from the network"}, + } { + var out bytes.Buffer + printStatus(newScreen(&out), "status", Status{Ready: true, Model: "flint", Version: "test", Context: 4096, Listen: tc.listen, Endpoint: tc.endpoint}) + if tc.want == "" && strings.Contains(out.String(), "Listen") { + t.Fatalf("loopback status %q shows a listen line:\n%s", tc.listen, out.String()) + } + if tc.want != "" && !strings.Contains(out.String(), tc.want) { + t.Fatalf("status for %q lacks %q:\n%s", tc.listen, tc.want, out.String()) + } + } +} + func TestServeLifecycle(t *testing.T) { if runtime.GOOS != "darwin" || runtime.GOARCH != "arm64" { t.Skip("serving requires Apple Silicon")