Add host binding option for server and update related documentation

README.md

@@ -114,6 +114,16 @@ The Anthropic-compatible API is available at
 `http://127.0.0.1:11500` and send the token from `rec auth token` as a bearer
 token or as `x-api-key`. The model is `flint`.
 
+By default the server binds to loopback only. To expose it on your network for
+other machines to reach, start it on all interfaces:
+
+```bash
+rec serve --host 0.0.0.0
+```
+
+The endpoint then listens on every interface; connect through the machine's IP
+address, for example `http://192.168.1.42:11500/v1`. Keep the API key required
+and restrict the interface with a firewall or VPN when binding to `0.0.0.0`.
 
 ## Commands
 

cli/cli.go

@@ -91,11 +91,25 @@ func run(args []string) error {
 	flags := flag.NewFlagSet(command, flag.ContinueOnError)
 	manifest := flags.String("manifest", "", "")
 	runtimeDir := flags.String("runtime-dir", "", "")
+	quant := flags.String("quant", QuantInt4, "")
 	var contextTokens int
+	var tempVal float64
+	var presenceVal float64
+	var repeatVal float64
+	var tempPtr *float64
+	var presencePtr *float64
+	var repeatPtr *float64
 	port := 11500
+	host := "127.0.0.1"
 	if command == "serve" {
 		flags.IntVar(&contextTokens, "context", 0, "")
 		flags.IntVar(&port, "port", port, "")
+		flags.StringVar(&host, "host", host, "")
+		flags.Float64Var(&tempVal, "temperature", 1.0, "")
+		flags.Float64Var(&presenceVal, "presence-penalty", 0.0, "")
+		flags.Float64Var(&repeatVal, "repeat-penalty", 1.0, "")
+		flags.Float64Var(&repeatVal, "repeat_penalty", 1.0, "")
+		flags.Float64Var(&repeatVal, "repetition-penalty", 1.0, "")
 	}
 	flags.SetOutput(io.Discard)
 	if e := flags.Parse(rest); e != nil {
@@ -107,14 +121,36 @@ func run(args []string) error {
 	if port < 1 || port > 65535 {
 		return fmt.Errorf("port must be between 1 and 65535")
 	}
-	m, e := loadManifest(*manifest)
+	normalizedQuant, quantErr := validateQuant(*quant)
+	if quantErr != nil {
+		return quantErr
+	}
+	m, e := loadManifest(*manifest, normalizedQuant)
 	if e == nil && command == "serve" {
 		flags.Visit(func(f *flag.Flag) {
 			if f.Name == "context" {
 				m, e = withContext(m, contextTokens)
 			}
+			if f.Name == "temperature" {
+				tempPtr = &tempVal
+			}
+			if f.Name == "presence-penalty" {
+				presencePtr = &presenceVal
+			}
+			if f.Name == "repeat-penalty" || f.Name == "repeat_penalty" || f.Name == "repetition-penalty" {
+				repeatPtr = &repeatVal
+			}
 		})
 	}
+	if tempPtr != nil && (*tempPtr < 0 || *tempPtr > 2) {
+		return fmt.Errorf("temperature must be between 0 and 2")
+	}
+	if presencePtr != nil && (*presencePtr < -2 || *presencePtr > 2) {
+		return fmt.Errorf("presence-penalty must be between -2 and 2")
+	}
+	if repeatPtr != nil && (*repeatPtr <= 0 || *repeatPtr > 5) {
+		return fmt.Errorf("repeat-penalty must be greater than 0 and at most 5")
+	}
 	if command == "doctor" {
 		return doctor(p, m, e)
 	}
@@ -139,7 +175,7 @@ func run(args []string) error {
 		}
 		return e
 	}
-	return serve(ctx, p, m, *runtimeDir, port, os.Stderr, background)
+	return serve(ctx, p, m, *runtimeDir, host, port, tempPtr, presencePtr, repeatPtr, os.Stderr, background)
 }
 
 func doctor(p Paths, m Manifest, manifestErr error) error {

cli/context_test.go

@@ -53,32 +53,32 @@ func TestExistingInstanceUsesEffectiveProfile(t *testing.T) {
 	m := Manifest{Model: "flint", Revision: "test", RuntimeSHA: "runtime", Context: 4096}
 	s := Status{Ready: true, Model: m.Model, Revision: m.Revision, Runtime: m.RuntimeSHA,
 		Context: m.Context, Instance: instanceKey(m), Endpoint: "http://127.0.0.1:11500/v1"}
-	if !sameInstance(s, m, 11500) {
+	if !sameInstance(s, m, "127.0.0.1", 11500) {
 		t.Fatal("identical instance rejected")
 	}
 	explicit, _ := withContext(m, 4096)
-	if !sameInstance(s, explicit, 11500) {
+	if !sameInstance(s, explicit, "127.0.0.1", 11500) {
 		t.Fatal("explicit default rejected")
 	}
 	changed := m
 	changed.Context = 16384
-	if sameInstance(s, changed, 11500) {
+	if sameInstance(s, changed, "127.0.0.1", 11500) {
 		t.Fatal("reused a smaller context")
 	}
 	changed = m
 	changed.MinRAM++
-	if sameInstance(s, changed, 11500) {
+	if sameInstance(s, changed, "127.0.0.1", 11500) {
 		t.Fatal("reused different serving settings")
 	}
 	s.Instance = ""
-	if sameInstance(s, m, 11500) {
+	if sameInstance(s, m, "127.0.0.1", 11500) {
 		t.Fatal("legacy server without profile must restart")
 	}
 }
 
 func TestContextReachesEngine(t *testing.T) {
 	m := releaseManifest(t)
-	_, args, _ := launchCommand(Paths{}, m, "/runtime", "/model/flint-int4.json", 11500)
+	_, args, _ := launchCommand(Paths{}, m, "/runtime", "/model/flint-int4.json", "127.0.0.1", 11500, nil, nil, nil)
 	found := false
 	for i, arg := range args {
 		if arg == "--context-limit" && i+1 < len(args) {

cli/engine.go

@@ -6,12 +6,23 @@ import (
 	"strconv"
 )
 
-func launchCommand(p Paths, m Manifest, runtimeDir, model string, port int) (string, []string, []string) {
-	return filepath.Join(runtimeDir, runtimeServer), []string{
+func launchCommand(p Paths, m Manifest, runtimeDir, model string, host string, port int, temp *float64, presence *float64, repeat *float64) (string, []string, []string) {
+	args := []string{
 		"serve", m.Model, "--model-dir", filepath.Dir(model), "--manifest", model,
 		"--port", strconv.Itoa(port), "--context-limit", strconv.Itoa(m.Context),
 		"--api-key-file", filepath.Join(p.Config, "api-key"),
-	}, []string{"HF_HUB_OFFLINE=1", "TOKENIZERS_PARALLELISM=false"}
+		"--host", host,
+	}
+	if temp != nil {
+		args = append(args, "--temperature", fmt.Sprintf("%g", *temp))
+	}
+	if presence != nil {
+		args = append(args, "--presence-penalty", fmt.Sprintf("%g", *presence))
+	}
+	if repeat != nil {
+		args = append(args, "--repeat-penalty", fmt.Sprintf("%g", *repeat))
+	}
+	return filepath.Join(runtimeDir, runtimeServer), args, []string{"HF_HUB_OFFLINE=1", "TOKENIZERS_PARALLELISM=false"}
 }
 
 func checkAppleChip(name string) error {

cli/help.go

@@ -14,6 +14,11 @@ type commandHelp struct {
 	options             [][2]string
 }
 
+var quantOption = [2]string{"--quant FORMAT", "Model quantization: int4 (default, 48 GiB+ RAM) or int6 (64 GiB+ RAM)"}
+var temperatureOption = [2]string{"--temperature NUM", "Sampling temperature between 0 and 2 (default: 1.0)"}
+var presenceOption = [2]string{"--presence-penalty NUM", "Presence penalty between -2 and 2 (default: 0.0)"}
+var repeatOption = [2]string{"--repeat-penalty NUM", "Repetition penalty greater than 0 (default: 1.0)"}
+
 var developmentOptions = [][2]string{
 	{"--manifest FILE", "Use this manifest instead of the bundled one"},
 	{"--runtime-dir DIR", "Use this runtime instead of the bundled one; needs --manifest"},
@@ -22,17 +27,26 @@ var developmentOptions = [][2]string{
 var commands = []commandHelp{
 	{"launch", "pi|opencode", "Start the server if needed and open a harness", [][2]string{
 		{"--args", "Print the harness command instead of running it; needs a running server"},
+		temperatureOption,
+		presenceOption,
+		repeatOption,
+		quantOption,
 	}},
 	{"exec", "pi|opencode ARGS", "Run a harness command, such as installing a plugin", nil},
 	{"serve", "[MODEL]", "Start the model server in this terminal", append([][2]string{
 		{"--context TOKENS", "Total input and output tokens (default: set by the release)"},
 		{"--port PORT", "Local port (default: 11500)"},
+		{"--host HOST", "Network interface to bind the server to (default: 127.0.0.1)"},
+		temperatureOption,
+		presenceOption,
+		repeatOption,
+		quantOption,
 	}, developmentOptions...)},
 	{"status", "", "Show whether the server is running", nil},
 	{"stop", "", "Stop the server", nil},
 	{"auth token", "", "Print the local API key", nil},
-	{"pull", "[MODEL]", "Download the model without starting the server", developmentOptions},
-	{"doctor", "", "Check your hardware and installation", developmentOptions},
+	{"pull", "[MODEL]", "Download the model without starting the server", append([][2]string{quantOption}, developmentOptions...)},
+	{"doctor", "", "Check your hardware and installation", append([][2]string{quantOption}, developmentOptions...)},
 	{"update", "[-y]", "Update rec to the latest release", [][2]string{{"-y", "Do not ask for confirmation"}}},
 	{"uninstall", "[-y]", "Remove rec, its models and its settings", [][2]string{{"-y", "Do not ask for confirmation"}}},
 	{"--version", "", "Print the version", nil},