diff --git a/README.md b/README.md index bc35ceee0..cdb4e1c7e 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,21 @@ docker run --rm --device /dev/kfd --device /dev/dri \ ghcr.io/luce-org/lucebox-hub:rocm ``` +The container picks the GPU the model fits on (a discrete card before an integrated one, else the largest) and sizes the context from that GPU's memory. `serve` is the default command, so `luce_server` flags can follow the image name directly (or `serve`); they replace the values the container would pass. `--target-device`, `--max-ctx` and `--profile` all work: + +```bash +# Show the GPUs, the model, and the device auto placement would use +docker run --rm -v ... ghcr.io/luce-org/lucebox-hub:rocm devices + +# DeepSeek V4 on Strix Halo with its qualified profile (DSpark drafter in models/draft/) +docker run --rm -p 8000:8080 -v ... ghcr.io/luce-org/lucebox-hub:rocm --profile ds4-strix + +# Pin a device +docker run --rm -p 8000:8080 -v ... ghcr.io/luce-org/lucebox-hub:rocm --target-device hip:1 +``` + +Environment variables such as `LUCE_TARGET`, `LUCE_TARGET_DEVICE`, `LUCE_MAX_CTX` and `LUCE_ARGS` cover the same settings for compose files; see the header of [`server/scripts/entrypoint.sh`](server/scripts/entrypoint.sh). + ## Run the Server This quick start runs the R9700 profile above. The complete flag reference is in the [server guide](server/README.md#server-parameter-reference). diff --git a/docs/image-input.md b/docs/image-input.md index f6e88c753..bac610f18 100644 --- a/docs/image-input.md +++ b/docs/image-input.md @@ -48,28 +48,27 @@ hf download Lucebox/DeepSeek-V4-Flash-0731-ROCmFP3 \ hf download Lucebox/DeepSeek-V4-Flash-0731-DSpark-GGUF \ DeepSeek-V4-Flash-0731-DSpark-draft-Q4RMFP4-denseF16.gguf --local-dir models -LUCE_DS4_SPEC=1 \ -LUCE_DS4_DRAFT=models/DeepSeek-V4-Flash-0731-DSpark-draft-Q4RMFP4-denseF16.gguf \ LUCE_DS4_SPARSE_DECODE_FLASH=1 \ ./server/build-hip/luce_server models/DeepSeek-V4-Flash-Vision-Exp-ROCMFPX-MIX-STRIX.gguf \ - --target-device hip:0 --max-ctx 131072 --chunk 8192 \ - --cache-type-k q4_0 --cache-type-v q4_0 \ - --ds4-fused-decode --ds4-fused-verify-f16-kv \ - --ds4-expert-top-k 6 --ds4-prefill sparse \ + --draft models/DeepSeek-V4-Flash-0731-DSpark-draft-Q4RMFP4-denseF16.gguf \ + --target-device auto --profile ds4-strix \ --mmproj models/DeepSeek-V4-Flash-Vision-Exp-mmproj-BF16.gguf \ --port 8216 ``` -`hip:0` must be the Strix Halo; on a host with a discrete GPU too, expose the -Strix Halo alone with `HIP_VISIBLE_DEVICES`. This is the text model's published -launch plus `--mmproj`: the Vision file replaces +The model must run on the Strix Halo. `--target-device auto` picks it when the +discrete GPU cannot hold the model, as with an R9700 (32 GB); with a larger +discrete GPU, name the Strix Halo with `--target-device hip:N` instead +(`luce_server --list-devices` shows the choice). +This is the text model's published launch (`--profile ds4-strix`) plus `--mmproj`: the Vision file replaces `DeepSeek-V4-Flash-0731-ROCMFPX-MIX-STRIX.gguf` for text as well and decodes at least as fast (numbers below). For R9700 + Strix Halo see [DS4V](#ds4v) below. With an R9700 in the same box, run the image encoder there while the model -stays on the Strix Halo: expose both GPUs, point `--target-device` at the Strix -Halo and add `--mmproj-device` with the R9700 (on lucebox6, without -`HIP_VISIBLE_DEVICES`, that is `--target-device hip:1 --mmproj-device hip:0`). +stays on the Strix Halo: expose both GPUs and add `--mmproj-device` with the +R9700. `--target-device auto` already puts the model on the Strix Halo (on +lucebox6, without `HIP_VISIBLE_DEVICES`, that is `hip:1`, so add +`--mmproj-device hip:0`). The encoder then runs about twice as fast and streams each image into prefill as soon as it is encoded, so the Strix Halo never waits for the next one: diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index d5d95d706..287617912 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -603,6 +603,7 @@ add_library(luce_common STATIC src/common/backend_factory.cpp src/common/feature_gate.cpp src/placement/placement_config.cpp + src/placement/device_select.cpp src/common/layer_split_utils.cpp src/common/ddtree.cpp src/common/peer_access.cpp @@ -1966,15 +1967,15 @@ if(LUCE_TESTS) list(APPEND _raw_unit_test_targets test_moe_input_ready) endif() - # The production CUDA and ROCm images share this entrypoint. Keep native - # server defaults intact unless an operator explicitly supplies an env - # override; otherwise container launches can silently disable features - # that are enabled by the C++ server itself. - if(UNIX AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_entrypoint_cache_defaults.sh") + # The production CUDA and ROCm images share this entrypoint. The test + # drives it against a fake luce_server: native defaults stay with the + # server, operator flags pass through and win, and device, context and + # draft choices follow the probed hardware and model architecture. + if(UNIX AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_entrypoint.sh") add_test( - NAME server_unit_entrypoint_cache_defaults + NAME server_unit_entrypoint COMMAND bash - "${CMAKE_CURRENT_SOURCE_DIR}/test/test_entrypoint_cache_defaults.sh" + "${CMAKE_CURRENT_SOURCE_DIR}/test/test_entrypoint.sh" "${CMAKE_CURRENT_SOURCE_DIR}/scripts/entrypoint.sh") endif() @@ -2245,6 +2246,7 @@ if(LUCE_TESTS) add_executable(test_feature_gate test/test_feature_gate.cpp) target_sources(test_feature_gate PRIVATE test/test_backend_plan.cpp + test/test_launch_policy.cpp src/common/backend_plan.cpp src/common/feature_gate.cpp src/kv_quant.cpp diff --git a/server/README.md b/server/README.md index d95f32f4b..1129c1659 100644 --- a/server/README.md +++ b/server/README.md @@ -226,17 +226,21 @@ RTX mixed-hardware notes before running long prompts. The command shape is `luce_server [options]`. The first positional argument selects the target weights. `--model-name` only changes the name reported by the API; it does not select a model file. +`luce_server --list-devices [model.gguf]` prints every GPU with its `backend:N` index, architecture and memory, and, given a model, the device `--target-device auto` would choose. + +`--profile ` applies a qualified hardware and model configuration: `ds4-strix` (DeepSeek V4 on Strix Halo) or `ds4-r9700-strix` (DeepSeek V4 with experts split between an R9700 and Strix Halo). Flags on the command line replace the profile's value, and environment variables that are already set keep theirs. The startup log lists what the profile applied. + ### Core server | Option | Default | Purpose | |---|---|---| -| `--draft ` | none | Draft model for speculative decode. | +| `--draft ` | none | Draft model for speculative decode: DFlash for Qwen, Gemma and Laguna, DSpark for DeepSeek V4. | | `--host ` | `0.0.0.0` | Bind address. | | `--port ` | `8080` | Listen port. | -| `--max-ctx ` | `131072` | Maximum context length. | +| `--max-ctx ` | `8192` | Maximum context length. | | `--max-tokens ` | model card | Legacy alias for `--default-max-tokens`. | | `--default-max-tokens ` | model card or `16000` | Output cap when a request omits a token limit. | -| `--model-name ` | `dflash` | API alias returned by `/v1/models` and responses. It does not change the loaded weights. | +| `--model-name ` | `luce` | API alias returned by `/v1/models` and responses. It does not change the loaded weights. | | `--chat-template-file ` | model default | Jinja chat-template override. | | `--no-cors` | CORS enabled | Disable CORS headers. | @@ -263,8 +267,9 @@ The command shape is `luce_server [options]`. The first positional | Option | Default | Purpose | |---|---|---| -| `--target-device ` | `auto:0` | Place the target on a CUDA or HIP device. | -| `--draft-device ` | `auto:0` | Place the draft on a CUDA or HIP device. | +| `--target-device ` | `auto:0` or `LUCE_TARGET_DEVICE` | Place the target on a CUDA or HIP device. `auto` picks a GPU the model fits on (discrete before integrated, then the lowest index), else the largest GPU. | +| `--draft-device ` | `auto:0` | Place the draft on a CUDA or HIP device. DeepSeek V4 and `--target-device auto` default to the target GPU. | +| `--expert-device ` | none | DeepSeek V4: keep dense work and hot experts on the target and run the remaining routed experts on this GPU in the same process. | | `--target-devices ` | one device | Select multiple target devices, such as `cuda:0,cuda:1`. | | `--target-split-mode layer\|tensor` | `layer` | Select the multi-GPU target strategy. | | `--target-layer-split ` | none | Optional layer-split weights. | diff --git a/server/docs/DS4.md b/server/docs/DS4.md index b4b2174a9..1e37bac05 100644 --- a/server/docs/DS4.md +++ b/server/docs/DS4.md @@ -120,8 +120,8 @@ options are available: HC, attention, MoE, and the output projection on the GPU and avoids per-layer host round trips. On HIP this option requests a monolithic model load because the fused graph must reference every expert tensor directly. - If that allocation fails, the backend logs the fallback and continues with - hybrid expert placement and layered decode. + If that allocation fails, startup fails; use `luce_server --list-devices + ` to find a GPU that holds the model, or `--target-device auto`. - Adaptive qtype-105/106 experts work in monolithic mode and across two GPUs using the same runtime. The loader gives each GPU's compact tensor the decode-table rows for the experts it owns. CPU expert offload and mixed @@ -150,14 +150,28 @@ options are available: formats' different error profile crosses the threshold. Serve adaptive artifacts at the model default. -For the validated single-device Strix Halo profile: +For the validated single-device Strix Halo profile (`--profile ds4-strix` +expands to the flags listed in +[RECOMMENDED_SETUPS.md](RECOMMENDED_SETUPS.md#deepseek-v4-on-strix-halo)): ```bash ./server/build-hip/luce_server /opt/models/DeepSeek-V4-Flash.gguf \ - --target-device hip:0 \ - --ds4-fused-decode + --draft /opt/models/DeepSeek-V4-Flash-DSpark-draft.gguf \ + --target-device auto \ + --profile ds4-strix ``` +`--draft` on a DeepSeek V4 target loads the DSpark drafter and +`--draft-device` places it (default: the target GPU). The environment +spelling `LUCE_DS4_SPEC=1 LUCE_DS4_DRAFT=` with `LUCE_DS4_DRAFT_GPU` and +`LUCE_DS4_DRAFT_BACKEND` still works and is used when `--draft` is absent. An +explicit `--draft` that cannot be loaded stops startup; the environment +spelling keeps its autoregressive fallback. DSpark and both profiles serve one +request at a time: paged concurrent serving (`--max-concurrency` above 1) +decodes autoregressively with exact prefill and rejects `--draft`, sparse +prefill and fused decode (see +[Strix Halo concurrent serving](#strix-halo-concurrent-serving)). + ### In-process heterogeneous expert parallel The Lucebox path keeps dense target work, selected hot experts, and the local @@ -185,36 +199,28 @@ tokens per step with the fused verifier, and prefills in batched sparse mode. It needs no calibration files. ```bash -export LUCE_DS4_MOE_TP=1 LUCE_DS4_MOE_TP_INPROC=1 LUCE_DS4_MOE_TP_GPU=1 -export LUCE_EXPERT_BUDGET_MB=14350 -export LUCE_DS4_SPEC=1 LUCE_DS4_DRAFT=/path/to/deepseek4-dspark-draft.gguf -export LUCE_DS4_DRAFT_GPU=0 LUCE_DS4_SPEC_Q=5 LUCE_DS4_Q5_VERIFY=1 -export LUCE_DS4_FUSED_VERIFY=1 LUCE_DS4_FUSED_HYBRID_DECODE=1 -export LUCE_DS4_PINNED_ROLLBACK=1 LUCE_DS4_GPU_ARGMAX_VERIFY=1 -export LUCE_DS4_DRAFT_CONTEXT_KV_CACHE=1 -export LUCE_DS4_TP_ROUTE_PREFORK=1 LUCE_DS4_TP_DEVICE_JOIN=1 LUCE_DS4_TP_DEVICE_JOIN_SPLIT=1 -export LUCE_DS4_TP_FUSED_HC_JOIN=1 LUCE_DS4_TP_MAIN_ROUTE_WEIGHTS=1 -export LUCE_DS4_TP_COARSE_OWNER=1 LUCE_DS4_TP_NATIVE_ROUTE_WIDTH=1 -export LUCE_DS4_TP_MASKED_ROUTES=1 LUCE_DS4_TP_GROUPED_MMVQ=1 -export LUCE_DS4_TP_CAPTURE_CACHE_SLOTS=4 -export LUCE_MOE_TP_DYNAMIC_ROUTE_BALANCE=1 LUCE_MOE_TP_DYNAMIC_MAIN_SLOTS_X4=13 -export LUCE_MOE_DUPLICATE_HOT_ON_COLD=1 LUCE_MOE_FULL_COLD_PARALLEL=1 -export LUCE_MOE_PREFILL_PERSISTENT_OWNER_ALLOC=1 -export LUCE_DS4_HYBRID_PREFILL_GPU_HC=1 LUCE_DS4_HYBRID_PREFILL_EAGER=1 -export GGML_CUDA_BATCH_PEER_COPIES=1 -export LUCE_MMID_GROUPED=1 LUCE_MMID_GROUPED_TYPES=8 LUCE_MMID_GROUPED_DEVICE=1 -export LUCE_CUDA_MMVQ_MOE_ROWS_PER_BLOCK=2 LUCE_CUDA_MMVQ_MOE_FP3_PACKED24=1 LUCE_CUDA_MMVQ_FP4_X4=1 -export LUCE_DS4_DIRECT_INDEXER_TOPK=1 GGML_DS4_TOPK_BLOCK_RADIX=1 -export LUCE_DS4_MIX_MMQ_PREFILL=1 LUCE_CUDA_I32_REPEAT=1 -export ROCBLAS_USE_HIPBLASLT=0 - ./server/build-hip-dual/luce_server /path/to/deepseek4-target.gguf \ - --target-device hip:0 --peer-access \ - --max-ctx 18432 --chunk 2048 \ - --ds4-fused-decode --ds4-expert-top-k 6 \ - --ds4-prefill sparse + --draft /path/to/deepseek4-dspark-draft.gguf \ + --profile ds4-r9700-strix ``` +`--profile ds4-r9700-strix` expands to `--target-device hip:0 +--expert-device hip:1 --peer-access --max-ctx 18432 --chunk 2048 +--ds4-fused-decode --ds4-expert-top-k 6 --ds4-prefill sparse` and installs +the 37 tuning variables of the qualified recipe (`LUCE_EXPERT_BUDGET_MB=14350`, +`LUCE_DS4_SPEC_Q=5`, `LUCE_DS4_Q5_VERIFY=1`, the `LUCE_DS4_TP_*` join and +routing switches, and the grouped MMVQ kernels; the full list is in +`src/server/launch_profiles.h`). Flags on the command line replace the +profile's value, and variables already set in the environment keep theirs; +the startup log names each one it kept. The profile's `--expert-device` is a +flag like any other, so it sets the `LUCE_DS4_MOE_TP*` variables below; pass +`--expert-device` to move the experts. If the R9700 and Strix Halo enumerate +the other way round, pass `--target-device hip:1 --expert-device hip:0` +(`luce_server --list-devices` shows the order). + +`--expert-device ` is the flag form of `LUCE_DS4_MOE_TP=1 +LUCE_DS4_MOE_TP_INPROC=1 LUCE_DS4_MOE_TP_GPU= LUCE_DS4_MOE_TP_BACKEND=`. + Measured with this command on an R9700 + Strix Halo machine (fixed-codebook ROCmFPx target, 18432-token context, greedy, every output deterministic across repeats and identical between streaming and non-streaming): @@ -235,9 +241,8 @@ path exists only for sparse prefill. Top-4 routing (`--ds4-expert-top-k 4`) is a further approximation that raises decode speed; omit it when the model-default top-6 route is required. -The minimal activation (`LUCE_DS4_MOE_TP=1`, `LUCE_DS4_MOE_TP_INPROC=1`, -`LUCE_DS4_MOE_TP_GPU=1`, `LUCE_EXPERT_BUDGET_MB=11700`, -`LUCE_MMVQ_MAX_NCOLS=4`) runs the same model with the fused decode and verify +The minimal activation (`--target-device hip:0 --expert-device hip:1` with +`LUCE_EXPERT_BUDGET_MB=11700` and `LUCE_MMVQ_MAX_NCOLS=4`) runs the same model with the fused decode and verify paths off and decodes at 12-15 tok/s; it is only useful to check placement. #### Radeon RX 7900 XT + Strix Halo, true top-k-6 @@ -347,10 +352,6 @@ ctest --test-dir server/build-cuda-hip -R mixed_cuda_hip --output-on-failure ``` ```bash -export LUCE_DS4_MOE_TP=1 -export LUCE_DS4_MOE_TP_INPROC=1 -export LUCE_DS4_MOE_TP_BACKEND=cuda -export LUCE_DS4_MOE_TP_GPU=0 # cuda:0 (RTX 3090) export LUCE_DS4_MOE_TP_CONCENTRATE_COLD=1 export LUCE_DS4_TP_SCHEDULE_BRANCHES=1 export LUCE_DS4_TP_TARGETED_JOIN_SPLIT=1 @@ -358,12 +359,15 @@ export GGML_BATCH_PEER_COPIES=1 # Start conservatively and tune from the startup placement and memory logs; # the usable budget depends on the model, placement policy, and free VRAM. export LUCE_EXPERT_BUDGET_MB=85000 -export LUCE_DS4_DRAFT=/path/to/dspark-draft.gguf +# The drafter runs on the peer runtime, which --draft-device cannot name +# without the IPC path, so its placement stays in the environment. export LUCE_DS4_DRAFT_BACKEND=cuda export LUCE_DS4_DRAFT_GPU=0 ./server/build-cuda-hip/luce_server /path/to/deepseek4-target.gguf \ --target-device hip:0 \ + --expert-device cuda:0 \ + --draft /path/to/dspark-draft.gguf \ --ds4-prefill sparse ``` @@ -388,6 +392,11 @@ DeepSeek4 paged concurrency supports two resident HIP deployments: secondary owns the remaining materialized experts. The runtime accepts up to 6 lanes, but concurrency 5–6 is not qualified for this configuration. +Paged serving decodes autoregressively: it rejects `--draft` (and +`LUCE_DS4_SPEC`), sparse prefill and fused decode, so it does not combine +with `--profile ds4-strix` or `ds4-r9700-strix`. The container entrypoint +leaves out its DSpark drafter when `--max-concurrency` is above 1. + The heterogeneous mode is route-level expert parallelism. It is not an explicit `--target-device hip:0,hip:1` layer split and does not use a remote target shard or host-streamed experts. @@ -440,15 +449,12 @@ cmake --build server/build-hip -j For the R9700 + Strix Halo path, build `server/build-hip-dual` for `gfx1151;gfx1201` as shown in [In-process heterogeneous expert parallel](#in-process-heterogeneous-expert-parallel). -Then expose the R9700 first and select the static in-process expert split. -The grouped expert setting below is qualified at concurrency 1–4 with -ROCmFP2 gate/up and ROCmFP3 down weights: +Then target the R9700 and give the static in-process expert split to Strix +Halo (`luce_server --list-devices` shows which index each has). The grouped +expert setting below is qualified at concurrency 1–4 with ROCmFP2 gate/up and +ROCmFP3 down weights: ```bash -export HIP_VISIBLE_DEVICES=, -export LUCE_DS4_MOE_TP=1 -export LUCE_DS4_MOE_TP_INPROC=1 -export LUCE_DS4_MOE_TP_GPU=1 export LUCE_EXPERT_BUDGET_MB=11700 export LUCE_DS4_TP_BATCH_SPLIT_COPIES=1 export LUCE_DS4_TP_GROUPED_MMVQ=1 @@ -456,6 +462,7 @@ export LUCE_DS4_TP_GROUPED_MMVQ=1 ./server/build-hip-dual/luce_server \ /path/to/models/DeepSeek-V4-Flash.gguf \ --target-device hip:0 \ + --expert-device hip:1 \ --peer-access \ --paged-attention \ --max-concurrency 4 \ @@ -534,7 +541,7 @@ The runtime logs the chosen split with a `[deepseek4-split] auto-split:` banner. | `LUCE_DS4_CUDA_LAYERS` | Override the auto-split heuristic and pin the first `N` DeepSeek4 layers to CUDA. The remaining `43 - N` layers run on the Halo shard. | | `LUCE_DS4_TIMING` | Enable DS4 timing logs for local, paged, and layer-split execution. Paged rounds report full-graph build, input upload, compute, and readback time; leave unset for normal runs. | | `LUCE_DS4_ROCTX` | HIP-only, default-off semantic ROCTX ranges for an external rocprof trace. The library is loaded dynamically only when set to `1`, `true`, `yes`, or `on`. | -| `LUCE_DS4_SPEC` / `LUCE_DS4_DRAFT` | Enable DSpark and select its GGUF. | +| `LUCE_DS4_SPEC` / `LUCE_DS4_DRAFT` | Enable DSpark and select its GGUF when `--draft` is absent. | | `LUCE_DS4_DRAFT_BACKEND` / `LUCE_DS4_DRAFT_GPU` | Backend and device for the in-process drafter. | | `LUCE_DS4_MOE_TP` | Enable routed-expert partitioning. | | `LUCE_DS4_MOE_TP_INPROC` | Use two local GPU backends instead of an expert IPC worker. | @@ -553,12 +560,12 @@ The runtime logs the chosen split with a `[deepseek4-split] auto-split:` banner. | `LUCE_DS4_HOTNESS_CSV` | Optional per-layer routing profile for hot placement. | | `LUCE_DS4_TP_GROUPED_MMVQ` | Opt in to grouped expert MMVQ for `n_tokens > 1`, replacing tokenwise ROCmFP2 gate/up dispatch. The paged R9700 + Strix profile is qualified at concurrency 1–4; the flag itself does not enforce topology or lane limits. `LUCE_MOE_TP_GROUPED_MMVQ` is the model-neutral name and takes precedence. | | `LUCE_DS4_TP_BATCH_SPLIT_COPIES` | Establish destination readiness once per scheduler split without combining backend copy dependencies. The qualified dual-ROCm launcher enables this exact path. | -| `GGML_BATCH_PEER_COPIES` | Additionally combine HIP peer-copy dependency publication. The old `GGML_CUDA_BATCH_PEER_COPIES` spelling remains an alias. Keep these event-batching variables unset for the exact qualified profile. | +| `GGML_BATCH_PEER_COPIES` | Additionally combine HIP peer-copy dependency publication. The old `GGML_CUDA_BATCH_PEER_COPIES` spelling remains an alias. Keep these event-batching variables unset for the qualified RX 7900 XT launcher (`serve_ds4_dual_rocm_128k.sh`); the R9700 + Strix profile (`--profile ds4-r9700-strix`) sets `GGML_CUDA_BATCH_PEER_COPIES=1`. | | `LUCE_DS4_TP_CRITICAL_PATH_PLACEMENT` | Use the routing profile and measured owner-rate ratio to minimize the predicted two-owner MoE critical path instead of maximizing aggregate hot-hit rate. Requires `LUCE_DS4_HOTNESS_CSV`. | | `LUCE_DS4_TP_MAIN_TO_PEER_RATE` | Relative main/peer routed-expert rate used by critical-path placement. It must be finite and greater than zero; the default is `3.4`. | | `LUCE_DS4_TP_BALANCE_MIN_HOT` | Minimum hot experts retained on every routed layer by critical-path placement. Defaults to `0`. | -| `LUCE_DS4_Q5_VERIFY` | AMD q=5 fused verifier. Defaults to `1` on `gfx1151` when `LUCE_DS4_SPEC` is set, together with `LUCE_DS4_FUSED_VERIFY=1` and `LUCE_DS4_ADAPTIVE_WIDTH=1`; set `0` to restore the q<=4 verifier. It also selects the qualified MMVQ width and verifier-cache defaults when they are not explicitly overridden. | -| `LUCE_DS4_ADAPTIVE_WIDTH` | Acceptance-and-cost verify-width controller. Defaults to `1` on `gfx1151` with `LUCE_DS4_SPEC`; chooses q2 to q5 per step (q5 is the cap) from measured acceptance and per-width cost. Set `0` for a fixed width. | +| `LUCE_DS4_Q5_VERIFY` | AMD q=5 fused verifier. Defaults to `1` on `gfx1151` when DSpark is enabled (`--draft` or `LUCE_DS4_SPEC`), together with `LUCE_DS4_FUSED_VERIFY=1` and `LUCE_DS4_ADAPTIVE_WIDTH=1`; set `0` to restore the q<=4 verifier. It also selects the qualified MMVQ width and verifier-cache defaults when they are not explicitly overridden. | +| `LUCE_DS4_ADAPTIVE_WIDTH` | Acceptance-and-cost verify-width controller. Defaults to `1` on `gfx1151` with DSpark enabled; chooses q2 to q5 per step (q5 is the cap) from measured acceptance and per-width cost. Set `0` for a fixed width. | | `LUCE_DS4_CONFIDENCE_WIDTH` | With the adaptive width and a drafter that carries a confidence head, the width of every step is chosen from the head's per-candidate scores (three depths on the q5 verifier; the fourth is learned from target feedback), calibrated online per depth against the target's actual acceptance. Defaults on; set `0` to fall back to the learned-acceptance policy. `LUCE_DS4_TIMING=1` prints the per-depth calibration (predicted, actual, applied scale) after every request. | | `LUCE_DS4_DIRECT_CONTIGUOUS_CAUSAL` | Analytic causal window for layer-major sliding-window layers instead of the quadratic mask upload. Part of the `gfx1151` sparse-prefill defaults (measured +10% prefill, identical output); set `0` to restore the explicit mask. | | `LUCE_DS4_INDEXER_F16_Q` / `LUCE_DS4_PREFILL_F16_KV_ALL` | F16 indexer queries and F16 selected-KV transport for sparse prefill. Part of the `gfx1151` sparse-prefill defaults; set `0` to restore F32. | @@ -637,14 +644,13 @@ python server/scripts/convert_dflash_to_gguf.py \ Run the converted drafter against a DeepSeek4 target with: ```bash -export LUCE_DS4_SPEC=1 export LUCE_DS4_FUSED_VERIFY=1 # Experimental, single HIP target only; may change generated tokens: # export LUCE_DS4_SPARSE_DECODE_FLASH=1 -export LUCE_DS4_DRAFT=/path/to/dflash-draft.gguf export LUCE_DS4_SPEC_Q=4 ./server/build-hip/luce_server /path/to/deepseek4-target.gguf \ + --draft /path/to/dflash-draft.gguf \ --target-device hip:0 \ --ds4-fused-verify-f16-kv \ --ds4-fused-decode @@ -735,7 +741,7 @@ ROCmFP2 matvecs with three or more query rows reuse each activation across four output rows on gfx1151. Set `LUCE_ROCMFP2_ROW4=0` to restore the two-row schedule. Narrow F16 projections retain the shared MMVF dispatch policy. -`LUCE_DS4_FUSED_VERIFY=1` is the throughput profile; it is the default on `gfx1151` when `LUCE_DS4_SPEC` is set and opt-in elsewhere. Its persistent +`LUCE_DS4_FUSED_VERIFY=1` is the throughput profile; it is the default on `gfx1151` when DSpark is enabled and opt-in elsewhere. Its persistent whole-model GPU graph uses stable padded reduction shapes, so near-tied greedy logits can select a different token than the normal causal verifier even at temperature 0. Leave it unset when comparing against the normal verifier, or @@ -746,8 +752,9 @@ fused verification nor the separate `--ds4-expert-top-k 4` approximation should be presented as byte-identical AR. DSpark can verify against in-process heterogeneous expert placement. The -drafter remains local to its selected GPU backend; a failed draft load is -reported and falls back to normal autoregressive decode. The target cache and +drafter remains local to its selected GPU backend. A drafter given with +`--draft` that fails to load stops startup; with the `LUCE_DS4_SPEC` spelling +the failure is reported and decode falls back to autoregressive. The target cache and sampler stay on the main backend while routed target experts execute on their configured owners. `--ds4-expert-top-k 4` remains a separate approximate policy; omit it to retain the model's default six routed experts. @@ -958,9 +965,9 @@ that fallback and `LUCE_DS4_ADAPTIVE_WIDTH=0` restores a fixed width. Re-run workload-level speed and quality checks before enabling it on another target or drafter. -The qualified `gfx1151` launch is the plain one: `LUCE_DS4_SPEC=1`, -`LUCE_DS4_DRAFT=`, the release CLI with `--chunk 8192` -and a 128K context. Every kernel and policy default above is installed by the +The qualified `gfx1151` launch is the plain one: `--draft --profile ds4-strix`, which is the release CLI with `--chunk 8192` and +a 128K context. Every kernel and policy default above is installed by the device profile at start, and the published Strix Halo numbers (8K 320 / 42 prefill / decode tok/s, 123K 284 / 36, code and math suites 39 tok/s at q5, prose 25 at q2) are measured exactly that way. Use `--chunk 8192` on the diff --git a/server/docs/ENVIRONMENT.md b/server/docs/ENVIRONMENT.md index 947064096..45a41774d 100644 --- a/server/docs/ENVIRONMENT.md +++ b/server/docs/ENVIRONMENT.md @@ -33,7 +33,7 @@ consolidation of this list into CLI flags is tracked as follow-up work. | `LUCE_MMID_GROUPED` | unset | Grouped MUL_MAT_ID kernel for small verify batches; candidate for CLI promotion. | | `LUCE_MMID_GROUPED_TYPES` | 7 | Grouped-kernel type mask; bit 3 (`8`) opts ROCmFP2/ROCmFP3 into the path. | | `LUCE_MMID_GROUPED_DEVICE` | -1 | Optional zero-based device restriction; unset/-1 applies to every eligible device. | -| `LUCE_DS4_MOE_TP` / `LUCE_DS4_MOE_TP_INPROC` | unset | BURN-IN: enable DeepSeek4 route-owner expert parallelism in one process. | +| `LUCE_DS4_MOE_TP` / `LUCE_DS4_MOE_TP_INPROC` | unset | BURN-IN: enable DeepSeek4 route-owner expert parallelism in one process. Prefer the CLI: `--expert-device ` sets both with the device and backend. | | `LUCE_DS4_MOE_TP_BACKEND` / `LUCE_MOE_TP_BACKEND` | peer runtime in a mixed build; compiled runtime otherwise | Select the in-process cold expert owner backend. | | `LUCE_DS4_MOE_TP_GPU` | peer backend device 0 in a mixed build; other local device otherwise | Device index within the cold DeepSeek4 expert backend. | | `LUCE_DS4_MOE_TP_CONCENTRATE_COLD` | unset | BURN-IN: use complete peer-owned expert layers to reduce cross-runtime joins; falls back when the placement would exceed the target budget. | @@ -80,6 +80,8 @@ consolidation of this list into CLI flags is tracked as follow-up work. | `LUCE_PREFIX_CACHE_SLOTS` | 32 | Container-entrypoint equivalent of `--prefix-cache-slots`; not read directly by the native binary. | | `LUCE_PREFILL_CACHE_SLOTS` | 0 | Container-entrypoint equivalent of `--prefill-cache-slots`; not read directly by the native binary. | | `LUCE_MMPROJ` | unset | Container-entrypoint equivalent of `--mmproj` (vision projector path, enables image input); not read directly by the native binary. | +| `LUCE_TARGET_DEVICE` | unset (`auto` in the container) | Target device (`backend:gpu` or `auto`) used when neither `--target-device`, `--target-devices` nor a `--profile` names one. | +| `LUCE_PROFILE` / `LUCE_ARGS` | unset | Container entrypoint only: `--profile` name, and extra `luce_server` flags split on whitespace. | | `LUCE_PREFILL_POOL_TRIM_TOKENS` | unset | OPT-IN: trim cached allocations from legacy CUDA/HIP device pools at completed Qwen3.5 prefill chunk boundaries after each configured token interval. Intended for long, shape-changing prefills on non-VMM devices; each trim synchronizes the target backend and retires captured graphs. | | `LUCE_SPLIT_FAST_ROLLBACK` | unset | OPT-IN: exact F32 checkpoints and replay-free rollback for local qwen35 target layer splits. Prefer `--target-split-fast-rollback`; adds checkpoint VRAM (~1.65 GiB for the measured Qwen3.6-27B q=16 split). | | `LUCE_STALL_TOOL_PREFIX` | unset | OPT-IN: recover a stalled tool call by injecting the prepared tool prefix when generation stops after an action suffix. | @@ -107,6 +109,7 @@ consolidation of this list into CLI flags is tracked as follow-up work. - `LUCE_ADAPTIVE_SPEC_WIDTH` - adaptive_spec_width.h - `LUCE_ADAPTIVE_WIDTH_MIN` - adaptive_verify_width.h - `LUCE_ADAPTIVE_WIDTH_THETA` - adaptive_verify_width.h +- `LUCE_ARGS` - scripts/entrypoint.sh (extra `luce_server` flags, split on whitespace) - `LUCE_COLD_THREADS` - moe_expert_compute_cpu.cpp - `LUCE_CUDA_BACKEND_PATH` - dynamic_backend.cpp - `LUCE_CUDA_MMVF_NARROW_F16` - ggml-cuda/mmvf.cu @@ -299,6 +302,7 @@ consolidation of this list into CLI flags is tracked as follow-up work. - `LUCE_PREFILL_POOL_TRIM_TOKENS` - qwen35_backend.cpp (OPT-IN: trim legacy device pools during long prefills) - `LUCE_PREFILL_TIMING` - qwen35_backend.cpp (DEBUG: per-ubatch prefill build/alloc/compute timing) - `LUCE_PREFIX_CACHE_SLOTS` - scripts/entrypoint.sh (maps to `--prefix-cache-slots`) +- `LUCE_PROFILE` - scripts/entrypoint.sh (maps to `--profile`) - `LUCE_QWEN35MOE_CACHE_SLOTS` - qwen35moe_backend.cpp - `LUCE_QWEN35MOE_HOTNESS` - qwen35moe_backend.cpp - `LUCE_QWEN35MOE_NEXT_PLACEMENT_OUT` - qwen35moe_backend.cpp @@ -332,6 +336,7 @@ consolidation of this list into CLI flags is tracked as follow-up work. - `LUCE_SPLIT_FAST_ROLLBACK` - chain_rollback_policy.h - `LUCE_STALL_TOOL_PREFIX` - http_server.cpp - `LUCE_SV_DEBUG` - qwen35_backend.cpp +- `LUCE_TARGET_DEVICE` - server_main.cpp (default for `--target-device`) - `LUCE_TARGET_SHARD_IPC_SHARED_BYTES` - target_shard_ipc.cpp - `LUCE_TARGET_SHARD_IPC_TRANSPORT` - target_shard_ipc.cpp - `LUCE_TOPK_PROFILE` - geometric_draft_topk_cuda.cu diff --git a/server/docs/RECOMMENDED_SETUPS.md b/server/docs/RECOMMENDED_SETUPS.md index b3ce596ef..d3ac6b3a6 100644 --- a/server/docs/RECOMMENDED_SETUPS.md +++ b/server/docs/RECOMMENDED_SETUPS.md @@ -14,27 +14,23 @@ Entries are `luce_server` arguments unless the cell contains another command. Pa | **Gemma 4 26B-A4B or 31B** | `--target-device cuda:0`
`--draft-device cuda:0`
`--kvflash auto` | `--target-device hip:0`
`--draft-device hip:0`
`--kvflash auto` | `--target-device hip:0`
`--draft-device hip:0`
`--kvflash auto` | | **Qwen 3.5 0.8B Megakernel** | `uv run --directory optimizations/megakernel python final_bench.py --backend bf16` | Not supported | Not supported | -If a machine has both Strix Halo and an R9700, set `HIP_VISIBLE_DEVICES=` before using an R9700-only profile. The selected card is then `hip:0` inside the process. +`luce_server --list-devices [model.gguf]` prints each GPU with its `backend:N` index, architecture and memory. On a machine with both Strix Halo and an R9700, use it to pick the index for `--target-device` and `--draft-device`, or pass `--target-device auto` to place the model on a GPU it fits on (discrete first). ## DeepSeek V4 on Strix Halo -The plain launch is the qualified one. The `gfx1151` device profile installs every kernel and policy default at start (fused five-row verifier, verify width from the DSpark confidence head, sparse prefill kernels), so there is nothing to tune. Use the adaptive ROCmFPX artifact with all six routed experts and `--chunk 8192` on the 128 GB part. Measured this way: 42 tok/s decode and 320 tok/s prefill at 8K, 36 tok/s at 123K, 39 tok/s on code and math, 25 tok/s on prose ([PR #729](https://github.com/Luce-Org/lucebox/pull/729); details in the [DeepSeek V4 guide](DS4.md#experimental-amd-q5-verifier)). +`--profile ds4-strix` is the qualified launch: it sets `--max-ctx 131072 --chunk 8192 --ds4-fused-decode --ds4-fused-verify-f16-kv --ds4-expert-top-k 6 --ds4-prefill sparse`, and the `gfx1151` device profile installs every kernel and policy default at start (fused five-row verifier, verify width from the DSpark confidence head, sparse prefill kernels), so there is nothing to tune. Any flag you pass replaces the profile's value. Use the adaptive ROCmFPX artifact with all six routed experts on the 128 GB part. Measured this way: 42 tok/s decode and 320 tok/s prefill at 8K, 36 tok/s at 123K, 39 tok/s on code and math, 25 tok/s on prose ([PR #729](https://github.com/Luce-Org/lucebox/pull/729); details in the [DeepSeek V4 guide](DS4.md#experimental-amd-q5-verifier)). ```bash # LUCE_DS4_SPARSE_DECODE_FLASH=1 stays an explicit experimental opt-in # (single HIP target; may change generated tokens — see DS4.md). -LUCE_DS4_SPEC=1 \ -LUCE_DS4_DRAFT=/path/to/DeepSeek-V4-Flash-0731-DSpark-draft-Q4RMFP4-denseF16.gguf \ luce_server /path/to/DeepSeek-V4-Flash-0731-ROCMFPX-MIX-STRIX.gguf \ - --target-device hip:0 \ - --max-ctx 131072 \ - --chunk 8192 \ - --cache-type-k q4_0 --cache-type-v q4_0 \ - --ds4-fused-decode --ds4-fused-verify-f16-kv \ - --ds4-expert-top-k 6 \ - --ds4-prefill sparse + --draft /path/to/DeepSeek-V4-Flash-0731-DSpark-draft-Q4RMFP4-denseF16.gguf \ + --target-device auto \ + --profile ds4-strix ``` +`--draft` loads the DSpark drafter; the older `LUCE_DS4_SPEC=1 LUCE_DS4_DRAFT=` spelling still works. `--target-device auto` resolves to the Strix Halo when it is the only GPU that holds the model; on a Strix-only machine `hip:0` is the same device. + The earlier fixed-width recipe behind the [blog post](https://www.lucebox.com/blog/deepseek-v4-flash-0731) still runs, but it pins verify width 4 and exact prefill and misses the fast path. ## Multi-GPU @@ -42,6 +38,7 @@ The earlier fixed-width recipe behind the [blog post](https://www.lucebox.com/bl | Hardware and model | Configuration | Validation | |---|---|---| | **2x RTX 3090 + Qwen 3.8 27B** | `--target-devices cuda:0,cuda:1`
`--target-split-mode tensor`
`--peer-access`
`--cache-type-k q4_0`
`--cache-type-v q4_0`
`--verify-width 8`
Use the Qwen 3.8 DFlash2 drafter. | [PR #637](https://github.com/Luce-Org/lucebox/pull/637) | +| **R9700 + Strix Halo + DeepSeek V4** | `--profile ds4-r9700-strix`
`--draft `
Dense work, hot experts and the drafter on the R9700 (`hip:0`); remaining experts on Strix Halo (`--expert-device hip:1`). | [DeepSeek V4 guide](DS4.md#in-process-heterogeneous-expert-parallel) | | **RX 7900 XT + Strix Halo + DeepSeek V4** | Build for `gfx1100;gfx1151`, then run [`serve_ds4_dual_rocm_128k.sh`](../scripts/serve_ds4_dual_rocm_128k.sh) with the target and DSpark paths. The checked-in profile uses all six routed experts. | [PR #604](https://github.com/Luce-Org/lucebox/pull/604) | The original setup matrix was introduced in [PR #602](https://github.com/Luce-Org/lucebox/pull/602). Keep a setting here only when it is still a useful starting point; measured claims belong beside their exact benchmark or qualification link. diff --git a/server/scripts/entrypoint.sh b/server/scripts/entrypoint.sh index 9f8c2524d..c661805ee 100755 --- a/server/scripts/entrypoint.sh +++ b/server/scripts/entrypoint.sh @@ -1,543 +1,369 @@ #!/usr/bin/env bash -# In-container ENTRYPOINT for lucebox-hub. +# In-container ENTRYPOINT for lucebox-hub (CUDA and ROCm images). # -# Normal path: the host-side `lucebox` CLI has already populated every -# LUCE_* env var from its detection / autotune sweep, so this script -# just resolves paths and execs the native luce_server binary. +# docker run IMAGE [serve] [luce_server flags...] start the server (default) +# docker run IMAGE devices list GPUs and the auto choice +# docker run IMAGE shell bash inside the container +# docker run IMAGE [args...] run any other command # -# Fallback path: a user runs the image directly (`docker run --gpus all -# ghcr.io/luce-org/lucebox-hub:cuda12`) with no env-var prep. We then do a -# minimal VRAM-tiered autotune — same tiers as `lucebox autotune`, kept in -# sync by hand. Anything more elaborate (driver-version probes, AMD paths, -# lspci fallbacks) belongs in the host CLI, not here. +# `serve` finds the model files under models/, then execs luce_server with the +# flags below. Any luce_server flag can follow `serve`, or come first, and it +# replaces the value the entrypoint would pass, e.g. +# +# docker run ... IMAGE --profile ds4-strix +# docker run ... IMAGE serve --target-device hip:1 --max-ctx 65536 +# +# Environment (all optional): +# LUCE_TARGET target GGUF (default: the one >5 GB file in models/) +# LUCE_DRAFT draft file or directory (default: models/draft; +# "none" runs without a draft) +# LUCE_TARGET_DEVICE backend:gpu or auto (default: auto, which picks +# a GPU the model fits on; see `devices`) +# LUCE_PROFILE luce_server --profile name +# LUCE_ARGS extra luce_server flags, split on whitespace +# LUCE_MAX_CTX context length (default: from the GPU's memory) +# LUCE_HOST, LUCE_PORT listen address (default: 0.0.0.0:8080) +# LUCE_BUDGET DDTree budget (default: 22) +# LUCE_LAZY 1 = request-scoped draft (needs LUCE_PREFILL_DRAFTER) +# LUCE_CACHE_TYPE_K/V KV cache types +# LUCE_PREFILL_MODE off|auto|always PFlash compression (default: off) +# LUCE_PREFILL_KEEP, LUCE_PREFILL_THRESHOLD, LUCE_PREFILL_DRAFTER +# LUCE_PREFIX_CACHE_SLOTS, LUCE_PREFILL_CACHE_SLOTS +# LUCE_DEFAULT_MAX_TOKENS, LUCE_MODEL_NAME, LUCE_THINK_MAX, LUCE_FA_WINDOW, +# LUCE_MMPROJ +# Other LUCE_* variables reach luce_server unchanged. set -euo pipefail -# Honor a pre-set LUCE_DIR (used by the host-side smoke tests to drive -# the entrypoint with a synthetic models/draft layout). In the shipped -# image this var is unset, so the fallback is the normal install prefix. +# Honor a pre-set LUCE_DIR (used by the entrypoint tests to drive a synthetic +# models/draft layout). In the shipped image this var is unset. LUCE_DIR="${LUCE_DIR:-/opt/lucebox-hub/server}" +: "${LUCE_SERVER_BIN:=$LUCE_DIR/build/luce_server}" -info() { printf '\033[1;34m[INFO]\033[0m %s\n' "$*"; } -warn() { printf '\033[1;33m[WARN]\033[0m %s\n' "$*"; } +info() { printf '\033[1;34m[INFO]\033[0m %s\n' "$*" >&2; } +warn() { printf '\033[1;33m[WARN]\033[0m %s\n' "$*" >&2; } die() { printf '\033[1;31m[ERROR]\033[0m %s\n' "$*" >&2; exit 1; } -# ── arg dispatch ─────────────────────────────────────────────────────────── -# `serve` (default) — start the OpenAI-compatible server. -# `shell` — drop into bash inside the container (debug). -# `lucebox` — dispatch to the Python CLI. Any subcommand -# `lucebox.sh` doesn't handle on the host arrives here -# (check, config, pull, print-run, smoke, …). -# `python` or anything else -# — pass through to exec, so `docker run … python -m foo` -# still works for dev. -SUBCMD="${1:-serve}" -[ $# -gt 0 ] && shift || true +# ── target ───────────────────────────────────────────────────────────────── +# Targets are 10-100 GB; drafts and projectors are 1-4 GB and live under +# models/draft/. When several targets are present we refuse to guess: picking +# the wrong one silently has produced wrong benchmark numbers before. +target_candidates() { + [ -d "$LUCE_DIR/models" ] || return 0 + find -L "$LUCE_DIR/models" -maxdepth 4 -type f -name '*.gguf' -size +5G \ + -not -path '*/draft/*' -not -iname '*mmproj*' -printf '%p\n' 2>/dev/null | sort +} +resolve_target() { + : "${LUCE_TARGET:=}" + if [ -z "$LUCE_TARGET" ]; then + local candidates=() + mapfile -t candidates < <(target_candidates) + case "${#candidates[@]}" in + 0) ;; + 1) LUCE_TARGET="${candidates[0]}" + info "Auto-detected target: $(basename "$LUCE_TARGET")" ;; + *) warn "Multiple candidate target GGUFs in $LUCE_DIR/models:" + local c + for c in "${candidates[@]}"; do warn " $c"; done + die "Ambiguous target: set LUCE_TARGET= to one of the candidates above." ;; + esac + fi + if [ -z "$LUCE_TARGET" ] || [ ! -f "$LUCE_TARGET" ]; then + die "No target GGUF found. Mount a model dir: -v /host/models:/opt/lucebox-hub/server/models, or set LUCE_TARGET=." + fi +} -LUCEBOX_PKG="/opt/lucebox-hub" +# ── device probe ─────────────────────────────────────────────────────────── +# luce_server reports the GPUs of this image's runtime (CUDA or ROCm), the +# model architecture, and the device `--target-device auto` would pick. Its +# lines look like: +# device hip:0 arch=gfx1201 type=discrete total_mib=32624 free_mib=... name=... +# model arch=deepseek4 size_mib=89740 kv_mib=0 path=... +# auto hip:1 fits=no total_mib=98304 reason=... +PROBE="" +probe_field() { # probe_field (first matching line) + awk -v kind="$1" -v key="$2" ' + $1 == kind { for (i = 2; i <= NF; i++) if (index($i, key "=") == 1) { + print substr($i, length(key) + 2); exit } }' <<<"$PROBE" +} +device_total_mib() { # device_total_mib + awk -v dev="$1" '$1 == "device" && $2 == dev { + for (i = 3; i <= NF; i++) if (index($i, "total_mib=") == 1) { + print substr($i, 11); exit } }' <<<"$PROBE" +} -case "$SUBCMD" in - lucebox) - # --no-sync: the venv was fully populated at image build time - # (`uv sync --no-editable` in the Dockerfile). Skipping the env - # consistency check here prevents hatch-vcs from firing its - # `_version.py` write hook against the read-only workspace source - # dirs and crashing the entire subcommand. - exec uv run --no-sync --directory "$LUCEBOX_PKG" python -m lucebox "$@" - ;; - shell) - exec /bin/bash "$@" - ;; - serve|"") - : # fall through to server startup below - ;; - *) - exec "$SUBCMD" "$@" - ;; -esac +# Last value of a flag in an argv list, e.g. the --target-device a user passed. +arg_value() { # arg_value + local flag="$1" value="" prev="" a + shift + for a in "$@"; do + [ "$prev" = "$flag" ] && value="$a" + prev="$a" + done + printf '%s' "$value" +} +has_arg() { # has_arg + local flag="$1" a + shift + for a in "$@"; do [ "$a" = "$flag" ] && return 0; done + return 1 +} -# ── HOST_INFO (host-identity capture) ────────────────────────────────────── -# Write /opt/lucebox-hub/HOST_INFO as JSON before exec'ing luce_server. -# The C++ server reads this file at startup and surfaces the parsed JSON -# under /props.host. Mirrors the IMAGE_INFO pattern (server_main.cpp -# read_image_info) but in JSON instead of KEY=VALUE — host facts have -# nested structure (gpu array, multi-field per GPU) that doesn't fit a -# flat KEY=VALUE layout. -# -# Inputs: the LUCEBOX_HOST_* env vars set by the host wrapper's -# probe_host(). When none are set (e.g. someone ran `docker run` directly, -# bypassing lucebox.sh), we still write a stub `{"source":"unknown", ...}` -# so the C++ side doesn't have to special-case missing-vs-blank. -# -# Failure is never fatal — the host_info file is informational. A -# write-failure (read-only FS, etc.) gets a warning and we continue. -write_host_info() { - local target="/opt/lucebox-hub/HOST_INFO" - local tmp="${target}.tmp.$$" - local collected_at - collected_at=$(date -u +%FT%TZ 2>/dev/null || echo "") - # If any LUCEBOX_HOST_* var was supplied, the source is "lucebox.sh" - # (the host wrapper probed and forwarded these via -e). Otherwise the - # container was launched outside the wrapper — we still emit a stub - # so the C++ side can read /props.host without special-casing missing. - local source_tag="unknown" - local collector_tag="entrypoint.sh" - if [ -n "${LUCEBOX_HOST_OS_PRETTY:-}" ] \ - || [ -n "${LUCEBOX_HOST_KERNEL:-}" ] \ - || [ -n "${LUCEBOX_HOST_GPU_LIST_CSV:-}" ] \ - || [ -n "${LUCEBOX_HOST_CPU_MODEL:-}" ]; then - source_tag="lucebox.sh" - collector_tag="lucebox.sh" - fi +# ── draft ────────────────────────────────────────────────────────────────── +# Drafts are architecture-specific (a Qwen3.6 DFlash draft crashes a Gemma +# target and vice versa), so a draft directory is searched with the target's +# family patterns first. DeepSeek V4 only takes its DSpark drafter. +resolve_draft() { # sets DRAFT_ARG + local arch="$1" + DRAFT_ARG="" + : "${LUCE_DRAFT:=$LUCE_DIR/models/draft}" + case "$LUCE_DRAFT" in none|off|"") return ;; esac - if ! _build_host_info_json "$source_tag" "$collector_tag" "$collected_at" > "$tmp" 2>/dev/null; then - warn "Failed to build HOST_INFO JSON — skipping" - rm -f "$tmp" 2>/dev/null || true - return 0 + # Common host layouts link ~/models/qwen3.6-27b-dflash instead of draft/. + if [ "$LUCE_DRAFT" = "$LUCE_DIR/models/draft" ] && [ ! -e "$LUCE_DRAFT" ]; then + local cand + for cand in "$LUCE_DIR/models/qwen3.6-27b-dflash" \ + "$LUCE_DIR/models/Qwen3.6-27B-DFlash" \ + "$LUCE_DIR/models/dflash"; do + if [ -e "$cand" ]; then LUCE_DRAFT="$cand"; break; fi + done fi - if ! mv -f "$tmp" "$target" 2>/dev/null; then - warn "Failed to write $target (continuing without it)" - rm -f "$tmp" 2>/dev/null || true - return 0 + if [ -f "$LUCE_DRAFT" ]; then + DRAFT_ARG="$LUCE_DRAFT" + return fi - info "Wrote $target (source=$source_tag)" -} - -# Build the HOST_INFO JSON on stdout. Real JSON escape via python3 (always -# present in the runtime image — uv pulls it in for the venv stage) with -# a bash fallback for parsing emergencies (broken venv, debug invocations -# from a minimal base). The bash fallback covers the realistic char set -# that leaks from lscpu / /etc/os-release / nvidia-smi (backslash, quote, -# newline, tab, CR); the python path covers every JSON-illegal char -# including the full U+0000-U+001F control range, so a misbehaved upstream -# can't silently invalidate the entire HOST_INFO and turn /props.host -# into null on the C++ side (which silently drops a parse-failed block). -_json_escape() { - # Read from $1, emit on stdout. No quotes around the result — the - # caller wraps with `"..."`. - if command -v python3 >/dev/null 2>&1; then - # json.dumps emits `"…escaped…"`; strip the surrounding quotes - # so callers can keep their existing `"..."` wrap convention. - python3 -c ' -import json, sys -out = json.dumps(sys.argv[1]) -sys.stdout.write(out[1:-1]) -' "$1" + if [ ! -d "$LUCE_DRAFT" ]; then + [ "$LUCE_DRAFT" = "$LUCE_DIR/models/draft" ] || + warn "Draft path $LUCE_DRAFT not found — running without draft" return fi - local s="$1" - s="${s//\\/\\\\}" - s="${s//\"/\\\"}" - s="${s//$'\n'/\\n}" - s="${s//$'\r'/\\r}" - s="${s//$'\t'/\\t}" - printf '%s' "$s" -} -# Emit a JSON value for a string field. Empty input → JSON null. Caller -# embeds the result directly (no leading/trailing whitespace). -_json_str_or_null() { - if [ -z "${1:-}" ]; then - printf 'null' + local target_name family=() generic=() + target_name="$(basename "$LUCE_TARGET" .gguf | tr 'A-Z' 'a-z')" + if [ "$arch" = deepseek4 ]; then + family=('*dspark*.gguf') else - printf '"%s"' "$(_json_escape "$1")" + case "$target_name" in + *gemma-4-26b*|*gemma4-26b*) family=('*gemma*4*26b*dflash*.gguf' '*dflash*gemma*4*26b*.gguf') ;; + *gemma-4-31b*|*gemma4-31b*) family=('*gemma*4*31b*dflash*.gguf' '*dflash*gemma*4*31b*.gguf') ;; + *gemma-4*|*gemma4*) family=('*gemma*4*dflash*.gguf' '*dflash*gemma*4*.gguf') ;; + *qwen3.6*|*qwen36*) family=('dflash-draft-3.6-*.gguf' '*qwen*3.6*dflash*.gguf') ;; + esac + generic=('dflash-draft-*.gguf' '*dflash*.gguf' '*.gguf' 'model.safetensors' '*.safetensors') fi -} -# Emit a JSON value for an integer field. Empty / non-numeric → null. -_json_int_or_null() { - local v="${1:-}" - if [ -z "$v" ] || ! [[ "$v" =~ ^-?[0-9]+$ ]]; then - printf 'null' - else - printf '%s' "$v" - fi + # Projectors are never drafts, and DSpark drafters only serve DeepSeek V4. + local exclude=(-not -iname '*mmproj*') + [ "$arch" = deepseek4 ] || exclude+=(-not -iname '*dspark*') + local pattern file + for pattern in "${family[@]}" "${generic[@]}"; do + # Sorted so the pick does not depend on filesystem order. + file="$(find -L "$LUCE_DRAFT" -maxdepth 4 -type f -iname "$pattern" \ + "${exclude[@]}" -print 2>/dev/null | sort | head -n 1)" + if [ -n "$file" ]; then + DRAFT_ARG="$file" + info "Resolved draft dir $LUCE_DRAFT → $DRAFT_ARG (pattern: $pattern)" + return + fi + done + warn "No draft for $(basename "$LUCE_TARGET") in $LUCE_DRAFT — running without draft" } -# Parse the LUCEBOX_HOST_GPU_LIST_CSV (whatever -# `nvidia-smi --query-gpu=index,uuid,pci.bus_id,name,compute_cap,memory.total,power.limit -# --format=csv,noheader` produced on the host) into a JSON -# array. Empty CSV → "[]". Each row becomes one object. -_emit_gpu_array() { - local csv="${LUCEBOX_HOST_GPU_LIST_CSV:-}" - if [ -z "$csv" ]; then - printf '[]' - return - fi - local out="[" first=1 - while IFS= read -r line; do - [ -z "$line" ] && continue - # Trim surrounding whitespace from each field. nvidia-smi prints - # `0, GPU-abc..., 00000000:01:00.0, NVIDIA RTX 5090, 12.0, 24576 MiB, 175.00 W`. - # Some driver builds emit bare `,` delimiters with no trailing space — - # split on `,` alone and trim whitespace per field so both forms parse. - local idx uuid pci name cc mem plimit - IFS=',' read -r idx uuid pci name cc mem plimit <<<"$line" - idx=$(printf '%s' "$idx" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') - uuid=$(printf '%s' "$uuid" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') - pci=$(printf '%s' "$pci" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') - name=$(printf '%s' "$name" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') - cc=$(printf '%s' "$cc" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') - mem=$(printf '%s' "$mem" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') - plimit=$(printf '%s' "$plimit" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') - # Strip units. "24576 MiB" → 24576; "175.00 W" → 175 (truncate). - local mem_mib vram_gb power_w - mem_mib=$(printf '%s' "$mem" | awk '{print $1+0}') - vram_gb="" - if [ -n "$mem_mib" ] && [ "$mem_mib" -gt 0 ] 2>/dev/null; then - vram_gb=$((mem_mib / 1024)) - fi - power_w=$(printf '%s' "$plimit" | awk '{printf "%d", $1+0.5}') - if [ "$first" = "1" ]; then - first=0 - else - out+="," +# ── dispatch ─────────────────────────────────────────────────────────────── +SUBCMD="${1:-serve}" +case "$SUBCMD" in + serve) [ $# -eq 0 ] || shift ;; + -*) ;; # bare luce_server flags: serve with them + devices) + shift + target=("${LUCE_TARGET:-}") + if [ -z "${target[0]}" ]; then + mapfile -t target < <(target_candidates) + if [ "${#target[@]}" -gt 1 ]; then + warn "Several target GGUFs in $LUCE_DIR/models; set LUCE_TARGET to include one in the listing." + target=() + fi fi - out+="{\"index\":$(_json_int_or_null "$idx"),\"uuid\":$(_json_str_or_null "$uuid")," - out+="\"pci_bus_id\":$(_json_str_or_null "$pci"),\"name\":$(_json_str_or_null "$name")," - out+="\"sm\":$(_json_str_or_null "$cc"),\"vram_gb\":$(_json_int_or_null "$vram_gb")," - out+="\"power_limit_w\":$(_json_int_or_null "$power_w")}" - done <<<"$csv" - out+="]" - printf '%s' "$out" -} + exec "$LUCE_SERVER_BIN" --list-devices "${target[@]}" "$@" + ;; + shell) + shift + exec /bin/bash "$@" + ;; + *) + exec "$@" + ;; +esac +USER_ARGS=("$@") +EXTRA_ARGS=() +if [ -n "${LUCE_ARGS:-}" ]; then + read -r -a EXTRA_ARGS <<<"$LUCE_ARGS" +fi +ALL_ARGS=("${EXTRA_ARGS[@]}" "${USER_ARGS[@]}") -_build_host_info_json() { - local source_tag="$1" collector_tag="$2" collected_at="$3" - printf '{' - printf '"os_pretty":%s,' "$(_json_str_or_null "${LUCEBOX_HOST_OS_PRETTY:-}")" - printf '"kernel":%s,' "$(_json_str_or_null "${LUCEBOX_HOST_KERNEL:-}")" - printf '"wsl_version":%s,' "$(_json_str_or_null "${LUCEBOX_HOST_WSL_VERSION:-}")" - printf '"docker_version":%s,' "$(_json_str_or_null "${LUCEBOX_HOST_DOCKER_VERSION:-}")" - printf '"nvidia_driver":%s,' "$(_json_str_or_null "${LUCEBOX_HOST_DRIVER_VERSION:-}")" - printf '"nvidia_ctk_version":%s,' "$(_json_str_or_null "${LUCEBOX_HOST_NVIDIA_CTK_VERSION:-}")" - printf '"cpu_model":%s,' "$(_json_str_or_null "${LUCEBOX_HOST_CPU_MODEL:-}")" - printf '"nproc":%s,' "$(_json_int_or_null "${LUCEBOX_HOST_NPROC:-}")" - printf '"ram_gb":%s,' "$(_json_int_or_null "${LUCEBOX_HOST_RAM_GB:-}")" - printf '"gpus":%s,' "$(_emit_gpu_array)" - printf '"cuda_visible_devices":%s,' "$(_json_str_or_null "${LUCEBOX_HOST_CUDA_VISIBLE_DEVICES:-}")" - printf '"source":%s,' "$(_json_str_or_null "$source_tag")" - printf '"collector":%s,' "$(_json_str_or_null "$collector_tag")" - printf '"collected_at":%s' "$(_json_str_or_null "$collected_at")" - printf '}\n' -} +# These mapped to flags luce_server no longer accepts; forwarding them made +# the server exit with "unknown option". +for retired in LUCE_THINK_SOFT_CLOSE_MIN_RATIO LUCE_DEBUG_THINKING_LOGITS; do + [ -n "${!retired:-}" ] && warn "$retired is no longer supported by luce_server and is ignored" +done -write_host_info +[ -x "$LUCE_SERVER_BIN" ] || die "luce_server binary missing at $LUCE_SERVER_BIN (image build failed?)" +resolve_target + +# The server reads LUCE_TARGET_DEVICE whenever no device flag or profile names +# one, so the default never overrides an explicit placement. +export LUCE_TARGET_DEVICE="${LUCE_TARGET_DEVICE:-auto}" + +PROBE="$("$LUCE_SERVER_BIN" --list-devices "$LUCE_TARGET" 2>/dev/null || true)" +MODEL_ARCH="$(probe_field model arch)" +GPU_COUNT="$(grep -c '^device ' <<<"$PROBE" || true)" +if [ -z "$PROBE" ] || [ "$GPU_COUNT" = 0 ]; then + warn "No GPU visible to luce_server. CUDA: --gpus all. ROCm: --device /dev/kfd --device /dev/dri --group-add video --group-add render." +fi -# ── detect ───────────────────────────────────────────────────────────────── -# nvidia-smi is always present here (--gpus all wires the driver in). -GPU_VRAM_GB=0 -if command -v nvidia-smi &>/dev/null; then - if mem_mib=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null \ - | head -1) && [ -n "$mem_mib" ]; then - GPU_VRAM_GB=$((mem_mib / 1024)) +# A --profile in the arguments replaces LUCE_PROFILE; luce_server takes one. +PROFILE_ARG="$(arg_value --profile "${ALL_ARGS[@]}")" +PROFILE="${PROFILE_ARG:-${LUCE_PROFILE:-}}" + +# The device the model will run on, for sizing: an explicit single device, else +# the auto choice. A profile may name its own device, so with a profile only an +# explicit one is known here. Multi-device placements (--target-devices) keep +# the server's own defaults and are reported as given. +TARGET_DEVICES="$(arg_value --target-devices "${ALL_ARGS[@]}")" +USED_DEVICE="" +if [ -z "$TARGET_DEVICES" ]; then + USED_DEVICE="$(arg_value --target-device "${ALL_ARGS[@]}")" + if [ -z "$USED_DEVICE" ] && [ "$LUCE_TARGET_DEVICE" != auto ]; then + USED_DEVICE="$LUCE_TARGET_DEVICE" + fi + if [ -z "$PROFILE" ] && { [ -z "$USED_DEVICE" ] || [ "$USED_DEVICE" = auto ]; }; then + USED_DEVICE="$(awk '$1 == "auto" { print $2; exit }' <<<"$PROBE")" fi + [ "$USED_DEVICE" = auto ] && USED_DEVICE="" fi -GPU_COUNT=0 -if command -v nvidia-smi &>/dev/null; then - GPU_COUNT=$(nvidia-smi -L 2>/dev/null | awk '/^GPU /{n++} END{print n+0}') || GPU_COUNT=0 +TARGET_DESC="${TARGET_DEVICES:-${USED_DEVICE:-chosen by luce_server}}" +GPU_MIB=0 +if [ -n "$USED_DEVICE" ]; then + GPU_MIB="$(device_total_mib "$USED_DEVICE")" + GPU_MIB="${GPU_MIB:-0}" fi +[ "$GPU_COUNT" -gt 1 ] 2>/dev/null && + info "$GPU_COUNT GPUs visible; target $TARGET_DESC (LUCE_TARGET_DEVICE=$LUCE_TARGET_DEVICE; list with: docker run ... devices)" -# ── fallback autotune (only fills unset env) ─────────────────────────────── -# Keep these tiers in lockstep with lucebox::autotune_env on the host. The -# divergence we accept is the lower-VRAM error tier — the host CLI refuses -# to start there with a clear message; here we just warn and let the server -# decide whether it can load. +# DeepSeek V4 serves concurrent requests (paged attention) autoregressively, +# so it takes no drafter there. +PAGED=0 +CONCURRENCY="$(arg_value --max-concurrency "${ALL_ARGS[@]}")" +if has_arg --paged-attention "${ALL_ARGS[@]}" || [ "${CONCURRENCY:-1}" -gt 1 ] 2>/dev/null; then + PAGED=1 +fi -if [ "$GPU_VRAM_GB" -gt 0 ]; then +# ── context size from GPU memory ─────────────────────────────────────────── +# Only when the operator set neither LUCE_MAX_CTX nor a profile: a profile's +# --max-ctx is part of its qualified configuration. An explicit --max-ctx flag +# still gets the memory-based draft settings, but its context is the one used. +MAX_CTX_ARG="$(arg_value --max-ctx "${ALL_ARGS[@]}")" +LAZY_EXPLICIT="${LUCE_LAZY:-}" +GPU_GB=$((GPU_MIB / 1024)) +if [ -z "${LUCE_MAX_CTX:-}" ] && [ -z "$PROFILE" ] && [ "$GPU_GB" -gt 0 ]; then IS_WSL=0 if grep -qi microsoft /proc/version 2>/dev/null || [ -e /proc/sys/fs/binfmt_misc/WSLInterop ]; then IS_WSL=1 fi - if [ "$GPU_VRAM_GB" -lt 12 ]; then + if [ "$GPU_GB" -lt 12 ]; then + LUCE_MAX_CTX=4096; : "${LUCE_LAZY:=1}" + warn "GPU memory ${GPU_GB} GB < 12 GB — a 27B target is unlikely to fit" + elif [ "$GPU_GB" -lt 22 ]; then + LUCE_MAX_CTX=32768; : "${LUCE_LAZY:=1}" + elif [ "$GPU_GB" -lt 32 ]; then : "${LUCE_LAZY:=1}" - : "${LUCE_MAX_CTX:=4096}" - warn "VRAM ${GPU_VRAM_GB} GB < 12 GB — 27B target unlikely to fit" - elif [ "$GPU_VRAM_GB" -lt 22 ]; then - : "${LUCE_LAZY:=1}" - : "${LUCE_MAX_CTX:=32768}" - elif [ "$GPU_VRAM_GB" -lt 32 ]; then - : "${LUCE_LAZY:=1}" - if [ "$IS_WSL" = "1" ]; then - : "${LUCE_BUDGET:=16}" - : "${LUCE_MAX_CTX:=65536}" + if [ "$IS_WSL" = 1 ]; then + LUCE_MAX_CTX=65536; : "${LUCE_BUDGET:=16}" else - : "${LUCE_MAX_CTX:=98304}" + LUCE_MAX_CTX=98304 fi else - : "${LUCE_MAX_CTX:=131072}" + LUCE_MAX_CTX=131072 fi fi +[ -n "$PROFILE" ] || : "${LUCE_MAX_CTX:=16384}" -: "${LUCE_BIN:=$LUCE_DIR/build/test_dflash}" -: "${LUCE_SERVER_BIN:=$LUCE_DIR/build/luce_server}" -: "${LUCE_HOST:=0.0.0.0}" -: "${LUCE_PORT:=8080}" -: "${LUCE_BUDGET:=22}" -: "${LUCE_MAX_CTX:=16384}" -: "${LUCE_LAZY:=0}" -: "${LUCE_CACHE_TYPE_K:=}" -: "${LUCE_CACHE_TYPE_V:=}" -: "${LUCE_VERBOSE:=0}" -: "${LUCE_TARGET:=}" -: "${LUCE_DRAFT:=$LUCE_DIR/models/draft}" -: "${LUCE_PREFILL_MODE:=off}" -: "${LUCE_PREFILL_KEEP:=0.05}" -: "${LUCE_PREFILL_THRESHOLD:=32000}" -: "${LUCE_PREFILL_DRAFTER:=}" -# Optional server default for requests that omit max_tokens. When unset, -# the C++ server uses the model-card default. -: "${LUCE_DEFAULT_MAX_TOKENS:=}" -# Optional advertised model name for /v1/models (also selects the matching -# share/model_cards/.json). When unset, the C++ server uses its default -# ("luce"). Lets an operator surface the real model id without a wrapper. -: "${LUCE_MODEL_NAME:=}" -# Phase-1 (thinking) cap when a request opts into thinking. Default mirrors -# antirez/ds4 ds4_eval.c: think_max_tokens = max_tokens(16000) - hard_limit -# reply budget(512) = 15488. The server's own hardcoded default is 10000; -# overriding here aligns ds4-eval and similar reasoning benches with upstream. -: "${LUCE_THINK_MAX:=15488}" -# Soft-close thinking termination dial (PR #326). Lets the AR loop force -# early when the close-token logit comes within this probability -# ratio of the chosen-token logit. Range [0.0, 1.0]; 0.0 = disabled (server -# default, byte-identical to pre-change behavior). 0.5 = close when close -# is within 2× of chosen; 0.9 = aggressive (close when close is within -# ~10% of chosen). Only emitted to the server CLI when nonzero so unset -# reproduces the server's own default. Qwen3.5/3.6 AR path only in v1. -: "${LUCE_THINK_SOFT_CLOSE_MIN_RATIO:=0.0}" -# Diagnostic: when "1", forward --debug-thinking-logits to the server so -# the AR loop emits per-step [soft-trace] lines for fitting a sliding- -# ratio curve. Heavy stderr; operator-only. Default off. -: "${LUCE_DEBUG_THINKING_LOGITS:=0}" -# Flash-attention sliding-window on full-attention layers. 0 = server's -# stock full attention. Sparse decode windows (e.g. 2048-8192) bound -# the compute on long prompts for gemma4's hybrid iSWA without changing -# the KV footprint. Only emitted to the server CLI when nonzero so -# unset reproduces the server's own default unchanged. -: "${LUCE_FA_WINDOW:=0}" - -# ── auto-detect target ───────────────────────────────────────────────────── -# Target .gguf is typically 10-30 GB (Q4_K_M). Drafts are 1-2 GB (Q8_0 / Q4) -# and live under models/draft/ or have a dflash- prefix. The 5 GB threshold -# excludes drafts cleanly without needing to parse GGUF arch metadata. -# -# CRITICAL UX RULE: when multiple candidate targets exist, we DO NOT silently -# pick one based on filename pattern. That hid a real bug for the matrix -# bench — a hardcoded Qwen3.6 preference made the container run the wrong -# model when both gemma4 and qwen3.6 GGUFs were present, and the operator -# only noticed when the bench numbers came out wrong. Either set -# LUCE_TARGET=... explicitly, or have exactly one .gguf in models/. -if [ -z "$LUCE_TARGET" ] && [ -d "$LUCE_DIR/models" ]; then - # Collect candidates: .gguf files ≥5 GB (target-sized), excluding - # anything under models/draft/. Sort alphabetically for determinism. - mapfile -t TARGET_CANDIDATES < <( - find -L "$LUCE_DIR/models" -maxdepth 4 -type f -name '*.gguf' \ - -size +5G \ - -not -path '*/draft/*' \ - -printf '%p\n' 2>/dev/null \ - | sort - ) - case "${#TARGET_CANDIDATES[@]}" in - 0) - : # fall through to the missing-target die below - ;; - 1) - LUCE_TARGET="${TARGET_CANDIDATES[0]}" - info "Auto-detected target: $(basename "$LUCE_TARGET")" - ;; - *) - # Refuse to guess: silently picking the wrong target has burned - # us before (bench numbers come out wrong, only noticed after the - # fact). Force the operator to disambiguate via LUCE_TARGET. - warn "Multiple candidate target GGUFs in $LUCE_DIR/models. Refusing to auto-select." - warn "Set LUCE_TARGET= to choose one. Candidates:" - for c in "${TARGET_CANDIDATES[@]}"; do - warn " $c" - done - die "Ambiguous target: set LUCE_TARGET= from the candidates above." - ;; - esac -fi - -if [ -z "$LUCE_TARGET" ] || [ ! -f "$LUCE_TARGET" ]; then - die "No target GGUF found. Mount a model dir: -v /host/models:/opt/lucebox-hub/server/models, or set LUCE_TARGET=." -fi -[ -x "$LUCE_SERVER_BIN" ] || die "luce_server binary missing at $LUCE_SERVER_BIN (image build failed?)" - -# Qwen3.6 DFlash drafters use sliding-window attention in the draft. Some GGUFs -# carry this metadata directly; keep the documented env override as the startup -# default so older drafts behave like the autotune-sweep path. +# Qwen3.6 DFlash drafters use sliding-window attention; older GGUFs lack the +# metadata, so keep the documented default for them. case "$(basename "$LUCE_TARGET")" in *Qwen3.6*|*qwen3.6*) if [ -z "${LUCE_DRAFT_SWA:-}" ]; then export LUCE_DRAFT_SWA=2048 - info "Autotune: LUCE_DRAFT_SWA=2048 (Qwen3.6 draft SWA)" - fi - ;; + info "LUCE_DRAFT_SWA=2048 (Qwen3.6 draft SWA)" + fi ;; esac -# Common host layouts use ~/models/qwen3.6-27b-dflash as an absolute symlink -# rather than a literal models/draft directory. If the default is absent, find -# that draft before deciding to run without DFlash. -if [ "$LUCE_DRAFT" = "$LUCE_DIR/models/draft" ] && [ ! -e "$LUCE_DRAFT" ]; then - for cand in "$LUCE_DIR/models/qwen3.6-27b-dflash" \ - "$LUCE_DIR/models/Qwen3.6-27B-DFlash" \ - "$LUCE_DIR/models/dflash"; do - if [ -e "$cand" ]; then - LUCE_DRAFT="$cand" - break - fi - done +if [ "$MODEL_ARCH" = deepseek4 ] && [ "$PAGED" = 1 ]; then + DRAFT_ARG="" + info "DeepSeek V4 concurrent serving decodes autoregressively; not loading a DSpark drafter" +else + resolve_draft "$MODEL_ARCH" fi -# Draft: directory holding GGUF/safetensors, or a direct draft file. -# The native luce_server expects --draft to be a FILE path (not a dir). -# If LUCE_DRAFT points at a directory, resolve it to a draft GGUF inside. -# -# Draft files are arch-specific: a draft trained for qwen3.6 has a fc -# weight shape that only divides evenly into the qwen3.6 target's hidden -# size, and crashes hard at spec-decode time when fed gemma4 (or vice -# versa) — see Gemma4Backend draft-incompatibility check. So when the -# draft dir contains multiple drafts (e.g. a host with both qwen3.6 and -# gemma4 drafts pre-downloaded), pick the one whose filename matches the -# target's family. Falls back to the generic dflash-draft-*.gguf pattern -# (legacy qwen3.6-only behavior) when the target family is unknown. -DRAFT_ARG="$LUCE_DRAFT" -if [ -d "$LUCE_DRAFT" ]; then - # Derive a target-family hint from the target filename. Matching the - # GGUF arch metadata would be cleaner but requires parsing the header - # in shell; the filename convention is enforced upstream by the - # publish-side dflash quantize scripts. - TARGET_BASENAME="$(basename "$LUCE_TARGET" .gguf 2>/dev/null)" - # Use -iname (case-insensitive) throughout so both naming conventions - # work: legacy "dflash-gemma-4-31b-*.gguf" and the Lucebox HF repo's - # "gemma-4-31B-it-DFlash-q8_0.gguf". Glob list is family-specific first, - # then generic dflash-draft-*.gguf legacy, then last-resort *.gguf. - # The 31B match in the Lucebox repo uses capital B in the filename — - # -iname handles that without needing to enumerate every case form. - case "$(echo "$TARGET_BASENAME" | tr 'A-Z' 'a-z')" in - *gemma-4-26b*|*gemma4-26b*) - FAMILY_GLOBS=('*gemma*4*26b*dflash*.gguf' '*dflash*gemma*4*26b*.gguf') ;; - *gemma-4-31b*|*gemma4-31b*) - FAMILY_GLOBS=('*gemma*4*31b*dflash*.gguf' '*dflash*gemma*4*31b*.gguf') ;; - *gemma-4*|*gemma4*) - FAMILY_GLOBS=('*gemma*4*dflash*.gguf' '*dflash*gemma*4*.gguf') ;; - *qwen3.6*|*qwen36*) - FAMILY_GLOBS=('dflash-draft-3.6-*.gguf' '*qwen*3.6*dflash*.gguf') ;; - *) - FAMILY_GLOBS=() ;; +if [ "$MODEL_ARCH" = deepseek4 ] && [ -z "$PROFILE" ] && [ "$PAGED" = 0 ]; then + case "$(awk -v dev="$USED_DEVICE" '$1 == "device" && $2 == dev { print $3 }' <<<"$PROBE")" in + arch=gfx1151) info "DeepSeek V4 on Strix Halo: add --profile ds4-strix for the qualified serving profile" ;; esac - - DRAFT_FILE="" - # Track which glob actually matched so the info() log can show whether - # we picked via the family-specific pattern or fell back to a generic - # one. Initialize empty up front — `set -u` will fire if we read the - # var without an assignment having run, and the for-loop below may - # exit on the very first iteration without entering the body. - DRAFT_FAMILY_GLOB="" - # Family-specific globs first (most specific). Then the legacy - # `dflash-draft-*.gguf` for single-draft setups. Then the generic - # `*.gguf` / safetensors fallbacks. - GENERIC_GLOBS=('dflash-draft-*.gguf' '*dflash*.gguf' '*.gguf' 'model.safetensors' '*.safetensors') - family_count="${#FAMILY_GLOBS[@]}" - i=0 - for pattern in "${FAMILY_GLOBS[@]}" "${GENERIC_GLOBS[@]}"; do - # Sort matches lexicographically so the pick is deterministic across - # filesystems (find's traversal order is filesystem-dependent without - # an explicit sort). First lexicographic match wins. - DRAFT_FILE="$(find -L "$LUCE_DRAFT" -maxdepth 4 -type f -iname "$pattern" -print 2>/dev/null | sort | head -n 1)" - if [ -n "$DRAFT_FILE" ]; then - # Mark the family-specific match so the log line below can - # distinguish "matched on family hint" from "generic fallback". - if [ "$i" -lt "$family_count" ]; then - DRAFT_FAMILY_GLOB="$pattern" - fi - break - fi - i=$((i + 1)) - done - # Defensive: every read of DRAFT_FAMILY_GLOB below must survive `set -u` - # even if the init on line ~257 was somehow skipped (e.g. a future refactor - # that moves the init out of this block, or a partial-rewrite during a - # rebase that drops it). Coalesce-to-empty inline so a regression can't - # re-trip the unbound-variable crash that fired on the sindri sweep with - # multiple target GGUFs in models/ (commit a87bb93 was a partial fix — - # the recurrence proved that "initialize once at the top of the block" - # is too easy to undo). Cost: zero bytes at runtime. - DRAFT_FAMILY_GLOB="${DRAFT_FAMILY_GLOB:-}" - if [ -n "$DRAFT_FILE" ] && [ -f "$DRAFT_FILE" ]; then - DRAFT_ARG="$DRAFT_FILE" - if [ -n "$DRAFT_FAMILY_GLOB" ]; then - info "Resolved draft dir $LUCE_DRAFT → $DRAFT_ARG (target family: $DRAFT_FAMILY_GLOB)" - else - info "Resolved draft dir $LUCE_DRAFT → $DRAFT_ARG" - fi - else - warn "No DFlash draft GGUF/safetensors in draft dir $LUCE_DRAFT — running without draft" - DRAFT_ARG="" - fi -elif [ -n "$LUCE_DRAFT" ] && [ ! -f "$LUCE_DRAFT" ]; then - warn "Draft path $LUCE_DRAFT not found — running without draft" - DRAFT_ARG="" fi -[ "$GPU_COUNT" -gt 1 ] && warn "${GPU_COUNT} GPUs detected — native server layer sharding is not auto-enabled" +# ── build + exec ─────────────────────────────────────────────────────────── +: "${LUCE_HOST:=0.0.0.0}" +: "${LUCE_PORT:=8080}" +: "${LUCE_BUDGET:=22}" +: "${LUCE_THINK_MAX:=15488}" # ds4_eval.c: max_tokens(16000) - reply budget(512) +: "${LUCE_PREFILL_MODE:=off}" -# ── build + exec native server ──────────────────────────────────────────── CMD=("$LUCE_SERVER_BIN" "$LUCE_TARGET" --host "$LUCE_HOST" --port "$LUCE_PORT" - --max-ctx "$LUCE_MAX_CTX" --think-max-tokens "$LUCE_THINK_MAX") +[ -n "$PROFILE" ] && [ -z "$PROFILE_ARG" ] && CMD+=(--profile "$PROFILE") +[ -n "${LUCE_MAX_CTX:-}" ] && [ -z "$MAX_CTX_ARG" ] && CMD+=(--max-ctx "$LUCE_MAX_CTX") -# Keep cache defaults owned by luce_server. In particular, omitting -# LUCE_PREFIX_CACHE_SLOTS preserves the native nonzero default instead of -# silently disabling multi-turn prefix reuse in the container. Explicit -# values, including 0 as an operator opt-out, are forwarded unchanged. +# Cache defaults belong to luce_server: omitting the variable keeps the native +# default, and explicit values (including 0) are forwarded. [ -n "${LUCE_PREFIX_CACHE_SLOTS:-}" ] && CMD+=(--prefix-cache-slots "$LUCE_PREFIX_CACHE_SLOTS") [ -n "${LUCE_PREFILL_CACHE_SLOTS:-}" ] && CMD+=(--prefill-cache-slots "$LUCE_PREFILL_CACHE_SLOTS") -[ -n "$DRAFT_ARG" ] && CMD+=(--draft "$DRAFT_ARG") -[ -n "$DRAFT_ARG" ] && CMD+=(--ddtree --ddtree-budget "$LUCE_BUDGET") -[ -n "$LUCE_DEFAULT_MAX_TOKENS" ] && CMD+=(--default-max-tokens "$LUCE_DEFAULT_MAX_TOKENS") -[ -n "$LUCE_MODEL_NAME" ] && CMD+=(--model-name "$LUCE_MODEL_NAME") -[ -n "${LUCE_MMPROJ:-}" ] && CMD+=(--mmproj "$LUCE_MMPROJ") -# `--lazy-draft` is silently dropped by the C++ server unless both -# `--prefill-drafter` and `--draft` are present (look for the runtime -# warning `--lazy-draft ignored: requires both --prefill-drafter and -# --draft`). Warn loudly here when the operator's config asked for lazy -# but we're about to drop it — sweeping past the silent no-op was the -# fingerprint left in every sindri decode-tuning docker.stderr. -if [ "$LUCE_LAZY" = "1" ]; then - if [ -z "$DRAFT_ARG" ] || [ -z "$LUCE_PREFILL_DRAFTER" ]; then - warn "LUCE_LAZY=1 ignored: requires both LUCE_DRAFT and LUCE_PREFILL_DRAFTER (see entrypoint.sh comment). Continuing without --lazy-draft." - else +if [ -n "$DRAFT_ARG" ]; then + CMD+=(--draft "$DRAFT_ARG") + # DeepSeek V4 verifies DSpark proposals itself; DDTree is a DFlash mode. + [ "$MODEL_ARCH" = deepseek4 ] || CMD+=(--ddtree --ddtree-budget "$LUCE_BUDGET") +fi +[ -n "${LUCE_DEFAULT_MAX_TOKENS:-}" ] && CMD+=(--default-max-tokens "$LUCE_DEFAULT_MAX_TOKENS") +[ -n "${LUCE_MODEL_NAME:-}" ] && CMD+=(--model-name "$LUCE_MODEL_NAME") +[ -n "${LUCE_MMPROJ:-}" ] && CMD+=(--mmproj "$LUCE_MMPROJ") +[ -n "${LUCE_CACHE_TYPE_K:-}" ] && CMD+=(--cache-type-k "$LUCE_CACHE_TYPE_K") +[ -n "${LUCE_CACHE_TYPE_V:-}" ] && CMD+=(--cache-type-v "$LUCE_CACHE_TYPE_V") +[ "${LUCE_FA_WINDOW:-0}" -gt 0 ] 2>/dev/null && CMD+=(--fa-window "$LUCE_FA_WINDOW") + +# --lazy-draft parks a decode draft only while PFlash compresses a prompt. +if [ "${LUCE_LAZY:-0}" = 1 ]; then + if [ -n "$DRAFT_ARG" ] && [ -n "${LUCE_PREFILL_DRAFTER:-}" ]; then CMD+=(--lazy-draft) + elif [ "$LAZY_EXPLICIT" = 1 ]; then + warn "LUCE_LAZY=1 ignored: requires a draft and LUCE_PREFILL_DRAFTER" fi fi -[ -n "$LUCE_CACHE_TYPE_K" ] && CMD+=(--cache-type-k "$LUCE_CACHE_TYPE_K") -[ -n "$LUCE_CACHE_TYPE_V" ] && CMD+=(--cache-type-v "$LUCE_CACHE_TYPE_V") -[ "$LUCE_FA_WINDOW" -gt 0 ] 2>/dev/null && CMD+=(--fa-window "$LUCE_FA_WINDOW") -# Soft-close ratio: emit only when nonzero. The default-string compare -# guards against the floating-point quirks of `[` numeric tests for -# values like 0.0/0/0.00 — anything non-"0.0" passes through to the -# server, which clamps to [0,1] itself. -case "$LUCE_THINK_SOFT_CLOSE_MIN_RATIO" in - 0|0.0|0.00|0.000) ;; # disabled — don't emit - *) CMD+=(--think-soft-close-min-ratio "$LUCE_THINK_SOFT_CLOSE_MIN_RATIO") ;; -esac -[ "$LUCE_DEBUG_THINKING_LOGITS" = "1" ] && CMD+=(--debug-thinking-logits) -if [ "$LUCE_PREFILL_MODE" != "off" ]; then - [ -n "$LUCE_PREFILL_DRAFTER" ] || die "LUCE_PREFILL_MODE=$LUCE_PREFILL_MODE requires LUCE_PREFILL_DRAFTER" +if [ "$LUCE_PREFILL_MODE" != off ]; then + [ -n "${LUCE_PREFILL_DRAFTER:-}" ] || die "LUCE_PREFILL_MODE=$LUCE_PREFILL_MODE requires LUCE_PREFILL_DRAFTER" [ -f "$LUCE_PREFILL_DRAFTER" ] || die "Prefill drafter not found at $LUCE_PREFILL_DRAFTER" CMD+=(--prefill-compression "$LUCE_PREFILL_MODE" - --prefill-keep-ratio "$LUCE_PREFILL_KEEP" - --prefill-threshold "$LUCE_PREFILL_THRESHOLD" + --prefill-keep-ratio "${LUCE_PREFILL_KEEP:-0.05}" + --prefill-threshold "${LUCE_PREFILL_THRESHOLD:-32000}" --prefill-drafter "$LUCE_PREFILL_DRAFTER") fi -info "lucebox-hub container starting (target=$(basename "$LUCE_TARGET"), max_ctx=$LUCE_MAX_CTX, budget=$LUCE_BUDGET, lazy=$LUCE_LAZY)" +# Operator flags go last: luce_server keeps the last value of a repeated flag. +CMD+=("${ALL_ARGS[@]}") + +info "lucebox-hub starting: target=$(basename "$LUCE_TARGET") arch=${MODEL_ARCH:-unknown} device=$TARGET_DESC max_ctx=${MAX_CTX_ARG:-${LUCE_MAX_CTX:-profile}}${PROFILE:+ profile=$PROFILE}" cd "$LUCE_DIR" exec "${CMD[@]}" diff --git a/server/src/common/backend_factory.cpp b/server/src/common/backend_factory.cpp index 1202dca6c..933a8bfb8 100644 --- a/server/src/common/backend_factory.cpp +++ b/server/src/common/backend_factory.cpp @@ -371,7 +371,9 @@ std::unique_ptr construct_backend( cfg.model_path = model.path; cfg.mmproj_path = model.mmproj_path.value_or(""); cfg.mmproj_gpu = model.mmproj_device ? model.mmproj_device->gpu : -1; + cfg.draft_path = speculation.draft_path.value_or(""); cfg.device = placement.target; + cfg.draft_device = placement.draft; cfg.stream_fd = execution.stream_fd; cfg.max_ctx = placement.target.max_ctx; cfg.chunk = execution.chunk; diff --git a/server/src/common/feature_gate.cpp b/server/src/common/feature_gate.cpp index d3a703880..e16f08c49 100644 --- a/server/src/common/feature_gate.cpp +++ b/server/src/common/feature_gate.cpp @@ -400,7 +400,11 @@ std::vector collect_feature_warnings( const std::string & arch) { std::vector out; - const bool split = args.device.is_layer_split(); + // create_backend() also hands deepseek4 to its layer-split adapter for a + // remote target shard on one local device; that adapter drops the same + // options as an explicit split. + const bool split = args.device.is_layer_split() || + (arch == "deepseek4" && args.remote_target_shard.enabled()); // Each entry pairs a requested option with the capability predicate for // the field create_backend() would have to forward for it to take effect. diff --git a/server/src/common/model_capabilities.h b/server/src/common/model_capabilities.h index 44a6c42d0..4e8813dde 100644 --- a/server/src/common/model_capabilities.h +++ b/server/src/common/model_capabilities.h @@ -77,7 +77,7 @@ inline constexpr ArchCapabilities kArchCapabilities[] = { {"laguna", true, false, false, true, kMono, kMono, kMono, kNever, kNever,kNever, kNever}, {"qwen3", false, false, true, false, kNever, kNever, kNever, kNever, kNever,kNever, kNever}, {"gemma4", true, false, false, false, kMono, kNever, kNever, kNever, kBoth,kNever, kNever}, - {"deepseek4", true, false, true, false, kNever, kNever, kNever, kNever, kNever,kNever, kMono}, + {"deepseek4", true, false, true, false, kMono, kNever, kNever, kNever, kNever,kNever, kMono}, }; inline constexpr std::size_t kArchCount = diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index 1b40aa756..0f5d13470 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -246,14 +246,14 @@ static bool is_gfx_device(int gpu, const char * arch) { #endif } -static bool configure_dspark_mmvq_defaults(int gpu) { +static bool configure_dspark_mmvq_defaults(int gpu, bool spec) { if (env_flag_enabled("LUCE_DS4_Q6_VERIFY")) { std::fprintf(stderr, "[deepseek4] q=6 verification is unsupported; use q=5\n"); return false; } #if defined(LUCE_BACKEND_HIP) || defined(GGML_USE_HIP) - if (!env_flag_enabled("LUCE_DS4_SPEC")) { + if (!spec) { return true; } @@ -350,9 +350,10 @@ static bool configure_dspark_mmvq_defaults(int gpu) { // The gfx1151 dense ROCmFP4 weight-reuse kernel preserves single-column // arithmetic through that width. Keep those projections on MMVQ while // honoring an explicit LUCE_MMVQ_MAX_NCOLS setting. -static void configure_gfx1151_paged_mmvq_default(int gpu, bool paged_attention) { +static void configure_gfx1151_paged_mmvq_default(int gpu, bool paged_attention, + bool spec) { #if defined(LUCE_BACKEND_HIP) || defined(GGML_USE_HIP) - if (!paged_attention || env_flag_enabled("LUCE_DS4_SPEC") || + if (!paged_attention || spec || !is_gfx_device(gpu, "gfx1151")) { return; } @@ -418,9 +419,9 @@ static bool apply_gfx1151_profile_defaults( // test_failed_init_preserves_sparse_opt_in asserts init() leaves it alone). // Returns false when a default could not be installed, so init() never // continues with a partially applied verifier profile. -static bool configure_gfx1151_dspark_verifier_defaults(int gpu) { +static bool configure_gfx1151_dspark_verifier_defaults(int gpu, bool spec) { #if defined(LUCE_BACKEND_HIP) || defined(GGML_USE_HIP) - if (!env_flag_enabled("LUCE_DS4_SPEC") || + if (!spec || !is_gfx_device(gpu, "gfx1151")) { return true; } @@ -1625,17 +1626,24 @@ bool DeepSeek4Backend::load_spec_drafter() { ggml_backend_t draft_backend = backend_; int draft_gpu = cfg_.device.gpu; - if (const char * gpu = std::getenv("LUCE_DS4_DRAFT_GPU")) { - draft_gpu = std::max(0, std::atoi(gpu)); - } const bool separate_draft_stream = env_flag_enabled("LUCE_DS4_DRAFT_SEPARATE_STREAM"); PlacementBackend draft_kind = PlacementBackend::Auto; - if (!ds4_draft_backend(draft_kind)) { - std::fprintf(stderr, - "[deepseek4] invalid LUCE_DS4_DRAFT_BACKEND; " - "expected cuda or hip\n"); - return false; + // server_main makes an explicit --draft-device concrete (auto:N included), + // so an auto backend here means "not placed": environment, then target. + if (cfg_.draft_device.backend != PlacementBackend::Auto) { + draft_kind = cfg_.draft_device.backend; + draft_gpu = cfg_.draft_device.gpu; + } else { + if (const char * gpu = std::getenv("LUCE_DS4_DRAFT_GPU")) { + draft_gpu = std::max(0, std::atoi(gpu)); + } + if (!ds4_draft_backend(draft_kind)) { + std::fprintf(stderr, + "[deepseek4] invalid LUCE_DS4_DRAFT_BACKEND; " + "expected cuda or hip\n"); + return false; + } } const PlacementBackend target_kind = placement_backend_of(backend_); if (draft_kind != target_kind || draft_gpu != cfg_.device.gpu || @@ -1795,6 +1803,9 @@ bool DeepSeek4Backend::supports_batched_spec_feature_capture( } bool DeepSeek4Backend::init() { + // --draft selects DSpark directly; LUCE_DS4_SPEC + LUCE_DS4_DRAFT remain + // the environment spelling of the same request. + spec_requested_ = !cfg_.draft_path.empty() || env_flag_enabled("LUCE_DS4_SPEC"); if (cfg_.paged_attention) { const PlacementBackend target_backend = cfg_.device.backend == PlacementBackend::Auto @@ -1830,16 +1841,17 @@ bool DeepSeek4Backend::init() { // Install the gfx1151 DSpark verifier profile first: the MMVQ crossover // below reads LUCE_DS4_Q5_VERIFY. - if (!configure_gfx1151_dspark_verifier_defaults(cfg_.device.gpu)) { + if (!configure_gfx1151_dspark_verifier_defaults(cfg_.device.gpu, spec_requested_)) { return false; } // The shared MMVQ/MMQ crossover defaults to q=3 for NVIDIA. On gfx1151, // DSpark q=4 is faster through MMVQ. Keep AR and other devices unchanged, // and preserve LUCE_MMVQ_MAX_NCOLS as an explicit override. - if (!configure_dspark_mmvq_defaults(cfg_.device.gpu)) { + if (!configure_dspark_mmvq_defaults(cfg_.device.gpu, spec_requested_)) { return false; } - configure_gfx1151_paged_mmvq_default(cfg_.device.gpu, cfg_.paged_attention); + configure_gfx1151_paged_mmvq_default(cfg_.device.gpu, cfg_.paged_attention, + spec_requested_); if (!configure_gfx1151_sparse_prefill_kernel_defaults( cfg_.device.gpu, cfg_.prefill_mode)) { return false; @@ -1853,7 +1865,7 @@ bool DeepSeek4Backend::init() { cfg_.prefill_mode != PrefillAttentionMode::Exact || cfg_.fused_decode || cfg_.fused_verify_f16_kv || env_flag_enabled("LUCE_DS4_FUSED_DECODE") || - env_flag_enabled("LUCE_DS4_SPEC"))) { + spec_requested_)) { std::fprintf(stderr, "[deepseek4] paged serving requires 1..%d local slots, exact " "prefill, and autoregressive non-fused decode\n", @@ -1987,7 +1999,16 @@ bool DeepSeek4Backend::init() { prefill_attention_mode_name(cfg_.prefill_mode), moe_hybrid_ ? " [hybrid]" : ""); - if (!cfg_.paged_attention && env_flag_enabled("LUCE_DS4_SPEC")) { + if (!cfg_.paged_attention && !cfg_.draft_path.empty()) { + // An explicit --draft is part of the launch contract: fail rather + // than silently serving without the requested drafter. + spec_draft_path_ = cfg_.draft_path; + if (!load_spec_drafter()) { + std::fprintf(stderr, "[deepseek4] --draft %s could not be loaded as a " + "DSpark drafter\n", cfg_.draft_path.c_str()); + return false; + } + } else if (!cfg_.paged_attention && spec_requested_) { const char * dp = std::getenv("LUCE_DS4_DRAFT"); if (dp && *dp) { spec_draft_path_ = dp; diff --git a/server/src/deepseek4/deepseek4_backend.h b/server/src/deepseek4/deepseek4_backend.h index 729dd13ee..5b4d47204 100644 --- a/server/src/deepseek4/deepseek4_backend.h +++ b/server/src/deepseek4/deepseek4_backend.h @@ -157,7 +157,9 @@ class DeepSeek4Backend : public ModelBackend { // safe only when this matches cache_.cur_pos. int last_logits_pos_ = -1; - // DSpark speculative decode (opt-in: LUCE_DS4_SPEC=1 + LUCE_DS4_DRAFT=). + // DSpark speculative decode (opt-in: --draft , or + // LUCE_DS4_SPEC=1 + LUCE_DS4_DRAFT=). + bool spec_requested_ = false; bool spec_enabled_ = false; bool spec_drafter_parked_ = false; std::string spec_draft_path_; diff --git a/server/src/deepseek4/deepseek4_internal.h b/server/src/deepseek4/deepseek4_internal.h index 7e7432da7..bf5a45388 100644 --- a/server/src/deepseek4/deepseek4_internal.h +++ b/server/src/deepseek4/deepseek4_internal.h @@ -372,7 +372,9 @@ struct DeepSeek4BackendConfig { std::string model_path; std::string mmproj_path; int mmproj_gpu = -1; // vision encoder GPU; -1 = the target's + std::string draft_path; // DSpark drafter; empty falls back to LUCE_DS4_DRAFT DevicePlacement device; + DevicePlacement draft_device; // auto backend: LUCE_DS4_DRAFT_GPU/_BACKEND, else the target int stream_fd = -1; int chunk = 512; // prefill chunk size PrefillAttentionMode prefill_mode = PrefillAttentionMode::Exact; diff --git a/server/src/placement/device_select.cpp b/server/src/placement/device_select.cpp new file mode 100644 index 000000000..c684b432f --- /dev/null +++ b/server/src/placement/device_select.cpp @@ -0,0 +1,121 @@ +#include "device_select.h" + +#include "common/gguf_inspect.h" +#include "common/gpu_runtime_compat.h" +#include "ggml-cuda.h" +#include "gguf.h" +#include "kv_quant.h" + +#include +#include +#include +#include + +namespace luce::common { + +std::vector enumerate_gpu_devices(bool query_free) { + std::vector devices; + const int count = ggml_backend_cuda_get_device_count(); + for (int i = 0; i < count; ++i) { + GpuDeviceInfo info; + info.index = i; + char description[256] = {}; + ggml_backend_cuda_get_device_description(i, description, sizeof(description)); + info.name = description; + if (query_free) { + size_t free_bytes = 0; + size_t total_bytes = 0; + ggml_backend_cuda_get_device_memory(i, &free_bytes, &total_bytes); + info.free_bytes = free_bytes; + } + cudaDeviceProp prop{}; + if (cudaGetDeviceProperties(&prop, i) == cudaSuccess) { + info.total_bytes = prop.totalGlobalMem; + info.integrated = prop.integrated != 0; +#if defined(LUCE_BACKEND_HIP) || defined(GGML_USE_HIP) + info.arch = prop.gcnArchName; + const size_t features = info.arch.find(':'); + if (features != std::string::npos) info.arch.resize(features); +#else + info.arch = "sm_" + std::to_string(prop.major) + std::to_string(prop.minor); +#endif + } + devices.push_back(std::move(info)); + } + return devices; +} + +uint64_t gguf_model_bytes(const std::string & path) { + namespace fs = std::filesystem; + std::error_code ec; + const uint64_t first = fs::file_size(path, ec); + if (ec) return 0; + + // llama.cpp split naming: -00001-of-00003.gguf + static const std::regex split_re(R"(^(.*)-(\d{5})-of-(\d{5})\.gguf$)"); + std::smatch match; + const std::string file = fs::path(path).filename().string(); + if (!std::regex_match(file, match, split_re)) return first; + + const int parts = std::stoi(match[3].str()); + uint64_t total = 0; + for (int part = 1; part <= parts; ++part) { + char suffix[32]; + std::snprintf(suffix, sizeof(suffix), "-%05d-of-%05d.gguf", part, parts); + const fs::path sibling = + fs::path(path).parent_path() / (match[1].str() + suffix); + const uint64_t size = fs::file_size(sibling, ec); + if (ec) return 0; + total += size; + } + return total; +} + +namespace { + +// A u32 header value, or 0 when the key is missing or has another type (some +// architectures store per-layer arrays; those are not estimated). +uint32_t gguf_u32(const gguf_context * gctx, const std::string & key) { + const int64_t id = gguf_find_key(gctx, key.c_str()); + if (id < 0 || gguf_get_kv_type(gctx, id) != GGUF_TYPE_UINT32) return 0; + return gguf_get_val_u32(gctx, id); +} + +} // namespace + +uint64_t gguf_kv_cache_bytes(const std::string & path, int max_ctx, + ggml_type cache_type_k, ggml_type cache_type_v) { + if (max_ctx <= 0) return 0; + gguf_init_params params{}; + params.no_alloc = true; + gguf_context * gctx = gguf_init_from_file(path.c_str(), params); + if (!gctx) return 0; + + uint64_t bytes = 0; + const int64_t arch_id = gguf_find_key(gctx, "general.architecture"); + const std::string arch = arch_id >= 0 ? gguf_get_val_str(gctx, arch_id) : ""; + if (arch == "qwen35" || arch == "qwen35moe") { + const std::string pre = arch + "."; + uint32_t n_layer = 0; + std::string error; + const uint32_t n_head_kv = gguf_u32(gctx, pre + "attention.head_count_kv"); + const uint32_t key_length = gguf_u32(gctx, pre + "attention.key_length"); + const uint32_t value_length = gguf_u32(gctx, pre + "attention.value_length"); + if (n_head_kv && key_length && + derive_effective_target_layer_count( + arch, gguf_u32(gctx, pre + "block_count"), + gguf_u32(gctx, pre + "nextn_predict_layers"), n_layer, error)) { + ggml_type kv_k = GGML_TYPE_Q4_0, kv_v = GGML_TYPE_Q4_0; + luce::resolve_kv_types(kv_k, kv_v, cache_type_k, cache_type_v); + bytes = luce::kv_reservation_bytes_per_token( + (int) n_layer, (int) gguf_u32(gctx, pre + "full_attention_interval"), + (int) n_head_kv, kv_k, (int) key_length, + kv_v, (int) (value_length ? value_length : key_length)) * + (uint64_t) max_ctx; + } + } + gguf_free(gctx); + return bytes; +} + +} // namespace luce::common diff --git a/server/src/placement/device_select.h b/server/src/placement/device_select.h new file mode 100644 index 000000000..45f295f68 --- /dev/null +++ b/server/src/placement/device_select.h @@ -0,0 +1,126 @@ +// Target device discovery and `--target-device auto` selection. +// +// Enumeration talks to the compiled GPU runtime; the selection policy is a +// pure function over the enumerated facts so it can be tested without a GPU. + +#pragma once + +#include "ggml.h" +#include "placement_backend.h" +#include "placement_config.h" + +#include +#include +#include + +namespace luce::common { + +struct GpuDeviceInfo { + int index = -1; + std::string name; + std::string arch; // gfx arch on HIP, sm_XY on CUDA + bool integrated = false; + uint64_t total_bytes = 0; + uint64_t free_bytes = 0; // filled only when queried +}; + +// Devices of the compiled backend, in runtime order (the order hip:N and +// cuda:N refer to). Reading free memory opens a runtime context on every +// device, so it is opt-in; selection uses total memory only. +std::vector enumerate_gpu_devices(bool query_free = false); + +// Bytes the target weights need: the GGUF file size, summed over every part +// of a split GGUF. Returns 0 when the file or any of its parts cannot be read. +uint64_t gguf_model_bytes(const std::string & path); + +// Bytes the KV cache of one max_ctx sequence needs, for models whose cache +// follows from the GGUF header: the Qwen3.5/3.6 hybrids, where only the +// full-attention layers carry KV (kv_reservation_bytes_per_token), with the +// cache types the backend resolves from the overrides (GGML_TYPE_COUNT: none) +// and LUCE_KV_*. Returns 0 for other families, whose backends size their own +// caches (DeepSeek4 compresses it, Gemma4 uses a sliding window), and for +// unreadable files: the flat margin below is all they get. +uint64_t gguf_kv_cache_bytes(const std::string & path, int max_ctx, + ggml_type cache_type_k = GGML_TYPE_COUNT, + ggml_type cache_type_v = GGML_TYPE_COUNT); + +// Weights, the estimated KV cache (0 when unknown), and a margin for compute +// buffers, runtime overhead and any cache the estimate does not cover: a tenth +// of the weights, kept between 2 and 4 GiB. +inline uint64_t auto_device_required_bytes(uint64_t model_bytes, uint64_t kv_bytes = 0) { + const uint64_t min_margin = 2ull << 30; + const uint64_t max_margin = 4ull << 30; + uint64_t margin = model_bytes / 10; + if (margin < min_margin) margin = min_margin; + if (margin > max_margin) margin = max_margin; + return model_bytes + kv_bytes + margin; +} + +struct AutoDeviceChoice { + int index = -1; // -1: no device available + bool fits = false; // the model fits resident on the choice + std::string reason; +}; + +// Prefer a device the whole model fits on, discrete GPUs before integrated +// ones (they have their own memory bandwidth), then runtime order. When no +// device fits, take the largest one: backends with expert or layer offload +// can still run there, and it gives the monolithic loaders the best chance. +inline AutoDeviceChoice choose_auto_target_device( + const std::vector & devices, + uint64_t model_bytes, + uint64_t kv_bytes = 0) +{ + AutoDeviceChoice choice; + if (devices.empty()) { + choice.reason = "no GPU devices found"; + return choice; + } + const uint64_t need = auto_device_required_bytes(model_bytes, kv_bytes); + const GpuDeviceInfo * best = nullptr; + for (const GpuDeviceInfo & device : devices) { + if (device.total_bytes < need) continue; + if (!best || (best->integrated && !device.integrated)) best = &device; + } + if (best) { + choice.index = best->index; + choice.fits = true; + choice.reason = best->integrated + ? "first integrated GPU that fits the model" + : "first discrete GPU that fits the model"; + return choice; + } + for (const GpuDeviceInfo & device : devices) { + if (!best || device.total_bytes > best->total_bytes) best = &device; + } + choice.index = best->index; + choice.reason = "no GPU holds the model with headroom; using the largest one"; + return choice; +} + +// Draft placement before architecture defaults apply, with one precedence: +// an explicit --draft-device, then the architecture's own default (DeepSeek4 +// reads LUCE_DS4_DRAFT_GPU/_BACKEND), then the target GPU. +// - An explicit placement becomes concrete: auto:N names GPU N of the +// compiled backend, so no backend mistakes it for "unspecified". +// - An unplaced drafter keeps the auto backend, which backends read as +// "unspecified". With an auto-placed target it takes the target's GPU +// index, for backends that read only the index. +inline DevicePlacement resolve_draft_placement(DevicePlacement draft, + bool draft_explicit, + const DevicePlacement & target, + bool target_auto) { + if (draft_explicit) { + if (draft.backend == PlacementBackend::Auto) { + draft.backend = compiled_placement_backend(); + } + return draft; + } + if (target_auto) { + draft.backend = PlacementBackend::Auto; + draft.gpu = target.gpu; + } + return draft; +} + +} // namespace luce::common diff --git a/server/src/server/launch_profiles.h b/server/src/server/launch_profiles.h new file mode 100644 index 000000000..805325f1c --- /dev/null +++ b/server/src/server/launch_profiles.h @@ -0,0 +1,156 @@ +// Named launch profiles (`--profile `). +// +// A profile is a qualified hardware + model configuration: the server flags +// and the tuning environment a documented recipe needs, under one name. +// Everything a profile sets is a default. Flags given on the command line +// replace the profile's value for that flag, and environment variables that +// are already set keep their value, so a profile never overrides an explicit +// choice. + +#pragma once + +#include +#include +#include + +namespace luce::server { + +struct LaunchProfileFlag { + const char * flag; + const char * value; // nullptr for boolean flags +}; + +struct LaunchProfileEnv { + const char * name; + const char * value; +}; + +struct LaunchProfile { + const char * name; + const char * summary; + std::vector flags; + std::vector env; +}; + +inline const std::vector & launch_profiles() { + static const std::vector profiles = { + { + "ds4-strix", + "DeepSeek V4 Flash on one Strix Halo (gfx1151); the gfx1151 device " + "profile supplies the kernel defaults", + { + {"--max-ctx", "131072"}, + {"--chunk", "8192"}, + {"--ds4-fused-decode", nullptr}, + {"--ds4-fused-verify-f16-kv", nullptr}, + {"--ds4-expert-top-k", "6"}, + {"--ds4-prefill", "sparse"}, + }, + {}, + }, + { + "ds4-r9700-strix", + "DeepSeek V4 Flash with dense work and hot experts on an R9700 " + "(gfx1201) and the remaining experts on Strix Halo (gfx1151)", + { + {"--target-device", "hip:0"}, + {"--expert-device", "hip:1"}, + {"--peer-access", nullptr}, + {"--max-ctx", "18432"}, + {"--chunk", "2048"}, + {"--ds4-fused-decode", nullptr}, + {"--ds4-expert-top-k", "6"}, + {"--ds4-prefill", "sparse"}, + }, + { + {"LUCE_EXPERT_BUDGET_MB", "14350"}, + {"LUCE_DS4_SPEC_Q", "5"}, + {"LUCE_DS4_Q5_VERIFY", "1"}, + {"LUCE_DS4_FUSED_VERIFY", "1"}, + {"LUCE_DS4_FUSED_HYBRID_DECODE", "1"}, + {"LUCE_DS4_PINNED_ROLLBACK", "1"}, + {"LUCE_DS4_GPU_ARGMAX_VERIFY", "1"}, + {"LUCE_DS4_DRAFT_CONTEXT_KV_CACHE", "1"}, + {"LUCE_DS4_TP_ROUTE_PREFORK", "1"}, + {"LUCE_DS4_TP_DEVICE_JOIN", "1"}, + {"LUCE_DS4_TP_DEVICE_JOIN_SPLIT", "1"}, + {"LUCE_DS4_TP_FUSED_HC_JOIN", "1"}, + {"LUCE_DS4_TP_MAIN_ROUTE_WEIGHTS", "1"}, + {"LUCE_DS4_TP_COARSE_OWNER", "1"}, + {"LUCE_DS4_TP_NATIVE_ROUTE_WIDTH", "1"}, + {"LUCE_DS4_TP_MASKED_ROUTES", "1"}, + {"LUCE_DS4_TP_GROUPED_MMVQ", "1"}, + {"LUCE_DS4_TP_CAPTURE_CACHE_SLOTS", "4"}, + {"LUCE_MOE_TP_DYNAMIC_ROUTE_BALANCE", "1"}, + {"LUCE_MOE_TP_DYNAMIC_MAIN_SLOTS_X4", "13"}, + {"LUCE_MOE_DUPLICATE_HOT_ON_COLD", "1"}, + {"LUCE_MOE_FULL_COLD_PARALLEL", "1"}, + {"LUCE_MOE_PREFILL_PERSISTENT_OWNER_ALLOC", "1"}, + {"LUCE_DS4_HYBRID_PREFILL_GPU_HC", "1"}, + {"LUCE_DS4_HYBRID_PREFILL_EAGER", "1"}, + {"GGML_CUDA_BATCH_PEER_COPIES", "1"}, + {"LUCE_MMID_GROUPED", "1"}, + {"LUCE_MMID_GROUPED_TYPES", "8"}, + {"LUCE_MMID_GROUPED_DEVICE", "1"}, + {"LUCE_CUDA_MMVQ_MOE_ROWS_PER_BLOCK", "2"}, + {"LUCE_CUDA_MMVQ_MOE_FP3_PACKED24", "1"}, + {"LUCE_CUDA_MMVQ_FP4_X4", "1"}, + {"LUCE_DS4_DIRECT_INDEXER_TOPK", "1"}, + {"GGML_DS4_TOPK_BLOCK_RADIX", "1"}, + {"LUCE_DS4_MIX_MMQ_PREFILL", "1"}, + {"LUCE_CUDA_I32_REPEAT", "1"}, + {"ROCBLAS_USE_HIPBLASLT", "0"}, + }, + }, + }; + return profiles; +} + +inline const LaunchProfile * find_launch_profile(const std::string & name) { + for (const LaunchProfile & profile : launch_profiles()) { + if (name == profile.name) return &profile; + } + return nullptr; +} + +inline std::string launch_profile_names() { + std::string names; + for (const LaunchProfile & profile : launch_profiles()) { + if (!names.empty()) names += ", "; + names += profile.name; + } + return names; +} + +// Flags that select the same setting. A profile's --target-device yields to +// an explicit --target-devices as well as to an explicit --target-device. +inline bool launch_flags_overlap(const std::string & a, const std::string & b) { + auto target = [](const std::string & f) { + return f == "--target-device" || f == "--target-devices"; + }; + return a == b || (target(a) && target(b)); +} + +// Profile flags the command line has not already set, flattened into argv +// tokens. `given` holds the tokens of the model block the profile applies to. +inline std::vector launch_profile_args( + const LaunchProfile & profile, + const std::vector & given) +{ + std::vector out; + for (const LaunchProfileFlag & entry : profile.flags) { + bool explicit_flag = false; + for (const std::string & token : given) { + if (launch_flags_overlap(token, entry.flag)) { + explicit_flag = true; + break; + } + } + if (explicit_flag) continue; + out.emplace_back(entry.flag); + if (entry.value) out.emplace_back(entry.value); + } + return out; +} + +} // namespace luce::server diff --git a/server/src/server/server_main.cpp b/server/src/server/server_main.cpp index 10818a54c..ce8296bcd 100644 --- a/server/src/server/server_main.cpp +++ b/server/src/server/server_main.cpp @@ -13,9 +13,11 @@ #include "http_server.h" #include "chat_template.h" +#include "launch_profiles.h" #include "model_card.h" #include "common/backend_factory.h" #include "common/chain_rollback_policy.h" +#include "common/gguf_inspect.h" #include "common/layer_split_utils.h" #include "common/model_capabilities.h" #include "common/spark_corpus.h" @@ -27,6 +29,7 @@ #include "engine/luce_engine.h" #include "placement/pflash_placement.h" #include "placement/draft_residency.h" +#include "placement/device_select.h" #include "kvflash_pager.h" #include "kv_quant.h" @@ -40,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -77,24 +81,38 @@ static bool parse_double_list(const char * value, std::vector & out) { static void print_usage(const char * prog) { std::fprintf(stderr, "Usage: %s [options]\n" + " %s --list-devices [model.gguf]\n" "\n" "Options:\n" + " --profile Apply a qualified launch profile (%s).\n" + " Explicit flags and already-set environment\n" + " variables take precedence over the profile.\n" " --model Begin a model block; set placement with --target-device.\n" " --load-balancing Enable primary-first fallback (disabled by default).\n" " --load-balancing-primary-gpu Select the primary model by its target device.\n" " Defaults to the first block; request model names\n" " do not change generation routing.\n" - " --draft Draft model for speculative decode\n" + " --draft Draft model for speculative decode (DFlash for Qwen,\n" + " Gemma and Laguna; DSpark for DeepSeek4)\n" " --mmproj Vision projector GGUF: enables image input (Qwen3.5/3.8, DS4V)\n" " --mmproj-device hip:N Run the DS4V image encoder on another GPU (one-GPU layout)\n" " --port Listen port (default: 8080)\n" " --host Bind address (default: 0.0.0.0)\n" - " --max-ctx Max context length (default: 131072)\n" + " --max-ctx Max context length (default: 8192)\n" " --max-tokens Default max output tokens (legacy alias for\n" " --default-max-tokens; loses to --default-max-tokens\n" " when both are passed)\n" - " --target-device Target device (default: auto:0)\n" - " --draft-device Draft device (default: auto:0)\n" + " --target-device \n" + " Target device (default: auto:0, the first\n" + " GPU). auto picks a GPU the model fits on,\n" + " discrete before integrated, else the\n" + " largest; see --list-devices. Env default:\n" + " LUCE_TARGET_DEVICE\n" + " --draft-device Draft device (default: auto:0; DeepSeek4\n" + " and --target-device auto: the target GPU)\n" + " --expert-device DeepSeek4: keep dense work and hot experts on\n" + " the target and run the remaining routed\n" + " experts on this GPU in process\n" " --draft-ipc-bin Remote backend IPC daemon for mixed backends\n" " --draft-ipc-work-dir Remote draft IPC scratch directory\n" " --draft-ipc-ring-cap Remote draft feature ring capacity\n" @@ -247,7 +265,7 @@ static void print_usage(const char * prog) { " --freq Enable expert frequency tracking + print analysis at shutdown\n" " --collect-routing Log binary routing data (hidden states + expert IDs)\n" " for MLP predictor training (see scripts/train_predictor.py)\n" - "\n", prog); + "\n", prog, prog, luce::server::launch_profile_names().c_str()); } // Own everything borrowed by a model's HTTP/scheduler context. Shutdown must @@ -291,6 +309,11 @@ struct ModelOptions { std::string cache_type_v; // explicit --cache-type-v override bool fast_rollback_forced_off = false; bool target_split_fast_rollback_cli = false; + bool target_device_auto = false; // --target-device auto + bool target_device_from_env = false; // the device came from LUCE_TARGET_DEVICE + bool draft_device_set = false; // --draft-device given explicitly + std::optional expert_device; // --expert-device + const luce::server::LaunchProfile * profile = nullptr; // --profile; env installed at load // Track which thinking-budget tunables the operator set via CLI. // Those values win over the model card (spec §3.1: "Explicit CLI @@ -350,7 +373,7 @@ static int parse_model_options(int argc, char ** argv, ModelOptions & model, std::fprintf(stderr, "[server] %s belongs in the first model block: there is one listener\n", argv[i]); return 2; } - if (load_balancing && (option == "--peer-access" || + if (load_balancing && (option == "--peer-access" || option == "--expert-device" || option == "--no-fast-rollback" || option == "--target-split-fast-rollback" || option == "--adaptive-experts" || option == "--specla" || option == "--specla-top-k" || option.rfind("--kvflash", 0) == 0 || @@ -358,7 +381,13 @@ static int parse_model_options(int argc, char ** argv, ModelOptions & model, std::fprintf(stderr, "[server] %s changes process-wide policy and cannot be scoped to a model\n", argv[i]); return 2; } - if (std::strcmp(argv[i], "--draft") == 0 && i + 1 < argc) { + if (std::strcmp(argv[i], "--draft") == 0) { + // An empty path would silently start without speculation, while + // --draft promises a drafter or a failed start. + if (i + 1 >= argc || argv[i + 1][0] == '\0') { + std::fprintf(stderr, "[server] --draft needs a draft model path\n"); + return 2; + } bargs.draft_path = argv[++i]; } else if (std::strcmp(argv[i], "--mmproj") == 0) { if (i + 1 >= argc) { @@ -394,8 +423,11 @@ static int parse_model_options(int argc, char ** argv, ModelOptions & model, return 2; } target_device_seen = true; - if (!parse_placement_device(argv[++i], bargs.device)) { - std::fprintf(stderr, "[server] bad --target-device value (expected backend:gpu)\n"); + const char * value = argv[++i]; + model.target_device_auto = std::strcmp(value, "auto") == 0; + if (!model.target_device_auto && + !parse_placement_device(value, bargs.device)) { + std::fprintf(stderr, "[server] bad --target-device value (expected backend:gpu or auto)\n"); return 2; } } else if (std::strcmp(argv[i], "--draft-swa") == 0 && i + 1 < argc) { @@ -418,6 +450,19 @@ static int parse_model_options(int argc, char ** argv, ModelOptions & model, std::fprintf(stderr, "[server] bad --draft-device value (expected backend:gpu)\n"); return 2; } + model.draft_device_set = true; + } else if (std::strcmp(argv[i], "--expert-device") == 0 && i + 1 < argc) { + DevicePlacement expert; + if (!parse_placement_device(argv[++i], expert)) { + std::fprintf(stderr, "[server] bad --expert-device value (expected backend:gpu)\n"); + return 2; + } + model.expert_device = expert; + } else if (std::strcmp(argv[i], "--profile") == 0) { + // main() expands profiles before blocks are parsed. + std::fprintf(stderr, "[server] --profile needs a profile name (%s)\n", + luce::server::launch_profile_names().c_str()); + return 2; } else if (std::strcmp(argv[i], "--draft-ipc-bin") == 0 && i + 1 < argc) { bargs.remote_draft.ipc_bin = argv[++i]; } else if (std::strcmp(argv[i], "--draft-ipc-work-dir") == 0 && i + 1 < argc) { @@ -816,6 +861,37 @@ static int parse_model_options(int argc, char ** argv, ModelOptions & model, return 2; } } + // LUCE_TARGET_DEVICE supplies the device when the block names none (a + // profile's --target-device counts as naming one). The container sets it + // to auto; native launches keep auto:0 unless they export it. + if (!target_device_seen && !target_devices_seen && !load_balancing) { + const char * env_device = std::getenv("LUCE_TARGET_DEVICE"); + if (env_device && *env_device) { + model.target_device_from_env = true; + model.target_device_auto = std::strcmp(env_device, "auto") == 0; + if (!model.target_device_auto && + !parse_placement_device(env_device, bargs.device)) { + std::fprintf(stderr, + "[server] bad LUCE_TARGET_DEVICE value '%s' (expected backend:gpu or auto)\n", + env_device); + return 2; + } + } + } + if (model.target_device_auto && model.expert_device) { + std::fprintf(stderr, + "[server] --expert-device needs an explicit --target-device for the dense work%s\n", + model.target_device_from_env + ? " (LUCE_TARGET_DEVICE=auto, which the container sets, does not name one)" + : ""); + return 2; + } + if (model.target_device_auto && load_balancing) { + std::fprintf(stderr, + "[server] --target-device auto is unavailable with --load-balancing; " + "give each model block an explicit backend:gpu\n"); + return 2; + } if (bargs.specla_top_k_explicit && !bargs.specla_mode) { std::fprintf(stderr, "[server] --specla-top-k requires --specla\n"); return 2; @@ -858,7 +934,132 @@ static int parse_model_options(int argc, char ** argv, ModelOptions & model, return 0; } +static double bytes_to_gib(uint64_t bytes) { + return (double) bytes / (1024.0 * 1024.0 * 1024.0); +} + +// The KV cache a model block's context needs on its GPU, with its cache-type +// flags (already validated) as overrides; 0 when the family is not estimated. +static uint64_t model_kv_cache_bytes(const ModelOptions & model) { + auto override_type = [](const std::string & name) { + return name.empty() ? GGML_TYPE_COUNT : luce::parse_kv_type(name.c_str()); + }; + return gguf_kv_cache_bytes(model.bargs.model_path, model.bargs.device.max_ctx, + override_type(model.cache_type_k), + override_type(model.cache_type_v)); +} + +// --target-device auto: bind the model block to the GPU the policy picks. +static bool resolve_auto_target_device(ModelOptions & model) { + const std::vector devices = enumerate_gpu_devices(); + const uint64_t model_bytes = gguf_model_bytes(model.bargs.model_path); + if (model_bytes == 0) { + std::fprintf(stderr, "[server] --target-device auto: cannot read model %s\n", + model.bargs.model_path.c_str()); + return false; + } + // One sequence at --max-ctx must fit; paged serving sizes its pool from + // whatever memory is left. + const uint64_t kv_bytes = model_kv_cache_bytes(model); + const AutoDeviceChoice choice = choose_auto_target_device(devices, model_bytes, kv_bytes); + if (choice.index < 0) { + std::fprintf(stderr, "[server] --target-device auto: %s\n", choice.reason.c_str()); + return false; + } + DevicePlacement & target = model.bargs.device; + target.backend = compiled_placement_backend(); + target.gpu = choice.index; + const GpuDeviceInfo & device = devices[(size_t) choice.index]; + std::fprintf(stderr, + "[server] --target-device auto: %s (%s, %s, %.1f GiB) for a %.1f GiB model" + " + %.1f GiB KV at %d tokens: %s\n", + placement_device_name(target).c_str(), device.name.c_str(), device.arch.c_str(), + bytes_to_gib(device.total_bytes), bytes_to_gib(model_bytes), bytes_to_gib(kv_bytes), + model.bargs.device.max_ctx, choice.reason.c_str()); + return true; +} + +// After a failed load on a fixed device, name the device auto would pick when +// the fixed one is too small for the model and a different one is available. +static void print_target_device_hint(const std::string & model_path, + const DevicePlacement & target, + uint64_t kv_bytes) { + if (target.is_multi_device()) return; + const std::vector devices = enumerate_gpu_devices(); + const uint64_t model_bytes = gguf_model_bytes(model_path); + if (devices.size() < 2 || model_bytes == 0) return; + if (target.gpu < 0 || (size_t) target.gpu >= devices.size()) return; + const GpuDeviceInfo & current = devices[(size_t) target.gpu]; + if (current.total_bytes >= auto_device_required_bytes(model_bytes, kv_bytes)) return; + const AutoDeviceChoice choice = choose_auto_target_device(devices, model_bytes, kv_bytes); + if (choice.index < 0 || choice.index == target.gpu) return; + const GpuDeviceInfo & better = devices[(size_t) choice.index]; + const char * backend = placement_backend_name(compiled_placement_backend()); + std::fprintf(stderr, + "[server] hint: the %.1f GiB model is too large for %s:%d (%s, %.1f GiB). " + "%s:%d (%s, %s, %.1f GiB) is the better fit: pass --target-device %s:%d " + "or --target-device auto (see --list-devices).\n", + bytes_to_gib(model_bytes), backend, target.gpu, current.name.c_str(), + bytes_to_gib(current.total_bytes), backend, choice.index, better.name.c_str(), + better.arch.c_str(), bytes_to_gib(better.total_bytes), backend, choice.index); +} + +// --expert-device: DeepSeek4 in-process expert parallelism. The flag is the +// command-line spelling of LUCE_DS4_MOE_TP=1 LUCE_DS4_MOE_TP_INPROC=1 +// LUCE_DS4_MOE_TP_GPU= LUCE_DS4_MOE_TP_BACKEND=. +static bool apply_expert_device(const DevicePlacement & expert, + const BackendPlan & plan) { + const DevicePlacement & target = plan.placement().target; + if (plan.arch() != "deepseek4") { + std::fprintf(stderr, + "[server] --expert-device is only valid for deepseek4 models (detected '%s')\n", + plan.arch().c_str()); + return false; + } + if (target.is_multi_device() || plan.placement().remote_target_shard.enabled()) { + std::fprintf(stderr, "[server] --expert-device requires one local --target-device\n"); + return false; + } + const PlacementBackend compiled = compiled_placement_backend(); + const PlacementBackend target_backend = + target.backend == PlacementBackend::Auto ? compiled : target.backend; + const PlacementBackend expert_backend = + expert.backend == PlacementBackend::Auto ? compiled : expert.backend; + if (expert_backend == target_backend && expert.gpu == target.gpu) { + std::fprintf(stderr, "[server] --expert-device must differ from the target device\n"); + return false; + } + const std::string gpu = std::to_string(expert.gpu); + set_environment_variable("LUCE_DS4_MOE_TP", "1", true); + set_environment_variable("LUCE_DS4_MOE_TP_INPROC", "1", true); + set_environment_variable("LUCE_DS4_MOE_TP_GPU", gpu.c_str(), true); + set_environment_variable("LUCE_DS4_MOE_TP_BACKEND", + placement_backend_name(expert_backend), true); + return true; +} + +// Install a profile's environment defaults. Variables that are already set +// keep their value, and the log names them. +static void apply_launch_profile_env(const luce::server::LaunchProfile & profile) { + if (profile.env.empty()) return; + int applied = 0; + std::string kept_env; + for (const luce::server::LaunchProfileEnv & env : profile.env) { + if (std::getenv(env.name)) { + kept_env += std::string(kept_env.empty() ? "" : ", ") + env.name; + continue; + } + set_environment_variable(env.name, env.value, false); + ++applied; + } + std::fprintf(stderr, "[server] profile %s: %d environment defaults applied%s%s\n", + profile.name, applied, + kept_env.empty() ? "" : "; kept explicit ", + kept_env.c_str()); +} + static int load_model(ModelOptions & model, LoadedModel & loaded, bool multi_model) { + if (model.profile) apply_launch_profile_env(*model.profile); if (!model.adaptive_experts_tau.empty()) set_environment_variable("LUCE_ADAPTIVE_K_TAU", model.adaptive_experts_tau.c_str(), false); if (!model.kvflash_pool.empty()) @@ -914,6 +1115,13 @@ static int load_model(ModelOptions & model, LoadedModel & loaded, bool multi_mod } } + if (model.target_device_auto && !resolve_auto_target_device(model)) { + return 1; + } + bargs.draft_device = resolve_draft_placement( + bargs.draft_device, model.draft_device_set, bargs.device, + model.target_device_auto); + // Explicit --cache-type-* overrides enter the request here; the qwen35 // env/default resolution runs inside prepare_backend() once the model // architecture is known. Other families still consume their env vars. @@ -1194,9 +1402,16 @@ static int load_model(ModelOptions & model, LoadedModel & loaded, bool multi_mod arch.c_str()); } } + if (model.expert_device && !apply_expert_device(*model.expert_device, backend_plan)) { + return 2; + } auto backend_owner = create_backend(backend_plan); if (!backend_owner) { std::fprintf(stderr, "[server] backend creation failed\n"); + if (!model.expert_device) { + print_target_device_hint(backend_model.path, backend_placement.target, + model_kv_cache_bytes(model)); + } return 1; } ModelBackend * backend = backend_owner.get(); @@ -1633,7 +1848,109 @@ static int load_model(ModelOptions & model, LoadedModel & loaded, bool multi_mod return 0; } +// `luce_server --list-devices [model.gguf]`: one line per GPU in the order +// backend:N refers to, then, for a model, its size and the auto choice. Lines +// are " key=value ...", with the free-text field last. +static int list_devices(const char * model_path) { + const std::vector devices = enumerate_gpu_devices(/*query_free=*/true); + const char * backend = placement_backend_name(compiled_placement_backend()); + for (const GpuDeviceInfo & device : devices) { + std::printf("device %s:%d arch=%s type=%s total_mib=%llu free_mib=%llu name=%s\n", + backend, device.index, device.arch.empty() ? "unknown" : device.arch.c_str(), + device.integrated ? "integrated" : "discrete", + (unsigned long long) (device.total_bytes >> 20), + (unsigned long long) (device.free_bytes >> 20), device.name.c_str()); + } + if (!model_path) return devices.empty() ? 1 : 0; + + const uint64_t model_bytes = gguf_model_bytes(model_path); + if (model_bytes == 0) { + std::fprintf(stderr, "[server] cannot read model %s\n", model_path); + return 1; + } + const GgufModelInfo info = inspect_gguf_model_info(model_path); + // The KV cache at the default --max-ctx, as a plain launch would size it. + const uint64_t kv_bytes = gguf_kv_cache_bytes(model_path, DevicePlacement{}.max_ctx); + std::printf("model arch=%s size_mib=%llu kv_mib=%llu path=%s\n", + info.arch.empty() ? "unknown" : info.arch.c_str(), + (unsigned long long) (model_bytes >> 20), (unsigned long long) (kv_bytes >> 20), + model_path); + const AutoDeviceChoice choice = choose_auto_target_device(devices, model_bytes, kv_bytes); + if (choice.index < 0) { + std::printf("auto none reason=%s\n", choice.reason.c_str()); + return 1; + } + std::printf("auto %s:%d fits=%s total_mib=%llu reason=%s\n", backend, choice.index, + choice.fits ? "yes" : "no", + (unsigned long long) (devices[(size_t) choice.index].total_bytes >> 20), + choice.reason.c_str()); + return 0; +} + +// Replace `--profile ` in a model block with the profile's flags, placed +// right after the model path. Tokens the block already sets are left out, so +// explicit flags win in any order. The profile's environment is installed by +// load_model(), so a block that is never loaded changes nothing. +static bool expand_launch_profile(std::vector & block, + std::vector> & storage, + bool load_balancing, + const luce::server::LaunchProfile *& profile) { + profile = nullptr; + std::vector kept; + for (size_t i = 0; i < block.size(); ++i) { + if (std::strcmp(block[i], "--profile") != 0) { + kept.push_back(block[i]); + continue; + } + if (profile || i + 1 >= block.size()) { + std::fprintf(stderr, "[server] --profile takes one profile name per model (%s)\n", + luce::server::launch_profile_names().c_str()); + return false; + } + profile = luce::server::find_launch_profile(block[++i]); + if (!profile) { + std::fprintf(stderr, "[server] unknown --profile '%s' (available: %s)\n", + block[i], luce::server::launch_profile_names().c_str()); + return false; + } + } + if (!profile) return true; + if (load_balancing && !profile->env.empty()) { + std::fprintf(stderr, "[server] --profile %s sets process-wide environment and " + "cannot be scoped to a model\n", profile->name); + return false; + } + + const std::vector given(kept.begin() + 1, kept.end()); + const std::vector args = luce::server::launch_profile_args(*profile, given); + std::vector expanded(kept.begin(), kept.end()); + // kept[0] is argv[0]; the model path follows unless the block has none. + const size_t insert_at = expanded.size() > 1 && expanded[1][0] != '-' ? 2 : 1; + std::vector inserted; + std::string shown; + for (const std::string & arg : args) { + storage.push_back(std::make_unique(arg)); + inserted.push_back(storage.back()->data()); + shown += " " + arg; + } + expanded.insert(expanded.begin() + insert_at, inserted.begin(), inserted.end()); + block = std::move(expanded); + + std::fprintf(stderr, "[server] profile %s: %s\n", profile->name, profile->summary); + std::fprintf(stderr, "[server] profile %s: flags%s\n", profile->name, + shown.empty() ? " (all set explicitly)" : shown.c_str()); + return true; +} + int main(int argc, char ** argv) { + if (argc >= 2 && std::strcmp(argv[1], "--list-devices") == 0) { + if (argc > 3) { + std::fprintf(stderr, "Usage: %s --list-devices [model.gguf]\n", argv[0]); + return 2; + } + return list_devices(argc == 3 ? argv[2] : nullptr); + } + // Reuse the existing per-model CLI and loader. Argument strings belong to // main's argv and outlive every backend, including factories borrowing paths. std::vector> model_args(1, {argv[0]}); @@ -1668,6 +1985,15 @@ int main(int argc, char ** argv) { std::fprintf(stderr, "[server] --load-balancing requires at least two model blocks\n"); return 2; } + // Profile tokens are referenced by model_args for the process lifetime. + static std::vector> profile_storage; + std::vector profiles(model_args.size()); + for (size_t m = 0; m < model_args.size(); ++m) { + if (!expand_launch_profile(model_args[m], profile_storage, load_balancing, + profiles[m])) { + return 2; + } + } std::vector options(model_args.size()); std::set names; for (size_t m = 0; m < model_args.size(); ++m) { @@ -1680,6 +2006,7 @@ int main(int argc, char ** argv) { options[m].sconfig.model_name.c_str()); return ret; } + options[m].profile = profiles[m]; const auto & name = options[m].sconfig.model_name; if (load_balancing && (name.empty() || name == "auto" || !names.insert(name).second)) { std::fprintf(stderr, "[server] model block %zu: --model-name must be unique, nonempty and different from auto (got '%s')\n", m + 1, name.c_str()); @@ -1716,7 +2043,8 @@ int main(int argc, char ** argv) { } std::fprintf(stderr, "[server] load balancing %s; primary=%s target=%s\n", load_balancing ? "enabled" : "disabled", options.front().sconfig.model_name.c_str(), - placement_device_name(options.front().bargs.device).c_str()); + options.front().target_device_auto + ? "auto" : placement_device_name(options.front().bargs.device).c_str()); if (load_balancing) { const auto & listener = options.front().sconfig; diff --git a/server/test/test_entrypoint.sh b/server/test/test_entrypoint.sh new file mode 100755 index 000000000..4fec3246a --- /dev/null +++ b/server/test/test_entrypoint.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +# Drives scripts/entrypoint.sh against a fake luce_server that records its argv +# and answers --list-devices from FAKE_PROBE. + +set -euo pipefail + +ENTRYPOINT="${1:?usage: test_entrypoint.sh }" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +TARGET="$TMP_DIR/model.gguf" +FAKE_SERVER="$TMP_DIR/luce_server" +touch "$TARGET" +mkdir -p "$TMP_DIR/drafts" +touch "$TMP_DIR/drafts/dflash-draft-3.6-q8_0.gguf" \ + "$TMP_DIR/drafts/DeepSeek-V4-Flash-DSpark-draft.gguf" \ + "$TMP_DIR/drafts/model-mmproj-BF16.gguf" + +cat >"$FAKE_SERVER" <<'FAKE' +#!/usr/bin/env bash +if [ "${1:-}" = --list-devices ]; then + printf '%s\n' "${FAKE_PROBE:-}" + printf 'PROBE_ARG=%s\n' "$@" >&2 + exit 0 +fi +printf 'SERVER_ARG=%s\n' "$@" +printf 'ENV_LUCE_TARGET_DEVICE=%s\n' "${LUCE_TARGET_DEVICE:-}" +FAKE +chmod +x "$FAKE_SERVER" + +DUAL_AMD_DS4='device hip:0 arch=gfx1201 type=discrete total_mib=32624 free_mib=32000 name=AMD Radeon AI PRO R9700 +device hip:1 arch=gfx1151 type=integrated total_mib=98304 free_mib=98000 name=AMD Radeon Graphics +model arch=deepseek4 size_mib=93741 path=/models/ds4.gguf +auto hip:1 fits=yes total_mib=98304 reason=first integrated GPU that fits the model' +DUAL_AMD_QWEN='device hip:0 arch=gfx1201 type=discrete total_mib=32624 free_mib=32000 name=AMD Radeon AI PRO R9700 +device hip:1 arch=gfx1151 type=integrated total_mib=98304 free_mib=98000 name=AMD Radeon Graphics +model arch=qwen35 size_mib=13592 path=/models/qwen.gguf +auto hip:0 fits=yes total_mib=32624 reason=first discrete GPU that fits the model' +RTX_24G='device cuda:0 arch=sm_86 type=discrete total_mib=24576 free_mib=24000 name=NVIDIA GeForce RTX 3090 +model arch=qwen35 size_mib=15000 path=/models/qwen.gguf +auto cuda:0 fits=yes total_mib=24576 reason=first discrete GPU that fits the model' + +run_entrypoint() { # run_entrypoint [VAR=value ...] -- [entrypoint args...] + local env_args=() unset_args=() var + while [ $# -gt 0 ] && [ "$1" != -- ]; do env_args+=("$1"); shift; done + [ $# -gt 0 ] && shift + # The entrypoint reads many LUCE_* variables (LUCE_BUDGET, LUCE_PORT, ...); + # none from the caller's shell may reach it. + for var in $(compgen -e); do + case "$var" in LUCE_*) unset_args+=(-u "$var") ;; esac + done + env "${unset_args[@]}" \ + LUCE_DIR="$TMP_DIR" \ + LUCE_TARGET="$TARGET" \ + LUCE_DRAFT="$TMP_DIR/no-draft" \ + LUCE_SERVER_BIN="$FAKE_SERVER" \ + "${env_args[@]}" \ + bash "$ENTRYPOINT" "$@" 2>"$TMP_DIR/stderr" +} + +fail() { + echo "FAIL: $*" >&2 + echo "--- entrypoint stderr of the last run:" >&2 + cat "$TMP_DIR/stderr" >&2 + exit 1 +} + +has_pair() { # has_pair + awk -v f="SERVER_ARG=$2" -v v="SERVER_ARG=$3" ' + prev == f && $0 == v { found = 1 } { prev = $0 } + END { exit(found ? 0 : 1) }' <<<"$1" +} +has_flag() { grep -Fxq "SERVER_ARG=$2" <<<"$1"; } +last_value() { # value after the last occurrence of a flag + awk -v f="SERVER_ARG=$2" 'prev == f { v = $0 } { prev = $0 } + END { sub(/^SERVER_ARG=/, "", v); print v }' <<<"$1" +} + +# ── native cache defaults stay with luce_server ───────────────────────────── +out="$(run_entrypoint --)" +for flag in --prefix-cache-slots --prefill-cache-slots; do + has_flag "$out" "$flag" && fail "entrypoint overrides the native cache default with $flag" +done +out="$(run_entrypoint LUCE_PREFIX_CACHE_SLOTS=0 --)" +has_pair "$out" --prefix-cache-slots 0 || fail "explicit LUCE_PREFIX_CACHE_SLOTS=0 dropped" +out="$(run_entrypoint LUCE_PREFIX_CACHE_SLOTS=4 LUCE_PREFILL_CACHE_SLOTS=2 -- serve)" +has_pair "$out" --prefix-cache-slots 4 || fail "LUCE_PREFIX_CACHE_SLOTS not forwarded" +has_pair "$out" --prefill-cache-slots 2 || fail "LUCE_PREFILL_CACHE_SLOTS not forwarded" + +# ── no arguments serves, with auto device selection by default ───────────── +out="$(run_entrypoint FAKE_PROBE="$DUAL_AMD_DS4" --)" +grep -Fxq "ENV_LUCE_TARGET_DEVICE=auto" <<<"$out" || fail "LUCE_TARGET_DEVICE=auto not exported" +has_pair "$out" --max-ctx 131072 || fail "context not sized from the auto device (96 GB)" +out="$(run_entrypoint FAKE_PROBE="$DUAL_AMD_DS4" LUCE_TARGET_DEVICE=hip:0 --)" +grep -Fxq "ENV_LUCE_TARGET_DEVICE=hip:0" <<<"$out" || fail "explicit LUCE_TARGET_DEVICE replaced" +has_pair "$out" --max-ctx 98304 || fail "context not sized from LUCE_TARGET_DEVICE (31.9 GB)" +out="$(run_entrypoint FAKE_PROBE="$RTX_24G" --)" +has_pair "$out" --max-ctx 98304 || fail "24 GB tier not applied" +out="$(run_entrypoint FAKE_PROBE="" --)" +has_pair "$out" --max-ctx 16384 || fail "fallback context without a probe" + +# ── server flags pass through and win ────────────────────────────────────── +out="$(run_entrypoint FAKE_PROBE="$DUAL_AMD_DS4" -- serve --target-device hip:1 --max-ctx 4096)" +has_pair "$out" --target-device hip:1 || fail "--target-device not forwarded" +[ "$(last_value "$out" --max-ctx)" = 4096 ] || fail "operator --max-ctx does not win" +[ "$(grep -c '^SERVER_ARG=--max-ctx$' <<<"$out")" = 1 ] || + fail "entrypoint forwards its own --max-ctx next to the operator's" +grep -q "max_ctx=4096" "$TMP_DIR/stderr" || fail "startup log does not show the operator's --max-ctx" +out="$(run_entrypoint FAKE_PROBE="$DUAL_AMD_DS4" -- --target-devices hip:0,hip:1)" +has_pair "$out" --target-devices hip:0,hip:1 || fail "--target-devices not forwarded" +grep -q "device=hip:0,hip:1" "$TMP_DIR/stderr" || fail "startup log does not show the --target-devices placement" +grep -q "target hip:1 " "$TMP_DIR/stderr" && fail "startup log claims the auto device with --target-devices" +out="$(run_entrypoint FAKE_PROBE="$DUAL_AMD_QWEN" LUCE_DRAFT="$TMP_DIR/drafts" LUCE_BUDGET=7 LUCE_PORT=9999 --)" +has_pair "$out" --ddtree-budget 7 || fail "LUCE_BUDGET passed to run_entrypoint is ignored" +out="$(run_entrypoint FAKE_PROBE="$DUAL_AMD_DS4" -- --chunk 2048)" +has_pair "$out" --chunk 2048 || fail "bare flags do not start the server" +out="$(run_entrypoint FAKE_PROBE="$DUAL_AMD_DS4" LUCE_ARGS="--chunk 1024 --peer-access" --)" +has_pair "$out" --chunk 1024 || fail "LUCE_ARGS not split into flags" +has_flag "$out" --peer-access || fail "LUCE_ARGS boolean flag lost" + +# ── profiles own their context size ──────────────────────────────────────── +out="$(run_entrypoint FAKE_PROBE="$DUAL_AMD_DS4" -- --profile ds4-strix)" +has_pair "$out" --profile ds4-strix || fail "--profile not forwarded" +has_flag "$out" --max-ctx && fail "entrypoint --max-ctx overrides the profile" +grep -q "device=hip:1" "$TMP_DIR/stderr" && fail "entrypoint reports the auto device over the profile's" +out="$(run_entrypoint FAKE_PROBE="$DUAL_AMD_DS4" LUCE_PROFILE=ds4-strix LUCE_MAX_CTX=65536 --)" +has_pair "$out" --profile ds4-strix || fail "LUCE_PROFILE not forwarded" +has_pair "$out" --max-ctx 65536 || fail "explicit LUCE_MAX_CTX dropped with a profile" + +out="$(run_entrypoint FAKE_PROBE="$DUAL_AMD_DS4" LUCE_PROFILE=ds4-strix -- --profile ds4-r9700-strix)" +[ "$(grep -Fxc "SERVER_ARG=--profile" <<<"$out")" = 1 ] || fail "two --profile flags reach luce_server" +has_pair "$out" --profile ds4-r9700-strix || fail "command-line --profile does not replace LUCE_PROFILE" + +# ── drafts follow the target architecture ────────────────────────────────── +out="$(run_entrypoint FAKE_PROBE="$DUAL_AMD_DS4" LUCE_DRAFT="$TMP_DIR/drafts" --)" +[ "$(last_value "$out" --draft)" = "$TMP_DIR/drafts/DeepSeek-V4-Flash-DSpark-draft.gguf" ] || + fail "DeepSeek V4 did not get its DSpark drafter" +has_flag "$out" --ddtree && fail "DDTree requested for a DSpark drafter" +out="$(run_entrypoint FAKE_PROBE="$DUAL_AMD_QWEN" LUCE_DRAFT="$TMP_DIR/drafts" --)" +[ "$(last_value "$out" --draft)" = "$TMP_DIR/drafts/dflash-draft-3.6-q8_0.gguf" ] || + fail "Qwen did not get the DFlash draft" +has_pair "$out" --ddtree-budget 22 || fail "DDTree budget missing for DFlash" +rm "$TMP_DIR/drafts/dflash-draft-3.6-q8_0.gguf" +out="$(run_entrypoint FAKE_PROBE="$DUAL_AMD_QWEN" LUCE_DRAFT="$TMP_DIR/drafts" --)" +has_flag "$out" --draft && fail "DSpark or mmproj file used as a Qwen draft" +out="$(run_entrypoint FAKE_PROBE="$DUAL_AMD_DS4" LUCE_DRAFT=none --)" +has_flag "$out" --draft && fail "LUCE_DRAFT=none still passed a draft" +# DeepSeek V4 paged (concurrent) serving is autoregressive and rejects a drafter. +out="$(run_entrypoint FAKE_PROBE="$DUAL_AMD_DS4" LUCE_DRAFT="$TMP_DIR/drafts" -- --max-concurrency 4)" +has_flag "$out" --draft && fail "DSpark drafter passed to concurrent DeepSeek V4" +out="$(run_entrypoint FAKE_PROBE="$DUAL_AMD_DS4" LUCE_DRAFT="$TMP_DIR/drafts" \ + LUCE_ARGS="--paged-attention" --)" +has_flag "$out" --draft && fail "DSpark drafter passed to paged DeepSeek V4" +out="$(run_entrypoint FAKE_PROBE="$DUAL_AMD_DS4" LUCE_DRAFT="$TMP_DIR/drafts" -- --max-concurrency 1)" +has_flag "$out" --draft || fail "single-lane DeepSeek V4 lost its DSpark drafter" + +# ── retired variables do not reach the server ────────────────────────────── +out="$(run_entrypoint LUCE_THINK_SOFT_CLOSE_MIN_RATIO=0.5 LUCE_DEBUG_THINKING_LOGITS=1 --)" +grep -q "soft-close\|debug-thinking" <<<"$out" && fail "retired flags forwarded" + +# ── devices subcommand ───────────────────────────────────────────────────── +probe_args="$(env LUCE_DIR="$TMP_DIR" LUCE_TARGET="$TARGET" LUCE_SERVER_BIN="$FAKE_SERVER" \ + bash "$ENTRYPOINT" devices 2>&1 >/dev/null)" +grep -Fxq "PROBE_ARG=$TARGET" <<<"$probe_args" || fail "devices did not pass the target" + +echo "entrypoint: PASS" diff --git a/server/test/test_entrypoint_cache_defaults.sh b/server/test/test_entrypoint_cache_defaults.sh deleted file mode 100755 index d6c3b8d52..000000000 --- a/server/test/test_entrypoint_cache_defaults.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -ENTRYPOINT="${1:?usage: test_entrypoint_cache_defaults.sh }" -TMP_DIR="$(mktemp -d)" -trap 'rm -rf "$TMP_DIR"' EXIT - -TARGET="$TMP_DIR/model.gguf" -FAKE_SERVER="$TMP_DIR/luce_server" -touch "$TARGET" - -cat >"$FAKE_SERVER" <<'EOF' -#!/usr/bin/env bash -printf 'SERVER_ARG=%s\n' "$@" -EOF -chmod +x "$FAKE_SERVER" - -run_entrypoint() { - env \ - LUCE_DIR="$TMP_DIR" \ - LUCE_TARGET="$TARGET" \ - LUCE_DRAFT="$TMP_DIR/no-draft" \ - LUCE_SERVER_BIN="$FAKE_SERVER" \ - "$@" \ - bash "$ENTRYPOINT" serve 2>/dev/null -} - -assert_arg_pair() { - local output="$1" - local flag="$2" - local value="$3" - if ! awk -v expected_flag="SERVER_ARG=$flag" \ - -v expected_value="SERVER_ARG=$value" ' - previous == expected_flag && $0 == expected_value { found = 1 } - { previous = $0 } - END { exit(found ? 0 : 1) } - ' <<<"$output"; then - echo "missing server argument: $flag $value" >&2 - exit 1 - fi -} - -default_output="$( - unset LUCE_PREFIX_CACHE_SLOTS LUCE_PREFILL_CACHE_SLOTS - run_entrypoint -)" -for flag in --prefix-cache-slots --prefill-cache-slots; do - if grep -Fq "SERVER_ARG=$flag" <<<"$default_output"; then - echo "entrypoint overrides the native cache default with $flag" >&2 - exit 1 - fi -done - -disabled_output="$(run_entrypoint LUCE_PREFIX_CACHE_SLOTS=0)" -assert_arg_pair "$disabled_output" --prefix-cache-slots 0 - -configured_output="$( - run_entrypoint \ - LUCE_PREFIX_CACHE_SLOTS=4 \ - LUCE_PREFILL_CACHE_SLOTS=2 -)" -assert_arg_pair "$configured_output" --prefix-cache-slots 4 -assert_arg_pair "$configured_output" --prefill-cache-slots 2 - -echo "entrypoint cache defaults: PASS" diff --git a/server/test/test_feature_gate.cpp b/server/test/test_feature_gate.cpp index 994024cdf..18fdafcfe 100644 --- a/server/test/test_feature_gate.cpp +++ b/server/test/test_feature_gate.cpp @@ -649,18 +649,27 @@ void test_feature_warnings_report_inert_draft() { args.model_path = "/nonexistent/model.gguf"; args.draft_path = "/nonexistent/draft.gguf"; - // qwen3 and deepseek4 never forward a draft model. + // qwen3 never forwards a draft model. CHECK(warns_about(warn_result(args, "qwen3"), "--draft")); - CHECK(warns_about(warn_result(args, "deepseek4"), "--draft")); - // laguna and gemma4 forward it only when monolithic. + // laguna, gemma4 and deepseek4 (DSpark) forward it only when monolithic. CHECK(!warns_about(warn_result(args, "laguna"), "--draft")); CHECK(!warns_about(warn_result(args, "gemma4"), "--draft")); + CHECK(!warns_about(warn_result(args, "deepseek4"), "--draft")); BackendArgs split = args; CHECK(parse_placement_device_list("cuda:0,cuda:1", split.device)); const std::vector w = collect_feature_warnings(split, "laguna"); CHECK(warns_about(w, "--draft")); CHECK(w[0].find("single-device placement") != std::string::npos); + const std::vector ds4 = collect_feature_warnings(split, "deepseek4"); + CHECK(warns_about(ds4, "--draft")); + CHECK(ds4[0].find("single-device placement") != std::string::npos); + + // A deepseek4 remote target shard runs the layer-split adapter too. + BackendArgs shard = args; + shard.remote_target_shard.ipc_bin = "/usr/bin/target-shard-ipc"; + CHECK(warns_about(collect_feature_warnings(shard, "deepseek4"), "--draft")); + CHECK(!warns_about(collect_feature_warnings(shard, "laguna"), "--draft")); } void test_feature_warnings_report_inert_decode_tunables() { diff --git a/server/test/test_launch_policy.cpp b/server/test/test_launch_policy.cpp new file mode 100644 index 000000000..fd5e99ff5 --- /dev/null +++ b/server/test/test_launch_policy.cpp @@ -0,0 +1,201 @@ +// Unit tests for launch policy: `--target-device auto` selection and +// `--profile` expansion. Both are pure functions over resolved facts, so +// they need no model file or GPU. + +#include "CppUnitTestFramework.hpp" +#include "placement/device_select.h" +#include "server/launch_profiles.h" + +#include +#include +#include +#include + +using namespace CppUnitTestFramework; +using namespace luce::common; +using namespace luce::server; + +namespace { + +constexpr uint64_t GiB = 1ull << 30; + +GpuDeviceInfo device(int index, uint64_t total_gib, bool integrated) { + GpuDeviceInfo info; + info.index = index; + info.total_bytes = total_gib * GiB; + info.integrated = integrated; + return info; +} + +bool contains_flag(const std::vector & args, const char * flag) { + for (const std::string & arg : args) { + if (arg == flag) return true; + } + return false; +} + +struct LaunchPolicyFixture : CommonFixture { + using CommonFixture::CommonFixture; + + void test_auto_device_prefers_discrete_gpu_that_fits() { + // R9700 (32 GiB, device 0) + Strix Halo (96 GiB, integrated, device 1). + const std::vector devices = { + device(0, 32, false), device(1, 96, true)}; + const AutoDeviceChoice qwen = choose_auto_target_device(devices, 15 * GiB); + CHECK(qwen.index == 0); + CHECK(qwen.fits); + + // The same pair enumerated the other way round still picks the dGPU. + const std::vector reversed = { + device(0, 96, true), device(1, 32, false)}; + CHECK(choose_auto_target_device(reversed, 15 * GiB).index == 1); + } + + void test_auto_device_moves_large_model_to_device_that_fits() { + const std::vector devices = { + device(0, 32, false), device(1, 120, true)}; + const AutoDeviceChoice ds4 = choose_auto_target_device(devices, 88 * GiB); + CHECK(ds4.index == 1); + CHECK(ds4.fits); + } + + void test_auto_device_falls_back_to_largest() { + // Nothing holds 100 GiB of weights; the largest device is still the + // right one to try. + const std::vector devices = { + device(0, 32, false), device(1, 96, true)}; + const AutoDeviceChoice big = choose_auto_target_device(devices, 100 * GiB); + CHECK(big.index == 1); + CHECK(!big.fits); + + // Issue #712: 91.5 GiB of DeepSeek V4 weights on R9700 + Strix Halo. + const AutoDeviceChoice ds4 = choose_auto_target_device( + devices, 91 * GiB + GiB / 2); + CHECK(ds4.index == 1); + CHECK(ds4.fits); + + CHECK(choose_auto_target_device({}, GiB).index == -1); + } + + void test_auto_device_keeps_first_of_equal_discrete_gpus() { + const std::vector devices = { + device(0, 24, false), device(1, 24, false)}; + CHECK(choose_auto_target_device(devices, 10 * GiB).index == 0); + } + + void test_required_bytes_margin() { + CHECK(auto_device_required_bytes(GiB) == 3 * GiB); + CHECK(auto_device_required_bytes(30 * GiB) == 33 * GiB); + CHECK(auto_device_required_bytes(100 * GiB) == 104 * GiB); + } + + void test_auto_device_counts_the_kv_cache() { + // 26 GiB of weights fit a 32 GiB card with the flat margin alone, but + // not with the KV cache of a long context on top. + const std::vector devices = { + device(0, 32, false), device(1, 96, true)}; + CHECK(auto_device_required_bytes(26 * GiB, 5 * GiB) == + 26 * GiB + 5 * GiB + 26 * GiB / 10); + CHECK(choose_auto_target_device(devices, 26 * GiB).index == 0); + const AutoDeviceChoice long_ctx = choose_auto_target_device(devices, 26 * GiB, 5 * GiB); + CHECK(long_ctx.index == 1); + CHECK(long_ctx.fits); + } + + void test_draft_placement_precedence() { + DevicePlacement target; + target.backend = compiled_placement_backend(); + target.gpu = 1; + DevicePlacement unplaced; // auto:0, the parser default + + // Explicit auto:N names GPU N of the compiled backend. + DevicePlacement explicit_auto; + explicit_auto.gpu = 1; + const DevicePlacement a = resolve_draft_placement(explicit_auto, true, target, false); + CHECK(a.backend == compiled_placement_backend()); + CHECK(a.gpu == 1); + + // An explicit placement is kept even with an auto-placed target. + DevicePlacement explicit_hip; + explicit_hip.backend = compiled_placement_backend(); + explicit_hip.gpu = 0; + const DevicePlacement b = resolve_draft_placement(explicit_hip, true, target, true); + CHECK(b.gpu == 0); + + // Unplaced drafters stay on the auto backend so backend defaults + // (LUCE_DS4_DRAFT_GPU/_BACKEND) still apply; with an auto target the + // index follows the target. + const DevicePlacement c = resolve_draft_placement(unplaced, false, target, true); + CHECK(c.backend == PlacementBackend::Auto); + CHECK(c.gpu == 1); + const DevicePlacement d = resolve_draft_placement(unplaced, false, target, false); + CHECK(d.backend == PlacementBackend::Auto); + CHECK(d.gpu == 0); + } + + void test_profiles_are_well_formed() { + std::set names; + for (const LaunchProfile & profile : launch_profiles()) { + CHECK(names.insert(profile.name).second); + CHECK(find_launch_profile(profile.name) == &profile); + std::set env; + for (const LaunchProfileEnv & entry : profile.env) { + CHECK(env.insert(entry.name).second); + } + for (const LaunchProfileFlag & flag : profile.flags) { + CHECK(std::strncmp(flag.flag, "--", 2) == 0); + } + } + CHECK(find_launch_profile("missing") == nullptr); + } + + void test_profile_flags_yield_to_explicit_flags() { + const LaunchProfile * profile = find_launch_profile("ds4-r9700-strix"); + CHECK(profile != nullptr); + + const std::vector all = launch_profile_args(*profile, {}); + CHECK(contains_flag(all, "--expert-device")); + CHECK(contains_flag(all, "--target-device")); + + const std::vector explicit_args = { + "--max-ctx", "65536", "--target-devices", "hip:0,hip:1"}; + const std::vector merged = + launch_profile_args(*profile, explicit_args); + CHECK(!contains_flag(merged, "--max-ctx")); + // --target-devices and --target-device select the same setting. + CHECK(!contains_flag(merged, "--target-device")); + CHECK(contains_flag(merged, "--chunk")); + CHECK(!contains_flag(merged, "65536")); + } + + void test_profile_replaces_documented_recipe() { + // The qualified R9700 + Strix recipe in docs/DS4.md, minus the variables + // that --expert-device and --draft now express as flags. + const LaunchProfile * profile = find_launch_profile("ds4-r9700-strix"); + CHECK(profile != nullptr); + for (const LaunchProfileEnv & entry : profile->env) { + const std::string name = entry.name; + CHECK(name != "LUCE_DS4_MOE_TP"); + CHECK(name != "LUCE_DS4_MOE_TP_INPROC"); + CHECK(name != "LUCE_DS4_MOE_TP_GPU"); + CHECK(name != "LUCE_DS4_SPEC"); + CHECK(name != "LUCE_DS4_DRAFT"); + } + CHECK(profile->env.size() == 37); + } +}; + +} // namespace + +TEST_CASE(LaunchPolicyFixture, launch_policy_suite) { + test_auto_device_prefers_discrete_gpu_that_fits(); + test_auto_device_moves_large_model_to_device_that_fits(); + test_auto_device_falls_back_to_largest(); + test_auto_device_keeps_first_of_equal_discrete_gpus(); + test_required_bytes_margin(); + test_auto_device_counts_the_kv_cache(); + test_draft_placement_precedence(); + test_profiles_are_well_formed(); + test_profile_flags_yield_to_explicit_flags(); + test_profile_replaces_documented_recipe(); +} diff --git a/server/test/test_server_unit.cpp b/server/test/test_server_unit.cpp index 370ba8868..e6831a2e5 100644 --- a/server/test/test_server_unit.cpp +++ b/server/test/test_server_unit.cpp @@ -38,6 +38,7 @@ #include "common/layer_split_kvflash.h" #include "common/layer_split_utils.h" #include "common/kvflash_pager.h" +#include "placement/device_select.h" #include "placement/draft_residency.h" #include "common/gguf_bounds.h" #include "common/gguf_inspect.h" @@ -188,6 +189,41 @@ TEST_CASE(ServerUnitFixture, test_placement_device_rejects_gpu_index_overflow) { TEST_ASSERT(!parse_placement_device_list("hip:0,hip:4294967296", device)); } +// --target-device auto sizes the KV cache from the GGUF header for the +// Qwen3.5/3.6 hybrids: only full-attention layers, excluding NextN blocks. +TEST_CASE(ServerUnitFixture, test_gguf_kv_cache_bytes_counts_full_attention_layers) { + auto write = [](const char * arch, const char * name) { + gguf_context * g = gguf_init_empty(); + const std::string pre = std::string(arch) + "."; + gguf_set_val_str(g, "general.architecture", arch); + gguf_set_val_u32(g, (pre + "block_count").c_str(), 9); + gguf_set_val_u32(g, (pre + "nextn_predict_layers").c_str(), 1); + gguf_set_val_u32(g, (pre + "full_attention_interval").c_str(), 4); + gguf_set_val_u32(g, (pre + "attention.head_count_kv").c_str(), 2); + gguf_set_val_u32(g, (pre + "attention.key_length").c_str(), 128); + gguf_set_val_u32(g, (pre + "attention.value_length").c_str(), 128); + const std::string path = test_tmp_path(name).string(); + gguf_write_to_file(g, path.c_str(), /*only_meta=*/true); + gguf_free(g); + return path; + }; + const std::string qwen = write("qwen35", "luce_test_kv_qwen35.gguf"); + // 8 target layers / interval 4 = 2 full-attention layers, 2 KV heads. + const uint64_t q4_row = ggml_row_size(GGML_TYPE_Q4_0, 128); + TEST_ASSERT(gguf_kv_cache_bytes(qwen, 1000, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0) == + 2ull * 2 * (q4_row + q4_row) * 1000); + const uint64_t f16_row = ggml_row_size(GGML_TYPE_F16, 128); + TEST_ASSERT(gguf_kv_cache_bytes(qwen, 1000, GGML_TYPE_F16, GGML_TYPE_F16) == + 2ull * 2 * (f16_row + f16_row) * 1000); + TEST_ASSERT(gguf_kv_cache_bytes(qwen, 0, GGML_TYPE_F16, GGML_TYPE_F16) == 0); + + // Families that size their own caches are not estimated. + TEST_ASSERT(gguf_kv_cache_bytes(write("deepseek4", "luce_test_kv_ds4.gguf"), 1000, + GGML_TYPE_F16, GGML_TYPE_F16) == 0); + TEST_ASSERT(gguf_kv_cache_bytes(test_tmp_path("luce_test_kv_missing.gguf").string(), + 1000) == 0); +} + TEST_CASE(ServerUnitFixture, test_api_format_names_are_total) { CHECK(std::string(api_format_name(ApiFormat::OPENAI_CHAT)) == "chat"); CHECK(std::string(api_format_name(ApiFormat::ANTHROPIC)) == "anthropic"); diff --git a/variables.md b/variables.md index 4319281ea..cb76e7e5e 100644 --- a/variables.md +++ b/variables.md @@ -1,6 +1,6 @@ # Environment Variables Reference -Summary of `LUCE_*` / `LUCE_*` environment variables recognized across the +Summary of `LUCE_*` environment variables recognized across the codebase, grouped by subsystem. Most are runtime toggles read via `getenv` / `os.environ`; a few are build/compile-time or harness knobs (noted where relevant). @@ -29,21 +29,24 @@ Untagged variables are operational tuning knobs. |---|---| | `LUCE_HOST` | Server bind host. | | `LUCE_PORT` | Server bind port. | -| `LUCE_BIN` / `LUCE_SERVER_BIN` | Path to the server binary (harness/scripts). | +| `LUCE_BIN` | 🧪 **test/bench** Path to `test_dflash` for the bench scripts. | +| `LUCE_SERVER_BIN` | Path to `luce_server` (container entrypoint, harness). | | `LUCE_BIN_AR` | Alternate/AR binary path for benchmarks. | | `LUCE_DIR` | Base working directory. | | `LUCE_SHARE_DIR` | Static/share asset directory served by the HTTP server. | | `LUCE_MODEL_CARDS_DIR` | Directory of model-card definitions. | | `LUCE_MODEL_NAME` | Model name/identifier. | | `LUCE_TOKENIZER` | Tokenizer path/identifier. | -| `LUCE_TARGET` | Target model path/spec. | -| `LUCE_DRAFT` | Draft model path/spec. | +| `LUCE_TARGET` | Target model path (container entrypoint, `run.py`). | +| `LUCE_DRAFT` | Draft file or directory (container entrypoint, `run.py`); `none` disables the container draft. | +| `LUCE_TARGET_DEVICE` | Default `--target-device` (`backend:gpu` or `auto`) when no flag or profile names one; the container sets `auto`. | +| `LUCE_PROFILE` / `LUCE_ARGS` | Container entrypoint: `--profile` name and extra `luce_server` flags. | | `LUCE_IMAGE_INFO_PATH` | Path to image/build info metadata. | -| `LUCE_MAX_CONTEXT` / `LUCE_MAX_CTX` | Maximum context length. | +| `LUCE_MAX_CTX` | Container entrypoint `--max-ctx` (default: sized from the GPU's memory). | +| `LUCE_MAX_CONTEXT` | KV sizing override for laguna and qwen35moe expert placement. | | `LUCE_DEFAULT_MAX_TOKENS` | Default generation token cap. | | `LUCE_IGNORE_EOS` | Ignore EOS token during generation. | -| `LUCE_LAZY` | Lazy model/weight loading. | -| `LUCE_VERBOSE` | 🐛 **debug** Verbose logging. | +| `LUCE_LAZY` | Container entrypoint: `1` adds `--lazy-draft` (needs a draft and `LUCE_PREFILL_DRAFTER`). | ## GPU / backend placement @@ -291,12 +294,10 @@ Untagged variables are operational tuning knobs. | Variable | Purpose | |---|---| | `LUCE_THINK_MAX` | Max thinking tokens. | -| `LUCE_THINK_SOFT_CLOSE_MIN_RATIO` | Soft-close ratio for thinking blocks. | -| `LUCE_DEBUG_THINKING_LOGITS` | 🐛 **debug** Debug thinking logits. | | `LUCE_DEGENERATE_RUN_TOKENS` | Degenerate-run token threshold. | | `LUCE_STALL_TOOL_PREFIX` | Tool-call stall prefix handling. | | `LUCE_MIN_TOKENS` | Minimum generated tokens. | -| `LUCE_BUDGET` | Token/compute budget. | +| `LUCE_BUDGET` | Container entrypoint `--ddtree-budget` (default 22). | | `LUCE_ANTHROPIC_RAW_SYSTEM` / `LUCE_ANTHROPIC_RAW_USER` | Pass raw system/user content on the Anthropic-compatible path. | ## Profiling / debug instrumentation @@ -326,7 +327,7 @@ Untagged variables are operational tuning knobs. Runtime C/C++ variables can be re-listed with: ```sh -grep -rE 'getenv\("DFLASH[A-Z0-9_]*"\)' server/src +grep -rE 'getenv\("LUCE_[A-Z0-9_]*"\)' server/src ``` See `server/docs/ENVIRONMENT.md` for the canonical generated inventory and the