From 457cf8a974f1042119adf8355035ea3b7c207b4e Mon Sep 17 00:00:00 2001 From: mrciffa Date: Wed, 23 Sep 2026 16:23:23 +0200 Subject: [PATCH 1/2] feat(deepseek4): run the vision encoder on a second GPU and stream images into prefill --mmproj-device hip:N loads the DS4V encoder on another GPU in the one-GPU layout (the R9700 next to a Strix Halo holding the model). Its scratch is charged to that GPU. The encoder then runs image by image on a background thread and publishes each image's rows; prefill waits per chunk only for the images that chunk contains, so the Strix Halo prefills image k while the R9700 encodes image k+1. Failure or cancellation releases waiters and the thread is joined before the request, shutdown or park returns. Requests may carry up to 16 images (one shared constant for the HTTP transport, DS4V and Qwen3.5), and each request logs its encode time. lucebox6, Strix Halo decoder, published DSpark launch, ChartQA charts, time to first token, Strix encoder -> R9700 encoder streamed: 1 image 2.97 -> 2.94 s, 4 images 11.2 -> 9.1 s, 8 images 27.7 -> 19.2 s, 16 images (4,358 tokens) 51.8 -> 34.6 s. Answers identical to the sequential encode. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/image-input.md | 21 +- server/src/common/backend_args.h | 3 + server/src/common/backend_factory.cpp | 1 + server/src/common/backend_factory.h | 1 + server/src/common/backend_plan.cpp | 1 + server/src/common/feature_gate.cpp | 8 + server/src/common/image_prompt.h | 5 + server/src/deepseek4/deepseek4_backend.cpp | 183 ++++++++++++++---- server/src/deepseek4/deepseek4_backend.h | 9 + .../deepseek4/deepseek4_image_assembly.cpp | 4 +- .../src/deepseek4/deepseek4_image_prompt.cpp | 4 +- server/src/deepseek4/deepseek4_image_spans.h | 3 +- server/src/deepseek4/deepseek4_internal.h | 1 + server/src/qwen35/qwen35_backend_images.cpp | 3 +- server/src/server/image_input.h | 2 +- server/src/server/server_main.cpp | 8 + server/test/test_ds4v_image_integration.cpp | 4 +- server/test/test_ds4v_image_prompt.cpp | 3 +- 18 files changed, 219 insertions(+), 45 deletions(-) diff --git a/docs/image-input.md b/docs/image-input.md index d4a264b9d..6bc77336b 100644 --- a/docs/image-input.md +++ b/docs/image-input.md @@ -66,6 +66,25 @@ launch plus `--mmproj`: the Vision file replaces `DeepSeek-V4-Flash-0731-ROCMFPX-MIX-STRIX.gguf` for text as well and decodes at least as fast (numbers below). For R9700 + Strix Halo see [DS4V](#ds4v) below. +With an R9700 in the same box, run the image encoder there while the model +stays on the Strix Halo: expose both GPUs, point `--target-device` at the Strix +Halo and add `--mmproj-device` with the R9700 (on lucebox6, without +`HIP_VISIBLE_DEVICES`, that is `--target-device hip:1 --mmproj-device hip:0`). +The encoder then runs about twice as fast and streams each image into prefill +as soon as it is encoded, so the Strix Halo never waits for the next one: + +| Images | Prompt tokens | Encoder on the Strix Halo | Encoder on the R9700 | +| --- | --- | --- | --- | +| 1 | 126 | 2.97 s | 2.94 s | +| 4 | 942 | 11.2 s | 9.1 s | +| 8 | 2,262 | 27.7 s | 19.2 s | +| 16 | 4,358 | 51.8 s | 34.6 s | + +Time to the first token, with the published launch above and ChartQA charts. +Answers are identical in both layouts. `--mmproj-device` applies to this one-GPU +layout; with the experts split across both GPUs the encoder already runs on the +R9700. + ### Send an image ```bash @@ -101,7 +120,7 @@ Use `POST /v1/chat/completions` with user-message content parts in display order Only base64 JPEG/PNG data URLs are supported. Remote URLs, images outside user content arrays, and image parts through other API formats are rejected. A -request carries at most four images, 16 MiB encoded each and 32 MiB combined. +request carries at most 16 images, 16 MiB encoded each and 32 MiB combined. Decoder pixel and aspect limits also apply. A model's image marker cannot be supplied as ordinary text. diff --git a/server/src/common/backend_args.h b/server/src/common/backend_args.h index 8c2a295ff..0c0907530 100644 --- a/server/src/common/backend_args.h +++ b/server/src/common/backend_args.h @@ -56,6 +56,9 @@ struct BackendArgs { // Optional: vision projector .gguf (deepseek4 only) std::optional mmproj_path; + // Optional: GPU for the vision encoder when it should not share the + // target's (deepseek4, one-GPU layout only). + std::optional mmproj_device; // Device placement DevicePlacement device; diff --git a/server/src/common/backend_factory.cpp b/server/src/common/backend_factory.cpp index 6e3dea2b5..1202dca6c 100644 --- a/server/src/common/backend_factory.cpp +++ b/server/src/common/backend_factory.cpp @@ -370,6 +370,7 @@ std::unique_ptr construct_backend( DeepSeek4BackendConfig cfg; cfg.model_path = model.path; cfg.mmproj_path = model.mmproj_path.value_or(""); + cfg.mmproj_gpu = model.mmproj_device ? model.mmproj_device->gpu : -1; cfg.device = placement.target; cfg.stream_fd = execution.stream_fd; cfg.max_ctx = placement.target.max_ctx; diff --git a/server/src/common/backend_factory.h b/server/src/common/backend_factory.h index 11a53b809..ab884ac80 100644 --- a/server/src/common/backend_factory.h +++ b/server/src/common/backend_factory.h @@ -37,6 +37,7 @@ class BackendPlan final { struct Model { std::string path; std::optional mmproj_path; + std::optional mmproj_device; GgufModelInfo metadata; }; diff --git a/server/src/common/backend_plan.cpp b/server/src/common/backend_plan.cpp index adf3c7c9f..afa390691 100644 --- a/server/src/common/backend_plan.cpp +++ b/server/src/common/backend_plan.cpp @@ -131,6 +131,7 @@ BackendPreparation BackendPlanBuilder::resolve( BackendPlan plan; plan.model_.path = std::move(args.model_path); plan.model_.mmproj_path = std::move(args.mmproj_path); + plan.model_.mmproj_device = std::move(args.mmproj_device); plan.model_.metadata = std::move(model); plan.placement_.target = std::move(args.device); diff --git a/server/src/common/feature_gate.cpp b/server/src/common/feature_gate.cpp index fadbb6d40..7e371856e 100644 --- a/server/src/common/feature_gate.cpp +++ b/server/src/common/feature_gate.cpp @@ -59,6 +59,14 @@ std::string check_feature_compatibility( return "--mmproj with DeepSeek4 requires a HIP backend"; } } + if (args.mmproj_device.has_value()) { + if (!args.mmproj_path.has_value() || arch != "deepseek4" || + args.mmproj_device->backend != PlacementBackend::Hip || + args.mmproj_device->gpu == args.device.gpu) { + return "--mmproj-device needs --mmproj, a DeepSeek4 target, and a HIP GPU " + "other than the target's"; + } + } // ── PFlash enablement × drafter model if (admission.pflash_enabled && diff --git a/server/src/common/image_prompt.h b/server/src/common/image_prompt.h index 2692b6f70..44e46af11 100644 --- a/server/src/common/image_prompt.h +++ b/server/src/common/image_prompt.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -7,6 +8,10 @@ namespace luce::common { +// Most images one request may carry, for every vision backend and the HTTP +// transport. Each image still has its own token and byte bounds. +inline constexpr size_t MAX_REQUEST_IMAGES = 16; + struct EncodedImage { std::string mime_type; std::vector bytes; diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index d45d9b762..587ab2e01 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -34,6 +34,8 @@ #include #include #include +#include +#include #include #include #include @@ -61,6 +63,7 @@ class DeepSeek4ImagePrompt final : public ImagePromptPayload { bool embed_chunk(const CpuEmbedder & embedder, size_t position, int count, float * output) const { if (!output || count <= 0 || embedder.n_embd <= 0) return false; + if (!wait_for_images(position, size_t(count))) return false; std::vector result; std::string error; const bool ok = vision::embed_image_prompt_chunk( @@ -74,12 +77,43 @@ class DeepSeek4ImagePrompt final : public ImagePromptPayload { return ok; } + // Streaming encode: images arrive in prompt order. A chunk waits only for + // the images it contains; failure or cancellation releases every waiter. + bool wait_for_images(size_t position, size_t count) const { + size_t needed = 0; + for (size_t i = 0; i < spans_.size(); ++i) { + if (spans_[i].block_begin < position + count && spans_[i].block_end > position) needed = i + 1; + } + std::unique_lock lock(stream_mutex_); + stream_ready_.wait(lock, [&] { return stream_failed_ || ready_ >= needed; }); + return ready_ >= needed; + } + void publish_image(size_t index, std::vector rows) const { + std::lock_guard lock(stream_mutex_); + materialized_[index] = std::move(rows); + ready_ = index + 1; + stream_ready_.notify_all(); + } + void fail_stream() const { + std::lock_guard lock(stream_mutex_); + stream_failed_ = true; + stream_ready_.notify_all(); + } + bool complete() const { + std::lock_guard lock(stream_mutex_); + return !materialized_.empty() && ready_ == materialized_.size(); + } + const DeepSeek4Backend * const owner_; const vision::PreparedImagePrompt prepared_; const std::vector encoded_; const std::shared_ptr lease_; std::vector spans_; mutable std::vector> materialized_; + mutable std::mutex stream_mutex_; + mutable std::condition_variable stream_ready_; + mutable size_t ready_ = 0; + mutable bool stream_failed_ = false; }; namespace { @@ -1065,7 +1099,7 @@ bool DeepSeek4Backend::prepare_images( return false; } try { - if (images.size() > 4) { + if (images.size() > MAX_REQUEST_IMAGES) { error = "too many images in request"; return false; } @@ -1115,6 +1149,10 @@ bool DeepSeek4Backend::prepare_images( } } +void DeepSeek4Backend::join_image_stream() { + if (image_stream_.joinable()) image_stream_.join(); +} + bool DeepSeek4Backend::materialize_images(const DeepSeek4ImagePrompt & images, const DaemonIO & io, std::string & error) { if (io.is_cancelled()) return false; @@ -1141,6 +1179,7 @@ bool DeepSeek4Backend::materialize_images(const DeepSeek4ImagePrompt & images, name, released); }; trim_pool(backend_, "primary"); + if (vision_backend_) trim_pool(vision_backend_, "vision"); if (expert_backend_ && expert_backend_ != backend_) { trim_pool(expert_backend_, "expert"); } @@ -1148,7 +1187,7 @@ bool DeepSeek4Backend::materialize_images(const DeepSeek4ImagePrompt & images, trim_pool(spec_backend_, "spec"); } auto reserves = image_reserves_; - const uint64_t resident_workspace = + const uint64_t resident_workspace = !vision_backend_ && (vision::detail::hip_bias_launches(backend_) || vision::detail::hip_av_launches(backend_)) ? vision::detail::hip_bias_workspace(backend_) : 0; if (resident_workspace > vision::SCRATCH_RESERVATION) { @@ -1156,8 +1195,9 @@ bool DeepSeek4Backend::materialize_images(const DeepSeek4ImagePrompt & images, return false; } // Current free memory already reflects weights, KV, optional drafter, - // snapshots and retained backend pools. Charge only upcoming work here. - reserves.primary_future_bytes = vision::SCRATCH_RESERVATION - resident_workspace + + // snapshots and retained backend pools. Charge only upcoming work here; + // an encoder on its own GPU charges nothing to the target. + reserves.primary_future_bytes = (vision_backend_ ? 0 : vision::SCRATCH_RESERVATION - resident_workspace) + 128ULL * 1024 * 1024; if (!moe_hybrid_) { uint64_t free_bytes = 0; @@ -1182,37 +1222,74 @@ bool DeepSeek4Backend::materialize_images(const DeepSeek4ImagePrompt & images, gib(report.host_required_bytes), gib(report.host_available_bytes), error.c_str()); return false; } - if (!images.materialized_.empty()) return true; + if (images.complete()) return true; + const auto encode_one = [this](const vision::PromptImage & image, vision::ImageRaster & raster, + std::string & encode_error) { + std::vector patches(image.input.patches_bf16.size()); + for (size_t i = 0; i < patches.size(); ++i) { + const uint32_t bits = uint32_t(image.input.patches_bf16[i]) << 16; + std::memcpy(&patches[i], &bits, sizeof(bits)); + } + vision::VisionOutput output; + if (!vision_->encode(patches, + {int(image.input.plan.vit_rows), int(image.input.plan.vit_cols)}, + output, encode_error)) return false; + if (output.rows <= 0 || output.columns != w_.n_embd) { + encode_error = "vision output shape differs from decoder dimensions"; + return false; + } + raster = {size_t(output.rows), size_t(output.columns), std::move(output.embeddings)}; + return true; + }; + vision::ImageSentinels sentinels; + if (!vision_->sentinel(vision::Sentinel::Start, sentinels.start, error) || + !vision_->sentinel(vision::Sentinel::Pad, sentinels.pad, error) || + !vision_->sentinel(vision::Sentinel::Newline, sentinels.newline, error) || + !vision_->sentinel(vision::Sentinel::End, sentinels.end, error)) return false; + if (vision_backend_) { + // The encoder has its own GPU: encode image k+1 there while the target + // prefills image k. Prefill waits per chunk in embed_chunk. + join_image_stream(); + { + std::lock_guard lock(images.stream_mutex_); + images.materialized_.assign(images.prepared_.images.size(), {}); + images.ready_ = 0; + images.stream_failed_ = false; + } + image_stream_ = std::thread([this, &images, &io, encode_one, sentinels] { + const auto t0 = Clock::now(); + bool ok = true; + std::string stream_error; + for (size_t i = 0; ok && i < images.prepared_.images.size(); ++i) { + vision::ImageRows one; + ok = vision::materialize_image_rows({images.prepared_.images[i]}, sentinels, + size_t(w_.n_embd), encode_one, [&] { return io.is_cancelled(); }, + one, stream_error) && one.size() == 1; + if (ok) images.publish_image(i, std::move(one.front())); + } + vision_->release_scratch(); + if (!ok) { + std::fprintf(stderr, "[deepseek4] streaming image encode stopped: %s\n", + stream_error.empty() ? "cancelled" : stream_error.c_str()); + images.fail_stream(); + } + std::fprintf(stderr, "[deepseek4] images encoded in %.0f ms on the --mmproj-device GPU (streamed)\n", + elapsed_s(t0) * 1000.0); + }); + return true; + } struct ReleaseScratch { vision::VisionRuntime & runtime; ~ReleaseScratch() { runtime.release_scratch(); } } release{*vision_}; try { - vision::ImageSentinels sentinels; - if (!vision_->sentinel(vision::Sentinel::Start, sentinels.start, error) || - !vision_->sentinel(vision::Sentinel::Pad, sentinels.pad, error) || - !vision_->sentinel(vision::Sentinel::Newline, sentinels.newline, error) || - !vision_->sentinel(vision::Sentinel::End, sentinels.end, error)) return false; - return vision::materialize_image_rows( - images.prepared_.images, sentinels, size_t(w_.n_embd), - [&](const vision::PromptImage & image, vision::ImageRaster & raster, - std::string & encode_error) { - std::vector patches(image.input.patches_bf16.size()); - for (size_t i = 0; i < patches.size(); ++i) { - const uint32_t bits = uint32_t(image.input.patches_bf16[i]) << 16; - std::memcpy(&patches[i], &bits, sizeof(bits)); - } - vision::VisionOutput output; - if (!vision_->encode(patches, - {int(image.input.plan.vit_rows), int(image.input.plan.vit_cols)}, - output, encode_error)) return false; - if (output.rows <= 0 || output.columns != w_.n_embd) { - encode_error = "vision output shape differs from decoder dimensions"; - return false; - } - raster = {size_t(output.rows), size_t(output.columns), std::move(output.embeddings)}; - return true; - }, [&] { return io.is_cancelled(); }, images.materialized_, error); + vision::ImageRows rows; + if (!vision::materialize_image_rows(images.prepared_.images, sentinels, size_t(w_.n_embd), + encode_one, [&] { return io.is_cancelled(); }, rows, error)) return false; + std::lock_guard lock(images.stream_mutex_); + images.materialized_ = std::move(rows); + images.ready_ = images.materialized_.size(); + return true; } catch (const std::bad_alloc &) { error = "image materialization allocation failed"; return false; @@ -1234,7 +1311,7 @@ bool DeepSeek4Backend::init_single_gpu_vision() { reserves.primary_domain = properties.integrated || std::getenv("GGML_CUDA_ENABLE_UNIFIED_MEMORY") ? vision::ImageMemoryDomain::HostShared : vision::ImageMemoryDomain::Dedicated; reserves.primary_future_bytes = estimate_ds4_cache_bytes(w_, cfg_.max_ctx > 0 ? cfg_.max_ctx : 8192) + - vision::SCRATCH_RESERVATION + 256ULL * 1024 * 1024; + (vision_backend_ ? 0 : vision::SCRATCH_RESERVATION) + 256ULL * 1024 * 1024; uint64_t free_bytes = 0; std::string error; const bool admitted = vision::check_deepseek4_image_single_gpu_admission( @@ -1267,15 +1344,32 @@ bool DeepSeek4Backend::load_vision() { return false; } } + if (cfg_.mmproj_gpu >= 0 && cfg_.mmproj_gpu != cfg_.device.gpu && !vision_backend_) { + // The encoder only hands host rows to the decoder, so it can run on a + // faster idle GPU (an R9700 next to a Strix Halo holding the model). + vision_backend_ = ggml_backend_cuda_init(cfg_.mmproj_gpu); + size_t free_bytes = 0, total_bytes = 0; + if (vision_backend_) { + ggml_backend_dev_memory(ggml_backend_get_device(vision_backend_), &free_bytes, &total_bytes); + } + if (!vision_backend_ || !vision::detail::hip_bias_workspace(vision_backend_) || + free_bytes < vision::SCRATCH_RESERVATION + 2ULL * 1024 * 1024 * 1024) { + std::fprintf(stderr, "[deepseek4] --mmproj-device hip:%d is unavailable, lacks the DS4V " + "vision ops, or has under 4 GiB free\n", cfg_.mmproj_gpu); + return false; + } + } + ggml_backend_t vision_owner = vision_backend_ ? vision_backend_ : backend_; auto runtime = std::make_unique(); std::string error; - if (!runtime->load(cfg_.mmproj_path, backend_, w_.n_embd, w_.n_vocab, error)) { + if (!runtime->load(cfg_.mmproj_path, vision_owner, w_.n_embd, w_.n_vocab, error)) { std::fprintf(stderr, "[deepseek4] projector load failed: %s\n", error.c_str()); return false; } std::fprintf(stderr, - "[deepseek4] vision weights=%.3f GiB scratch reservation=%.3f GiB before expert placement\n", - gib(runtime->weight_bytes()), gib(vision::SCRATCH_RESERVATION)); + "[deepseek4] vision weights=%.3f GiB scratch reservation=%.3f GiB on %s\n", + gib(runtime->weight_bytes()), gib(vision::SCRATCH_RESERVATION), + vision_backend_ ? "the --mmproj-device GPU" : "the target GPU, before expert placement"); vision_ = std::move(runtime); return true; } @@ -1338,6 +1432,11 @@ bool DeepSeek4Backend::load_model() { "on one GPU or with in-process expert owners on two distinct GPUs\n"); return false; } + if (cfg_.mmproj_gpu >= 0 && tp.requested) { + std::fprintf(stderr, "[deepseek4] --mmproj-device is for the one-GPU layout; with two GPUs " + "the encoder already runs on the primary\n"); + return false; + } if (!vision::detail::hip_bias_workspace(backend_)) { std::fprintf(stderr, "[deepseek4] --mmproj needs the DS4V vision ops, which this build lacks " "(hipBLASLt was not found when ggml-hip was configured)\n"); @@ -2473,6 +2572,7 @@ bool DeepSeek4Backend::park(ParkTarget target) { } moe_placement_ = {}; moe_decode_placement_ = {}; + join_image_stream(); vision_.reset(); free_deepseek4_weights(w_); parked_ = true; @@ -3198,6 +3298,11 @@ GenerateResult DeepSeek4Backend::generate_from_state( } const auto * images = dynamic_cast(req.images.get()); + // A streaming encode reads the payload and the IO: finish it before either ends. + struct ImageStreamJoin { + DeepSeek4Backend * backend; + ~ImageStreamJoin() { backend->join_image_stream(); } + } image_stream_join{this}; if (req.images) { if (!images || images->owner_ != this || !images->matches(req.prompt) || kv_offset != 0 || req.snap_slot >= 0 || req.snap_pos >= 0 || @@ -3208,7 +3313,13 @@ GenerateResult DeepSeek4Backend::generate_from_state( return result; } std::string error; - if (!materialize_images(*images, out_io, error)) { + const auto encode_t0 = Clock::now(); + const bool encoded = materialize_images(*images, out_io, error); + if (!vision_backend_) { + std::fprintf(stderr, "[deepseek4] images encoded in %.0f ms on the target GPU\n", + elapsed_s(encode_t0) * 1000.0); + } + if (!encoded) { if (out_io.is_cancelled()) { result.succeed(); return result; } result.fail(GenerateErrorCode::PrefillFailed, error.empty() ? "image materialization failed" : error); return result; @@ -3719,6 +3830,7 @@ void DeepSeek4Backend::maybe_save_routing_stats() { } void DeepSeek4Backend::shutdown() { + join_image_stream(); maybe_save_routing_stats(); free_drafter(); for (int i = 0; i < PREFIX_SLOTS; i++) { @@ -3739,6 +3851,7 @@ void DeepSeek4Backend::shutdown() { moe_placement_ = {}; moe_decode_placement_ = {}; vision_.reset(); + if (vision_backend_) { ggml_backend_free(vision_backend_); vision_backend_ = nullptr; } free_deepseek4_weights(w_); if (snap_backend_) { ggml_backend_free(snap_backend_); snap_backend_ = nullptr; } if (backend_) { ggml_backend_free(backend_); backend_ = nullptr; } diff --git a/server/src/deepseek4/deepseek4_backend.h b/server/src/deepseek4/deepseek4_backend.h index 4e45ca153..68dad4204 100644 --- a/server/src/deepseek4/deepseek4_backend.h +++ b/server/src/deepseek4/deepseek4_backend.h @@ -16,6 +16,8 @@ #include "deepseek4_internal.h" #include "deepseek4_dspark.h" #include "deepseek4_vision.h" + +#include #include "deepseek4_image_prompt.h" #include "deepseek4_image_assembly.h" #include "deepseek4_image_admission.h" @@ -131,6 +133,11 @@ class DeepSeek4Backend : public ModelBackend { bool image_capable_ = false; bool cache_has_images_ = false; std::unique_ptr vision_; + // Owned backend for the vision encoder when --mmproj-device names a GPU + // other than the target's; null when the encoder shares backend_. + ggml_backend_t vision_backend_ = nullptr; + // Encodes images on vision_backend_ while prefill consumes them. + std::thread image_stream_; vision::ImageRequestGate image_request_gate_; vision::ImageAdmissionReserves image_reserves_; @@ -195,6 +202,8 @@ class DeepSeek4Backend : public ModelBackend { const DeepSeek4ImagePrompt * images = nullptr); bool load_vision(); bool init_single_gpu_vision(); + // Waits for a streaming image encode started by materialize_images. + void join_image_stream(); bool materialize_images(const DeepSeek4ImagePrompt & images, const DaemonIO & io, std::string & error); diff --git a/server/src/deepseek4/deepseek4_image_assembly.cpp b/server/src/deepseek4/deepseek4_image_assembly.cpp index 7a25ad456..204107476 100644 --- a/server/src/deepseek4/deepseek4_image_assembly.cpp +++ b/server/src/deepseek4/deepseek4_image_assembly.cpp @@ -104,7 +104,7 @@ bool materialize_image_rows(const std::vector & images, ImageRows & output, std::string & error) { error.clear(); try { - require(!images.empty() && images.size() <= 4 && bool(encode), "invalid image encode request"); + require(!images.empty() && images.size() <= DS4V_MAX_IMAGES && bool(encode), "invalid image encode request"); cancelled(is_cancelled); sentinels_valid(sentinels, dimension); uint64_t previous_end = 0; @@ -145,7 +145,7 @@ bool embed_image_prompt_chunk(const PreparedImagePrompt & prompt, const ImageRow try { require(bool(prompt) && vocabulary > 0 && count && position <= prompt.tokens.size() && count <= prompt.tokens.size() - position && rows.size() == prompt.images.size() && - prompt.images.size() <= 4, "invalid mixed embedding request"); + prompt.images.size() <= DS4V_MAX_IMAGES, "invalid mixed embedding request"); const size_t end = position + count; uint64_t previous_end = 0; for (size_t i = 0; i < prompt.images.size(); ++i) { diff --git a/server/src/deepseek4/deepseek4_image_prompt.cpp b/server/src/deepseek4/deepseek4_image_prompt.cpp index 70e02d661..d3d7dac0f 100644 --- a/server/src/deepseek4/deepseek4_image_prompt.cpp +++ b/server/src/deepseek4/deepseek4_image_prompt.cpp @@ -1,4 +1,5 @@ #include "deepseek4_image_prompt.h" +#include "../common/image_prompt.h" #include #include #include @@ -21,7 +22,8 @@ PreparedImagePrompt prepare_image_prompt(const std::vector & token limits.output_reserve>limits.context_capacity || limits.max_expanded_tokens==0 || limits.max_expanded_tokens>MAX_PREPARED_PROMPT_TOKENS) return fail(ImagePromptError::InvalidLimits,"invalid context, output reserve, or prompt bound"); - if (images.size()>4) return fail(ImagePromptError::ImageCount,"at most four images are supported"); + if (images.size()>common::MAX_REQUEST_IMAGES) + return fail(ImagePromptError::ImageCount,"too many images in request"); if (tokens.size()>limits.max_expanded_tokens) return fail(ImagePromptError::TokenLimit,"rendered tokens exceed prompt bound"); size_t markers=0; diff --git a/server/src/deepseek4/deepseek4_image_spans.h b/server/src/deepseek4/deepseek4_image_spans.h index 3b4d1ed6e..5dde1b6b6 100644 --- a/server/src/deepseek4/deepseek4_image_spans.h +++ b/server/src/deepseek4/deepseek4_image_spans.h @@ -1,11 +1,12 @@ // DS4V limits for the shared image span helpers. #pragma once +#include "../common/image_prompt.h" #include "../common/vision/image_spans.h" namespace luce::vision { -inline constexpr size_t DS4V_MAX_IMAGES = 4; +inline constexpr size_t DS4V_MAX_IMAGES = common::MAX_REQUEST_IMAGES; inline constexpr uint64_t DS4V_MAX_IMAGE_BLOCK_TOKENS = 384; inline bool valid_image_spans(ImageSpanView spans, uint64_t prompt_size) { diff --git a/server/src/deepseek4/deepseek4_internal.h b/server/src/deepseek4/deepseek4_internal.h index 5872ff703..48c20aba5 100644 --- a/server/src/deepseek4/deepseek4_internal.h +++ b/server/src/deepseek4/deepseek4_internal.h @@ -369,6 +369,7 @@ struct DeepSeek4Head4Tail2Routes { struct DeepSeek4BackendConfig { std::string model_path; std::string mmproj_path; + int mmproj_gpu = -1; // vision encoder GPU; -1 = the target's DevicePlacement device; int stream_fd = -1; int chunk = 512; // prefill chunk size diff --git a/server/src/qwen35/qwen35_backend_images.cpp b/server/src/qwen35/qwen35_backend_images.cpp index 9020ebaee..abe0616b2 100644 --- a/server/src/qwen35/qwen35_backend_images.cpp +++ b/server/src/qwen35/qwen35_backend_images.cpp @@ -3,6 +3,7 @@ // in qwen35_backend.cpp. #include "qwen35_backend.h" +#include "common/image_prompt.h" #include "common/vision/image_decode.h" #include "qwen35_image_request.h" @@ -14,7 +15,7 @@ namespace luce::common { namespace { -constexpr size_t MAX_IMAGES_PER_REQUEST = 4; // the server's transport limit +constexpr size_t MAX_IMAGES_PER_REQUEST = common::MAX_REQUEST_IMAGES; } bool Qwen35Backend::load_vision() { diff --git a/server/src/server/image_input.h b/server/src/server/image_input.h index 16c770de6..be1531f55 100644 --- a/server/src/server/image_input.h +++ b/server/src/server/image_input.h @@ -17,7 +17,7 @@ inline constexpr size_t MAX_IMAGE_BYTES = 16 * 1024 * 1024; struct ImageInputLimits { size_t image_bytes = MAX_IMAGE_BYTES; size_t request_bytes = 32 * 1024 * 1024; - size_t image_count = 4; + size_t image_count = MAX_REQUEST_IMAGES; }; struct ImageRequestPolicy { diff --git a/server/src/server/server_main.cpp b/server/src/server/server_main.cpp index a70e05081..10818a54c 100644 --- a/server/src/server/server_main.cpp +++ b/server/src/server/server_main.cpp @@ -86,6 +86,7 @@ static void print_usage(const char * prog) { " do not change generation routing.\n" " --draft Draft model for speculative decode\n" " --mmproj Vision projector GGUF: enables image input (Qwen3.5/3.8, DS4V)\n" + " --mmproj-device hip:N Run the DS4V image encoder on another GPU (one-GPU layout)\n" " --port Listen port (default: 8080)\n" " --host Bind address (default: 0.0.0.0)\n" " --max-ctx Max context length (default: 131072)\n" @@ -365,6 +366,13 @@ static int parse_model_options(int argc, char ** argv, ModelOptions & model, return 2; } bargs.mmproj_path = argv[++i]; + } else if (std::strcmp(argv[i], "--mmproj-device") == 0 && i + 1 < argc) { + DevicePlacement vision_device; + if (!parse_placement_device(argv[++i], vision_device)) { + std::fprintf(stderr, "[server] bad --mmproj-device value (expected hip:gpu)\n"); + return 2; + } + bargs.mmproj_device = vision_device; } else if (std::strcmp(argv[i], "--port") == 0 && i + 1 < argc) { sconfig.port = std::atoi(argv[++i]); } else if (std::strcmp(argv[i], "--host") == 0 && i + 1 < argc) { diff --git a/server/test/test_ds4v_image_integration.cpp b/server/test/test_ds4v_image_integration.cpp index 45cb7967f..6d798b8fb 100644 --- a/server/test/test_ds4v_image_integration.cpp +++ b/server/test/test_ds4v_image_integration.cpp @@ -69,8 +69,8 @@ void validation_and_lookup() { require(valid_image_spans({}, 0), "empty prompt/view is valid"); require(!image_block_at({}, 0), "empty lookup is null"); require(!valid_image_spans({nullptr, 1}, 100), "nonempty null view rejected"); - std::array too_many{}; - require(!valid_image_spans({too_many.data(), too_many.size()}, 100), "more than four images rejected"); + std::array too_many{}; + require(!valid_image_spans({too_many.data(), too_many.size()}, 100), "too many images rejected"); const std::vector spans{{10, 13, 18, 20}, {20, 20, 25, 25}, {30, 31, 33, 35}}; require(valid_image_spans(view(spans), 35), "adjacent and separated blocks valid"); require(!image_block_at(view(spans), 9), "text before block excluded"); diff --git a/server/test/test_ds4v_image_prompt.cpp b/server/test/test_ds4v_image_prompt.cpp index f24901a53..e588e6590 100644 --- a/server/test/test_ds4v_image_prompt.cpp +++ b/server/test/test_ds4v_image_prompt.cpp @@ -1,4 +1,5 @@ #include "deepseek4_image_prompt.h" +#include "common/image_prompt.h" #include #include #include @@ -63,7 +64,7 @@ int main() { for (int kind=0;kind<5;++kind) rejected(prepare_image_prompt({vocab+kind},{}),ImagePromptError::InvalidToken); rejected(prepare_image_prompt({marker},{image},{},{129279,marker}),ImagePromptError::InvalidContract); rejected(prepare_image_prompt({marker},{image},{},{vocab,marker-1}),ImagePromptError::InvalidContract); - rejected(prepare_image_prompt(std::vector(5,marker),std::vector(5,image)),ImagePromptError::ImageCount); + rejected(prepare_image_prompt(std::vector(luce::common::MAX_REQUEST_IMAGES+1,marker),std::vector(luce::common::MAX_REQUEST_IMAGES+1,image)),ImagePromptError::ImageCount); auto bad=image; bad.plan.resized_width++; rejected(prepare_image_prompt({marker,marker},{image,bad}),ImagePromptError::InvalidPlan); bad=image; bad.plan.aligner_rows=2; From 88e54cff48968b48387f8893e58111666d98c3f1 Mon Sep 17 00:00:00 2001 From: mrciffa Date: Thu, 24 Sep 2026 13:34:08 +0200 Subject: [PATCH 2/2] fix(deepseek4): keep exceptions inside the image stream thread A throw in the --mmproj-device encoder thread would terminate the server; it now fails the stream so prefill stops waiting. Encode-time logs print only on success, and joins the system includes. Co-Authored-By: Claude Opus 5.5 (1M context) --- server/src/deepseek4/deepseek4_backend.cpp | 22 +++++++++++++++------- server/src/deepseek4/deepseek4_backend.h | 3 +-- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index 587ab2e01..d5f238858 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -1260,18 +1261,25 @@ bool DeepSeek4Backend::materialize_images(const DeepSeek4ImagePrompt & images, const auto t0 = Clock::now(); bool ok = true; std::string stream_error; - for (size_t i = 0; ok && i < images.prepared_.images.size(); ++i) { - vision::ImageRows one; - ok = vision::materialize_image_rows({images.prepared_.images[i]}, sentinels, - size_t(w_.n_embd), encode_one, [&] { return io.is_cancelled(); }, - one, stream_error) && one.size() == 1; - if (ok) images.publish_image(i, std::move(one.front())); + try { + for (size_t i = 0; ok && i < images.prepared_.images.size(); ++i) { + vision::ImageRows one; + ok = vision::materialize_image_rows({images.prepared_.images[i]}, sentinels, + size_t(w_.n_embd), encode_one, [&] { return io.is_cancelled(); }, + one, stream_error) && one.size() == 1; + if (ok) images.publish_image(i, std::move(one.front())); + } + } catch (const std::exception & e) { + // Nothing may escape the thread: fail the stream so prefill stops waiting. + ok = false; + stream_error = e.what(); } vision_->release_scratch(); if (!ok) { std::fprintf(stderr, "[deepseek4] streaming image encode stopped: %s\n", stream_error.empty() ? "cancelled" : stream_error.c_str()); images.fail_stream(); + return; } std::fprintf(stderr, "[deepseek4] images encoded in %.0f ms on the --mmproj-device GPU (streamed)\n", elapsed_s(t0) * 1000.0); @@ -3315,7 +3323,7 @@ GenerateResult DeepSeek4Backend::generate_from_state( std::string error; const auto encode_t0 = Clock::now(); const bool encoded = materialize_images(*images, out_io, error); - if (!vision_backend_) { + if (encoded && !vision_backend_) { std::fprintf(stderr, "[deepseek4] images encoded in %.0f ms on the target GPU\n", elapsed_s(encode_t0) * 1000.0); } diff --git a/server/src/deepseek4/deepseek4_backend.h b/server/src/deepseek4/deepseek4_backend.h index 68dad4204..51ea90169 100644 --- a/server/src/deepseek4/deepseek4_backend.h +++ b/server/src/deepseek4/deepseek4_backend.h @@ -16,8 +16,6 @@ #include "deepseek4_internal.h" #include "deepseek4_dspark.h" #include "deepseek4_vision.h" - -#include #include "deepseek4_image_prompt.h" #include "deepseek4_image_assembly.h" #include "deepseek4_image_admission.h" @@ -30,6 +28,7 @@ #include #include #include +#include #include namespace luce::common {