diff --git a/docs/image-input.md b/docs/image-input.md index d4a264b9d..6bc77336b 100644 --- a/docs/image-input.md +++ b/docs/image-input.md @@ -66,6 +66,25 @@ launch plus `--mmproj`: the Vision file replaces `DeepSeek-V4-Flash-0731-ROCMFPX-MIX-STRIX.gguf` for text as well and decodes at least as fast (numbers below). For R9700 + Strix Halo see [DS4V](#ds4v) below. +With an R9700 in the same box, run the image encoder there while the model +stays on the Strix Halo: expose both GPUs, point `--target-device` at the Strix +Halo and add `--mmproj-device` with the R9700 (on lucebox6, without +`HIP_VISIBLE_DEVICES`, that is `--target-device hip:1 --mmproj-device hip:0`). +The encoder then runs about twice as fast and streams each image into prefill +as soon as it is encoded, so the Strix Halo never waits for the next one: + +| Images | Prompt tokens | Encoder on the Strix Halo | Encoder on the R9700 | +| --- | --- | --- | --- | +| 1 | 126 | 2.97 s | 2.94 s | +| 4 | 942 | 11.2 s | 9.1 s | +| 8 | 2,262 | 27.7 s | 19.2 s | +| 16 | 4,358 | 51.8 s | 34.6 s | + +Time to the first token, with the published launch above and ChartQA charts. +Answers are identical in both layouts. `--mmproj-device` applies to this one-GPU +layout; with the experts split across both GPUs the encoder already runs on the +R9700. + ### Send an image ```bash @@ -101,7 +120,7 @@ Use `POST /v1/chat/completions` with user-message content parts in display order Only base64 JPEG/PNG data URLs are supported. Remote URLs, images outside user content arrays, and image parts through other API formats are rejected. A -request carries at most four images, 16 MiB encoded each and 32 MiB combined. +request carries at most 16 images, 16 MiB encoded each and 32 MiB combined. Decoder pixel and aspect limits also apply. A model's image marker cannot be supplied as ordinary text. diff --git a/server/src/common/backend_args.h b/server/src/common/backend_args.h index 8c2a295ff..0c0907530 100644 --- a/server/src/common/backend_args.h +++ b/server/src/common/backend_args.h @@ -56,6 +56,9 @@ struct BackendArgs { // Optional: vision projector .gguf (deepseek4 only) std::optional mmproj_path; + // Optional: GPU for the vision encoder when it should not share the + // target's (deepseek4, one-GPU layout only). + std::optional mmproj_device; // Device placement DevicePlacement device; diff --git a/server/src/common/backend_factory.cpp b/server/src/common/backend_factory.cpp index 6e3dea2b5..1202dca6c 100644 --- a/server/src/common/backend_factory.cpp +++ b/server/src/common/backend_factory.cpp @@ -370,6 +370,7 @@ std::unique_ptr construct_backend( DeepSeek4BackendConfig cfg; cfg.model_path = model.path; cfg.mmproj_path = model.mmproj_path.value_or(""); + cfg.mmproj_gpu = model.mmproj_device ? model.mmproj_device->gpu : -1; cfg.device = placement.target; cfg.stream_fd = execution.stream_fd; cfg.max_ctx = placement.target.max_ctx; diff --git a/server/src/common/backend_factory.h b/server/src/common/backend_factory.h index 11a53b809..ab884ac80 100644 --- a/server/src/common/backend_factory.h +++ b/server/src/common/backend_factory.h @@ -37,6 +37,7 @@ class BackendPlan final { struct Model { std::string path; std::optional mmproj_path; + std::optional mmproj_device; GgufModelInfo metadata; }; diff --git a/server/src/common/backend_plan.cpp b/server/src/common/backend_plan.cpp index adf3c7c9f..afa390691 100644 --- a/server/src/common/backend_plan.cpp +++ b/server/src/common/backend_plan.cpp @@ -131,6 +131,7 @@ BackendPreparation BackendPlanBuilder::resolve( BackendPlan plan; plan.model_.path = std::move(args.model_path); plan.model_.mmproj_path = std::move(args.mmproj_path); + plan.model_.mmproj_device = std::move(args.mmproj_device); plan.model_.metadata = std::move(model); plan.placement_.target = std::move(args.device); diff --git a/server/src/common/feature_gate.cpp b/server/src/common/feature_gate.cpp index fadbb6d40..7e371856e 100644 --- a/server/src/common/feature_gate.cpp +++ b/server/src/common/feature_gate.cpp @@ -59,6 +59,14 @@ std::string check_feature_compatibility( return "--mmproj with DeepSeek4 requires a HIP backend"; } } + if (args.mmproj_device.has_value()) { + if (!args.mmproj_path.has_value() || arch != "deepseek4" || + args.mmproj_device->backend != PlacementBackend::Hip || + args.mmproj_device->gpu == args.device.gpu) { + return "--mmproj-device needs --mmproj, a DeepSeek4 target, and a HIP GPU " + "other than the target's"; + } + } // ── PFlash enablement × drafter model if (admission.pflash_enabled && diff --git a/server/src/common/image_prompt.h b/server/src/common/image_prompt.h index 2692b6f70..44e46af11 100644 --- a/server/src/common/image_prompt.h +++ b/server/src/common/image_prompt.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -7,6 +8,10 @@ namespace luce::common { +// Most images one request may carry, for every vision backend and the HTTP +// transport. Each image still has its own token and byte bounds. +inline constexpr size_t MAX_REQUEST_IMAGES = 16; + struct EncodedImage { std::string mime_type; std::vector bytes; diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index d45d9b762..d5f238858 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -34,6 +34,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -61,6 +64,7 @@ class DeepSeek4ImagePrompt final : public ImagePromptPayload { bool embed_chunk(const CpuEmbedder & embedder, size_t position, int count, float * output) const { if (!output || count <= 0 || embedder.n_embd <= 0) return false; + if (!wait_for_images(position, size_t(count))) return false; std::vector result; std::string error; const bool ok = vision::embed_image_prompt_chunk( @@ -74,12 +78,43 @@ class DeepSeek4ImagePrompt final : public ImagePromptPayload { return ok; } + // Streaming encode: images arrive in prompt order. A chunk waits only for + // the images it contains; failure or cancellation releases every waiter. + bool wait_for_images(size_t position, size_t count) const { + size_t needed = 0; + for (size_t i = 0; i < spans_.size(); ++i) { + if (spans_[i].block_begin < position + count && spans_[i].block_end > position) needed = i + 1; + } + std::unique_lock lock(stream_mutex_); + stream_ready_.wait(lock, [&] { return stream_failed_ || ready_ >= needed; }); + return ready_ >= needed; + } + void publish_image(size_t index, std::vector rows) const { + std::lock_guard lock(stream_mutex_); + materialized_[index] = std::move(rows); + ready_ = index + 1; + stream_ready_.notify_all(); + } + void fail_stream() const { + std::lock_guard lock(stream_mutex_); + stream_failed_ = true; + stream_ready_.notify_all(); + } + bool complete() const { + std::lock_guard lock(stream_mutex_); + return !materialized_.empty() && ready_ == materialized_.size(); + } + const DeepSeek4Backend * const owner_; const vision::PreparedImagePrompt prepared_; const std::vector encoded_; const std::shared_ptr lease_; std::vector spans_; mutable std::vector> materialized_; + mutable std::mutex stream_mutex_; + mutable std::condition_variable stream_ready_; + mutable size_t ready_ = 0; + mutable bool stream_failed_ = false; }; namespace { @@ -1065,7 +1100,7 @@ bool DeepSeek4Backend::prepare_images( return false; } try { - if (images.size() > 4) { + if (images.size() > MAX_REQUEST_IMAGES) { error = "too many images in request"; return false; } @@ -1115,6 +1150,10 @@ bool DeepSeek4Backend::prepare_images( } } +void DeepSeek4Backend::join_image_stream() { + if (image_stream_.joinable()) image_stream_.join(); +} + bool DeepSeek4Backend::materialize_images(const DeepSeek4ImagePrompt & images, const DaemonIO & io, std::string & error) { if (io.is_cancelled()) return false; @@ -1141,6 +1180,7 @@ bool DeepSeek4Backend::materialize_images(const DeepSeek4ImagePrompt & images, name, released); }; trim_pool(backend_, "primary"); + if (vision_backend_) trim_pool(vision_backend_, "vision"); if (expert_backend_ && expert_backend_ != backend_) { trim_pool(expert_backend_, "expert"); } @@ -1148,7 +1188,7 @@ bool DeepSeek4Backend::materialize_images(const DeepSeek4ImagePrompt & images, trim_pool(spec_backend_, "spec"); } auto reserves = image_reserves_; - const uint64_t resident_workspace = + const uint64_t resident_workspace = !vision_backend_ && (vision::detail::hip_bias_launches(backend_) || vision::detail::hip_av_launches(backend_)) ? vision::detail::hip_bias_workspace(backend_) : 0; if (resident_workspace > vision::SCRATCH_RESERVATION) { @@ -1156,8 +1196,9 @@ bool DeepSeek4Backend::materialize_images(const DeepSeek4ImagePrompt & images, return false; } // Current free memory already reflects weights, KV, optional drafter, - // snapshots and retained backend pools. Charge only upcoming work here. - reserves.primary_future_bytes = vision::SCRATCH_RESERVATION - resident_workspace + + // snapshots and retained backend pools. Charge only upcoming work here; + // an encoder on its own GPU charges nothing to the target. + reserves.primary_future_bytes = (vision_backend_ ? 0 : vision::SCRATCH_RESERVATION - resident_workspace) + 128ULL * 1024 * 1024; if (!moe_hybrid_) { uint64_t free_bytes = 0; @@ -1182,37 +1223,81 @@ bool DeepSeek4Backend::materialize_images(const DeepSeek4ImagePrompt & images, gib(report.host_required_bytes), gib(report.host_available_bytes), error.c_str()); return false; } - if (!images.materialized_.empty()) return true; + if (images.complete()) return true; + const auto encode_one = [this](const vision::PromptImage & image, vision::ImageRaster & raster, + std::string & encode_error) { + std::vector patches(image.input.patches_bf16.size()); + for (size_t i = 0; i < patches.size(); ++i) { + const uint32_t bits = uint32_t(image.input.patches_bf16[i]) << 16; + std::memcpy(&patches[i], &bits, sizeof(bits)); + } + vision::VisionOutput output; + if (!vision_->encode(patches, + {int(image.input.plan.vit_rows), int(image.input.plan.vit_cols)}, + output, encode_error)) return false; + if (output.rows <= 0 || output.columns != w_.n_embd) { + encode_error = "vision output shape differs from decoder dimensions"; + return false; + } + raster = {size_t(output.rows), size_t(output.columns), std::move(output.embeddings)}; + return true; + }; + vision::ImageSentinels sentinels; + if (!vision_->sentinel(vision::Sentinel::Start, sentinels.start, error) || + !vision_->sentinel(vision::Sentinel::Pad, sentinels.pad, error) || + !vision_->sentinel(vision::Sentinel::Newline, sentinels.newline, error) || + !vision_->sentinel(vision::Sentinel::End, sentinels.end, error)) return false; + if (vision_backend_) { + // The encoder has its own GPU: encode image k+1 there while the target + // prefills image k. Prefill waits per chunk in embed_chunk. + join_image_stream(); + { + std::lock_guard lock(images.stream_mutex_); + images.materialized_.assign(images.prepared_.images.size(), {}); + images.ready_ = 0; + images.stream_failed_ = false; + } + image_stream_ = std::thread([this, &images, &io, encode_one, sentinels] { + const auto t0 = Clock::now(); + bool ok = true; + std::string stream_error; + try { + for (size_t i = 0; ok && i < images.prepared_.images.size(); ++i) { + vision::ImageRows one; + ok = vision::materialize_image_rows({images.prepared_.images[i]}, sentinels, + size_t(w_.n_embd), encode_one, [&] { return io.is_cancelled(); }, + one, stream_error) && one.size() == 1; + if (ok) images.publish_image(i, std::move(one.front())); + } + } catch (const std::exception & e) { + // Nothing may escape the thread: fail the stream so prefill stops waiting. + ok = false; + stream_error = e.what(); + } + vision_->release_scratch(); + if (!ok) { + std::fprintf(stderr, "[deepseek4] streaming image encode stopped: %s\n", + stream_error.empty() ? "cancelled" : stream_error.c_str()); + images.fail_stream(); + return; + } + std::fprintf(stderr, "[deepseek4] images encoded in %.0f ms on the --mmproj-device GPU (streamed)\n", + elapsed_s(t0) * 1000.0); + }); + return true; + } struct ReleaseScratch { vision::VisionRuntime & runtime; ~ReleaseScratch() { runtime.release_scratch(); } } release{*vision_}; try { - vision::ImageSentinels sentinels; - if (!vision_->sentinel(vision::Sentinel::Start, sentinels.start, error) || - !vision_->sentinel(vision::Sentinel::Pad, sentinels.pad, error) || - !vision_->sentinel(vision::Sentinel::Newline, sentinels.newline, error) || - !vision_->sentinel(vision::Sentinel::End, sentinels.end, error)) return false; - return vision::materialize_image_rows( - images.prepared_.images, sentinels, size_t(w_.n_embd), - [&](const vision::PromptImage & image, vision::ImageRaster & raster, - std::string & encode_error) { - std::vector patches(image.input.patches_bf16.size()); - for (size_t i = 0; i < patches.size(); ++i) { - const uint32_t bits = uint32_t(image.input.patches_bf16[i]) << 16; - std::memcpy(&patches[i], &bits, sizeof(bits)); - } - vision::VisionOutput output; - if (!vision_->encode(patches, - {int(image.input.plan.vit_rows), int(image.input.plan.vit_cols)}, - output, encode_error)) return false; - if (output.rows <= 0 || output.columns != w_.n_embd) { - encode_error = "vision output shape differs from decoder dimensions"; - return false; - } - raster = {size_t(output.rows), size_t(output.columns), std::move(output.embeddings)}; - return true; - }, [&] { return io.is_cancelled(); }, images.materialized_, error); + vision::ImageRows rows; + if (!vision::materialize_image_rows(images.prepared_.images, sentinels, size_t(w_.n_embd), + encode_one, [&] { return io.is_cancelled(); }, rows, error)) return false; + std::lock_guard lock(images.stream_mutex_); + images.materialized_ = std::move(rows); + images.ready_ = images.materialized_.size(); + return true; } catch (const std::bad_alloc &) { error = "image materialization allocation failed"; return false; @@ -1234,7 +1319,7 @@ bool DeepSeek4Backend::init_single_gpu_vision() { reserves.primary_domain = properties.integrated || std::getenv("GGML_CUDA_ENABLE_UNIFIED_MEMORY") ? vision::ImageMemoryDomain::HostShared : vision::ImageMemoryDomain::Dedicated; reserves.primary_future_bytes = estimate_ds4_cache_bytes(w_, cfg_.max_ctx > 0 ? cfg_.max_ctx : 8192) + - vision::SCRATCH_RESERVATION + 256ULL * 1024 * 1024; + (vision_backend_ ? 0 : vision::SCRATCH_RESERVATION) + 256ULL * 1024 * 1024; uint64_t free_bytes = 0; std::string error; const bool admitted = vision::check_deepseek4_image_single_gpu_admission( @@ -1267,15 +1352,32 @@ bool DeepSeek4Backend::load_vision() { return false; } } + if (cfg_.mmproj_gpu >= 0 && cfg_.mmproj_gpu != cfg_.device.gpu && !vision_backend_) { + // The encoder only hands host rows to the decoder, so it can run on a + // faster idle GPU (an R9700 next to a Strix Halo holding the model). + vision_backend_ = ggml_backend_cuda_init(cfg_.mmproj_gpu); + size_t free_bytes = 0, total_bytes = 0; + if (vision_backend_) { + ggml_backend_dev_memory(ggml_backend_get_device(vision_backend_), &free_bytes, &total_bytes); + } + if (!vision_backend_ || !vision::detail::hip_bias_workspace(vision_backend_) || + free_bytes < vision::SCRATCH_RESERVATION + 2ULL * 1024 * 1024 * 1024) { + std::fprintf(stderr, "[deepseek4] --mmproj-device hip:%d is unavailable, lacks the DS4V " + "vision ops, or has under 4 GiB free\n", cfg_.mmproj_gpu); + return false; + } + } + ggml_backend_t vision_owner = vision_backend_ ? vision_backend_ : backend_; auto runtime = std::make_unique(); std::string error; - if (!runtime->load(cfg_.mmproj_path, backend_, w_.n_embd, w_.n_vocab, error)) { + if (!runtime->load(cfg_.mmproj_path, vision_owner, w_.n_embd, w_.n_vocab, error)) { std::fprintf(stderr, "[deepseek4] projector load failed: %s\n", error.c_str()); return false; } std::fprintf(stderr, - "[deepseek4] vision weights=%.3f GiB scratch reservation=%.3f GiB before expert placement\n", - gib(runtime->weight_bytes()), gib(vision::SCRATCH_RESERVATION)); + "[deepseek4] vision weights=%.3f GiB scratch reservation=%.3f GiB on %s\n", + gib(runtime->weight_bytes()), gib(vision::SCRATCH_RESERVATION), + vision_backend_ ? "the --mmproj-device GPU" : "the target GPU, before expert placement"); vision_ = std::move(runtime); return true; } @@ -1338,6 +1440,11 @@ bool DeepSeek4Backend::load_model() { "on one GPU or with in-process expert owners on two distinct GPUs\n"); return false; } + if (cfg_.mmproj_gpu >= 0 && tp.requested) { + std::fprintf(stderr, "[deepseek4] --mmproj-device is for the one-GPU layout; with two GPUs " + "the encoder already runs on the primary\n"); + return false; + } if (!vision::detail::hip_bias_workspace(backend_)) { std::fprintf(stderr, "[deepseek4] --mmproj needs the DS4V vision ops, which this build lacks " "(hipBLASLt was not found when ggml-hip was configured)\n"); @@ -2473,6 +2580,7 @@ bool DeepSeek4Backend::park(ParkTarget target) { } moe_placement_ = {}; moe_decode_placement_ = {}; + join_image_stream(); vision_.reset(); free_deepseek4_weights(w_); parked_ = true; @@ -3198,6 +3306,11 @@ GenerateResult DeepSeek4Backend::generate_from_state( } const auto * images = dynamic_cast(req.images.get()); + // A streaming encode reads the payload and the IO: finish it before either ends. + struct ImageStreamJoin { + DeepSeek4Backend * backend; + ~ImageStreamJoin() { backend->join_image_stream(); } + } image_stream_join{this}; if (req.images) { if (!images || images->owner_ != this || !images->matches(req.prompt) || kv_offset != 0 || req.snap_slot >= 0 || req.snap_pos >= 0 || @@ -3208,7 +3321,13 @@ GenerateResult DeepSeek4Backend::generate_from_state( return result; } std::string error; - if (!materialize_images(*images, out_io, error)) { + const auto encode_t0 = Clock::now(); + const bool encoded = materialize_images(*images, out_io, error); + if (encoded && !vision_backend_) { + std::fprintf(stderr, "[deepseek4] images encoded in %.0f ms on the target GPU\n", + elapsed_s(encode_t0) * 1000.0); + } + if (!encoded) { if (out_io.is_cancelled()) { result.succeed(); return result; } result.fail(GenerateErrorCode::PrefillFailed, error.empty() ? "image materialization failed" : error); return result; @@ -3719,6 +3838,7 @@ void DeepSeek4Backend::maybe_save_routing_stats() { } void DeepSeek4Backend::shutdown() { + join_image_stream(); maybe_save_routing_stats(); free_drafter(); for (int i = 0; i < PREFIX_SLOTS; i++) { @@ -3739,6 +3859,7 @@ void DeepSeek4Backend::shutdown() { moe_placement_ = {}; moe_decode_placement_ = {}; vision_.reset(); + if (vision_backend_) { ggml_backend_free(vision_backend_); vision_backend_ = nullptr; } free_deepseek4_weights(w_); if (snap_backend_) { ggml_backend_free(snap_backend_); snap_backend_ = nullptr; } if (backend_) { ggml_backend_free(backend_); backend_ = nullptr; } diff --git a/server/src/deepseek4/deepseek4_backend.h b/server/src/deepseek4/deepseek4_backend.h index 4e45ca153..51ea90169 100644 --- a/server/src/deepseek4/deepseek4_backend.h +++ b/server/src/deepseek4/deepseek4_backend.h @@ -28,6 +28,7 @@ #include #include #include +#include #include namespace luce::common { @@ -131,6 +132,11 @@ class DeepSeek4Backend : public ModelBackend { bool image_capable_ = false; bool cache_has_images_ = false; std::unique_ptr vision_; + // Owned backend for the vision encoder when --mmproj-device names a GPU + // other than the target's; null when the encoder shares backend_. + ggml_backend_t vision_backend_ = nullptr; + // Encodes images on vision_backend_ while prefill consumes them. + std::thread image_stream_; vision::ImageRequestGate image_request_gate_; vision::ImageAdmissionReserves image_reserves_; @@ -195,6 +201,8 @@ class DeepSeek4Backend : public ModelBackend { const DeepSeek4ImagePrompt * images = nullptr); bool load_vision(); bool init_single_gpu_vision(); + // Waits for a streaming image encode started by materialize_images. + void join_image_stream(); bool materialize_images(const DeepSeek4ImagePrompt & images, const DaemonIO & io, std::string & error); diff --git a/server/src/deepseek4/deepseek4_image_assembly.cpp b/server/src/deepseek4/deepseek4_image_assembly.cpp index 7a25ad456..204107476 100644 --- a/server/src/deepseek4/deepseek4_image_assembly.cpp +++ b/server/src/deepseek4/deepseek4_image_assembly.cpp @@ -104,7 +104,7 @@ bool materialize_image_rows(const std::vector & images, ImageRows & output, std::string & error) { error.clear(); try { - require(!images.empty() && images.size() <= 4 && bool(encode), "invalid image encode request"); + require(!images.empty() && images.size() <= DS4V_MAX_IMAGES && bool(encode), "invalid image encode request"); cancelled(is_cancelled); sentinels_valid(sentinels, dimension); uint64_t previous_end = 0; @@ -145,7 +145,7 @@ bool embed_image_prompt_chunk(const PreparedImagePrompt & prompt, const ImageRow try { require(bool(prompt) && vocabulary > 0 && count && position <= prompt.tokens.size() && count <= prompt.tokens.size() - position && rows.size() == prompt.images.size() && - prompt.images.size() <= 4, "invalid mixed embedding request"); + prompt.images.size() <= DS4V_MAX_IMAGES, "invalid mixed embedding request"); const size_t end = position + count; uint64_t previous_end = 0; for (size_t i = 0; i < prompt.images.size(); ++i) { diff --git a/server/src/deepseek4/deepseek4_image_prompt.cpp b/server/src/deepseek4/deepseek4_image_prompt.cpp index 70e02d661..d3d7dac0f 100644 --- a/server/src/deepseek4/deepseek4_image_prompt.cpp +++ b/server/src/deepseek4/deepseek4_image_prompt.cpp @@ -1,4 +1,5 @@ #include "deepseek4_image_prompt.h" +#include "../common/image_prompt.h" #include #include #include @@ -21,7 +22,8 @@ PreparedImagePrompt prepare_image_prompt(const std::vector & token limits.output_reserve>limits.context_capacity || limits.max_expanded_tokens==0 || limits.max_expanded_tokens>MAX_PREPARED_PROMPT_TOKENS) return fail(ImagePromptError::InvalidLimits,"invalid context, output reserve, or prompt bound"); - if (images.size()>4) return fail(ImagePromptError::ImageCount,"at most four images are supported"); + if (images.size()>common::MAX_REQUEST_IMAGES) + return fail(ImagePromptError::ImageCount,"too many images in request"); if (tokens.size()>limits.max_expanded_tokens) return fail(ImagePromptError::TokenLimit,"rendered tokens exceed prompt bound"); size_t markers=0; diff --git a/server/src/deepseek4/deepseek4_image_spans.h b/server/src/deepseek4/deepseek4_image_spans.h index 3b4d1ed6e..5dde1b6b6 100644 --- a/server/src/deepseek4/deepseek4_image_spans.h +++ b/server/src/deepseek4/deepseek4_image_spans.h @@ -1,11 +1,12 @@ // DS4V limits for the shared image span helpers. #pragma once +#include "../common/image_prompt.h" #include "../common/vision/image_spans.h" namespace luce::vision { -inline constexpr size_t DS4V_MAX_IMAGES = 4; +inline constexpr size_t DS4V_MAX_IMAGES = common::MAX_REQUEST_IMAGES; inline constexpr uint64_t DS4V_MAX_IMAGE_BLOCK_TOKENS = 384; inline bool valid_image_spans(ImageSpanView spans, uint64_t prompt_size) { diff --git a/server/src/deepseek4/deepseek4_internal.h b/server/src/deepseek4/deepseek4_internal.h index 5872ff703..48c20aba5 100644 --- a/server/src/deepseek4/deepseek4_internal.h +++ b/server/src/deepseek4/deepseek4_internal.h @@ -369,6 +369,7 @@ struct DeepSeek4Head4Tail2Routes { struct DeepSeek4BackendConfig { std::string model_path; std::string mmproj_path; + int mmproj_gpu = -1; // vision encoder GPU; -1 = the target's DevicePlacement device; int stream_fd = -1; int chunk = 512; // prefill chunk size diff --git a/server/src/qwen35/qwen35_backend_images.cpp b/server/src/qwen35/qwen35_backend_images.cpp index 9020ebaee..abe0616b2 100644 --- a/server/src/qwen35/qwen35_backend_images.cpp +++ b/server/src/qwen35/qwen35_backend_images.cpp @@ -3,6 +3,7 @@ // in qwen35_backend.cpp. #include "qwen35_backend.h" +#include "common/image_prompt.h" #include "common/vision/image_decode.h" #include "qwen35_image_request.h" @@ -14,7 +15,7 @@ namespace luce::common { namespace { -constexpr size_t MAX_IMAGES_PER_REQUEST = 4; // the server's transport limit +constexpr size_t MAX_IMAGES_PER_REQUEST = common::MAX_REQUEST_IMAGES; } bool Qwen35Backend::load_vision() { diff --git a/server/src/server/image_input.h b/server/src/server/image_input.h index 16c770de6..be1531f55 100644 --- a/server/src/server/image_input.h +++ b/server/src/server/image_input.h @@ -17,7 +17,7 @@ inline constexpr size_t MAX_IMAGE_BYTES = 16 * 1024 * 1024; struct ImageInputLimits { size_t image_bytes = MAX_IMAGE_BYTES; size_t request_bytes = 32 * 1024 * 1024; - size_t image_count = 4; + size_t image_count = MAX_REQUEST_IMAGES; }; struct ImageRequestPolicy { diff --git a/server/src/server/server_main.cpp b/server/src/server/server_main.cpp index a70e05081..10818a54c 100644 --- a/server/src/server/server_main.cpp +++ b/server/src/server/server_main.cpp @@ -86,6 +86,7 @@ static void print_usage(const char * prog) { " do not change generation routing.\n" " --draft Draft model for speculative decode\n" " --mmproj Vision projector GGUF: enables image input (Qwen3.5/3.8, DS4V)\n" + " --mmproj-device hip:N Run the DS4V image encoder on another GPU (one-GPU layout)\n" " --port Listen port (default: 8080)\n" " --host Bind address (default: 0.0.0.0)\n" " --max-ctx Max context length (default: 131072)\n" @@ -365,6 +366,13 @@ static int parse_model_options(int argc, char ** argv, ModelOptions & model, return 2; } bargs.mmproj_path = argv[++i]; + } else if (std::strcmp(argv[i], "--mmproj-device") == 0 && i + 1 < argc) { + DevicePlacement vision_device; + if (!parse_placement_device(argv[++i], vision_device)) { + std::fprintf(stderr, "[server] bad --mmproj-device value (expected hip:gpu)\n"); + return 2; + } + bargs.mmproj_device = vision_device; } else if (std::strcmp(argv[i], "--port") == 0 && i + 1 < argc) { sconfig.port = std::atoi(argv[++i]); } else if (std::strcmp(argv[i], "--host") == 0 && i + 1 < argc) { diff --git a/server/test/test_ds4v_image_integration.cpp b/server/test/test_ds4v_image_integration.cpp index 45cb7967f..6d798b8fb 100644 --- a/server/test/test_ds4v_image_integration.cpp +++ b/server/test/test_ds4v_image_integration.cpp @@ -69,8 +69,8 @@ void validation_and_lookup() { require(valid_image_spans({}, 0), "empty prompt/view is valid"); require(!image_block_at({}, 0), "empty lookup is null"); require(!valid_image_spans({nullptr, 1}, 100), "nonempty null view rejected"); - std::array too_many{}; - require(!valid_image_spans({too_many.data(), too_many.size()}, 100), "more than four images rejected"); + std::array too_many{}; + require(!valid_image_spans({too_many.data(), too_many.size()}, 100), "too many images rejected"); const std::vector spans{{10, 13, 18, 20}, {20, 20, 25, 25}, {30, 31, 33, 35}}; require(valid_image_spans(view(spans), 35), "adjacent and separated blocks valid"); require(!image_block_at(view(spans), 9), "text before block excluded"); diff --git a/server/test/test_ds4v_image_prompt.cpp b/server/test/test_ds4v_image_prompt.cpp index f24901a53..e588e6590 100644 --- a/server/test/test_ds4v_image_prompt.cpp +++ b/server/test/test_ds4v_image_prompt.cpp @@ -1,4 +1,5 @@ #include "deepseek4_image_prompt.h" +#include "common/image_prompt.h" #include #include #include @@ -63,7 +64,7 @@ int main() { for (int kind=0;kind<5;++kind) rejected(prepare_image_prompt({vocab+kind},{}),ImagePromptError::InvalidToken); rejected(prepare_image_prompt({marker},{image},{},{129279,marker}),ImagePromptError::InvalidContract); rejected(prepare_image_prompt({marker},{image},{},{vocab,marker-1}),ImagePromptError::InvalidContract); - rejected(prepare_image_prompt(std::vector(5,marker),std::vector(5,image)),ImagePromptError::ImageCount); + rejected(prepare_image_prompt(std::vector(luce::common::MAX_REQUEST_IMAGES+1,marker),std::vector(luce::common::MAX_REQUEST_IMAGES+1,image)),ImagePromptError::ImageCount); auto bad=image; bad.plan.resized_width++; rejected(prepare_image_prompt({marker,marker},{image,bad}),ImagePromptError::InvalidPlan); bad=image; bad.plan.aligner_rows=2;