Dockerfile.cuda

# syntax=docker/dockerfile:1

# Stage 1: Build environment
FROM nvidia/cuda:13.0.3-cudnn-devel-ubuntu24.04 AS builder
SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"]

ARG DEBIAN_FRONTEND=noninteractive
RUN <<HEREDOC
    for i in 1 2 3 4 5; do apt-get -o Acquire::Retries=3 update && break; echo "apt-get update failed (attempt $i/5), mirror may be syncing; retrying in 15s"; sleep 15; done
    apt-get install -y --no-install-recommends \
        curl \
        git \
        libssl-dev \
        pkg-config

    rm -rf /var/lib/apt/lists/*
HEREDOC

RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"
RUN rustup update nightly && rustup default nightly

WORKDIR /hanzo
COPY . .

ARG CUDA_COMPUTE_CAP=80
ARG WITH_FEATURES="cuda,cudnn,flash-attn"
# Portable, memory-bounded release build. These MUST be ENV, not ARG: cargo and
# build.rs read the process environment, so an ARG alone is invisible to them.
#   RUSTFLAGS=""       overrides the repo .cargo/config.toml `target-cpu=native`,
#                      which would otherwise bake the builder pod's CPU features
#                      into the image and SIGILL on any other host.
#   CARGO_BUILD_JOBS=2 caps concurrent rustc so the ARC pod does not OOM. The
#                      unbounded default (num_cpus) recycled the runner
#                      mid-compile: "the runner has received a shutdown signal".
#   CUDA_COMPUTE_CAP   is read by build.rs — as ARG-only every matrix leg
#                      silently compiled the default cap instead of its own.
ENV RUSTFLAGS="" \
    CARGO_INCREMENTAL=0 \
    CARGO_NET_RETRY=5 \
    CARGO_BUILD_JOBS=2 \
    RAYON_NUM_THREADS=4 \
    CUDA_COMPUTE_CAP=${CUDA_COMPUTE_CAP}
# Only the two binaries the runtime stage copies (hanzo-server, hanzo-bench) —
# not the whole workspace.
RUN cargo build --release -p hanzo-server -p hanzo-bench --features "${WITH_FEATURES}"


# Stage 2: Minimal runtime environment
FROM nvidia/cuda:13.0.3-cudnn-runtime-ubuntu24.04 AS runtime
SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"]

ARG DEBIAN_FRONTEND=noninteractive
RUN <<HEREDOC
    for i in 1 2 3 4 5; do apt-get -o Acquire::Retries=3 update && break; echo "apt-get update failed (attempt $i/5), mirror may be syncing; retrying in 15s"; sleep 15; done
    apt-get install -y --no-install-recommends \
        libomp-dev \
        ca-certificates \
        libssl-dev \
        curl \
        pkg-config
    # The MoE gate is held bitwise to its recorded reference logits, cut with cuBLAS 13.1.1.3:
    # cuBLAS picks the gate's GEMM kernel, so another build moves routing (lane.rs CUBLAS).
    apt-get install -y --no-install-recommends --allow-downgrades libcublas-13-0=13.1.1.3-1

    rm -rf /var/lib/apt/lists/*
HEREDOC

COPY --chmod=755 --from=builder /hanzo/target/release/hanzo-bench /usr/local/bin/hanzo-bench
COPY --chmod=755 --from=builder /hanzo/target/release/hanzo-server /usr/local/bin/hanzo-server
# Copy chat templates for users running models which may not include them
COPY --from=builder /hanzo/chat_templates /chat_templates

ENV HUGGINGFACE_HUB_CACHE=/data \
    PORT=80