Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,8 @@ drafts share it), `request_latency_seconds`, `time_to_first_token_seconds` and `
finished request's decode time: the CUDA engine's own figure, or first token to end on the Mac server; its `_sum`
over `generation_tokens_total` is the decode rate), `request_time_per_output_token_seconds`, `decode_rounds_total`,
and the `request_prefill_seconds` histogram, all under the `tensorfold:` prefix. TPOT already carries vLLM's name.
DeepSeek-V4.1 with `TF_DS_GRAPHS=1` adds `graph_fallbacks_total`: the CUDA graph captures that failed on rank 0,
whose work ran eagerly instead. Other engines, and DeepSeek-V4.1 without graphs, leave it out.
The readings vLLM also has are repeated under its names (`num_requests_running`, `num_requests_waiting`,
`kv_cache_usage_perc`, `spec_decode_num_draft_tokens_total`, `spec_decode_num_accepted_tokens_total`,
`e2e_request_latency_seconds`, `request_decode_time_seconds`, `request_prefill_time_seconds`) with identical values, so a dashboard copied from vLLM fills by swapping the
Expand Down
2 changes: 2 additions & 0 deletions zig/src/core/engine_api.zig
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,8 @@ pub const Status = struct {
prefill_tokens_per_second: f64 = 0,
/// Requests that gave a lane up to a later one; null when the engine does not count them.
preemptions: ?u64 = null,
/// CUDA graph captures that failed, their work run eagerly instead; null when the engine does not count them.
graph_fallbacks: ?u64 = null,
warming: bool = false,
/// Live streams written to the caller's buffer: tokens each holds.
streams: usize = 0,
Expand Down
2 changes: 2 additions & 0 deletions zig/src/core/lane_host.zig
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ pub const LaneHost = struct {
lone_job: ?*Job = null, // the job that driver holds now
cache: ?*pc.Store = null, // kept prompt states (engine thread only); the backend restores and saves them
memory: ?api.MemorySource = null, // the backend's memory counts; null: Engine.memory reports none
graph_fallbacks: ?*const std.atomic.Value(u64) = null, // the backend's failed graph captures (read at any time)
explain: ?api.Explain = null, // the backend's words for a request it refuses; null: the error's name
// the request at the queue's front waits for room in the backend's pool (its prompt pass refused ContextFull while
// other streams ran): admitted again once fewer than this many streams are active, or at full_retry (a backend
Expand Down Expand Up @@ -183,6 +184,7 @@ pub const LaneHost = struct {
.decode_tokens_per_second = @as(f64, @floatFromInt(tokens)) / 2.0,
.prefill_tokens_per_second = if (now - h.prefill_at <= window_ns) h.prefill_rate else 0,
.preemptions = 0,
.graph_fallbacks = if (h.graph_fallbacks) |c| c.load(.monotonic) else null,
.streams = n,
.generation_tokens = h.live_generated,
};
Expand Down
21 changes: 21 additions & 0 deletions zig/src/core/lane_host_test.zig
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,27 @@ test "a lane host reports its backend's memory counts, and none without them" {
try std.testing.expectEqual(@as(u32, 1), counts.resets);
}

test "a lane host reports its backend's failed graph captures, and none without a count" {
const gpa = std.testing.allocator;
var cfg = try lanes.Config.init(gpa, .{}, 1, 0);
defer cfg.deinit(gpa);
var target: lanes.fake.Fake = .{ .gpa = gpa };
defer target.deinit();
var clock: lanes.fake.FixedClock = .{};
var core = lanes.Engine.init(gpa, &cfg, target.backend(), clock.clock());
defer core.deinit();
var host = LaneHost.init(gpa, std.testing.io, &core, .{ .lanes = 1 });
var status: api.Status = .{};
var streams: [4]u32 = undefined;
host.engine().status(&status, &streams);
try std.testing.expect(status.graph_fallbacks == null);
var failed: std.atomic.Value(u64) = .init(0);
host.graph_fallbacks = &failed;
_ = failed.fetchAdd(3, .monotonic);
host.engine().status(&status, &streams);
try std.testing.expectEqual(@as(?u64, 3), status.graph_fallbacks);
}

test "a request the backend refuses fails alone, in the backend's words" {
const gpa = std.testing.allocator;
var cfg = try lanes.Config.init(gpa, .{ .exact_width = 8, .gpu_tokens = true, .hidden_rows = true }, 8, 7);
Expand Down
52 changes: 30 additions & 22 deletions zig/src/families/dsv41/draft.zig
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ const exl3_linear = @import("exl3_linear.zig");
const exl3_experts = @import("exl3_experts.zig");
const markov_tokens = @import("markov_tokens.zig");
const draft2d = @import("draft2d.zig");
const graph_fallback = @import("graph_fallback.zig");

const Engine = prompt.Engine;
const Chunk = prompt.Chunk;
Expand Down Expand Up @@ -113,44 +114,51 @@ const MarkovGather = struct {
/// The drafter pass's CUDA graphs (BatchDraftGraph), one a (streams, steps): every pointer it reads is a fixed
/// buffer and every per-pass value (ids, positions, slots, the pick lists) is uploaded ahead of the replay.
pub const PassGraphs = struct {
gpa: std.mem.Allocator,
map: std.AutoHashMapUnmanaged(Key, cuda.graph.Exec) = .empty,
lazy: Lazy,

pub const Key = struct { streams: u32, steps: u32 };
const Entry = struct { exec: cuda.graph.Exec };
const Lazy = graph_fallback.Lazy(graph_fallback.Cuda, Key, Entry);

pub fn init(gpa: std.mem.Allocator) PassGraphs {
return .{ .gpa = gpa };
return .{ .lazy = .init(gpa) };
}

pub fn deinit(g: *PassGraphs) void {
var it = g.map.valueIterator();
while (it.next()) |x| x.deinit();
g.map.deinit(g.gpa);
g.lazy.deinit();
g.* = undefined;
}

pub fn count(g: *const PassGraphs) usize {
return g.map.count();
return g.lazy.count();
}

fn run(g: *PassGraphs, dr: *Drafter, e: *const Engine, ch: *const Chunk, pool: *const Pool, N: usize, steps: usize) !void {
const key: Key = .{ .streams = @intCast(N), .steps = @intCast(steps) };
if (g.map.getPtr(key)) |x| return x.launchOn(e.s);
try cuda.graph.beginCapture(e.s, .thread_local);
dr.body(e, ch, pool, N, steps, null) catch |err| {
if (cuda.graph.endCapture(e.s)) |got| {
var x = got;
x.deinit();
} else |_| {}
return err;
};
var graph = try cuda.graph.endCapture(e.s);
defer graph.deinit();
var exec = try graph.instantiate();
errdefer exec.deinit();
try g.map.put(g.gpa, key, exec);
try exec.launchOn(e.s);
if (g.lazy.get(key)) |x| return x.exec.launchOn(e.s);
var site: Site = .{ .dr = dr, .e = e, .ch = ch, .pool = pool, .N = N, .steps = steps };
try g.lazy.capture(e.s, key, &site);
}

/// A pass as PassGraphs.Lazy's site (a capture that fails runs it eagerly: graph_fallback.zig).
const Site = struct {
dr: *Drafter,
e: *const Engine,
ch: *const Chunk,
pool: *const Pool,
N: usize,
steps: usize,

pub fn body(s: *Site) !void {
return s.dr.body(s.e, s.ch, s.pool, s.N, s.steps, null);
}

pub fn undo(_: *Site) void {} // the pass's host state is its inputs, uploaded before it

pub fn value(_: *Site, exec: cuda.graph.Exec) Entry {
return .{ .exec = exec };
}
};
};

pub const Drafter = struct {
Expand Down
2 changes: 2 additions & 0 deletions zig/src/families/dsv41/dsv41.zig
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ pub const round = @import("round.zig");
pub const round2d = @import("round2d.zig");
pub const draft2d = @import("draft2d.zig");
pub const markov_tokens = @import("markov_tokens.zig");
pub const graph_fallback = @import("graph_fallback.zig");
pub const draft = @import("draft.zig");
pub const sampling = @import("sampling.zig");
pub const model = @import("model.zig");
Expand Down Expand Up @@ -94,6 +95,7 @@ test {
refAll(round2d);
refAll(draft2d);
refAll(markov_tokens);
refAll(graph_fallback);
refAll(draft);
refAll(sampling);
refAll(model);
Expand Down
Loading