From a8c91e9a3721dc67b8df6a1555f47d85e1af9720 Mon Sep 17 00:00:00 2001 From: nuk3s <44327449+nuk3s@users.noreply.github.com> Date: Mon, 26 Oct 2026 08:44:06 -0400 Subject: [PATCH] fix(dsv41): run eagerly when a CUDA graph capture fails A failed capture used to fail the round and every live stream, and one that failed with a fork open left a flag set that broke every later capture until a restart (MiaAI-Lab/DeepSeek-v4.1-Flash-1017-2x-DGX-Sparks#65). A capture runs nothing, so the stretch or pass now puts its host state back and runs the same launches eagerly. A failed key stays eager for 1024 uses, doubling up to 65,536. Graphs.run joins an open fork first. New counter: tensorfold:graph_fallbacks_total. --- docs/api.md | 2 + zig/src/core/engine_api.zig | 2 + zig/src/core/lane_host.zig | 2 + zig/src/core/lane_host_test.zig | 21 ++ zig/src/families/dsv41/draft.zig | 52 ++-- zig/src/families/dsv41/dsv41.zig | 2 + zig/src/families/dsv41/graph_fallback.zig | 360 ++++++++++++++++++++++ zig/src/families/dsv41/model.zig | 8 +- zig/src/families/dsv41/native.zig | 18 ++ zig/src/families/dsv41/round.zig | 110 +++++-- zig/src/native/cuda.zig | 3 + zig/src/server/metrics.zig | 17 + 12 files changed, 550 insertions(+), 47 deletions(-) create mode 100644 zig/src/families/dsv41/graph_fallback.zig diff --git a/docs/api.md b/docs/api.md index 3032adb4a09..f127cd4e75b 100644 --- a/docs/api.md +++ b/docs/api.md @@ -325,6 +325,8 @@ drafts share it), `request_latency_seconds`, `time_to_first_token_seconds` and ` finished request's decode time: the CUDA engine's own figure, or first token to end on the Mac server; its `_sum` over `generation_tokens_total` is the decode rate), `request_time_per_output_token_seconds`, `decode_rounds_total`, and the `request_prefill_seconds` histogram, all under the `tensorfold:` prefix. TPOT already carries vLLM's name. +DeepSeek-V4.1 with `TF_DS_GRAPHS=1` adds `graph_fallbacks_total`: the CUDA graph captures that failed on rank 0, +whose work ran eagerly instead. Other engines, and DeepSeek-V4.1 without graphs, leave it out. The readings vLLM also has are repeated under its names (`num_requests_running`, `num_requests_waiting`, `kv_cache_usage_perc`, `spec_decode_num_draft_tokens_total`, `spec_decode_num_accepted_tokens_total`, `e2e_request_latency_seconds`, `request_decode_time_seconds`, `request_prefill_time_seconds`) with identical values, so a dashboard copied from vLLM fills by swapping the diff --git a/zig/src/core/engine_api.zig b/zig/src/core/engine_api.zig index d9d7406f0f5..30673723352 100644 --- a/zig/src/core/engine_api.zig +++ b/zig/src/core/engine_api.zig @@ -194,6 +194,8 @@ pub const Status = struct { prefill_tokens_per_second: f64 = 0, /// Requests that gave a lane up to a later one; null when the engine does not count them. preemptions: ?u64 = null, + /// CUDA graph captures that failed, their work run eagerly instead; null when the engine does not count them. + graph_fallbacks: ?u64 = null, warming: bool = false, /// Live streams written to the caller's buffer: tokens each holds. streams: usize = 0, diff --git a/zig/src/core/lane_host.zig b/zig/src/core/lane_host.zig index 83b94b84fc1..d7cc0c91c5d 100644 --- a/zig/src/core/lane_host.zig +++ b/zig/src/core/lane_host.zig @@ -40,6 +40,7 @@ pub const LaneHost = struct { lone_job: ?*Job = null, // the job that driver holds now cache: ?*pc.Store = null, // kept prompt states (engine thread only); the backend restores and saves them memory: ?api.MemorySource = null, // the backend's memory counts; null: Engine.memory reports none + graph_fallbacks: ?*const std.atomic.Value(u64) = null, // the backend's failed graph captures (read at any time) explain: ?api.Explain = null, // the backend's words for a request it refuses; null: the error's name // the request at the queue's front waits for room in the backend's pool (its prompt pass refused ContextFull while // other streams ran): admitted again once fewer than this many streams are active, or at full_retry (a backend @@ -183,6 +184,7 @@ pub const LaneHost = struct { .decode_tokens_per_second = @as(f64, @floatFromInt(tokens)) / 2.0, .prefill_tokens_per_second = if (now - h.prefill_at <= window_ns) h.prefill_rate else 0, .preemptions = 0, + .graph_fallbacks = if (h.graph_fallbacks) |c| c.load(.monotonic) else null, .streams = n, .generation_tokens = h.live_generated, }; diff --git a/zig/src/core/lane_host_test.zig b/zig/src/core/lane_host_test.zig index a8706236c11..dece4713f31 100644 --- a/zig/src/core/lane_host_test.zig +++ b/zig/src/core/lane_host_test.zig @@ -35,6 +35,27 @@ test "a lane host reports its backend's memory counts, and none without them" { try std.testing.expectEqual(@as(u32, 1), counts.resets); } +test "a lane host reports its backend's failed graph captures, and none without a count" { + const gpa = std.testing.allocator; + var cfg = try lanes.Config.init(gpa, .{}, 1, 0); + defer cfg.deinit(gpa); + var target: lanes.fake.Fake = .{ .gpa = gpa }; + defer target.deinit(); + var clock: lanes.fake.FixedClock = .{}; + var core = lanes.Engine.init(gpa, &cfg, target.backend(), clock.clock()); + defer core.deinit(); + var host = LaneHost.init(gpa, std.testing.io, &core, .{ .lanes = 1 }); + var status: api.Status = .{}; + var streams: [4]u32 = undefined; + host.engine().status(&status, &streams); + try std.testing.expect(status.graph_fallbacks == null); + var failed: std.atomic.Value(u64) = .init(0); + host.graph_fallbacks = &failed; + _ = failed.fetchAdd(3, .monotonic); + host.engine().status(&status, &streams); + try std.testing.expectEqual(@as(?u64, 3), status.graph_fallbacks); +} + test "a request the backend refuses fails alone, in the backend's words" { const gpa = std.testing.allocator; var cfg = try lanes.Config.init(gpa, .{ .exact_width = 8, .gpu_tokens = true, .hidden_rows = true }, 8, 7); diff --git a/zig/src/families/dsv41/draft.zig b/zig/src/families/dsv41/draft.zig index 5ac250b9dfe..a6bd9c75090 100644 --- a/zig/src/families/dsv41/draft.zig +++ b/zig/src/families/dsv41/draft.zig @@ -24,6 +24,7 @@ const exl3_linear = @import("exl3_linear.zig"); const exl3_experts = @import("exl3_experts.zig"); const markov_tokens = @import("markov_tokens.zig"); const draft2d = @import("draft2d.zig"); +const graph_fallback = @import("graph_fallback.zig"); const Engine = prompt.Engine; const Chunk = prompt.Chunk; @@ -113,44 +114,51 @@ const MarkovGather = struct { /// The drafter pass's CUDA graphs (BatchDraftGraph), one a (streams, steps): every pointer it reads is a fixed /// buffer and every per-pass value (ids, positions, slots, the pick lists) is uploaded ahead of the replay. pub const PassGraphs = struct { - gpa: std.mem.Allocator, - map: std.AutoHashMapUnmanaged(Key, cuda.graph.Exec) = .empty, + lazy: Lazy, pub const Key = struct { streams: u32, steps: u32 }; + const Entry = struct { exec: cuda.graph.Exec }; + const Lazy = graph_fallback.Lazy(graph_fallback.Cuda, Key, Entry); pub fn init(gpa: std.mem.Allocator) PassGraphs { - return .{ .gpa = gpa }; + return .{ .lazy = .init(gpa) }; } pub fn deinit(g: *PassGraphs) void { - var it = g.map.valueIterator(); - while (it.next()) |x| x.deinit(); - g.map.deinit(g.gpa); + g.lazy.deinit(); g.* = undefined; } pub fn count(g: *const PassGraphs) usize { - return g.map.count(); + return g.lazy.count(); } fn run(g: *PassGraphs, dr: *Drafter, e: *const Engine, ch: *const Chunk, pool: *const Pool, N: usize, steps: usize) !void { const key: Key = .{ .streams = @intCast(N), .steps = @intCast(steps) }; - if (g.map.getPtr(key)) |x| return x.launchOn(e.s); - try cuda.graph.beginCapture(e.s, .thread_local); - dr.body(e, ch, pool, N, steps, null) catch |err| { - if (cuda.graph.endCapture(e.s)) |got| { - var x = got; - x.deinit(); - } else |_| {} - return err; - }; - var graph = try cuda.graph.endCapture(e.s); - defer graph.deinit(); - var exec = try graph.instantiate(); - errdefer exec.deinit(); - try g.map.put(g.gpa, key, exec); - try exec.launchOn(e.s); + if (g.lazy.get(key)) |x| return x.exec.launchOn(e.s); + var site: Site = .{ .dr = dr, .e = e, .ch = ch, .pool = pool, .N = N, .steps = steps }; + try g.lazy.capture(e.s, key, &site); } + + /// A pass as PassGraphs.Lazy's site (a capture that fails runs it eagerly: graph_fallback.zig). + const Site = struct { + dr: *Drafter, + e: *const Engine, + ch: *const Chunk, + pool: *const Pool, + N: usize, + steps: usize, + + pub fn body(s: *Site) !void { + return s.dr.body(s.e, s.ch, s.pool, s.N, s.steps, null); + } + + pub fn undo(_: *Site) void {} // the pass's host state is its inputs, uploaded before it + + pub fn value(_: *Site, exec: cuda.graph.Exec) Entry { + return .{ .exec = exec }; + } + }; }; pub const Drafter = struct { diff --git a/zig/src/families/dsv41/dsv41.zig b/zig/src/families/dsv41/dsv41.zig index 908b3c1cba1..19638787323 100644 --- a/zig/src/families/dsv41/dsv41.zig +++ b/zig/src/families/dsv41/dsv41.zig @@ -46,6 +46,7 @@ pub const round = @import("round.zig"); pub const round2d = @import("round2d.zig"); pub const draft2d = @import("draft2d.zig"); pub const markov_tokens = @import("markov_tokens.zig"); +pub const graph_fallback = @import("graph_fallback.zig"); pub const draft = @import("draft.zig"); pub const sampling = @import("sampling.zig"); pub const model = @import("model.zig"); @@ -94,6 +95,7 @@ test { refAll(round2d); refAll(draft2d); refAll(markov_tokens); + refAll(graph_fallback); refAll(draft); refAll(sampling); refAll(model); diff --git a/zig/src/families/dsv41/graph_fallback.zig b/zig/src/families/dsv41/graph_fallback.zig new file mode 100644 index 00000000000..44cc81f2a81 --- /dev/null +++ b/zig/src/families/dsv41/graph_fallback.zig @@ -0,0 +1,360 @@ +//! CUDA graphs captured a key at a time; a capture that fails ran nothing, so its work runs eagerly instead. +const std = @import("std"); +const cuda = @import("cuda"); + +/// A failed key's first hold in uses (each one eager), doubled at each further failure up to `max_hold`. +pub const first_hold: u32 = 1024; +pub const max_hold: u32 = first_hold << 6; + +/// The uses a key runs eagerly after its `fails`-th failed capture (1 or more). +pub fn holdFor(fails: u32) u32 { + const doublings: u5 = @intCast(@min(fails -| 1, std.math.log2_int(u32, max_hold / first_hold))); + return first_hold << doublings; +} + +/// The keys whose capture failed: how many more uses each runs eagerly, and how often its capture failed. +pub fn Holds(comptime Key: type) type { + return struct { + const Self = @This(); + pub const Hold = struct { left: u32, fails: u32 }; + + map: std.AutoHashMapUnmanaged(Key, Hold) = .empty, + + pub fn deinit(h: *Self, gpa: std.mem.Allocator) void { + h.map.deinit(gpa); + } + + /// Whether `key` runs eagerly this time: each such use counts its hold down; at zero it is captured again. + pub fn eager(h: *Self, key: Key) bool { + const x = h.map.getPtr(key) orelse return false; + if (x.left == 0) return false; + x.left -= 1; + return true; + } + + /// Another failed capture of `key`: its new hold (fails 1 is its first failure, the one to log). + pub fn fail(h: *Self, gpa: std.mem.Allocator, key: Key) std.mem.Allocator.Error!Hold { + const got = try h.map.getOrPut(gpa, key); + if (!got.found_existing) got.value_ptr.* = .{ .left = 0, .fails = 0 }; + got.value_ptr.fails +|= 1; + got.value_ptr.left = holdFor(got.value_ptr.fails); + return got.value_ptr.*; + } + + /// `key` was captured: it is held no longer. + pub fn clear(h: *Self, key: Key) void { + _ = h.map.remove(key); + } + }; +} + +/// The graph calls a lazy capture makes, on CUDA (the host tests pass a fake with the same declarations). +pub const Cuda = struct { + pub const Stream = cuda.Stream; + pub const Graph = cuda.graph.Graph; + pub const Exec = cuda.graph.Exec; + + pub fn begin(s: Stream) !void { + return cuda.graph.beginCapture(s, .thread_local); + } + + pub fn end(s: Stream) !Graph { + return cuda.graph.endCapture(s); + } + + pub fn instantiate(g: Graph) !Exec { + return g.instantiate(); + } + + pub fn launch(x: Exec, s: Stream) !void { + return x.launchOn(s); + } + + pub fn dropGraph(g: *Graph) void { + g.deinit(); + } + + pub fn dropExec(x: *Exec) void { + x.deinit(); + } +}; + +/// A kind of work's graphs by key, each captured at its key's first use and kept (Value: the key's `exec` and more). +pub fn Lazy(comptime G: type, comptime Key: type, comptime Value: type) type { + return struct { + const Self = @This(); + + gpa: std.mem.Allocator, + map: std.AutoHashMapUnmanaged(Key, Value) = .empty, + holds: Holds(Key) = .{}, + failures: ?*std.atomic.Value(u64) = null, // the engine's failed captures (/metrics), when it counts them + warned: u64 = 0, // the failures logged: each key's first + + pub fn init(gpa: std.mem.Allocator) Self { + return .{ .gpa = gpa }; + } + + pub fn deinit(l: *Self) void { + var it = l.map.valueIterator(); + while (it.next()) |v| G.dropExec(&v.exec); + l.map.deinit(l.gpa); + l.holds.deinit(l.gpa); + l.* = undefined; + } + + /// The captured keys. + pub fn count(l: *const Self) usize { + return l.map.count(); + } + + /// The key's kept graph, to replay. + pub fn get(l: *Self, key: Key) ?*Value { + return l.map.getPtr(key); + } + + /// A key without a graph: the site's body() captured on `s`, kept as its value(exec), launched; eager if held. + pub fn capture(l: *Self, s: G.Stream, key: Key, site: anytype) anyerror!void { + if (l.holds.eager(key)) return site.body(); + // a failure up to here ran nothing on the device: the site's undo() puts its host state back + var exec = record(s, site) catch |err| return l.fallback(key, site, err); + const slot = l.map.getOrPut(l.gpa, key) catch |err| { + G.dropExec(&exec); + return l.fallback(key, site, err); + }; + slot.value_ptr.* = site.value(exec); + l.holds.clear(key); + try G.launch(exec, s); // (a failed launch may have started work: it is returned, never run again) + } + + /// The site's work captured and instantiated; a failure ends the capture first (its graph dropped). + fn record(s: G.Stream, site: anytype) anyerror!G.Exec { + try G.begin(s); + site.body() catch |err| { + if (G.end(s)) |got| { + var g = got; + G.dropGraph(&g); + } else |_| {} + return err; + }; + var graph = try G.end(s); + defer G.dropGraph(&graph); + return G.instantiate(graph); + } + + /// A failed capture of `key`: the site's host state back, the key held (its first failure logged), work eager. + fn fallback(l: *Self, key: Key, site: anytype, err: anyerror) anyerror!void { + site.undo(); + const hold = l.holds.fail(l.gpa, key) catch null; + if (l.failures) |c| _ = c.fetchAdd(1, .monotonic); + if (hold == null or hold.?.fails == 1) { + l.warned += 1; + std.log.warn("dsv41: a CUDA graph capture failed ({t}) at {any}: it runs eagerly for its next {d} uses, then is captured again (logged once a key)", .{ err, key, if (hold) |h| h.left else 0 }); + } + return site.body(); + } + }; +} + +/// A fake of the graph calls: each call's count, an error injected at a chosen call, the live graphs and execs. +const Fake = struct { + pub const Stream = *Fake; + pub const Graph = struct { f: *Fake }; + pub const Exec = struct { f: *Fake }; + pub const Call = enum { begin, end, instantiate, launch }; + + fail_at: ?Call = null, + fail_with: anyerror = error.CudaFailed, + capturing: bool = false, + calls: std.EnumArray(Call, u32) = .initFill(0), + graphs: i32 = 0, + execs: i32 = 0, + + fn step(f: *Fake, c: Call) !void { + f.calls.getPtr(c).* += 1; + if (f.fail_at == c) return f.fail_with; + } + + pub fn begin(f: *Fake) !void { + try f.step(.begin); + f.capturing = true; + } + + pub fn end(f: *Fake) !Graph { + f.capturing = false; // (the driver ends a capture also when it reports it invalidated) + try f.step(.end); + f.graphs += 1; + return .{ .f = f }; + } + + pub fn instantiate(g: Graph) !Exec { + try g.f.step(.instantiate); + g.f.execs += 1; + return .{ .f = g.f }; + } + + pub fn launch(x: Exec, f: *Fake) !void { + std.debug.assert(x.f == f); + try f.step(.launch); + } + + pub fn dropGraph(g: *Graph) void { + g.f.graphs -= 1; + } + + pub fn dropExec(x: *Exec) void { + x.f.execs -= 1; + } +}; + +const Kept = struct { exec: Fake.Exec }; + +/// A fake site: its host state (a counter its body bumps), its runs captured and eager, its undos, injected failures. +const FakeSite = struct { + f: *Fake, + state: u32 = 0, + eager_runs: u32 = 0, + undos: u32 = 0, + body_fails: ?anyerror = null, // under capture + eager_fails: ?anyerror = null, + + pub fn body(s: *FakeSite) !void { + s.state += 1; + if (s.f.capturing) { + if (s.body_fails) |e| return e; + } else { + s.eager_runs += 1; + if (s.eager_fails) |e| return e; + } + } + + pub fn undo(s: *FakeSite) void { + s.undos += 1; + s.state = 0; + } + + pub fn value(_: *FakeSite, exec: Fake.Exec) Kept { + return .{ .exec = exec }; + } +}; + +const TestKey = struct { rows: u32, bucket: u64 }; +const TestLazy = Lazy(Fake, TestKey, Kept); + +test "a capture that works is kept and launched once" { + var f: Fake = .{}; + var l: TestLazy = .init(std.testing.allocator); + defer l.deinit(); + var site: FakeSite = .{ .f = &f }; + const k: TestKey = .{ .rows = 4, .bucket = 1024 }; + try l.capture(&f, k, &site); + try std.testing.expectEqual(@as(usize, 1), l.count()); + try std.testing.expect(l.get(k) != null); + try std.testing.expectEqual(@as(u32, 1), f.calls.get(.launch)); + try std.testing.expectEqual(@as(u32, 0), site.eager_runs); + try std.testing.expectEqual(@as(i32, 0), f.graphs); // the graph is dropped once instantiated + try std.testing.expectEqual(@as(i32, 1), f.execs); +} + +test "a failure in the captured work, at begin, at end capture or at instantiate runs the work eagerly from its start" { + const Case = struct { at: ?Fake.Call, in_body: bool, with: anyerror }; + const cases = [_]Case{ + .{ .at = null, .in_body = true, .with = error.MissingTritonVariant }, // a launch the kit has no variant for + .{ .at = null, .in_body = true, .with = error.CudaFailed }, // cuStreamWaitEvent refused inside the capture + .{ .at = .end, .in_body = false, .with = error.CudaFailed }, // CUDA_ERROR_STREAM_CAPTURE_INVALIDATED (901) + .{ .at = .instantiate, .in_body = false, .with = error.CudaFailed }, // CUDA_ERROR_NOT_PERMITTED (800) + .{ .at = .instantiate, .in_body = false, .with = error.OutOfDeviceMemory }, + .{ .at = .begin, .in_body = false, .with = error.CudaFailed }, + }; + for (cases) |c| { + var f: Fake = .{ .fail_at = c.at, .fail_with = c.with }; + var l: TestLazy = .init(std.testing.allocator); + defer l.deinit(); + var failures: std.atomic.Value(u64) = .init(0); + l.failures = &failures; + var site: FakeSite = .{ .f = &f, .body_fails = if (c.in_body) c.with else null }; + const k: TestKey = .{ .rows = 16, .bucket = 262144 }; + try l.capture(&f, k, &site); + try std.testing.expectEqual(@as(u32, 1), site.undos); + try std.testing.expectEqual(@as(u32, 1), site.eager_runs); + try std.testing.expectEqual(@as(u32, 1), site.state); // the eager run started from the start state + try std.testing.expect(!f.capturing); // a begun capture is always ended + try std.testing.expectEqual(@as(u32, if (c.at == .begin) 0 else 1), f.calls.get(.end)); + try std.testing.expectEqual(@as(u32, 0), f.calls.get(.launch)); + try std.testing.expectEqual(@as(i32, 0), f.graphs); + try std.testing.expectEqual(@as(i32, 0), f.execs); + try std.testing.expectEqual(@as(usize, 0), l.count()); + try std.testing.expectEqual(@as(u64, 1), failures.load(.monotonic)); + try std.testing.expectEqual(first_hold, l.holds.map.get(k).?.left); + } +} + +test "a held key runs eagerly for its hold, then is captured again; a success ends the hold" { + var f: Fake = .{ .fail_at = .instantiate }; + var l: TestLazy = .init(std.testing.allocator); + defer l.deinit(); + var site: FakeSite = .{ .f = &f }; + const k: TestKey = .{ .rows = 2, .bucket = 8192 }; + try l.capture(&f, k, &site); + try std.testing.expectEqual(@as(u32, 1), f.calls.get(.begin)); + for (0..first_hold) |_| try l.capture(&f, k, &site); + try std.testing.expectEqual(@as(u32, 1), f.calls.get(.begin)); // no capture while held + try std.testing.expectEqual(first_hold + 1, site.eager_runs); + try std.testing.expectEqual(@as(u32, 1), site.undos); // a held use has nothing to undo + f.fail_at = null; + try l.capture(&f, .{ .rows = 3, .bucket = 8192 }, &site); // another key is not held + try std.testing.expectEqual(@as(u32, 2), f.calls.get(.begin)); + try l.capture(&f, k, &site); // its hold is over: captured again, and kept + try std.testing.expectEqual(@as(u32, 3), f.calls.get(.begin)); + try std.testing.expect(l.get(k) != null); + try std.testing.expect(l.holds.map.get(k) == null); + try std.testing.expectEqual(@as(u32, 2), f.calls.get(.launch)); +} + +test "each further failure of a key doubles its hold up to the most, and only its first is logged" { + try std.testing.expectEqual(first_hold, holdFor(1)); + try std.testing.expectEqual(first_hold * 2, holdFor(2)); + try std.testing.expectEqual(max_hold, holdFor(7)); + try std.testing.expectEqual(max_hold, holdFor(std.math.maxInt(u32))); + var f: Fake = .{ .fail_at = .end }; + var l: TestLazy = .init(std.testing.allocator); + defer l.deinit(); + var site: FakeSite = .{ .f = &f }; + const k: TestKey = .{ .rows = 1, .bucket = 1024 }; + var want: u32 = first_hold; + for (0..3) |i| { + try l.capture(&f, k, &site); + try std.testing.expectEqual(@as(u32, @intCast(i + 1)), l.holds.map.get(k).?.fails); + try std.testing.expectEqual(want, l.holds.map.get(k).?.left); + for (0..want) |_| try l.capture(&f, k, &site); + want *= 2; + } + try std.testing.expectEqual(@as(u64, 1), l.warned); + try l.capture(&f, .{ .rows = 1, .bucket = 2048 }, &site); + try std.testing.expectEqual(@as(u64, 2), l.warned); // a new key logs its own first failure +} + +test "an eager run after a failed capture returns its own error, and the key stays held" { + var f: Fake = .{ .fail_at = .end }; + var l: TestLazy = .init(std.testing.allocator); + defer l.deinit(); + var site: FakeSite = .{ .f = &f, .eager_fails = error.MissingTritonVariant }; + const k: TestKey = .{ .rows = 1, .bucket = 524288 }; + try std.testing.expectError(error.MissingTritonVariant, l.capture(&f, k, &site)); + try std.testing.expect(!f.capturing); + try std.testing.expectEqual(first_hold, l.holds.map.get(k).?.left); + try std.testing.expectError(error.MissingTritonVariant, l.capture(&f, k, &site)); // held: eager, no capture + try std.testing.expectEqual(@as(u32, 1), f.calls.get(.begin)); +} + +test "a failed launch of a kept graph is returned: its work may have started, so it does not run again" { + var f: Fake = .{ .fail_at = .launch }; + var l: TestLazy = .init(std.testing.allocator); + defer l.deinit(); + var site: FakeSite = .{ .f = &f }; + const k: TestKey = .{ .rows = 4, .bucket = 1024 }; + try std.testing.expectError(error.CudaFailed, l.capture(&f, k, &site)); + try std.testing.expectEqual(@as(u32, 0), site.eager_runs); + try std.testing.expectEqual(@as(u32, 0), site.undos); + try std.testing.expect(l.get(k) != null); // kept and alive: the next use replays it + try std.testing.expectEqual(@as(i32, 1), f.execs); +} diff --git a/zig/src/families/dsv41/model.zig b/zig/src/families/dsv41/model.zig index 8ffb29e77a0..dbec0019e59 100644 --- a/zig/src/families/dsv41/model.zig +++ b/zig/src/families/dsv41/model.zig @@ -103,6 +103,7 @@ pub const Model = struct { rd: round.Round, graphs: ?round.Graphs = null, pgraphs: ?draft.PassGraphs = null, // the drafter pass as CUDA graphs (with graphs) + graph_fallbacks: std.atomic.Value(u64) = .init(0), // both kinds' failed captures, each run eagerly (/metrics) dpool: ?draft.Pool = null, dr: ?draft.Drafter = null, tables: ?engram_io.Tables = null, @@ -190,6 +191,7 @@ pub const Model = struct { m.ring = null; m.graphs = null; m.pgraphs = null; + m.graph_fallbacks = .init(0); m.two = null; m.rdma_rings = null; const d = ctx.d; @@ -297,7 +299,10 @@ pub const Model = struct { } m.rd = try round.Round.init(&m.eng, &m.arena, a, o.pool); m.amax = try m.arena.take(round.max_rows * 4); - if (o.graphs) m.graphs = round.Graphs.init(gpa); + if (o.graphs) { + m.graphs = round.Graphs.init(gpa); + m.graphs.?.lazy.failures = &m.graph_fallbacks; + } if (o.side) try m.rd.useSide(&m.eng); if (o.prefetch) try m.rd.usePrefetch(&m.eng); if (o.drafts) { @@ -305,6 +310,7 @@ pub const Model = struct { m.dr = try draft.Drafter.init(&m.eng, &m.arena, sp); if (o.graphs) { m.pgraphs = draft.PassGraphs.init(gpa); + m.pgraphs.?.lazy.failures = &m.graph_fallbacks; m.dr.?.graphs = &m.pgraphs.?; } } diff --git a/zig/src/families/dsv41/native.zig b/zig/src/families/dsv41/native.zig index 27a4f90b88b..9f1715800ec 100644 --- a/zig/src/families/dsv41/native.zig +++ b/zig/src/families/dsv41/native.zig @@ -160,6 +160,12 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, ctx: *const cuda.Context, dir: [ return .{ .backend = st.lanes.backend(), .facts = st.lanes.facts(), .rows = @intCast(st.m.eng.round_rows), .stream_bytes = 0, .ctx = st, .deinit = deinitFn, .snaps = snaps, .vision = if (st.vision) |v| v.hook() else null }; } +/// This rank's failed CUDA graph captures, each run eagerly (/metrics; rank 1 logs its own); null without graphs. +pub fn graphFallbacks(ctx: *anyopaque) ?*const std.atomic.Value(u64) { + const st: *State = @ptrCast(@alignCast(ctx)); + return if (st.m.graphs != null) &st.m.graph_fallbacks else null; +} + fn deinitFn(ptr: *anyopaque) void { const st: *State = @ptrCast(@alignCast(ptr)); st.lanes.deinit(); // rank 1 hears done @@ -191,3 +197,15 @@ test "refusals name what to set" { try std.testing.expect(std.mem.indexOf(u8, explain(null, error.NoFreeSlot).?, "TF_DS_STREAMS") != null); try std.testing.expectEqual(@as(?[]const u8, null), explain(null, error.OutOfMemory)); } + +test "an engine with graphs counts its failed graph captures; one without reports no count" { + var m: model.Model = undefined; + m.graphs = null; + m.graph_fallbacks = .init(0); + var st: State = undefined; + st.m = &m; + try std.testing.expect(graphFallbacks(&st) == null); + m.graphs = round.Graphs.init(std.testing.allocator); + defer m.graphs.?.deinit(); + try std.testing.expect(graphFallbacks(&st).? == &m.graph_fallbacks); +} diff --git a/zig/src/families/dsv41/round.zig b/zig/src/families/dsv41/round.zig index 607b8430006..aad1e03a992 100644 --- a/zig/src/families/dsv41/round.zig +++ b/zig/src/families/dsv41/round.zig @@ -27,6 +27,7 @@ const exl3_experts2d = @import("exl3_experts2d.zig"); const engram = @import("engram.zig"); const exact = @import("exact.zig"); const ops_mod = @import("ops.zig"); +const graph_fallback = @import("graph_fallback.zig"); const Engine = prompt.Engine; const Chunk = prompt.Chunk; @@ -433,6 +434,20 @@ pub const Round = struct { rd.side_out = false; } + /// What a stretch changes on the host beside its Carry: the open forks and the profile's marks. + const Undo = struct { side_out: bool, pf_out: bool, marks: usize }; + + fn undoMark(rd: *const Round) Undo { + return .{ .side_out = rd.side_out, .pf_out = rd.pf_out, .marks = if (rd.timer) |t| t.n else 0 }; + } + + /// A failed capture's forks closed again: its events are in a dropped graph, and a later wait on one is refused. + fn undo(rd: *Round, u: Undo) void { + rd.side_out = u.side_out; + rd.pf_out = u.pf_out; + if (rd.timer) |t| t.n = u.marks; + } + fn g(rd: *const Round, k: Glue) u64 { return rd.glue + @as(u64, @backingInt(k)) * max_rows * 8; } @@ -839,32 +854,33 @@ const Carry = struct { /// The rounds' stretches as CUDA graphs (the served graph.py: one capture a rows and context bucket, the round cut /// before each Engram layer, whose rows the host reads between the stretches): a stretch is captured the first time /// its (rows, bucket, stretch) comes up and replayed after, with what it leaves. The same launches with the same -/// arguments as the eager round, so the same bytes. +/// arguments as the eager round, so the same bytes; a stretch whose capture fails runs eagerly (graph_fallback.zig). pub const Graphs = struct { - gpa: std.mem.Allocator, - map: std.AutoHashMapUnmanaged(Key, Entry) = .empty, + lazy: Lazy, pub const Key = struct { rows: u32, bucket: u64, stretch: u32 }; const Entry = struct { exec: cuda.graph.Exec, start: Carry, end: Carry }; + const Lazy = graph_fallback.Lazy(graph_fallback.Cuda, Key, Entry); pub fn init(gpa: std.mem.Allocator) Graphs { - return .{ .gpa = gpa }; + return .{ .lazy = .init(gpa) }; } pub fn deinit(g: *Graphs) void { - var it = g.map.valueIterator(); - while (it.next()) |en| en.exec.deinit(); - g.map.deinit(g.gpa); + g.lazy.deinit(); g.* = undefined; } /// The captured stretches. pub fn count(g: *const Graphs) usize { - return g.map.count(); + return g.lazy.count(); } fn run(g: *Graphs, key: Key, e: *const Engine, rd: *Round, ch: *const Chunk, cs: *const Caches, rings: []const u64, cy: *Carry, li: usize, end: usize) !void { - if (g.map.getPtr(key)) |en| { + // a replay or a capture starts joined: a failed eager stretch may have left a fork open (a plain event) + try rd.joinSide(e); + try rd.joinPrefetch(e); + if (g.lazy.get(key)) |en| { if (!std.meta.eql(en.start, cy.*)) return error.GraphStateMismatch; try en.exec.launchOn(e.s); cy.* = en.end; @@ -873,22 +889,39 @@ pub const Graphs = struct { rd.has_cblk = cy.has_cblk; return; } - const start = cy.*; - try cuda.graph.beginCapture(e.s, .thread_local); - stretch(e, rd, ch, cs, rings, cy, li, end, null) catch |err| { - if (cuda.graph.endCapture(e.s)) |got| { - var x = got; - x.deinit(); - } else |_| {} - return err; - }; - var graph = try cuda.graph.endCapture(e.s); - defer graph.deinit(); - var exec = try graph.instantiate(); - errdefer exec.deinit(); - try g.map.put(g.gpa, key, .{ .exec = exec, .start = start, .end = cy.* }); - try exec.launchOn(e.s); + var site: Site = .{ .e = e, .rd = rd, .ch = ch, .cs = cs, .rings = rings, .cy = cy, .li = li, .end = end, .start = cy.*, .mark = rd.undoMark() }; + try g.lazy.capture(e.s, key, &site); } + + /// A stretch as Graphs.Lazy's site: its launches, the host state it starts from, and its entry once captured. + const Site = struct { + e: *const Engine, + rd: *Round, + ch: *const Chunk, + cs: *const Caches, + rings: []const u64, + cy: *Carry, + li: usize, + end: usize, + start: Carry, + mark: Round.Undo, + + pub fn body(s: *Site) !void { + return stretch(s.e, s.rd, s.ch, s.cs, s.rings, s.cy, s.li, s.end, null); + } + + pub fn undo(s: *Site) void { + s.cy.* = s.start; + s.rd.kk = s.start.kk; + s.rd.has_cand = s.start.has_cand; + s.rd.has_cblk = s.start.has_cblk; + s.rd.undo(s.mark); + } + + pub fn value(s: *Site, exec: cuda.graph.Exec) Entry { + return .{ .exec = exec, .start = s.start, .end = s.cy.* }; + } + }; }; /// A round of these rows through every layer and the head: rd.logits [R, vocab] and rd.taps. The caches (ring, the @@ -1128,6 +1161,35 @@ test "context buckets as graph.py makes them" { try std.testing.expectEqual(@as(usize, 4096), bucketFor(5000, 4096)); } +test "a failed capture's undo puts the stretch's start back: its Carry, kk, the candidate flags, forks and marks" { + var t: PhaseTimer = undefined; + t.n = 3; + var rd: Round = undefined; + rd.timer = &t; + rd.side_out = false; + rd.pf_out = false; + rd.kk = 64; + rd.has_cand = false; + rd.has_cblk = false; + var cy: Carry = .{ .h = 1, .spare = 2, .pre = 3, .pre_f = 4, .kk = 64 }; + var e: Engine = undefined; + var site: Graphs.Site = .{ .e = &e, .rd = &rd, .ch = undefined, .cs = undefined, .rings = &.{}, .cy = &cy, .li = 0, .end = 1, .start = cy, .mark = rd.undoMark() }; + // what a stretch that failed in its capture leaves: forks open on the dropped graph's events, a later layer's state + rd.side_out = true; + rd.pf_out = true; + rd.kk = 512; + rd.has_cand = true; + rd.has_cblk = true; + t.n = 9; + cy = .{ .h = 2, .spare = 1, .pre = 4, .pre_f = 3, .ntap = 2, .sh = .{ .kk = 512, .pool = true }, .kk = 512, .has_cand = true }; + site.undo(); + try std.testing.expect(std.meta.eql(cy, site.start)); + try std.testing.expect(!rd.side_out and !rd.pf_out); // no later join waits on an event of the dropped graph + try std.testing.expectEqual(@as(usize, 64), rd.kk); + try std.testing.expect(!rd.has_cand and !rd.has_cblk); + try std.testing.expectEqual(@as(usize, 3), t.n); +} + /// error.NotPortedYet with its site on stderr (a request that reaches a path the port lacks names it in the log). fn notPorted(src: std.builtin.SourceLocation) error{NotPortedYet} { std.debug.print("{{\"not_ported\": \"{s}:{d}\"}}\n", .{ src.file, src.line }); diff --git a/zig/src/native/cuda.zig b/zig/src/native/cuda.zig index a6eae705cb0..76cf746e567 100644 --- a/zig/src/native/cuda.zig +++ b/zig/src/native/cuda.zig @@ -157,6 +157,7 @@ const Host = struct { lone: ?LoneRun = null, // the family's driver for a lone drafted stream, called with `family` cache: ?api.prompt_cache.Store = null, // kept prompt states, when the family keeps them snap_vt: api.prompt_cache.Snapshots.VTable = undefined, + graph_fallbacks: ?*const std.atomic.Value(u64) = null, // the family's failed graph captures, when it counts them fn close(p: *anyopaque) void { const h: *Host = @ptrCast(@alignCast(p)); @@ -203,6 +204,7 @@ const Host = struct { h.host.explain = explain; if (h.cache) |*store| h.host.cache = store; if (h.lone != null) h.host.lone = .{ .ctx = h, .run = loneRun, .sampled = true }; + h.host.graph_fallbacks = h.graph_fallbacks; try h.host.start(); } @@ -385,6 +387,7 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O capability, device, name, if (after.unified) ", memory shared with the host" else "", toGib(model), streams, if (streams == 1) "" else "s", toGib(loaded.stream_bytes), window, toGib(room), toGib(after.reserve), F.prompt_rows, if (segments > 1) try std.fmt.allocPrint(a, ", {d} staggered segments a call", .{segments}) else "", }); errdefer gpa.free(h.startup); + if (@hasDecl(F, "graphFallbacks")) h.graph_fallbacks = F.graphFallbacks(loaded.ctx); // a family that keeps prompt states (its snapshots) gets the prompt cache: states at its chunk ends if (@hasField(@TypeOf(loaded), "snaps")) if (loaded.snaps) |sn| { const want: u64 = @intFromFloat((o.prompt_cache_gib orelse 0.5) * gib); diff --git a/zig/src/server/metrics.zig b/zig/src/server/metrics.zig index 7d748cc4f64..48e12eb93a3 100644 --- a/zig/src/server/metrics.zig +++ b/zig/src/server/metrics.zig @@ -141,6 +141,7 @@ pub const Metrics = struct { try histogram(w, "request_prefill_time_seconds", "Seconds a finished request's prompt pass took, under vLLM's name.", snap.prefill); try gauge(w, "client_disconnections_total", "counter", "Requests a client left before the reply left the server.", snap.disconnects); if (status.preemptions) |p| try gauge(w, "preemptions_total", "counter", "Requests that had to give a lane up to a later one.", p); + if (status.graph_fallbacks) |n| try gauge(w, "graph_fallbacks_total", "counter", "CUDA graph captures that failed; their work ran eagerly instead (this rank's).", n); } }; @@ -323,3 +324,19 @@ test "request counter append failure frees its label" { try std.testing.expectEqual(@as(u64, 2), m.requests.items[0].count); try std.testing.expectEqual(allocations, failing.allocations); } + +test "render counts failed graph captures only for an engine that counts them" { + const gpa = std.testing.allocator; + var m: Metrics = .{ .gpa = gpa }; + var stub: RenderStub = .{}; + var none: std.Io.Writer.Allocating = .init(gpa); + defer none.deinit(); + try m.render(std.testing.io, &none.writer, stub.engine(), 4096); + try std.testing.expect(std.mem.indexOf(u8, none.written(), "graph_fallbacks") == null); + stub.status_value.graph_fallbacks = 3; + var some: std.Io.Writer.Allocating = .init(gpa); + defer some.deinit(); + try m.render(std.testing.io, &some.writer, stub.engine(), 4096); + try std.testing.expect(std.mem.indexOf(u8, some.written(), "# TYPE tensorfold:graph_fallbacks_total counter\n") != null); + try std.testing.expect(std.mem.indexOf(u8, some.written(), "tensorfold:graph_fallbacks_total 3\n") != null); +}