From b899e2df1bc565f191d901bd6513ab6639acb5b6 Mon Sep 17 00:00:00 2001 From: Jason Titus <870238+jasontitus@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:32:46 -0700 Subject: [PATCH 1/2] metal : split graphs into 4 command buffers on iOS iOS discards a command buffer that has run for about 5 s of GPU time when another GPU client (such as the display compositor) is waiting, and the graph fails with "Discarded (victim of GPU error/recovery)" (kIOGPUCommandBufferCallbackErrorInnocentVictim). With the default of one extra command buffer, the one holding ~90% of a 512-token prefill graph of a 27B model runs 5-7 s on an iPhone 17 Pro Max and failed in 10 of 50 runs; each recorded failure was that command buffer, discarded after 5.0 s. With 4 (1.6-2.3 s each) none of 10 runs failed and pp512 speed was unchanged. Output is bitwise identical for 1, 4 and 8 (M5 Max, full model). - default n_cb = 4 on iPhone-class OSes (iOS, iPadOS, visionOS, tvOS; not Mac Catalyst), 1 elsewhere; GGML_METAL_N_CB=1..8 overrides - with an abort callback set, use 1: only command buffers 0 and 1 are committed on that path, so the main thread's buffer (index n_cb) would never run with n_cb > 1 and synchronize would hang - ggml_metal_free releases all GGML_METAL_MAX_COMMAND_BUFFERS + 1 command buffers - warn above 4 instead of above 2 Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01YHqq1nTe46u7euASncLsee --- ggml/src/ggml-metal/ggml-metal-context.m | 20 +++++++++++--- ggml/src/ggml-metal/ggml-metal.cpp | 33 ++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-context.m b/ggml/src/ggml-metal/ggml-metal-context.m index 32d97cd5d0a..5dec6b47370 100644 --- a/ggml/src/ggml-metal/ggml-metal-context.m +++ b/ggml/src/ggml-metal/ggml-metal-context.m @@ -189,7 +189,8 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) { void ggml_metal_free(ggml_metal_t ctx) { GGML_LOG_INFO("%s: deallocating\n", __func__); - for (int i = 0; i < GGML_METAL_MAX_COMMAND_BUFFERS; ++i) { + // n_cb command buffers + the main thread's, which is index n_cb + for (int i = 0; i <= GGML_METAL_MAX_COMMAND_BUFFERS; ++i) { if (ctx->cmd_bufs[i].obj) { [ctx->cmd_bufs[i].obj release]; } @@ -661,11 +662,20 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx) { } void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { + // with an abort callback, the encoders commit only command buffers 0 and 1 (later ones only when capturing); + // the main thread's buffer is index n_cb, so with n_cb > 1 it would never run and synchronize would wait + // forever + if (ctx->abort_callback && n_cb > 1) { + GGML_LOG_WARN("%s: an abort callback allows only 1 extra command buffer; using 1 instead of %d\n", __func__, n_cb); + n_cb = 1; + } + if (ctx->n_cb != n_cb) { ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_COMMAND_BUFFERS); - if (ctx->n_cb > 2) { - GGML_LOG_WARN("%s: n_cb = %d, using n_cb > 2 is not recommended and can degrade the performance in some cases\n", __func__, n_cb); + // 4 is the default on iOS (ggml-metal.cpp) + if (ctx->n_cb > 4) { + GGML_LOG_WARN("%s: n_cb = %d, using n_cb > 4 is not recommended and can degrade the performance in some cases\n", __func__, n_cb); } } @@ -724,6 +734,10 @@ void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { void ggml_metal_set_abort_callback(ggml_metal_t ctx, ggml_abort_callback abort_callback, void * user_data) { ctx->abort_callback = abort_callback; ctx->abort_callback_data = user_data; + + if (abort_callback && ctx->n_cb > 1) { + ggml_metal_set_n_cb(ctx, ctx->n_cb); // clamps to 1 + } } bool ggml_metal_supports_family(ggml_metal_t ctx, int family) { diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp index a2fdde0b3ae..fea90206b21 100644 --- a/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp @@ -8,9 +8,12 @@ #include "ggml-metal-ops.h" #include "ggml-metal-tuning.h" +#include #include #include +#include + #define GGML_METAL_NAME "MTL" #define GGML_METAL_MAX_DEVICES 16 @@ -571,6 +574,32 @@ static void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) { ggml_metal_set_n_cb(ctx, n_cb); } +// The number of command buffers a graph is split into, besides the main thread's. +// +// On iPhone-class devices (iOS, iPadOS, visionOS, tvOS; not Mac Catalyst) it is 4. iOS discards a command buffer +// that has run for about 5 s of GPU time when another GPU client (such as the display compositor) is waiting, +// failing the graph with "Discarded (victim of GPU error/recovery)" (kIOGPUCommandBufferCallbackErrorInnocentVictim). +// With 1, the command buffer holding ~90% of a 512-token prefill graph of a 27B model runs 5-7 s on an iPhone 17 Pro +// Max and failed in 10 of 50 runs; with 4 (1.6-2.3 s each) none failed and prefill speed was unchanged. +// +// GGML_METAL_N_CB=1..8 overrides the default. +static int ggml_backend_metal_default_n_cb(void) { +#if TARGET_OS_IPHONE && !TARGET_OS_MACCATALYST + int n_cb = 4; +#else + int n_cb = 1; +#endif + if (const char * env = getenv("GGML_METAL_N_CB")) { + const int v = atoi(env); + if (v >= 1 && v <= 8) { + n_cb = v; + } else { + GGML_LOG_WARN("%s: ignoring GGML_METAL_N_CB=%s (expected 1..8)\n", __func__, env); + } + } + return n_cb; +} + static ggml_backend_i ggml_backend_metal_i = { /* .get_name = */ ggml_backend_metal_name, /* .free = */ ggml_backend_metal_free, @@ -614,7 +643,7 @@ ggml_backend_t ggml_backend_metal_init(void) { /* .context = */ ctx, }; - ggml_backend_metal_set_n_cb(backend, 1); + ggml_backend_metal_set_n_cb(backend, ggml_backend_metal_default_n_cb()); return backend; } @@ -709,7 +738,7 @@ static ggml_backend_t ggml_backend_metal_device_init_backend(ggml_backend_dev_t /* .context = */ ctx, }; - ggml_backend_metal_set_n_cb(backend, 1); + ggml_backend_metal_set_n_cb(backend, ggml_backend_metal_default_n_cb()); return backend; From 0ffb0c76bd5b1d24d2fa5767dc19d34d2ceb1887 Mon Sep 17 00:00:00 2001 From: Jason Titus <870238+jasontitus@users.noreply.github.com> Date: Fri, 25 Sep 2026 18:34:21 -0700 Subject: [PATCH 2/2] metal : shorten comments Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01YHqq1nTe46u7euASncLsee --- ggml/src/ggml-metal/ggml-metal-context.m | 4 +--- ggml/src/ggml-metal/ggml-metal.cpp | 11 ++--------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-context.m b/ggml/src/ggml-metal/ggml-metal-context.m index 5dec6b47370..7ea28118276 100644 --- a/ggml/src/ggml-metal/ggml-metal-context.m +++ b/ggml/src/ggml-metal/ggml-metal-context.m @@ -662,9 +662,7 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx) { } void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { - // with an abort callback, the encoders commit only command buffers 0 and 1 (later ones only when capturing); - // the main thread's buffer is index n_cb, so with n_cb > 1 it would never run and synchronize would wait - // forever + // with an abort callback only command buffers 0 and 1 are committed; the main one is index n_cb, so n_cb > 1 hangs if (ctx->abort_callback && n_cb > 1) { GGML_LOG_WARN("%s: an abort callback allows only 1 extra command buffer; using 1 instead of %d\n", __func__, n_cb); n_cb = 1; diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp index fea90206b21..a3826320c57 100644 --- a/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp @@ -574,15 +574,8 @@ static void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) { ggml_metal_set_n_cb(ctx, n_cb); } -// The number of command buffers a graph is split into, besides the main thread's. -// -// On iPhone-class devices (iOS, iPadOS, visionOS, tvOS; not Mac Catalyst) it is 4. iOS discards a command buffer -// that has run for about 5 s of GPU time when another GPU client (such as the display compositor) is waiting, -// failing the graph with "Discarded (victim of GPU error/recovery)" (kIOGPUCommandBufferCallbackErrorInnocentVictim). -// With 1, the command buffer holding ~90% of a 512-token prefill graph of a 27B model runs 5-7 s on an iPhone 17 Pro -// Max and failed in 10 of 50 runs; with 4 (1.6-2.3 s each) none failed and prefill speed was unchanged. -// -// GGML_METAL_N_CB=1..8 overrides the default. +// iOS discards a command buffer after ~5 s of GPU time when another GPU client waits (InnocentVictim), so split graphs into 4 there +// GGML_METAL_N_CB=1..8 overrides the default static int ggml_backend_metal_default_n_cb(void) { #if TARGET_OS_IPHONE && !TARGET_OS_MACCATALYST int n_cb = 4;