diff --git a/ggml/src/ggml-metal/ggml-metal-context.m b/ggml/src/ggml-metal/ggml-metal-context.m index 32d97cd5d0af..7ea281182760 100644 --- a/ggml/src/ggml-metal/ggml-metal-context.m +++ b/ggml/src/ggml-metal/ggml-metal-context.m @@ -189,7 +189,8 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) { void ggml_metal_free(ggml_metal_t ctx) { GGML_LOG_INFO("%s: deallocating\n", __func__); - for (int i = 0; i < GGML_METAL_MAX_COMMAND_BUFFERS; ++i) { + // n_cb command buffers + the main thread's, which is index n_cb + for (int i = 0; i <= GGML_METAL_MAX_COMMAND_BUFFERS; ++i) { if (ctx->cmd_bufs[i].obj) { [ctx->cmd_bufs[i].obj release]; } @@ -661,11 +662,18 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx) { } void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { + // with an abort callback only command buffers 0 and 1 are committed; the main one is index n_cb, so n_cb > 1 hangs + if (ctx->abort_callback && n_cb > 1) { + GGML_LOG_WARN("%s: an abort callback allows only 1 extra command buffer; using 1 instead of %d\n", __func__, n_cb); + n_cb = 1; + } + if (ctx->n_cb != n_cb) { ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_COMMAND_BUFFERS); - if (ctx->n_cb > 2) { - GGML_LOG_WARN("%s: n_cb = %d, using n_cb > 2 is not recommended and can degrade the performance in some cases\n", __func__, n_cb); + // 4 is the default on iOS (ggml-metal.cpp) + if (ctx->n_cb > 4) { + GGML_LOG_WARN("%s: n_cb = %d, using n_cb > 4 is not recommended and can degrade the performance in some cases\n", __func__, n_cb); } } @@ -724,6 +732,10 @@ void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { void ggml_metal_set_abort_callback(ggml_metal_t ctx, ggml_abort_callback abort_callback, void * user_data) { ctx->abort_callback = abort_callback; ctx->abort_callback_data = user_data; + + if (abort_callback && ctx->n_cb > 1) { + ggml_metal_set_n_cb(ctx, ctx->n_cb); // clamps to 1 + } } bool ggml_metal_supports_family(ggml_metal_t ctx, int family) { diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp index a2fdde0b3ae8..a3826320c575 100644 --- a/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp @@ -8,9 +8,12 @@ #include "ggml-metal-ops.h" #include "ggml-metal-tuning.h" +#include #include #include +#include + #define GGML_METAL_NAME "MTL" #define GGML_METAL_MAX_DEVICES 16 @@ -571,6 +574,25 @@ static void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) { ggml_metal_set_n_cb(ctx, n_cb); } +// iOS discards a command buffer after ~5 s of GPU time when another GPU client waits (InnocentVictim), so split graphs into 4 there +// GGML_METAL_N_CB=1..8 overrides the default +static int ggml_backend_metal_default_n_cb(void) { +#if TARGET_OS_IPHONE && !TARGET_OS_MACCATALYST + int n_cb = 4; +#else + int n_cb = 1; +#endif + if (const char * env = getenv("GGML_METAL_N_CB")) { + const int v = atoi(env); + if (v >= 1 && v <= 8) { + n_cb = v; + } else { + GGML_LOG_WARN("%s: ignoring GGML_METAL_N_CB=%s (expected 1..8)\n", __func__, env); + } + } + return n_cb; +} + static ggml_backend_i ggml_backend_metal_i = { /* .get_name = */ ggml_backend_metal_name, /* .free = */ ggml_backend_metal_free, @@ -614,7 +636,7 @@ ggml_backend_t ggml_backend_metal_init(void) { /* .context = */ ctx, }; - ggml_backend_metal_set_n_cb(backend, 1); + ggml_backend_metal_set_n_cb(backend, ggml_backend_metal_default_n_cb()); return backend; } @@ -709,7 +731,7 @@ static ggml_backend_t ggml_backend_metal_device_init_backend(ggml_backend_dev_t /* .context = */ ctx, }; - ggml_backend_metal_set_n_cb(backend, 1); + ggml_backend_metal_set_n_cb(backend, ggml_backend_metal_default_n_cb()); return backend;