metal : split graphs into 4 command buffers on iOS

ggml/src/ggml-metal/ggml-metal-context.m

@@ -189,7 +189,8 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) {
 void ggml_metal_free(ggml_metal_t ctx) {
     GGML_LOG_INFO("%s: deallocating\n", __func__);
 
-    for (int i = 0; i < GGML_METAL_MAX_COMMAND_BUFFERS; ++i) {
+    // n_cb command buffers + the main thread's, which is index n_cb
+    for (int i = 0; i <= GGML_METAL_MAX_COMMAND_BUFFERS; ++i) {
         if (ctx->cmd_bufs[i].obj) {
             [ctx->cmd_bufs[i].obj release];
         }
@@ -661,11 +662,18 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx) {
 }
 
 void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) {
+    // with an abort callback only command buffers 0 and 1 are committed; the main one is index n_cb, so n_cb > 1 hangs
+    if (ctx->abort_callback && n_cb > 1) {
+        GGML_LOG_WARN("%s: an abort callback allows only 1 extra command buffer; using 1 instead of %d\n", __func__, n_cb);
+        n_cb = 1;
+    }
+
     if (ctx->n_cb != n_cb) {
         ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_COMMAND_BUFFERS);
 
-        if (ctx->n_cb > 2) {
-            GGML_LOG_WARN("%s: n_cb = %d, using n_cb > 2 is not recommended and can degrade the performance in some cases\n", __func__, n_cb);
+        // 4 is the default on iOS (ggml-metal.cpp)
+        if (ctx->n_cb > 4) {
+            GGML_LOG_WARN("%s: n_cb = %d, using n_cb > 4 is not recommended and can degrade the performance in some cases\n", __func__, n_cb);
         }
     }
 
@@ -724,6 +732,10 @@ void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) {
 void ggml_metal_set_abort_callback(ggml_metal_t ctx, ggml_abort_callback abort_callback, void * user_data) {
     ctx->abort_callback = abort_callback;
     ctx->abort_callback_data = user_data;
+
+    if (abort_callback && ctx->n_cb > 1) {
+        ggml_metal_set_n_cb(ctx, ctx->n_cb); // clamps to 1
+    }
 }
 
 bool ggml_metal_supports_family(ggml_metal_t ctx, int family) {

ggml/src/ggml-metal/ggml-metal.cpp

@@ -8,9 +8,12 @@
 #include "ggml-metal-ops.h"
 #include "ggml-metal-tuning.h"
 
+#include <cstdlib>
 #include <mutex>
 #include <string>
 
+#include <TargetConditionals.h>
+
 #define GGML_METAL_NAME "MTL"
 #define GGML_METAL_MAX_DEVICES 16
 
@@ -571,6 +574,25 @@ static void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) {
     ggml_metal_set_n_cb(ctx, n_cb);
 }
 
+// iOS discards a command buffer after ~5 s of GPU time when another GPU client waits (InnocentVictim), so split graphs into 4 there
+// GGML_METAL_N_CB=1..8 overrides the default
+static int ggml_backend_metal_default_n_cb(void) {
+#if TARGET_OS_IPHONE && !TARGET_OS_MACCATALYST
+    int n_cb = 4;
+#else
+    int n_cb = 1;
+#endif
+    if (const char * env = getenv("GGML_METAL_N_CB")) {
+        const int v = atoi(env);
+        if (v >= 1 && v <= 8) {
+            n_cb = v;
+        } else {
+            GGML_LOG_WARN("%s: ignoring GGML_METAL_N_CB=%s (expected 1..8)\n", __func__, env);
+        }
+    }
+    return n_cb;
+}
+
 static ggml_backend_i ggml_backend_metal_i = {
     /* .get_name                = */ ggml_backend_metal_name,
     /* .free                    = */ ggml_backend_metal_free,
@@ -614,7 +636,7 @@ ggml_backend_t ggml_backend_metal_init(void) {
         /* .context   = */ ctx,
     };
 
-    ggml_backend_metal_set_n_cb(backend, 1);
+    ggml_backend_metal_set_n_cb(backend, ggml_backend_metal_default_n_cb());
 
     return backend;
 }
@@ -709,7 +731,7 @@ static ggml_backend_t ggml_backend_metal_device_init_backend(ggml_backend_dev_t
         /* .context   = */ ctx,
     };
 
-    ggml_backend_metal_set_n_cb(backend, 1);
+    ggml_backend_metal_set_n_cb(backend, ggml_backend_metal_default_n_cb());
 
     return backend;