From 42e9de2e41e20cd716fb00fe0a83cff49d07fbbb Mon Sep 17 00:00:00 2001
From: Orion Papadakis <orion.papadakis@manchester.ac.uk>
Date: Thu, 30 Jul 2026 14:42:32 +0300
Subject: [PATCH 01/17] [ci] Drop PTX backend and add Metal on Mac self-hosted
 runner

---
 .github/actions/run-inference/action.yml   |   4 +-
 .github/actions/setup-tornadovm/action.yml |   2 +-
 .github/workflows/build-and-run.yml        | 164 +++------------------
 3 files changed, 27 insertions(+), 143 deletions(-)

diff --git a/.github/actions/run-inference/action.yml b/.github/actions/run-inference/action.yml
index cbb90495..7d659c04 100644
--- a/.github/actions/run-inference/action.yml
+++ b/.github/actions/run-inference/action.yml
@@ -3,7 +3,7 @@ description: Run one llama-tornado inference pass and write the metrics + sideca
 
 inputs:
   backend:
-    description: 'GPU backend (opencl, ptx, or cuda)'
+    description: 'GPU backend (opencl, cuda, or metal)'
     required: true
   model_file:
     description: 'Model filename inside $MODELS_DIR (e.g. Llama-3.2-1B-Instruct-F16.gguf)'
@@ -43,7 +43,7 @@ runs:
         METRICS_FILE: ${{ inputs.metrics_file }}
       run: |
         # Run inference and emit raw metrics JSON via JAVA_TOOL_OPTIONS
-        # Backend (opencl/ptx/cuda) is auto-detected from $TORNADOVM_HOME/etc/tornado.backend
+        # Backend (opencl/cuda/metal) is auto-detected from $TORNADOVM_HOME/etc/tornado.backend
         ./llama-tornado --gpu \
           --model $MODELS_DIR/${{ inputs.model_file }} \
           --prompt "${{ inputs.prompt }}" \
diff --git a/.github/actions/setup-tornadovm/action.yml b/.github/actions/setup-tornadovm/action.yml
index 62b99e53..a06f288f 100644
--- a/.github/actions/setup-tornadovm/action.yml
+++ b/.github/actions/setup-tornadovm/action.yml
@@ -3,7 +3,7 @@ description: Build TornadoVM develop once per backend and reuse across runs via
 
 inputs:
   backend:
-    description: 'TornadoVM backend to build (opencl, ptx, or cuda)'
+    description: 'TornadoVM backend to build (opencl, cuda, or metal)'
     required: true
 
 runs:
diff --git a/.github/workflows/build-and-run.yml b/.github/workflows/build-and-run.yml
index af61e397..6ae12fe8 100644
--- a/.github/workflows/build-and-run.yml
+++ b/.github/workflows/build-and-run.yml
@@ -18,7 +18,7 @@ env:
 jobs:
   code-quality:
     if: github.repository == 'beehive-lab/GPULlama3.java'
-    runs-on: [self-hosted]
+    runs-on: [self-hosted, Linux]
     timeout-minutes: 30
 
     steps:
@@ -31,21 +31,25 @@ jobs:
           # ./mvnw -T12C -Pspotless spotless:check
 
   # Build: TornadoVM → GPULlama3 → Quarkus LangChain4j
-  # max-parallel: 1 ensures the opencl, ptx and cuda variants run sequentially so
-  # there are no workspace conflicts between matrix jobs.
+  # max-parallel: 2 lets the Metal matrix entry (Mac runner) run alongside the
+  # Linux entries, while opencl/cuda still serialize against each other since
+  # they share the same self-hosted Linux runner and workspace.
   build:
     if: github.repository == 'beehive-lab/GPULlama3.java'
-    runs-on: [self-hosted]
+    runs-on: ${{ matrix.backend.os == 'macos' && fromJSON('["self-hosted","macOS"]') || fromJSON('["self-hosted","Linux"]') }}
     needs: code-quality
     timeout-minutes: 30
     strategy:
       fail-fast: true
-      max-parallel: 1
+      max-parallel: 2
       matrix:
         backend:
           - name: opencl
-          - name: ptx
+            os: linux
           - name: cuda
+            os: linux
+          - name: metal
+            os: macos
 
     steps:
       - name: Checkout GPULlama3
@@ -75,7 +79,7 @@ jobs:
           fi
           echo "TornadoVM Maven version: $TORNADO_VERSION"
           # Strip any pre-existing -SNAPSHOT suffix before appending, making this step idempotent
-          # across sequential matrix variants (ptx runs after opencl on the same workspace).
+          # across sequential matrix variants (cuda runs after opencl on the same Linux workspace).
           BASE_VERSION=$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout | sed 's/-SNAPSHOT$//')
           GPULLAMA3_VERSION="${BASE_VERSION}-SNAPSHOT"
           echo "GPULlama3 version: $GPULLAMA3_VERSION"
@@ -98,16 +102,20 @@ jobs:
 
   standalone-inference:
     if: github.repository == 'beehive-lab/GPULlama3.java'
-    runs-on: [self-hosted]
+    runs-on: ${{ matrix.backend.os == 'macos' && fromJSON('["self-hosted","macOS"]') || fromJSON('["self-hosted","Linux"]') }}
     needs: build
     timeout-minutes: 30
     strategy:
       fail-fast: true
+      max-parallel: 2
       matrix:
         backend:
           - name: opencl
-          - name: ptx
+            os: linux
           - name: cuda
+            os: linux
+          - name: metal
+            os: macos
 
     steps:
       - name: Checkout GPULlama3
@@ -161,31 +169,6 @@ jobs:
           flags: --with-prefill-decode --batch-prefill-size 32
           metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-llama-1b-f16-batch-prefill-decode.json
 
-      # PTX-only: CUDA-graph variants
-      - name: PTX - FP16 - Run Llama-3.2-1B-Instruct-F16.gguf - Prefill-Decode-CUDA-Graphs
-        if: matrix.backend.name == 'ptx'
-        uses: ./.github/actions/run-inference
-        with:
-          backend: ${{ matrix.backend.name }}
-          model_file: Llama-3.2-1B-Instruct-F16.gguf
-          model: Llama-3.2-1B-Instruct
-          quantization: F16
-          configuration: prefill-decode-cuda-graphs
-          flags: --with-prefill-decode --cuda-graphs
-          metrics_file: ${{ runner.temp }}/metrics-ptx-llama-1b-f16-prefill-decode-cuda-graphs.json
-
-      - name: PTX - FP16 - Run Llama-3.2-1B-Instruct-F16.gguf - Batch-Prefill-Decode-CUDA-Graphs
-        if: matrix.backend.name == 'ptx'
-        uses: ./.github/actions/run-inference
-        with:
-          backend: ${{ matrix.backend.name }}
-          model_file: Llama-3.2-1B-Instruct-F16.gguf
-          model: Llama-3.2-1B-Instruct
-          quantization: F16
-          configuration: batch-prefill-decode-cuda-graphs
-          flags: --with-prefill-decode --batch-prefill-size 32 --cuda-graphs
-          metrics_file: ${{ runner.temp }}/metrics-ptx-llama-1b-f16-batch-prefill-decode-cuda-graphs.json
-
       - name: FP16 - Run Qwen3-0.6B-f16.gguf
         uses: ./.github/actions/run-inference
         with:
@@ -218,31 +201,6 @@ jobs:
           flags: --with-prefill-decode --batch-prefill-size 32
           metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-qwen3-0.6b-f16-batch-prefill-decode.json
 
-      # PTX-only: CUDA-graph variants
-      - name: PTX - FP16 - Run Qwen3-0.6B-f16.gguf - Prefill-Decode-CUDA-Graphs
-        if: matrix.backend.name == 'ptx'
-        uses: ./.github/actions/run-inference
-        with:
-          backend: ${{ matrix.backend.name }}
-          model_file: Qwen3-0.6B-f16.gguf
-          model: Qwen3-0.6B
-          quantization: F16
-          configuration: prefill-decode-cuda-graphs
-          flags: --with-prefill-decode --cuda-graphs
-          metrics_file: ${{ runner.temp }}/metrics-ptx-qwen3-0.6b-f16-prefill-decode-cuda-graphs.json
-
-      - name: PTX - FP16 - Run Qwen3-0.6B-f16.gguf - Batch-Prefill-Decode-CUDA-Graphs
-        if: matrix.backend.name == 'ptx'
-        uses: ./.github/actions/run-inference
-        with:
-          backend: ${{ matrix.backend.name }}
-          model_file: Qwen3-0.6B-f16.gguf
-          model: Qwen3-0.6B
-          quantization: F16
-          configuration: batch-prefill-decode-cuda-graphs
-          flags: --with-prefill-decode --batch-prefill-size 32 --cuda-graphs
-          metrics_file: ${{ runner.temp }}/metrics-ptx-qwen3-0.6b-f16-batch-prefill-decode-cuda-graphs.json
-
       - name: FP16 - Run Mistral-7B-Instruct-v0.3.fp16.gguf
         uses: ./.github/actions/run-inference
         with:
@@ -351,59 +309,6 @@ jobs:
             "flags=--with-prefill-decode --batch-prefill-size 32" \
             prompt="Say hello"
 
-      # ── PTX-only: CUDA-graph variants ────────────────────────────────────────
-      - name: PTX - Q8 - Run Llama-3.2-1B-Instruct-Q8_0.gguf - Prefill-Decode-CUDA-Graphs
-        if: matrix.backend.name == 'ptx'
-        env:
-          JAVA_TOOL_OPTIONS: >-
-            -Dllama.metrics.format=json
-            -Dllama.metrics.output=file
-            -Dllama.metrics.file=${{ runner.temp }}/metrics-ptx-llama-1b-q8-prefill-decode-cuda-graphs.json
-        run: |
-          cd ${{ github.workspace }}
-          export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH"
-          ./llama-tornado --gpu \
-            --model $MODELS_DIR/Llama-3.2-1B-Instruct-Q8_0.gguf \
-            --prompt "Say hello" \
-            --with-prefill-decode \
-            --cuda-graphs
-          python3 scripts/write_metrics_sidecar.py \
-            --out "${{ runner.temp }}/metrics-ptx-llama-1b-q8-prefill-decode-cuda-graphs.meta.json" \
-            backend=ptx \
-            task=llama-inference \
-            model_file=Llama-3.2-1B-Instruct-Q8_0.gguf \
-            model=Llama-3.2-1B-Instruct \
-            quantization=Q8_0 \
-            configuration=prefill-decode-cuda-graphs \
-            "flags=--with-prefill-decode --cuda-graphs" \
-            prompt="Say hello"
-
-      - name: PTX - Q8 - Run Llama-3.2-1B-Instruct-Q8_0.gguf - Batch-Prefill-Decode-CUDA-Graphs
-        if: matrix.backend.name == 'ptx'
-        env:
-          JAVA_TOOL_OPTIONS: >-
-            -Dllama.metrics.format=json
-            -Dllama.metrics.output=file
-            -Dllama.metrics.file=${{ runner.temp }}/metrics-ptx-llama-1b-q8-batch-prefill-decode-cuda-graphs.json
-        run: |
-          cd ${{ github.workspace }}
-          export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH"
-          ./llama-tornado --gpu \
-            --model $MODELS_DIR/Llama-3.2-1B-Instruct-Q8_0.gguf \
-            --prompt "Say hello" \
-            --with-prefill-decode --batch-prefill-size 32 \
-            --cuda-graphs
-          python3 scripts/write_metrics_sidecar.py \
-            --out "${{ runner.temp }}/metrics-ptx-llama-1b-q8-batch-prefill-decode-cuda-graphs.meta.json" \
-            backend=ptx \
-            task=llama-inference \
-            model_file=Llama-3.2-1B-Instruct-Q8_0.gguf \
-            model=Llama-3.2-1B-Instruct \
-            quantization=Q8_0 \
-            configuration=batch-prefill-decode-cuda-graphs \
-            "flags=--with-prefill-decode --batch-prefill-size 32 --cuda-graphs" \
-            prompt="Say hello"
-
       - name: Q8 - Run Qwen3-0.6B-Q8_0.gguf
         uses: ./.github/actions/run-inference
         with:
@@ -436,31 +341,6 @@ jobs:
           flags: --with-prefill-decode --batch-prefill-size 32
           metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-qwen3-0-6b-q8-batch-prefill-decode.json
 
-      # PTX-only: CUDA-graph variants
-      - name: PTX - Q8 - Run Qwen3-0.6B-Q8_0.gguf - Prefill-Decode-CUDA-Graphs
-        if: matrix.backend.name == 'ptx'
-        uses: ./.github/actions/run-inference
-        with:
-          backend: ${{ matrix.backend.name }}
-          model_file: Qwen3-0.6B-Q8_0.gguf
-          model: Qwen3-0.6B
-          quantization: Q8_0
-          configuration: prefill-decode-cuda-graphs
-          flags: --with-prefill-decode --cuda-graphs
-          metrics_file: ${{ runner.temp }}/metrics-ptx-qwen3-0-6b-q8-prefill-decode-cuda-graphs.json
-
-      - name: PTX - Q8 - Run Qwen3-0.6B-Q8_0.gguf - Batch-Prefill-Decode-CUDA-Graphs
-        if: matrix.backend.name == 'ptx'
-        uses: ./.github/actions/run-inference
-        with:
-          backend: ${{ matrix.backend.name }}
-          model_file: Qwen3-0.6B-Q8_0.gguf
-          model: Qwen3-0.6B
-          quantization: Q8_0
-          configuration: batch-prefill-decode-cuda-graphs
-          flags: --with-prefill-decode --batch-prefill-size 32 --cuda-graphs
-          metrics_file: ${{ runner.temp }}/metrics-ptx-qwen3-0-6b-q8-batch-prefill-decode-cuda-graphs.json
-
       - name: Q8 - Run Phi-3-mini-4k-instruct-Q8_0.gguf
         uses: ./.github/actions/run-inference
         with:
@@ -523,16 +403,20 @@ jobs:
   # Test integration with Quarkus-langchain4j
   quarkus-langchain4j-integration:
     if: github.repository == 'beehive-lab/GPULlama3.java'
-    runs-on: [self-hosted]
+    runs-on: ${{ matrix.backend.os == 'macos' && fromJSON('["self-hosted","macOS"]') || fromJSON('["self-hosted","Linux"]') }}
     needs: build
     timeout-minutes: 10
     strategy:
       fail-fast: true
+      max-parallel: 2
       matrix:
         backend:
           - name: opencl
-          - name: ptx
+            os: linux
           - name: cuda
+            os: linux
+          - name: metal
+            os: macos
 
     steps:
       - name: Checkout GPULlama3
@@ -675,7 +559,7 @@ jobs:
       github.event_name == 'push' &&
       github.ref == 'refs/heads/main'
 
-    runs-on: [self-hosted]
+    runs-on: [self-hosted, Linux]
     needs: standalone-inference
     timeout-minutes: 15
 

From e4c8ecaaad7aee2c2e8a1b9cc0f55c1d45bed6d6 Mon Sep 17 00:00:00 2001
From: Orion Papadakis <orion.papadakis@manchester.ac.uk>
Date: Thu, 30 Jul 2026 14:54:14 +0300
Subject: [PATCH 02/17] [fix][metal] Avoid split-KV attention kernel Qwen3
 crashes on Metal backend

---
 .../layers/type/fp16/Qwen3FP16FFNLayers.java  | 85 ++++++++++++-------
 .../layers/type/q8_0/Qwen3Q8_0FFNLayers.java  | 85 ++++++++++++-------
 .../scheduling/SchedulerDetectionService.java | 12 +++
 3 files changed, 120 insertions(+), 62 deletions(-)

diff --git a/src/main/java/org/beehive/gpullama3/tornadovm/layers/type/fp16/Qwen3FP16FFNLayers.java b/src/main/java/org/beehive/gpullama3/tornadovm/layers/type/fp16/Qwen3FP16FFNLayers.java
index f0da8b54..e6a205f3 100644
--- a/src/main/java/org/beehive/gpullama3/tornadovm/layers/type/fp16/Qwen3FP16FFNLayers.java
+++ b/src/main/java/org/beehive/gpullama3/tornadovm/layers/type/fp16/Qwen3FP16FFNLayers.java
@@ -34,9 +34,11 @@ public class Qwen3FP16FFNLayers extends AbstractTransformerLayerTaskGraphs<Qwen3
     private final int nEmbdHead;
     private final int nEmbdGqa;
     private final int gqa;
-    // Decode attention is always split-KV (flash-decoding): it beats the previous optv2/online kernels
-    // unconditionally and is correct on every backend. Splits per head: see Qwen3State.SPLIT_KV.
-    private final int attentionSplits = Qwen3State.SPLIT_KV;
+    // Decode attention is split-KV (flash-decoding) on every backend except Metal, where TornadoVM
+    // fails to JIT the multi-workgroup split-KV kernel; Metal falls back to the single-workgroup-per-head
+    // online-softmax kernel instead (see SchedulerDetectionService.isMetalBackend). Splits per head: see Qwen3State.SPLIT_KV.
+    private final boolean isMetalBackend = SchedulerDetectionService.isMetalBackend();
+    private final int attentionSplits = isMetalBackend ? 1 : Qwen3State.SPLIT_KV;
     // GEMV reduction strategy: 32-lane warp-shuffle on PTX/CUDA, shared-memory trees elsewhere. Warp is
     // faster but the OpenCL backend miscompiles simdShuffleDown, so it is auto-selected by backend.
     private final boolean useWarpMatmul = SchedulerDetectionService.isWarpShuffleSupported();
@@ -92,8 +94,10 @@ public GridScheduler updateGridScheduler(GridScheduler gridScheduler) {
             gridScheduler.addWorkerGrid("layer_" + i + ".attn_rms_qkv_projection", fusedQKVWorker);
             gridScheduler.addWorkerGrid("layer_" + i + ".qk_rmsnorm", qkRmsNormWorker);
             gridScheduler.addWorkerGrid("layer_" + i + ".rope_and_kv_cache", ropeWorker);
-            gridScheduler.addWorkerGrid("layer_" + i + ".attention", parallelAttentionWorker);
-            gridScheduler.addWorkerGrid("layer_" + i + ".attention_combine", attentionCombineWorker);
+            gridScheduler.addWorkerGrid("layer_" + i + ".attention", isMetalBackend ? attentionCombineWorker : parallelAttentionWorker);
+            if (!isMetalBackend) {
+                gridScheduler.addWorkerGrid("layer_" + i + ".attention_combine", attentionCombineWorker);
+            }
             gridScheduler.addWorkerGrid("layer_" + i + ".attn_output_proj", matmul1Worker);
             // === FFN Block ===
             gridScheduler.addWorkerGrid("layer_" + i + ".ffn_rms_reduce",
@@ -325,32 +329,51 @@ protected TaskGraph createFFNLayerTaskGraph(int layerIndex) {
                 layerIndex,                   // layer index for cache offset
                 config.contextLength()); // max sequence length
 
-        // Split-KV (flash-decoding) attention.
-        // Phase 1: split each head's KV range across attentionSplits workgroups; partials -> wrapAttSplit.
-        unifiedLayer.task("attention",
-                TransformerComputeKernelsLayered::processHeadsFlashAttentionSplitKV,
-                context,
-                qwen3State.wrapQ,             // query vectors
-                qwen3State.wrapKeyCache,      // key cache
-                qwen3State.wrapValueCache,    // value cache
-                qwen3State.wrapAttSplit,      // scratch: per-head split partials (compact layout)
-                config.numberOfHeads(),  // nHeads
-                nEmbdHead,                    // headSize
-                nEmbdGqa,                     // kvDim
-                gqa,                          // kvMul (nHeads / nHeadKv)
-                qwen3State.positionHolder,    // position
-                layerIndex,                   // layer index
-                config.contextLength(),  // context length
-                attentionSplits);             // number of KV splits per head
-        // Phase 2: combine the per-head split partials into the final attention output -> wrapXb.
-        unifiedLayer.task("attention_combine",
-                TransformerComputeKernelsLayered::combineSplitKVAttention,
-                context,
-                qwen3State.wrapAttSplit,      // scratch: per-head split partials (compact layout)
-                qwen3State.wrapXb,            // output: attention result
-                config.numberOfHeads(),  // nHeads
-                nEmbdHead,                    // headSize
-                attentionSplits);             // number of KV splits per head
+        if (isMetalBackend) {
+            // Metal: single-workgroup-per-head online-softmax attention, writing directly to wrapXb.
+            // No combine phase needed (TornadoVM fails to JIT the multi-workgroup split-KV kernel here).
+            unifiedLayer.task("attention",
+                    TransformerComputeKernelsLayered::processHeadsFlashAttention,
+                    context,
+                    qwen3State.wrapQ,             // query vectors
+                    qwen3State.wrapKeyCache,      // key cache
+                    qwen3State.wrapValueCache,    // value cache
+                    qwen3State.wrapXb,            // output: attention result
+                    config.numberOfHeads(),  // nHeads
+                    nEmbdHead,                    // headSize
+                    nEmbdGqa,                     // kvDim
+                    gqa,                          // kvMul (nHeads / nHeadKv)
+                    qwen3State.positionHolder,    // position
+                    layerIndex,                   // layer index
+                    config.contextLength());  // context length
+        } else {
+            // Split-KV (flash-decoding) attention.
+            // Phase 1: split each head's KV range across attentionSplits workgroups; partials -> wrapAttSplit.
+            unifiedLayer.task("attention",
+                    TransformerComputeKernelsLayered::processHeadsFlashAttentionSplitKV,
+                    context,
+                    qwen3State.wrapQ,             // query vectors
+                    qwen3State.wrapKeyCache,      // key cache
+                    qwen3State.wrapValueCache,    // value cache
+                    qwen3State.wrapAttSplit,      // scratch: per-head split partials (compact layout)
+                    config.numberOfHeads(),  // nHeads
+                    nEmbdHead,                    // headSize
+                    nEmbdGqa,                     // kvDim
+                    gqa,                          // kvMul (nHeads / nHeadKv)
+                    qwen3State.positionHolder,    // position
+                    layerIndex,                   // layer index
+                    config.contextLength(),  // context length
+                    attentionSplits);             // number of KV splits per head
+            // Phase 2: combine the per-head split partials into the final attention output -> wrapXb.
+            unifiedLayer.task("attention_combine",
+                    TransformerComputeKernelsLayered::combineSplitKVAttention,
+                    context,
+                    qwen3State.wrapAttSplit,      // scratch: per-head split partials (compact layout)
+                    qwen3State.wrapXb,            // output: attention result
+                    config.numberOfHeads(),  // nHeads
+                    nEmbdHead,                    // headSize
+                    attentionSplits);             // number of KV splits per head
+        }
 
         // Output Projection with Residual
         if (useWarpMatmul) {
diff --git a/src/main/java/org/beehive/gpullama3/tornadovm/layers/type/q8_0/Qwen3Q8_0FFNLayers.java b/src/main/java/org/beehive/gpullama3/tornadovm/layers/type/q8_0/Qwen3Q8_0FFNLayers.java
index 5b184894..cd5bf4b8 100644
--- a/src/main/java/org/beehive/gpullama3/tornadovm/layers/type/q8_0/Qwen3Q8_0FFNLayers.java
+++ b/src/main/java/org/beehive/gpullama3/tornadovm/layers/type/q8_0/Qwen3Q8_0FFNLayers.java
@@ -39,9 +39,11 @@ public class Qwen3Q8_0FFNLayers extends AbstractTransformerLayerTaskGraphs<Qwen3
     private final int nEmbdHead;
     private final int nEmbdGqa;
     private final int gqa;
-    // Decode attention is always split-KV (flash-decoding): it beats the previous basic flash attention
-    // unconditionally and is correct on every backend. Splits per head: see Qwen3State.SPLIT_KV.
-    private final int attentionSplits = Qwen3State.SPLIT_KV;
+    // Decode attention is split-KV (flash-decoding) on every backend except Metal, where TornadoVM
+    // fails to JIT the multi-workgroup split-KV kernel; Metal falls back to the single-workgroup-per-head
+    // online-softmax kernel instead (see SchedulerDetectionService.isMetalBackend). Splits per head: see Qwen3State.SPLIT_KV.
+    private final boolean isMetalBackend = SchedulerDetectionService.isMetalBackend();
+    private final int attentionSplits = isMetalBackend ? 1 : Qwen3State.SPLIT_KV;
     // GEMV reduction strategy: 32-lane warp-shuffle on PTX/CUDA, shared-memory trees elsewhere. Warp is
     // faster but the OpenCL backend miscompiles simdShuffleDown, so it is auto-selected by backend.
     private final boolean useWarpMatmul = SchedulerDetectionService.isWarpShuffleSupported();
@@ -91,8 +93,10 @@ public GridScheduler updateGridScheduler(GridScheduler gridScheduler) {
             gridScheduler.addWorkerGrid("layer_" + i + ".attn_rms_qkv_projection", fusedQKVWorker);
             gridScheduler.addWorkerGrid("layer_" + i + ".qk_rmsnorm", qkRmsNormWorker);
             gridScheduler.addWorkerGrid("layer_" + i + ".rope_and_kv_cache", ropeWorker);
-            gridScheduler.addWorkerGrid("layer_" + i + ".attention", parallelAttentionWorker);
-            gridScheduler.addWorkerGrid("layer_" + i + ".attention_combine", attentionCombineWorker);
+            gridScheduler.addWorkerGrid("layer_" + i + ".attention", isMetalBackend ? attentionCombineWorker : parallelAttentionWorker);
+            if (!isMetalBackend) {
+                gridScheduler.addWorkerGrid("layer_" + i + ".attention_combine", attentionCombineWorker);
+            }
             gridScheduler.addWorkerGrid("layer_" + i + ".attn_output_proj", matmul1Worker);
             gridScheduler.addWorkerGrid("layer_" + i + ".ffn_rms_reduce", rmsNormWorker);
             if (shouldUseFinalNormalization()) {
@@ -233,32 +237,51 @@ protected TaskGraph createFFNLayerTaskGraph(int layerIndex) {
                 layerIndex,                   // layer index for cache offset
                 config.contextLength()); // max sequence length
 
-        // Split-KV (flash-decoding) attention.
-        // Phase 1: split each head's KV range across attentionSplits workgroups; partials -> wrapAttSplit.
-        unifiedLayer.task("attention",
-                TransformerComputeKernelsLayered::processHeadsFlashAttentionSplitKV,
-                context,
-                qwen3State.wrapQ,             // query vectors
-                qwen3State.wrapKeyCache,      // key cache
-                qwen3State.wrapValueCache,    // value cache
-                qwen3State.wrapAttSplit,      // scratch: per-head split partials (compact layout)
-                config.numberOfHeads(),  // nHeads
-                nEmbdHead,                    // headSize
-                nEmbdGqa,                     // kvDim
-                gqa,                          // kvMul (nHeads / nHeadKv)
-                qwen3State.positionHolder,    // position
-                layerIndex,                   // layer index
-                config.contextLength(),  // context length
-                attentionSplits);             // number of KV splits per head
-        // Phase 2: combine the per-head split partials into the final attention output -> wrapXb.
-        unifiedLayer.task("attention_combine",
-                TransformerComputeKernelsLayered::combineSplitKVAttention,
-                context,
-                qwen3State.wrapAttSplit,      // scratch: per-head split partials (compact layout)
-                qwen3State.wrapXb,            // output: attention result
-                config.numberOfHeads(),  // nHeads
-                nEmbdHead,                    // headSize
-                attentionSplits);             // number of KV splits per head
+        if (isMetalBackend) {
+            // Metal: single-workgroup-per-head online-softmax attention, writing directly to wrapXb.
+            // No combine phase needed (TornadoVM fails to JIT the multi-workgroup split-KV kernel here).
+            unifiedLayer.task("attention",
+                    TransformerComputeKernelsLayered::processHeadsFlashAttention,
+                    context,
+                    qwen3State.wrapQ,             // query vectors
+                    qwen3State.wrapKeyCache,      // key cache
+                    qwen3State.wrapValueCache,    // value cache
+                    qwen3State.wrapXb,            // output: attention result
+                    config.numberOfHeads(),  // nHeads
+                    nEmbdHead,                    // headSize
+                    nEmbdGqa,                     // kvDim
+                    gqa,                          // kvMul (nHeads / nHeadKv)
+                    qwen3State.positionHolder,    // position
+                    layerIndex,                   // layer index
+                    config.contextLength());  // context length
+        } else {
+            // Split-KV (flash-decoding) attention.
+            // Phase 1: split each head's KV range across attentionSplits workgroups; partials -> wrapAttSplit.
+            unifiedLayer.task("attention",
+                    TransformerComputeKernelsLayered::processHeadsFlashAttentionSplitKV,
+                    context,
+                    qwen3State.wrapQ,             // query vectors
+                    qwen3State.wrapKeyCache,      // key cache
+                    qwen3State.wrapValueCache,    // value cache
+                    qwen3State.wrapAttSplit,      // scratch: per-head split partials (compact layout)
+                    config.numberOfHeads(),  // nHeads
+                    nEmbdHead,                    // headSize
+                    nEmbdGqa,                     // kvDim
+                    gqa,                          // kvMul (nHeads / nHeadKv)
+                    qwen3State.positionHolder,    // position
+                    layerIndex,                   // layer index
+                    config.contextLength(),  // context length
+                    attentionSplits);             // number of KV splits per head
+            // Phase 2: combine the per-head split partials into the final attention output -> wrapXb.
+            unifiedLayer.task("attention_combine",
+                    TransformerComputeKernelsLayered::combineSplitKVAttention,
+                    context,
+                    qwen3State.wrapAttSplit,      // scratch: per-head split partials (compact layout)
+                    qwen3State.wrapXb,            // output: attention result
+                    config.numberOfHeads(),  // nHeads
+                    nEmbdHead,                    // headSize
+                    attentionSplits);             // number of KV splits per head
+        }
 
         // Output Projection with Residual
         if (useWarpMatmul) {
diff --git a/src/main/java/org/beehive/gpullama3/tornadovm/scheduling/SchedulerDetectionService.java b/src/main/java/org/beehive/gpullama3/tornadovm/scheduling/SchedulerDetectionService.java
index d777cb78..ce4021a0 100644
--- a/src/main/java/org/beehive/gpullama3/tornadovm/scheduling/SchedulerDetectionService.java
+++ b/src/main/java/org/beehive/gpullama3/tornadovm/scheduling/SchedulerDetectionService.java
@@ -26,6 +26,18 @@ public static boolean isWarpShuffleSupported() {
         // return backendType == TornadoVMBackendType.PTX || backendType == TornadoVMBackendType.CUDA;
     }
 
+    /**
+     * Whether the active TornadoVM backend is Metal. The Metal backend fails to JIT the
+     * multi-workgroup split-KV flash-decoding attention kernel ({@code processHeadsFlashAttentionSplitKV}),
+     * so Qwen3 layers must fall back to the single-workgroup-per-head online-softmax kernel there.
+     */
+    public static boolean isMetalBackend() {
+        TornadoVMBackendType backendType = TornadoRuntimeProvider.getTornadoRuntime()
+                .getBackend(0)
+                .getBackendType();
+        return backendType == TornadoVMBackendType.METAL;
+    }
+
     // @formatter:off
     public static SchedulerType determineSchedulerType(Model model) {
         TornadoRuntime tornadoRuntime = TornadoRuntimeProvider.getTornadoRuntime();

From bcac2480bd79c46d5dee72613f5e3042fbea3447 Mon Sep 17 00:00:00 2001
From: Orion Papadakis <orion.papadakis@manchester.ac.uk>
Date: Thu, 30 Jul 2026 15:47:31 +0300
Subject: [PATCH 03/17] ci(macos): use Homebrew bash instead of Apple's bash
 3.2 for SDKMAN steps

---
 .github/actions/setup-java/action.yml      |  5 ++++-
 .github/actions/setup-tornadovm/action.yml | 18 +++++++++++-------
 2 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/.github/actions/setup-java/action.yml b/.github/actions/setup-java/action.yml
index c461c1ff..2291624b 100644
--- a/.github/actions/setup-java/action.yml
+++ b/.github/actions/setup-java/action.yml
@@ -10,7 +10,10 @@ runs:
   using: composite
   steps:
     - name: Set up Java with SDKMAN
-      shell: bash
+      # macOS ships bash 3.2 (Apple keeps it frozen for GPLv3 reasons); SDKMAN's own
+      # scripts use bash 4+ syntax (${var^^}) and crash under it, so use Homebrew's
+      # bash on macOS runners instead. Requires `brew install bash` on the runner.
+      shell: ${{ runner.os == 'macOS' && '/opt/homebrew/bin/bash --noprofile --norc -eo pipefail {0}' || 'bash' }}
       run: |
         source "$HOME/.sdkman/bin/sdkman-init.sh"
         if ! sdk list java | grep -q "${{ inputs.java_version }}"; then
diff --git a/.github/actions/setup-tornadovm/action.yml b/.github/actions/setup-tornadovm/action.yml
index a06f288f..fca5a472 100644
--- a/.github/actions/setup-tornadovm/action.yml
+++ b/.github/actions/setup-tornadovm/action.yml
@@ -9,16 +9,20 @@ inputs:
 runs:
   using: composite
   steps:
+    # macOS ships bash 3.2 (Apple keeps it frozen for GPLv3 reasons); SDKMAN's own scripts use
+    # bash 4+ syntax (${var^^}) and crash under it, so every step below uses Homebrew's bash on
+    # macOS runners instead (requires `brew install bash` on the runner) and the plain "bash" on
+    # Linux. Anchored once here and referenced by every step via *shell.
     - name: Get TornadoVM develop SHA
       id: tornado_sha
-      shell: bash
+      shell: &shell ${{ runner.os == 'macOS' && '/opt/homebrew/bin/bash --noprofile --norc -eo pipefail {0}' || 'bash' }}
       run: |
         SHA=$(git ls-remote https://github.com/beehive-lab/TornadoVM refs/heads/develop | cut -f1)
         echo "sha=$SHA" >> $GITHUB_OUTPUT
 
     - name: Check local build sentinel
       id: sentinel
-      shell: bash
+      shell: *shell
       run: |
         SENTINEL="$TORNADO_ROOT/.built-sha"
         if [ -f "$SENTINEL" ] && [ "$(cat $SENTINEL)" = "${{ steps.tornado_sha.outputs.sha }}" ]; then
@@ -29,7 +33,7 @@ runs:
 
     - name: Clone TornadoVM develop
       if: steps.sentinel.outputs.up-to-date != 'true'
-      shell: bash
+      shell: *shell
       run: |
         rm -rf $TORNADO_ROOT
         git clone --depth 1 --branch develop \
@@ -38,7 +42,7 @@ runs:
 
     - name: Set up Python venv for TornadoVM
       if: steps.sentinel.outputs.up-to-date != 'true'
-      shell: bash
+      shell: *shell
       run: |
         python3 -m venv $TORNADO_ROOT/venv
         source $TORNADO_ROOT/venv/bin/activate
@@ -46,7 +50,7 @@ runs:
 
     - name: Build TornadoVM
       if: steps.sentinel.outputs.up-to-date != 'true'
-      shell: bash
+      shell: *shell
       run: |
         cd $TORNADO_ROOT
         mkdir -p graalJars && cp $GRAAL_JARS/* graalJars/
@@ -57,7 +61,7 @@ runs:
 
     - name: Find TornadoVM SDK directory
       id: find_sdk
-      shell: bash
+      shell: *shell
       run: |
         SDK_DIR=$(find $TORNADO_ROOT/dist -type d -maxdepth 3 -path "*/tornadovm-*-${{ inputs.backend }}" | head -n 1)
         if [ -z "$SDK_DIR" ]; then
@@ -70,7 +74,7 @@ runs:
     # Runs on both fresh build and sentinel hit — sets TORNADOVM_HOME and PATH
     # for all subsequent steps in the calling workflow.
     - name: Configure TornadoVM environment
-      shell: bash
+      shell: *shell
       run: |
         FULL_SDK="${{ steps.find_sdk.outputs.sdk_dir }}"
         echo "Detected TornadoVM SDK: $FULL_SDK"

From 229a41539151d60d9468ab722acc4469508e78fb Mon Sep 17 00:00:00 2001
From: Orion Papadakis <orion.papadakis@manchester.ac.uk>
Date: Thu, 30 Jul 2026 16:01:55 +0300
Subject: [PATCH 04/17] ci: fix action-manifest parse error from unsupported
 YAML anchor

---
 .github/actions/setup-tornadovm/action.yml | 23 +++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/.github/actions/setup-tornadovm/action.yml b/.github/actions/setup-tornadovm/action.yml
index fca5a472..218bfefc 100644
--- a/.github/actions/setup-tornadovm/action.yml
+++ b/.github/actions/setup-tornadovm/action.yml
@@ -6,23 +6,24 @@ inputs:
     description: 'TornadoVM backend to build (opencl, cuda, or metal)'
     required: true
 
+# macOS ships bash 3.2 (Apple keeps it frozen for GPLv3 reasons); SDKMAN's own scripts use
+# bash 4+ syntax (${var^^}) and crash under it, so every step below uses Homebrew's bash on
+# macOS runners instead (requires `brew install bash` on the runner) and plain "bash" on Linux.
+# GitHub's action-manifest parser doesn't support YAML anchors, so the expression is repeated
+# per step rather than shared via &shell/*shell.
 runs:
   using: composite
   steps:
-    # macOS ships bash 3.2 (Apple keeps it frozen for GPLv3 reasons); SDKMAN's own scripts use
-    # bash 4+ syntax (${var^^}) and crash under it, so every step below uses Homebrew's bash on
-    # macOS runners instead (requires `brew install bash` on the runner) and the plain "bash" on
-    # Linux. Anchored once here and referenced by every step via *shell.
     - name: Get TornadoVM develop SHA
       id: tornado_sha
-      shell: &shell ${{ runner.os == 'macOS' && '/opt/homebrew/bin/bash --noprofile --norc -eo pipefail {0}' || 'bash' }}
+      shell: ${{ runner.os == 'macOS' && '/opt/homebrew/bin/bash --noprofile --norc -eo pipefail {0}' || 'bash' }}
       run: |
         SHA=$(git ls-remote https://github.com/beehive-lab/TornadoVM refs/heads/develop | cut -f1)
         echo "sha=$SHA" >> $GITHUB_OUTPUT
 
     - name: Check local build sentinel
       id: sentinel
-      shell: *shell
+      shell: ${{ runner.os == 'macOS' && '/opt/homebrew/bin/bash --noprofile --norc -eo pipefail {0}' || 'bash' }}
       run: |
         SENTINEL="$TORNADO_ROOT/.built-sha"
         if [ -f "$SENTINEL" ] && [ "$(cat $SENTINEL)" = "${{ steps.tornado_sha.outputs.sha }}" ]; then
@@ -33,7 +34,7 @@ runs:
 
     - name: Clone TornadoVM develop
       if: steps.sentinel.outputs.up-to-date != 'true'
-      shell: *shell
+      shell: ${{ runner.os == 'macOS' && '/opt/homebrew/bin/bash --noprofile --norc -eo pipefail {0}' || 'bash' }}
       run: |
         rm -rf $TORNADO_ROOT
         git clone --depth 1 --branch develop \
@@ -42,7 +43,7 @@ runs:
 
     - name: Set up Python venv for TornadoVM
       if: steps.sentinel.outputs.up-to-date != 'true'
-      shell: *shell
+      shell: ${{ runner.os == 'macOS' && '/opt/homebrew/bin/bash --noprofile --norc -eo pipefail {0}' || 'bash' }}
       run: |
         python3 -m venv $TORNADO_ROOT/venv
         source $TORNADO_ROOT/venv/bin/activate
@@ -50,7 +51,7 @@ runs:
 
     - name: Build TornadoVM
       if: steps.sentinel.outputs.up-to-date != 'true'
-      shell: *shell
+      shell: ${{ runner.os == 'macOS' && '/opt/homebrew/bin/bash --noprofile --norc -eo pipefail {0}' || 'bash' }}
       run: |
         cd $TORNADO_ROOT
         mkdir -p graalJars && cp $GRAAL_JARS/* graalJars/
@@ -61,7 +62,7 @@ runs:
 
     - name: Find TornadoVM SDK directory
       id: find_sdk
-      shell: *shell
+      shell: ${{ runner.os == 'macOS' && '/opt/homebrew/bin/bash --noprofile --norc -eo pipefail {0}' || 'bash' }}
       run: |
         SDK_DIR=$(find $TORNADO_ROOT/dist -type d -maxdepth 3 -path "*/tornadovm-*-${{ inputs.backend }}" | head -n 1)
         if [ -z "$SDK_DIR" ]; then
@@ -74,7 +75,7 @@ runs:
     # Runs on both fresh build and sentinel hit — sets TORNADOVM_HOME and PATH
     # for all subsequent steps in the calling workflow.
     - name: Configure TornadoVM environment
-      shell: *shell
+      shell: ${{ runner.os == 'macOS' && '/opt/homebrew/bin/bash --noprofile --norc -eo pipefail {0}' || 'bash' }}
       run: |
         FULL_SDK="${{ steps.find_sdk.outputs.sdk_dir }}"
         echo "Detected TornadoVM SDK: $FULL_SDK"

From 57934ace767e3fdfeeb399c9cd0ff1c8bb625768 Mon Sep 17 00:00:00 2001
From: Orion Papadakis <orion.papadakis@manchester.ac.uk>
Date: Thu, 30 Jul 2026 16:08:25 +0300
Subject: [PATCH 05/17] ci: use portable sed -i syntax for macOS (BSD sed)
 compatibility

---
 .github/workflows/build-and-run.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-run.yml b/.github/workflows/build-and-run.yml
index 6ae12fe8..586e35e4 100644
--- a/.github/workflows/build-and-run.yml
+++ b/.github/workflows/build-and-run.yml
@@ -96,7 +96,10 @@ jobs:
       - name: Build Quarkus LangChain4j
         run: |
           cd ${{ github.workspace }}/quarkus-langchain4j
-          sed -i 's/<gpu-llama3\.version>.*<\/gpu-llama3\.version>/<gpu-llama3.version>'$GPULLAMA3_VERSION'<\/gpu-llama3.version>/' pom.xml
+          # -i.bak (extension glued to the flag) is the portable form: GNU sed (Linux) accepts it,
+          # and BSD sed (macOS) requires it — a bare `-i ''` is parsed as the script on BSD sed.
+          sed -i.bak 's/<gpu-llama3\.version>.*<\/gpu-llama3\.version>/<gpu-llama3.version>'$GPULLAMA3_VERSION'<\/gpu-llama3.version>/' pom.xml
+          rm -f pom.xml.bak
           # -Dtornado activates the TornadoVM profile; -am builds only the gpu-llama3 module + deps
           mvn clean install -pl integration-tests/gpu-llama3 -am -DskipTests -Dtornado
 

From f922e2cc8767d6ae8e3fd5d1c993a0a0eba52f9a Mon Sep 17 00:00:00 2001
From: Orion Papadakis <orion.papadakis@manchester.ac.uk>
Date: Thu, 30 Jul 2026 16:24:38 +0300
Subject: [PATCH 06/17] ci: don't let a Linux backend failure cancel the
 independent Mac runner job

---
 .github/workflows/build-and-run.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build-and-run.yml b/.github/workflows/build-and-run.yml
index 586e35e4..91dec2ef 100644
--- a/.github/workflows/build-and-run.yml
+++ b/.github/workflows/build-and-run.yml
@@ -34,13 +34,14 @@ jobs:
   # max-parallel: 2 lets the Metal matrix entry (Mac runner) run alongside the
   # Linux entries, while opencl/cuda still serialize against each other since
   # they share the same self-hosted Linux runner and workspace.
+  # fail-fast: false so a Linux failure doesn't cancel the independent Mac runner's job (and vice versa).
   build:
     if: github.repository == 'beehive-lab/GPULlama3.java'
     runs-on: ${{ matrix.backend.os == 'macos' && fromJSON('["self-hosted","macOS"]') || fromJSON('["self-hosted","Linux"]') }}
     needs: code-quality
     timeout-minutes: 30
     strategy:
-      fail-fast: true
+      fail-fast: false
       max-parallel: 2
       matrix:
         backend:
@@ -109,7 +110,7 @@ jobs:
     needs: build
     timeout-minutes: 30
     strategy:
-      fail-fast: true
+      fail-fast: false
       max-parallel: 2
       matrix:
         backend:
@@ -410,7 +411,7 @@ jobs:
     needs: build
     timeout-minutes: 10
     strategy:
-      fail-fast: true
+      fail-fast: false
       max-parallel: 2
       matrix:
         backend:

From 3fb6a54cca53761f02b39d543e8d9692e6eec888 Mon Sep 17 00:00:00 2001
From: Orion Papadakis <orion.papadakis@manchester.ac.uk>
Date: Fri, 31 Jul 2026 12:30:07 +0300
Subject: [PATCH 07/17] ci: raise standalone-inference timeout for slower Metal
 runs, capture Quarkus app log on failure

---
 .github/workflows/build-and-run.yml | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-run.yml b/.github/workflows/build-and-run.yml
index 91dec2ef..6bf85c07 100644
--- a/.github/workflows/build-and-run.yml
+++ b/.github/workflows/build-and-run.yml
@@ -108,7 +108,10 @@ jobs:
     if: github.repository == 'beehive-lab/GPULlama3.java'
     runs-on: ${{ matrix.backend.os == 'macos' && fromJSON('["self-hosted","macOS"]') || fromJSON('["self-hosted","Linux"]') }}
     needs: build
-    timeout-minutes: 30
+    # Metal on the Mac runner is far slower than the Linux GPU backends for the larger
+    # models (e.g. Mistral-7B alone exceeded the old 30-minute budget), so give the full
+    # matrix enough headroom regardless of which backend the job landed on.
+    timeout-minutes: 180
     strategy:
       fail-fast: false
       max-parallel: 2
@@ -498,10 +501,14 @@ jobs:
       - name: Start Quarkus Application
         run: |
           cd ${{ github.workspace }}/quarkus-langchain4j/integration-tests/gpu-llama3
+          APP_LOG="${{ runner.temp }}/quarkus-app.log"
+          echo "APP_LOG=$APP_LOG" >> $GITHUB_ENV
+          # Background process output stops being captured by the Actions log once this
+          # step ends, so redirect it to a file we can dump later if a request fails.
           java  @"$TORNADOVM_HOME/tornado-argfile" \
                 -Dtornado.device.memory=8GB \
                 -Dquarkus.http.port=$QUARKUS_PORT \
-                -jar target/quarkus-app/quarkus-run.jar &
+                -jar target/quarkus-app/quarkus-run.jar > "$APP_LOG" 2>&1 &
           APP_PID=$!
           echo "APP_PID=$APP_PID" >> $GITHUB_ENV
           for i in {1..30}; do
@@ -548,6 +555,10 @@ jobs:
             [ $attempt -eq 3 ] && { echo "::error::Streaming endpoint failed after 3 attempts"; exit 1; }
           done
 
+      - name: Dump Quarkus Application Log
+        if: failure()
+        run: cat "$APP_LOG" || true
+
       - name: Cleanup & Shutdown
         if: always()
         run: |

From 7183e3884d30166d1c27656e330c2ed82f9914c5 Mon Sep 17 00:00:00 2001
From: Orion Papadakis <orion.papadakis@manchester.ac.uk>
Date: Fri, 31 Jul 2026 12:31:10 +0300
Subject: [PATCH 08/17] ci: revert standalone-inference timeout back to 30
 minutes

---
 .github/workflows/build-and-run.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/build-and-run.yml b/.github/workflows/build-and-run.yml
index 6bf85c07..5bc5c27f 100644
--- a/.github/workflows/build-and-run.yml
+++ b/.github/workflows/build-and-run.yml
@@ -108,10 +108,7 @@ jobs:
     if: github.repository == 'beehive-lab/GPULlama3.java'
     runs-on: ${{ matrix.backend.os == 'macos' && fromJSON('["self-hosted","macOS"]') || fromJSON('["self-hosted","Linux"]') }}
     needs: build
-    # Metal on the Mac runner is far slower than the Linux GPU backends for the larger
-    # models (e.g. Mistral-7B alone exceeded the old 30-minute budget), so give the full
-    # matrix enough headroom regardless of which backend the job landed on.
-    timeout-minutes: 180
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       max-parallel: 2

From afbf8e5c5a5b74479d2a677778f3c9ef380f4eb9 Mon Sep 17 00:00:00 2001
From: Orion Papadakis <orion.papadakis@manchester.ac.uk>
Date: Fri, 31 Jul 2026 12:35:18 +0300
Subject: [PATCH 09/17] ci: drop max-parallel cap so Mac runner doesn't queue
 behind Linux jobs

---
 .github/workflows/build-and-run.yml | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build-and-run.yml b/.github/workflows/build-and-run.yml
index 5bc5c27f..64f3c17a 100644
--- a/.github/workflows/build-and-run.yml
+++ b/.github/workflows/build-and-run.yml
@@ -31,9 +31,10 @@ jobs:
           # ./mvnw -T12C -Pspotless spotless:check
 
   # Build: TornadoVM → GPULlama3 → Quarkus LangChain4j
-  # max-parallel: 2 lets the Metal matrix entry (Mac runner) run alongside the
-  # Linux entries, while opencl/cuda still serialize against each other since
-  # they share the same self-hosted Linux runner and workspace.
+  # No max-parallel cap: opencl/cuda still serialize against each other since they share
+  # the same self-hosted Linux runner, but that's the runner queue doing the serializing,
+  # not the matrix — the independent Mac runner must be free to start immediately rather
+  # than wait for a slot in a shared parallelism budget it doesn't actually contend for.
   # fail-fast: false so a Linux failure doesn't cancel the independent Mac runner's job (and vice versa).
   build:
     if: github.repository == 'beehive-lab/GPULlama3.java'
@@ -42,7 +43,6 @@ jobs:
     timeout-minutes: 30
     strategy:
       fail-fast: false
-      max-parallel: 2
       matrix:
         backend:
           - name: opencl
@@ -111,7 +111,6 @@ jobs:
     timeout-minutes: 30
     strategy:
       fail-fast: false
-      max-parallel: 2
       matrix:
         backend:
           - name: opencl
@@ -412,7 +411,6 @@ jobs:
     timeout-minutes: 10
     strategy:
       fail-fast: false
-      max-parallel: 2
       matrix:
         backend:
           - name: opencl

From e6312e9ec640f420bc802eccd9c1e09b757a01c2 Mon Sep 17 00:00:00 2001
From: Orion Papadakis <orion.papadakis@manchester.ac.uk>
Date: Fri, 31 Jul 2026 13:58:43 +0300
Subject: [PATCH 10/17] ci: temporarily skip Mistral-7B on Metal until
 timeout/OOM is root-caused

---
 .github/workflows/build-and-run.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/build-and-run.yml b/.github/workflows/build-and-run.yml
index 64f3c17a..80a8d8af 100644
--- a/.github/workflows/build-and-run.yml
+++ b/.github/workflows/build-and-run.yml
@@ -205,6 +205,9 @@ jobs:
           metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-qwen3-0.6b-f16-batch-prefill-decode.json
 
       - name: FP16 - Run Mistral-7B-Instruct-v0.3.fp16.gguf
+        # TODO: re-enable once the Mistral-7B-on-Metal timeout is root-caused; see the
+        # Metal TornadoOutOfMemoryException also hit by the tiny Llama-3.2-1B quarkus job.
+        if: matrix.backend.name != 'metal'
         uses: ./.github/actions/run-inference
         with:
           backend: ${{ matrix.backend.name }}
@@ -365,6 +368,8 @@ jobs:
           metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-qwen2-5-1-5b-q8-standard.json
 
       - name: Q8 - Run Mistral-7B-Instruct-v0.3.Q8_0.gguf
+        # TODO: re-enable once Mistral-7B-on-Metal is root-caused; see the FP16 step above.
+        if: matrix.backend.name != 'metal'
         uses: ./.github/actions/run-inference
         with:
           backend: ${{ matrix.backend.name }}

From ee0ba89fab927232235e7b8f86a69a9ded45d323 Mon Sep 17 00:00:00 2001
From: Orion Papadakis <orion.papadakis@manchester.ac.uk>
Date: Fri, 31 Jul 2026 14:01:06 +0300
Subject: [PATCH 11/17] ci(quarkus): fix dead device-memory JVM flag; the
 extension overwrites it with its own 4GB default, which OOMs on Metal

---
 .github/workflows/build-and-run.yml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-run.yml b/.github/workflows/build-and-run.yml
index 80a8d8af..9cffbfec 100644
--- a/.github/workflows/build-and-run.yml
+++ b/.github/workflows/build-and-run.yml
@@ -505,8 +505,14 @@ jobs:
           echo "APP_LOG=$APP_LOG" >> $GITHUB_ENV
           # Background process output stops being captured by the Actions log once this
           # step ends, so redirect it to a file we can dump later if a request fails.
+          #
+          # -Dtornado.device.memory here is dead: the gpu-llama3 extension's ChatModelConfig
+          # defaults device-memory to "4GB" and calls System.setProperty("tornado.device.memory", ...)
+          # itself during model init, overwriting whatever we pass as a bare JVM flag. On Metal
+          # that 4GB budget isn't enough even for Llama-3.2-1B in batch-prefill-decode mode
+          # (TornadoOutOfMemoryException), so override the actual Quarkus config property instead.
           java  @"$TORNADOVM_HOME/tornado-argfile" \
-                -Dtornado.device.memory=8GB \
+                -Dquarkus.langchain4j.gpu-llama3.chat-model.device-memory=8GB \
                 -Dquarkus.http.port=$QUARKUS_PORT \
                 -jar target/quarkus-app/quarkus-run.jar > "$APP_LOG" 2>&1 &
           APP_PID=$!

From b7fbbdd581d08d5c80c22092448215e56b5ed2fb Mon Sep 17 00:00:00 2001
From: Orion Papadakis <orion.papadakis@manchester.ac.uk>
Date: Fri, 31 Jul 2026 14:14:14 +0300
Subject: [PATCH 12/17] ci: use curl --max-time instead of external timeout
 (unavailable on macOS); skip Granite-3.2-2b-f16 on Metal (model not
 provisioned yet)

---
 .github/workflows/build-and-run.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-run.yml b/.github/workflows/build-and-run.yml
index 9cffbfec..c18cbdcb 100644
--- a/.github/workflows/build-and-run.yml
+++ b/.github/workflows/build-and-run.yml
@@ -238,6 +238,8 @@ jobs:
           metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-phi3-mini-fp16-standard.json
 
       - name: FP16 - Run Granite-3.2-2b-instruct-f16.gguf
+        # TODO: not yet provisioned under /opt/models on the Mac runner; re-enable once it is.
+        if: matrix.backend.name != 'metal'
         uses: ./.github/actions/run-inference
         with:
           backend: ${{ matrix.backend.name }}
@@ -551,7 +553,9 @@ jobs:
         run: |
           for attempt in 1 2 3; do
             echo "Attempt $attempt of 3 for streaming endpoint..."
-            HTTP_CODE=$(timeout 10s curl -s -o /dev/null -w "%{http_code}" http://localhost:$QUARKUS_PORT/chat/streaming)
+            # curl's own --max-time (not the external `timeout` command, which stock
+            # macOS doesn't ship) so this works the same on both runner OSes.
+            HTTP_CODE=$(curl -s --max-time 10 -o /dev/null -w "%{http_code}" http://localhost:$QUARKUS_PORT/chat/streaming)
             if [ "$HTTP_CODE" = "200" ]; then
               echo "SUCCESS: HTTP $HTTP_CODE"
               break

From 6e595467855d437f2b5ec48b8ff976a0aef34d4f Mon Sep 17 00:00:00 2001
From: Orion Papadakis <orion.papadakis@manchester.ac.uk>
Date: Fri, 28 Aug 2026 16:36:22 +0300
Subject: [PATCH 13/17] ci: skip the remaining unprovisioned Granite models on
 Metal, and drop the Metal Quarkus leg
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

standalone-inference (metal, macos) failed on:

    java.io.FileNotFoundException: Model file not found: /opt/models/granite-4.0-1b-F16.gguf

Granite-3.2-2b-f16 and both Mistral models already carry
`if: matrix.backend.name != 'metal'` for this reason, but Granite-4.0-1b (F16 and Q8) and
Granite-3.2-2b Q8 were missed, so the job died at the first of them. Guards them the same
way. Re-drop the guards once those models are provisioned on the macOS runner.

The Quarkus leg failed differently — curl exit 28, the endpoint never answered inside
--max-time and the app had to be reaped as an orphan. That is not worth chasing yet: the
Metal backend builds, but its output is wrong. In the same run, "Say hello" on
qwen2.5-1.5b-instruct-fp16 came back as

    inspace Nachrichtirst Libya version}%ereumspect Races stireger ello...

which no assertion caught because the inference steps do not check what is generated. So
the Metal leg is removed from the Quarkus job rather than left failing for a reason that
has nothing to do with Quarkus. build (metal, macos) still runs and still proves the
backend compiles.

Metal generating wrong text is a real bug and wants its own investigation; this only stops
CI from reporting it as two unrelated integration failures.
---
 .github/workflows/build-and-run.yml | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-run.yml b/.github/workflows/build-and-run.yml
index c18cbdcb..e04bd6dd 100644
--- a/.github/workflows/build-and-run.yml
+++ b/.github/workflows/build-and-run.yml
@@ -250,6 +250,8 @@ jobs:
           metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-granite-3-2-2b-f16-standard.json
 
       - name: FP16 - Run Granite-4.0-1b-F16.gguf
+        # Not provisioned on the macOS runner; same as the other Granite/Mistral models.
+        if: matrix.backend.name != 'metal'
         uses: ./.github/actions/run-inference
         with:
           backend: ${{ matrix.backend.name }}
@@ -382,6 +384,8 @@ jobs:
           metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-mistral-7b-q8-standard.json
 
       - name: Q8 - Run Granite-3.2-2b-instruct-Q8_0.gguf
+        # Not provisioned on the macOS runner; same as the other Granite/Mistral models.
+        if: matrix.backend.name != 'metal'
         uses: ./.github/actions/run-inference
         with:
           backend: ${{ matrix.backend.name }}
@@ -392,6 +396,8 @@ jobs:
           metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-granite-3-2-2b-q8-standard.json
 
       - name: Q8 - Run Granite-4.0-1b-Q8_0.gguf
+        # Not provisioned on the macOS runner; same as the other Granite/Mistral models.
+        if: matrix.backend.name != 'metal'
         uses: ./.github/actions/run-inference
         with:
           backend: ${{ matrix.backend.name }}
@@ -424,8 +430,9 @@ jobs:
             os: linux
           - name: cuda
             os: linux
-          - name: metal
-            os: macos
+          # No metal leg: the Metal backend builds, but its generated text is wrong
+          # (prompt "Say hello" returns unrelated tokens), so a streaming-endpoint check
+          # against it cannot mean anything yet. Re-add once Metal produces correct output.
 
     steps:
       - name: Checkout GPULlama3

From 3302f4d23cd20d524f1d95d13551da7f24d94eb6 Mon Sep 17 00:00:00 2001
From: Orion Papadakis <orion.papadakis@manchester.ac.uk>
Date: Fri, 28 Aug 2026 17:10:25 +0300
Subject: [PATCH 14/17] ci: launch the Quarkus app with --add-modules
 jdk.incubator.vector
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Both Quarkus legs failed with HTTP 500 on the blocking endpoint. The dumped application
log gives the reason:

    java.lang.NoClassDefFoundError: jdk/incubator/vector/VectorShape
        at org.beehive.gpullama3.tensor.standard.FloatTensor.<clinit>(FloatTensor.java:21)
        at org.beehive.gpullama3.tensor.GGUF.loadTensorsTornado(GGUF.java:170)

FloatTensor's static initializer reads VectorShape to decide whether the Vector API is
available, so the class fails to initialize without the incubator module on the module
path — and every subsequent request then fails with "Could not initialize class
FloatTensor". The tornado argfile does not add it, and this launch was the only place in
the workflow that did not pass it explicitly.

Nothing to do with the model, the backend or the endpoint: the app could never have loaded
a model in this configuration.
---
 .github/workflows/build-and-run.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/build-and-run.yml b/.github/workflows/build-and-run.yml
index e04bd6dd..7624f890 100644
--- a/.github/workflows/build-and-run.yml
+++ b/.github/workflows/build-and-run.yml
@@ -520,7 +520,13 @@ jobs:
           # itself during model init, overwriting whatever we pass as a bare JVM flag. On Metal
           # that 4GB budget isn't enough even for Llama-3.2-1B in batch-prefill-decode mode
           # (TornadoOutOfMemoryException), so override the actual Quarkus config property instead.
+          # --add-modules jdk.incubator.vector: FloatTensor's static initializer reads
+          # VectorShape, so without it every request dies with
+          # NoClassDefFoundError: jdk/incubator/vector/VectorShape and the endpoint 500s.
+          # The tornado argfile does not supply it; every other run in this workflow passes
+          # it explicitly.
           java  @"$TORNADOVM_HOME/tornado-argfile" \
+                --add-modules jdk.incubator.vector \
                 -Dquarkus.langchain4j.gpu-llama3.chat-model.device-memory=8GB \
                 -Dquarkus.http.port=$QUARKUS_PORT \
                 -jar target/quarkus-app/quarkus-run.jar > "$APP_LOG" 2>&1 &

From ca9dbcf5a20b0ff10a160d9ef1f416b3dad61099 Mon Sep 17 00:00:00 2001
From: Orion Papadakis <orion.papadakis@manchester.ac.uk>
Date: Fri, 28 Aug 2026 17:19:24 +0300
Subject: [PATCH 15/17] ci: skip Llama Q8_0 batch-prefill-decode on CUDA
 (TornadoVM address-lowering gap)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

    TornadoInternalError: unimplemented: address origin unimplemented:
        tornado.graal.compiler.nodes.calc.Mul
      at CUDAAddressLowering.lower (tornado.drivers.cuda 5.2.1-jdk21-dev)
      at CUDACompiler.emitFrontEnd

The CUDA backend cannot lower an address whose origin is a multiply, which the Q8_0
tensor-core batch-prefill kernel produces. Scope, from the same run:

  FP16 batch-prefill-decode, CUDA          pass  (Llama and Qwen3)
  Q8_0 standard and prefill-decode, CUDA   pass
  Q8_0 batch-prefill-decode, OpenCL        pass
  Q8_0 batch-prefill-decode, Metal         pass
  Q8_0 batch-prefill-decode, CUDA          FAIL

So it is specific to Q8_0 batch-prefill on that one backend. It is not a regression from
anything landing here: this PR touches CI, and #129 (merged earlier today) touches no Q8_0
file — the kernel predates both.

Guarding the one step that is proven to fail rather than the whole Q8_0 batch family, so
the next run says whether Qwen3's equivalent shares the gap. Worth reporting upstream to
TornadoVM; drop the guard when the backend can lower it.
---
 .github/workflows/build-and-run.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/build-and-run.yml b/.github/workflows/build-and-run.yml
index 7624f890..bc7b76c7 100644
--- a/.github/workflows/build-and-run.yml
+++ b/.github/workflows/build-and-run.yml
@@ -296,6 +296,12 @@ jobs:
             prompt="Say hello"
 
       - name: Q8 - Run Llama-3.2-1B-Instruct-Q8_0.gguf - Batch-Prefill-Decode
+        # Fails on the CUDA backend only (OpenCL and Metal both pass this step):
+        #   TornadoInternalError: unimplemented: address origin unimplemented: ...calc.Mul
+        #   at CUDAAddressLowering.lower (tornado.drivers.cuda 5.2.1-jdk21-dev)
+        # A TornadoVM CUDA address-lowering gap on the Q8_0 tensor-core batch-prefill
+        # kernel, not a change in this repository. Drop the guard once it is fixed.
+        if: matrix.backend.name != 'cuda'
         env:
           JAVA_TOOL_OPTIONS: >-
             -Dllama.metrics.format=json

From 007aec756f7ef9ccf10bc71defbcfe81f295b173 Mon Sep 17 00:00:00 2001
From: Orion Papadakis <orion.papadakis@manchester.ac.uk>
Date: Fri, 28 Aug 2026 17:30:32 +0300
Subject: [PATCH 16/17] ci: skip Qwen3 Q8_0 batch-prefill-decode on CUDA too
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The previous commit guarded only the Llama step, to let CI say whether Qwen3's equivalent
shared the gap. It does — same TornadoInternalError from CUDAAddressLowering. Both Q8_0
batch-prefill steps are now skipped on CUDA; every other Q8_0 configuration, and FP16
batch-prefill, still run there.
---
 .github/workflows/build-and-run.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/build-and-run.yml b/.github/workflows/build-and-run.yml
index bc7b76c7..ca19ba43 100644
--- a/.github/workflows/build-and-run.yml
+++ b/.github/workflows/build-and-run.yml
@@ -347,6 +347,8 @@ jobs:
           metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-qwen3-0-6b-q8-prefill-decode.json
 
       - name: Q8 - Run Qwen3-0.6B-Q8_0.gguf - Batch-Prefill-Decode
+        # Same CUDA address-lowering gap as the Llama Q8_0 batch-prefill step above.
+        if: matrix.backend.name != 'cuda'
         uses: ./.github/actions/run-inference
         with:
           backend: ${{ matrix.backend.name }}

From dbe8a56051a62cd65d2b269d290a2a6b8240417e Mon Sep 17 00:00:00 2001
From: Orion Papadakis <orion.papadakis@manchester.ac.uk>
Date: Fri, 28 Aug 2026 18:00:37 +0300
Subject: [PATCH 17/17] ci: link the CUDA Q8_0 batch-prefill skips to
 TornadoVM#1057

Reproduced locally against TornadoVM develop 7fcd11b10 (and 803428c7c, the SHA CI built):
the CUDA backend aborts compiling gemmMMAQKVQ8 with

    unimplemented: address origin unimplemented: ...calc.MulNode
      at CUDAAddressLowering.lower

while its FP16 sibling gemmMMAQKV, with identical tiling and mmaFragment use, compiles in
the same process. OpenCL and Metal both compile the Q8_0 kernel. Filed upstream as
beehive-lab/TornadoVM#1057; the guards come out when that is fixed.
---
 .github/workflows/build-and-run.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-run.yml b/.github/workflows/build-and-run.yml
index ca19ba43..5a873daf 100644
--- a/.github/workflows/build-and-run.yml
+++ b/.github/workflows/build-and-run.yml
@@ -300,7 +300,8 @@ jobs:
         #   TornadoInternalError: unimplemented: address origin unimplemented: ...calc.Mul
         #   at CUDAAddressLowering.lower (tornado.drivers.cuda 5.2.1-jdk21-dev)
         # A TornadoVM CUDA address-lowering gap on the Q8_0 tensor-core batch-prefill
-        # kernel, not a change in this repository. Drop the guard once it is fixed.
+        # kernel (gemmMMAQKVQ8), not a change in this repository. Reproduced on develop
+        # 7fcd11b10 and reported as beehive-lab/TornadoVM#1057. Drop the guard once it is fixed.
         if: matrix.backend.name != 'cuda'
         env:
           JAVA_TOOL_OPTIONS: >-
@@ -347,7 +348,8 @@ jobs:
           metrics_file: ${{ runner.temp }}/metrics-${{ matrix.backend.name }}-qwen3-0-6b-q8-prefill-decode.json
 
       - name: Q8 - Run Qwen3-0.6B-Q8_0.gguf - Batch-Prefill-Decode
-        # Same CUDA address-lowering gap as the Llama Q8_0 batch-prefill step above.
+        # Same CUDA address-lowering gap as the Llama Q8_0 batch-prefill step above
+        # (beehive-lab/TornadoVM#1057).
         if: matrix.backend.name != 'cuda'
         uses: ./.github/actions/run-inference
         with: