fix(run-engine): the repair clear drains the group set for keyless runs

clearMessageFromConcurrencySets now mirrors the group SREM like every
other release path, so a stuck keyless run cleared by the repair sweep
frees its total-concurrency slot instead of phantom-holding it until a
saturation reconcile. The plain gauge test pins the new tail: zeroed
CK fields plus the total pair.

Author: matt-aitken

internal-packages/run-engine/src/run-queue/index.ts

@@ -3112,8 +3112,9 @@ export class RunQueue {
     const messageKey = this.keys.messageKey(orgId, messageId);
     /**
      * Callers pass the bare TaskRun queue name plus its concurrencyKey; the run's
-     * slots live on the ck variant, and the tracked clear additionally mirrors the
-     * group set and counters that only keyed queues maintain.
+     * slots live on the ck variant. Both variants mirror the per-base-queue group
+     * set (keyed and keyless admits populate it); the tracked clear additionally
+     * maintains the counters that only keyed queues keep.
      */
     const fullQueue = concurrencyKey ? this.keys.queueKey(env, queue, concurrencyKey) : queue;
     const queueCurrentConcurrencyKey = this.keys.queueCurrentConcurrencyKey(
@@ -3159,6 +3160,7 @@ export class RunQueue {
       queueCurrentDequeuedKey,
       envCurrentDequeuedKey,
       messageKey,
+      this.keys.queueGroupConcurrencyKeyFromQueue(fullQueue),
       messageId,
       this.options.redis.keyPrefix ?? ""
     );
@@ -6238,22 +6240,27 @@ return results
     });
 
     this.redis.defineCommand("clearMessageFromConcurrencySets", {
-      numberOfKeys: 5,
+      numberOfKeys: 6,
       lua: `
 -- Keys:
 local queueCurrentConcurrencyKey = KEYS[1]
 local envCurrentConcurrencyKey = KEYS[2]
 local queueCurrentDequeuedKey = KEYS[3]
 local envCurrentDequeuedKey = KEYS[4]
 local messageKey = KEYS[5]
+local groupConcurrencyKey = KEYS[6]
 
 -- Args:
 local messageId = ARGV[1]
 local keyPrefix = ARGV[2]
 ${QUEUE_GATES_LUA_HELPERS}
 
--- Update the concurrency keys
-redis.call('SREM', queueCurrentConcurrencyKey, messageId)
+-- Update the concurrency keys. The groupConcurrency SREM mirrors the base SREM
+-- unconditionally (no flag check) so a disabled flag still drains the group set.
+local removedFromCurrentConcurrency = redis.call('SREM', queueCurrentConcurrencyKey, messageId)
+if removedFromCurrentConcurrency == 1 then
+  redis.call('SREM', groupConcurrencyKey, messageId)
+end
 redis.call('SREM', envCurrentConcurrencyKey, messageId)
 redis.call('SREM', queueCurrentDequeuedKey, messageId)
 redis.call('SREM', envCurrentDequeuedKey, messageId)
@@ -6473,6 +6480,7 @@ declare module "@internal/redis" {
       queueCurrentDequeuedKey: string,
       envCurrentDequeuedKey: string,
       messageKey: string,
+      groupConcurrencyKey: string,
       // args
       messageId: string,
       keyPrefix: string,

internal-packages/run-engine/src/run-queue/metrics.test.ts

@@ -156,9 +156,15 @@ describe("RunQueue queue-metrics emission", () => {
       for (const f of ["ql", "cc", "lim", "eql", "ec", "elim", "thr"]) {
         expect(gauge!.fields[f]).toBeDefined();
       }
-      // Non-CK scripts keep the 7-field gauge (no CK-health tail).
-      expect(gauge!.fields.ckq).toBeUndefined();
-      expect(gauge!.fields.ckw).toBeUndefined();
+      /**
+       * Non-CK scripts emit the full gauge tail too: zeroed CK-health fields (a base
+       * queue has no CK variants) followed by the total running/limit pair, so a
+       * keyless queue with a total limit still charts total concurrency.
+       */
+      expect(gauge!.fields.ckq).toBe("0");
+      expect(gauge!.fields.ckw).toBe("0");
+      expect(gauge!.fields.tcc).toBeDefined();
+      expect(gauge!.fields.tlim).toBeDefined();
 
       // Pins the dequeue script's sample-at-return wrapper: only the dequeue emits the
       // post-admission reading (running 1, queued 0); the enqueue gauge sees the inverse.