feat(run-engine): per-environment worker queue routing overrides

Admins can override worker queue routing per environment. A new org
feature flag, `workerQueueEnvironmentOverrides`, maps an environment
type (`PRODUCTION`, `STAGING`, `PREVIEW`) or a specific environment id
to a worker queue compatibility and channel, so every staging
environment in an org can be born onto the compute lane while production
stays on containers.

Compat and channel each resolve from the environment id entry, then the
environment type entry, then the org-wide `workerQueueCompatibility` and
`workerQueueChannel` flags, then the existing default. Development
environments are never routed. An environment compat of `compute` also
moves that environment onto its region's compute backing worker group
when the region has one, and `container` stops it from being moved
there, ahead of the org's compute migration setting. The org feature
flags dialog gets a table to edit these overrides.

Mono-RevId: c8a73661232d8df52694856c514ecaecf1c1e714

Author: myftija

apps/webapp/app/components/admin/FeatureFlagsDialog.tsx

@@ -21,8 +21,14 @@ import {
   NumberControl,
   StringControl,
   WorkerGroupControl,
+  WorkerQueueEnvironmentOverridesControl,
   type WorkerGroup,
 } from "./FlagControls";
+import {
+  incompleteOverrideScopes,
+  type OverrideEnvironment,
+  type WorkerQueueEnvironmentOverrides,
+} from "./workerQueueEnvironmentOverrides";
 
 type LoaderData = {
   org: { id: string; title: string; slug: string };
@@ -31,6 +37,7 @@ type LoaderData = {
   controlTypes: Record<string, FlagControlType>;
   workerGroupName?: string;
   workerGroups?: WorkerGroup[];
+  environments?: OverrideEnvironment[];
   isManagedCloud?: boolean;
 };
 
@@ -95,6 +102,9 @@ export function FeatureFlagsDialog({
   }, [saveFetcher.data]);
 
   const isDirty = stableStringify(overrides) !== stableStringify(initialOverrides);
+  const environmentOverrides = (overrides[FEATURE_FLAG.workerQueueEnvironmentOverrides] ??
+    {}) as WorkerQueueEnvironmentOverrides;
+  const hasIncompleteOverrides = incompleteOverrideScopes(environmentOverrides).length > 0;
 
   const setFlagValue = (key: string, value: unknown) => {
     setOverrides((prev) => ({ ...prev, [key]: value }));
@@ -187,6 +197,61 @@ export function FeatureFlagsDialog({
 
                 const isOverridden = key in overrides;
 
+                if (key === FEATURE_FLAG.workerQueueEnvironmentOverrides) {
+                  const enumOptions = (flag: string) => {
+                    const flagControl = data.controlTypes[flag];
+                    return flagControl?.type === "enum" ? flagControl.options : [];
+                  };
+                  return (
+                    <div
+                      key={key}
+                      className={cn(
+                        "flex flex-col gap-2 rounded-md border px-3 py-2.5",
+                        isOverridden
+                          ? "border-indigo-500/20 bg-indigo-500/5"
+                          : "border-transparent bg-background-hover"
+                      )}
+                    >
+                      <div className="flex items-start justify-between gap-2">
+                        <div className="min-w-0">
+                          <div
+                            className={cn(
+                              "truncate text-sm",
+                              isOverridden ? "text-text-bright" : "text-text-dimmed"
+                            )}
+                          >
+                            {key}
+                          </div>
+                          <div className="text-xs text-text-dimmed">
+                            Per field: environment id, then type, then the org flags above. A compat
+                            of compute or container here also decides the compute backing swap.
+                          </div>
+                        </div>
+                        <Button
+                          variant="minimal/small"
+                          onClick={() => unsetFlag(key)}
+                          className={cn(!isOverridden && "invisible")}
+                        >
+                          unset
+                        </Button>
+                      </div>
+                      <WorkerQueueEnvironmentOverridesControl
+                        value={environmentOverrides}
+                        environments={data.environments ?? []}
+                        compatOptions={enumOptions(FEATURE_FLAG.workerQueueCompatibility)}
+                        channelOptions={enumOptions(FEATURE_FLAG.workerQueueChannel)}
+                        onChange={(val) => {
+                          if (val === undefined) {
+                            unsetFlag(key);
+                          } else {
+                            setFlagValue(key, val);
+                          }
+                        }}
+                      />
+                    </div>
+                  );
+                }
+
                 return (
                   <div
                     key={key}
@@ -296,13 +361,21 @@ export function FeatureFlagsDialog({
           </details>
         )}
 
+        {hasIncompleteOverrides && (
+          <Callout variant="error">Each environment override needs a compat or a channel.</Callout>
+        )}
+
         {saveError && <Callout variant="error">{saveError}</Callout>}
 
         <DialogFooter>
           <Button variant="tertiary/small" onClick={() => onOpenChange(false)}>
             Cancel
           </Button>
-          <Button variant="primary/small" onClick={handleSave} disabled={!isDirty || isSaving}>
+          <Button
+            variant="primary/small"
+            onClick={handleSave}
+            disabled={!isDirty || isSaving || hasIncompleteOverrides}
+          >
             {isSaving ? "Saving..." : "Save changes"}
           </Button>
         </DialogFooter>

apps/webapp/app/components/admin/FlagControls.tsx

@@ -1,7 +1,19 @@
+import { useState } from "react";
 import { Switch } from "~/components/primitives/Switch";
 import { Select, SelectItem } from "~/components/primitives/Select";
 import { Input } from "~/components/primitives/Input";
+import { Button } from "~/components/primitives/Buttons";
 import { cn } from "~/utils/cn";
+import type { WorkerQueueEnvironmentOverride } from "~/v3/featureFlags";
+import {
+  availableOverrideScopes,
+  orderedOverrideScopes,
+  overrideScopeLabel,
+  removeOverride,
+  setOverrideField,
+  type OverrideEnvironment,
+  type WorkerQueueEnvironmentOverrides,
+} from "./workerQueueEnvironmentOverrides";
 
 export const UNSET_VALUE = "__unset__";
 
@@ -153,3 +165,130 @@ export function StringControl({
     />
   );
 }
+
+type Compat = WorkerQueueEnvironmentOverride["compat"];
+type Channel = WorkerQueueEnvironmentOverride["channel"];
+
+function fromEnumValue<T extends string>(val: string): T | undefined {
+  return val === UNSET_VALUE ? undefined : (val as T);
+}
+
+export function WorkerQueueEnvironmentOverridesControl({
+  value,
+  environments,
+  compatOptions,
+  channelOptions,
+  onChange,
+}: {
+  value: WorkerQueueEnvironmentOverrides;
+  environments: OverrideEnvironment[];
+  compatOptions: string[];
+  channelOptions: string[];
+  onChange: (val: WorkerQueueEnvironmentOverrides | undefined) => void;
+}) {
+  const [draftScope, setDraftScope] = useState(UNSET_VALUE);
+  const [draftCompat, setDraftCompat] = useState<Compat>(undefined);
+  const [draftChannel, setDraftChannel] = useState<Channel>(undefined);
+
+  const label = (scope: string) => overrideScopeLabel(scope, environments);
+  const availableScopes = availableOverrideScopes(value, environments);
+  const scopeItems = [UNSET_VALUE, ...availableScopes];
+  const canAdd =
+    availableScopes.includes(draftScope) &&
+    (draftCompat !== undefined || draftChannel !== undefined);
+
+  const addOverride = () => {
+    if (!canAdd) return;
+    const entry: WorkerQueueEnvironmentOverride = {};
+    if (draftCompat !== undefined) entry.compat = draftCompat;
+    if (draftChannel !== undefined) entry.channel = draftChannel;
+    onChange({ ...value, [draftScope]: entry });
+    setDraftScope(UNSET_VALUE);
+    setDraftCompat(undefined);
+    setDraftChannel(undefined);
+  };
+
+  return (
+    <div className="grid grid-cols-[minmax(0,1fr)_auto_auto_auto] items-center gap-x-2 gap-y-1.5">
+      <div className="text-xs text-text-dimmed">scope</div>
+      <div className="text-xs text-text-dimmed">compat</div>
+      <div className="text-xs text-text-dimmed">channel</div>
+      <div />
+
+      {orderedOverrideScopes(value, environments).map((scope) => {
+        const entry = value[scope];
+        const incomplete = entry.compat === undefined && entry.channel === undefined;
+        return (
+          <div key={scope} className="contents">
+            <div
+              className={cn("truncate text-sm", incomplete ? "text-error" : "text-text-bright")}
+              title={incomplete ? "Set compat or channel, or remove this row" : scope}
+            >
+              {label(scope)}
+            </div>
+            <EnumControl
+              value={entry.compat}
+              options={compatOptions}
+              onChange={(val) =>
+                onChange(
+                  setOverrideField(value, scope, "compat", fromEnumValue<NonNullable<Compat>>(val))
+                )
+              }
+              dimmed={false}
+            />
+            <EnumControl
+              value={entry.channel}
+              options={channelOptions}
+              onChange={(val) =>
+                onChange(
+                  setOverrideField(
+                    value,
+                    scope,
+                    "channel",
+                    fromEnumValue<NonNullable<Channel>>(val)
+                  )
+                )
+              }
+              dimmed={false}
+            />
+            <Button variant="minimal/small" onClick={() => onChange(removeOverride(value, scope))}>
+              remove
+            </Button>
+          </div>
+        );
+      })}
+
+      <Select
+        variant="tertiary/small"
+        value={draftScope}
+        setValue={setDraftScope}
+        items={scopeItems}
+        filter={(item, search) => label(item).toLowerCase().includes(search.toLowerCase())}
+        text={(val) => (val === UNSET_VALUE ? "add environment..." : label(val))}
+      >
+        {(items) =>
+          items.map((item) => (
+            <SelectItem key={item} value={item}>
+              {item === UNSET_VALUE ? "none" : label(item)}
+            </SelectItem>
+          ))
+        }
+      </Select>
+      <EnumControl
+        value={draftCompat}
+        options={compatOptions}
+        onChange={(val) => setDraftCompat(fromEnumValue<NonNullable<Compat>>(val))}
+        dimmed={draftScope === UNSET_VALUE}
+      />
+      <EnumControl
+        value={draftChannel}
+        options={channelOptions}
+        onChange={(val) => setDraftChannel(fromEnumValue<NonNullable<Channel>>(val))}
+        dimmed={draftScope === UNSET_VALUE}
+      />
+      <Button variant="secondary/small" onClick={addOverride} disabled={!canAdd}>
+        add
+      </Button>
+    </div>
+  );
+}

apps/webapp/app/components/admin/workerQueueEnvironmentOverrides.ts

@@ -0,0 +1,94 @@
+import {
+  WORKER_QUEUE_OVERRIDE_ENVIRONMENT_TYPES,
+  type WorkerQueueEnvironmentOverride,
+} from "~/v3/featureFlags";
+
+export type WorkerQueueEnvironmentOverrides = Record<string, WorkerQueueEnvironmentOverride>;
+
+export type OverrideEnvironment = {
+  id: string;
+  type: string;
+  branchName: string | null;
+  projectName: string;
+};
+
+const ENVIRONMENT_TYPE_LABELS: Record<string, string> = {
+  PRODUCTION: "Production",
+  STAGING: "Staging",
+  PREVIEW: "Preview",
+};
+
+export function overrideScopeLabel(
+  scope: string,
+  environments: readonly OverrideEnvironment[]
+): string {
+  const typeLabel = ENVIRONMENT_TYPE_LABELS[scope];
+  if (typeLabel) {
+    return `All ${typeLabel.toLowerCase()}`;
+  }
+
+  const environment = environments.find((environment) => environment.id === scope);
+  if (!environment) {
+    return `Unknown environment (${scope})`;
+  }
+
+  const name = ENVIRONMENT_TYPE_LABELS[environment.type] ?? environment.type;
+  return environment.branchName
+    ? `${environment.projectName} / ${name} (${environment.branchName})`
+    : `${environment.projectName} / ${name}`;
+}
+
+export function orderedOverrideScopes(
+  overrides: WorkerQueueEnvironmentOverrides,
+  environments: readonly OverrideEnvironment[]
+): string[] {
+  const known = [
+    ...WORKER_QUEUE_OVERRIDE_ENVIRONMENT_TYPES,
+    ...environments.map((environment) => environment.id),
+  ];
+  const scopes = Object.keys(overrides);
+  return [
+    ...known.filter((scope) => scopes.includes(scope)),
+    ...scopes.filter((scope) => !known.includes(scope)).sort(),
+  ];
+}
+
+export function availableOverrideScopes(
+  overrides: WorkerQueueEnvironmentOverrides,
+  environments: readonly OverrideEnvironment[]
+): string[] {
+  return [
+    ...WORKER_QUEUE_OVERRIDE_ENVIRONMENT_TYPES,
+    ...environments.map((environment) => environment.id),
+  ].filter((scope) => !(scope in overrides));
+}
+
+export function setOverrideField<K extends keyof WorkerQueueEnvironmentOverride>(
+  overrides: WorkerQueueEnvironmentOverrides,
+  scope: string,
+  field: K,
+  value: WorkerQueueEnvironmentOverride[K] | undefined
+): WorkerQueueEnvironmentOverrides {
+  const entry = { ...overrides[scope] };
+  if (value === undefined) {
+    delete entry[field];
+  } else {
+    entry[field] = value;
+  }
+  return { ...overrides, [scope]: entry };
+}
+
+export function removeOverride(
+  overrides: WorkerQueueEnvironmentOverrides,
+  scope: string
+): WorkerQueueEnvironmentOverrides | undefined {
+  const next = { ...overrides };
+  delete next[scope];
+  return Object.keys(next).length === 0 ? undefined : next;
+}
+
+export function incompleteOverrideScopes(overrides: WorkerQueueEnvironmentOverrides): string[] {
+  return Object.entries(overrides)
+    .filter(([, entry]) => entry.compat === undefined && entry.channel === undefined)
+    .map(([scope]) => scope);
+}