diff --git a/apps/hub/src/config/models.test.ts b/apps/hub/src/config/models.test.ts index 46cd88ee..6d8aa8db 100644 --- a/apps/hub/src/config/models.test.ts +++ b/apps/hub/src/config/models.test.ts @@ -1,5 +1,10 @@ import { expect, test } from "bun:test"; -import { MODEL_PROVIDERS, resolveModelChoice } from "./models.ts"; +import { + MODEL_PROVIDERS, + frozenModelChoice, + invalidModelChoice, + resolveModelChoice, +} from "./models.ts"; test("a project runs on the only connected provider's profile, a chosen one, or none", () => { expect( @@ -32,4 +37,70 @@ test("a project runs on the only connected provider's profile, a chosen one, or ).toEqual({ refused: "model_profile_required", }); + expect( + resolveModelChoice(MODEL_PROVIDERS, { profile: "openai_default" }, [ + "anthropic", + ]), + ).toMatchObject({ + choice: { profile: "anthropic_default" }, + }); + expect( + resolveModelChoice(MODEL_PROVIDERS, { profile: "xai_default" }, [ + "anthropic", + "openai", + ]), + ).toEqual({ + refused: "model_profile_required", + }); + expect( + frozenModelChoice(MODEL_PROVIDERS, { profile: "openai_default" }, [ + "anthropic", + ]), + ).toBeNull(); +}); + +test("invalidModelChoice refuses activating a profile whose provider is not connected", () => { + const custom = { + id: "custom_mix", + label: "Mix", + tiers: { + low: { provider: "anthropic", model: "claude-haiku-5-5", effort: "low" }, + medium: { + provider: "openai", + model: "gpt-6.1-terra", + effort: "medium", + }, + high: { + provider: "anthropic", + model: "claude-opus-5-5", + effort: "medium", + }, + max: { provider: "anthropic", model: "claude-opus-5-5", effort: "max" }, + }, + } as const; + + expect( + invalidModelChoice(MODEL_PROVIDERS, { profile: "anthropic_default" }, [ + "anthropic", + ]), + ).toBeUndefined(); + expect( + invalidModelChoice(MODEL_PROVIDERS, { profile: "openai_default" }, [ + "anthropic", + ]), + ).toBe("openai_default:low:unconnected:openai"); + expect( + invalidModelChoice( + MODEL_PROVIDERS, + { profile: "custom_mix", custom: [custom] }, + ["anthropic"], + ), + ).toBe("custom_mix:medium:unconnected:openai"); + expect( + invalidModelChoice( + MODEL_PROVIDERS, + { profile: "anthropic_default", custom: [custom] }, + ["anthropic"], + ), + ).toBeUndefined(); }); diff --git a/apps/hub/src/config/models.ts b/apps/hub/src/config/models.ts index 56db6473..44f2c312 100644 --- a/apps/hub/src/config/models.ts +++ b/apps/hub/src/config/models.ts @@ -386,13 +386,16 @@ export const PROFILES: readonly Profile[] = [ }, ), ]; -/** Why a binding cannot run: an unlisted provider or model, or a model without limits. */ +/** Why a binding cannot run: an unlisted or unconnected provider, an unlisted model, or no limits. */ function unrunnable( directory: ModelDirectory, binding: RoleBinding, + connected?: readonly string[], ): string | undefined { const provider = modelProvider(directory, binding.provider); if (!provider) return `provider:${binding.provider}`; + if (connected && !connected.includes(binding.provider)) + return `unconnected:${binding.provider}`; if (!provider.models.some((model) => model.id === binding.model)) return `model:${binding.provider}:${binding.model}`; if (!bindingLimits(directory, binding.provider, binding.model)) @@ -406,21 +409,23 @@ for (const profile of PROFILES) if (reason) throw new Error(`profile_unrunnable:${profile.id}:${reason}`); } -/** The first part of a choice the profiles cannot run: its profile or a made profile's tier. */ +/** The first part of a choice the profiles cannot run: its profile or the chosen profile's tier. */ export function invalidModelChoice( directory: ModelDirectory, choice: ModelChoice, + connected?: readonly string[], ): string | undefined { const custom = choice.custom ?? []; - const ids = [...PROFILES, ...custom].map((profile) => profile.id); + const profiles = [...PROFILES, ...custom]; + const ids = profiles.map((profile) => profile.id); if (new Set(ids).size !== ids.length) return "duplicate_profile"; if (choice.profile !== undefined && !ids.includes(choice.profile)) return choice.profile; // A made profile nobody runs may name a provider since removed; only the chosen one must run. - const chosen = custom.find((profile) => profile.id === choice.profile); + const chosen = profiles.find((profile) => profile.id === choice.profile); if (chosen) { for (const [tier, binding] of Object.entries(chosen.tiers)) { - const reason = unrunnable(directory, binding); + const reason = unrunnable(directory, binding, connected); if (reason) return `${chosen.id}:${tier}:${reason}`; } } @@ -431,13 +436,14 @@ export function invalidModelChoice( function chosenProfile( directory: ModelDirectory, choice: ModelChoice | null, + connected?: readonly string[], ): CustomProfile | undefined { const profile = choice?.custom?.find((entry) => entry.id === choice.profile) ?? PROFILES.find((entry) => entry.id === choice?.profile); return profile && Object.values(profile.tiers).every( - (binding) => !unrunnable(directory, binding), + (binding) => !unrunnable(directory, binding, connected), ) ? profile : undefined; @@ -470,7 +476,7 @@ export function resolveModelChoice( choice: ModelChoice | null, connected: readonly string[], ): { choice: ModelChoice } | { refused: ModelChoiceRefusal } { - const chosen = chosenProfile(directory, choice); + const chosen = chosenProfile(directory, choice, connected); if (chosen) return { choice: frozen(chosen) }; const providers = [...new Set(connected)]; const only = @@ -518,8 +524,9 @@ function frozen(profile: CustomProfile): ModelChoice { export function frozenModelChoice( directory: ModelDirectory, choice: ModelChoice | null, + connected?: readonly string[], ): ModelChoice | null { - const profile = chosenProfile(directory, choice); + const profile = chosenProfile(directory, choice, connected); return profile ? frozen(profile) : null; } diff --git a/apps/hub/src/projects/model-choice.ts b/apps/hub/src/projects/model-choice.ts index 2c341196..d833ec1d 100644 --- a/apps/hub/src/projects/model-choice.ts +++ b/apps/hub/src/projects/model-choice.ts @@ -52,22 +52,6 @@ async function personalChoice( .models.value; } -// A choice runs only when every tier's provider is connected in this workspace, not merely offered. -function frozenOn( - directory: ModelDirectory, - choice: ModelChoice | null, - connected: readonly string[], -): ModelChoice | null { - const kept = frozenModelChoice(directory, choice); - const profile = kept?.custom?.find((entry) => entry.id === kept.profile); - return profile && - Object.values(profile.tiers).every((binding) => - connected.includes(binding.provider), - ) - ? kept - : null; -} - /** * The choice a project's roles run on. A project keeps the profile it started with; one that has * none this server can run on a connected provider takes the caller's choice or the only connected @@ -81,11 +65,11 @@ export async function projectModelChoice( ): Promise { const { models } = await loadSettings(db, actor.tenantId, projectId); const connected = await connectedProviders(db, directory, actor.tenantId); - const kept = frozenOn(directory, models.value, connected); + const kept = frozenModelChoice(directory, models.value, connected); if (kept) return kept; const resolved = resolveModelChoice( directory, - frozenOn(directory, await personalChoice(db, actor), connected), + await personalChoice(db, actor), connected, ); if ("refused" in resolved) diff --git a/apps/hub/src/settings/service.ts b/apps/hub/src/settings/service.ts index 54dfccb7..b1499f89 100644 --- a/apps/hub/src/settings/service.ts +++ b/apps/hub/src/settings/service.ts @@ -26,6 +26,7 @@ import { } from "./policy.ts"; import { invalidPreference } from "../config/preferences.ts"; import { invalidModelChoice } from "../config/models.ts"; +import { connectedProviders } from "../projects/model-choice.ts"; import { modelDirectory } from "./custom-providers.ts"; import { digest } from "../lib/digest.ts"; import type { Actor } from "../lib/actor.ts"; @@ -90,11 +91,15 @@ export function createSettingsService(db: DB["db"]) { preference: unknown, }); const models = input.fields.models?.value; - const refused = - models && - invalidModelChoice(await modelDirectory(db, actor.tenantId), models); - if (refused) - throw new ProjectError(400, "unknown_model_choice", { choice: refused }); + if (models) { + const directory = await modelDirectory(db, actor.tenantId); + const connected = await connectedProviders(db, directory, actor.tenantId); + const refused = invalidModelChoice(directory, models, connected); + if (refused) + throw new ProjectError(400, "unknown_model_choice", { + choice: refused, + }); + } return input; } return { diff --git a/apps/web/src/pages/profile-select.test.tsx b/apps/web/src/pages/profile-select.test.tsx index 228f7750..c041d0cb 100644 --- a/apps/web/src/pages/profile-select.test.tsx +++ b/apps/web/src/pages/profile-select.test.tsx @@ -1,9 +1,15 @@ import { describe, expect, test } from "bun:test"; import { renderToStaticMarkup } from "react-dom/server"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import type { ModelCatalog, Tiers } from "@corbits-builder/schemas/models"; +import type { + ModelCatalog, + ModelChoice, + Tiers, +} from "@corbits-builder/schemas/models"; +import { wording } from "../config.ts"; import { keys } from "../queries/keys.ts"; import { NewProjectProfile } from "./profile-select.tsx"; +import { Profiles, available, editorProviders } from "./profiles.tsx"; function tiers(provider: string): Tiers { const binding = { provider, model: "m", effort: "medium" as const }; @@ -12,10 +18,60 @@ function tiers(provider: string): Tiers { function catalog(): ModelCatalog { return { - providers: [], - roles: [], - tiers: [], - efforts: [], + providers: [ + { + id: "xai", + plugin: "xai", + kind: "login", + label: "xAI", + hint: "xAI", + baseURL: "https://x.ai", + models: [{ id: "m", label: "Grok 4.7" }], + connections: [], + }, + { + id: "anthropic", + plugin: "anthropic", + kind: "api_key", + label: "Anthropic", + hint: "Anthropic", + baseURL: "https://api.anthropic.com", + models: [{ id: "m", label: "Claude" }], + connections: [ + { + id: "c1", + name: "Anthropic", + status: "connected", + credentialSet: true, + }, + ], + }, + { + id: "openai", + plugin: "openai-responses", + kind: "api_key", + label: "OpenAI", + hint: "OpenAI", + baseURL: "https://api.openai.com/v1", + models: [{ id: "m", label: "GPT" }], + connections: [ + { + id: "c2", + name: "OpenAI", + status: "connected", + credentialSet: true, + }, + ], + }, + ], + roles: [{ id: "coordinator", label: "Coordinator", hint: "", tier: "low" }], + tiers: [ + { id: "low", label: "Low" }, + { id: "medium", label: "Medium" }, + { id: "high", label: "High" }, + { id: "max", label: "Max" }, + ], + efforts: [{ id: "medium", label: "Medium" }], customApis: [], profiles: [ { @@ -56,6 +112,18 @@ function render(value: string | undefined, models = catalog()) { ); } +function renderProfiles(choice: ModelChoice | null, models = catalog()) { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false, staleTime: Infinity } }, + }); + client.setQueryData(keys.modelCatalog, models); + return renderToStaticMarkup( + + + , + ); +} + describe("NewProjectProfile", () => { test("selects the first runnable profile when none is automatic", () => { const initial = render(undefined); @@ -82,3 +150,52 @@ describe("NewProjectProfile", () => { expect(render(undefined, models)).toBe(""); }); }); + +describe("Profiles", () => { + test("offers Use only on profiles whose providers are connected", () => { + const html = renderProfiles({ profile: "anthropic_default" }); + expect(html).toContain(wording.profiles.unavailable); + expect(html).toContain(wording.profiles.active); + // Only GPT is runnable and not active; Grok is unavailable so it has no Use button. + expect(html.split(`>${wording.profiles.use}<`).length - 1).toBe(1); + }); + + test("falls back when the saved profile's provider is no longer connected", () => { + const models = catalog(); + models.automaticProfile = "anthropic_default"; + models.unresolved = { + code: "model_profile_required", + message: "Choose a connected profile.", + }; + const html = renderProfiles({ profile: "xai_default" }, models); + expect(html).toContain(wording.profiles.active); + expect(html).toContain(wording.profiles.detail("Claude")); + expect(html).not.toContain(wording.profiles.detail("Grok")); + }); + + test("shows the unresolved banner when the saved profile's provider is disconnected and none is automatic", () => { + const models = catalog(); + models.unresolved = { + code: "model_profile_required", + message: "Choose a connected profile.", + }; + const html = renderProfiles({ profile: "xai_default" }, models); + expect(html).not.toContain(wording.profiles.active); + expect(html).toContain("Choose a connected profile."); + }); + + test("lists only connected providers when building a custom profile", () => { + const models = catalog(); + expect(editorProviders(models).map((entry) => entry.id)).toEqual([ + "anthropic", + "openai", + ]); + expect(editorProviders(models, "xai").map((entry) => entry.id)).toEqual([ + "xai", + "anthropic", + "openai", + ]); + expect(available(models, tiers("anthropic"))).toBe(true); + expect(available(models, tiers("xai"))).toBe(false); + }); +}); diff --git a/apps/web/src/pages/profile-select.tsx b/apps/web/src/pages/profile-select.tsx index cb34aed0..d03fe621 100644 --- a/apps/web/src/pages/profile-select.tsx +++ b/apps/web/src/pages/profile-select.tsx @@ -19,7 +19,7 @@ import { wording } from "../config.ts"; import { keys } from "../queries/keys.ts"; import { workspaceTenantId } from "../api/auth.ts"; import { productClient } from "../api/transport.ts"; -import { connected } from "./profiles.jsx"; +import { available } from "./profiles.jsx"; import { useModelCatalog } from "./providers.jsx"; const copy = wording.profiles; @@ -46,9 +46,7 @@ function profileOptions(catalog: ModelCatalog, choice: ModelChoice | null) { return [ ...catalog.profiles.filter((profile) => profile.available), ...(choice?.custom ?? []).filter((profile) => - Object.values(profile.tiers).every((binding) => - connected(catalog, binding.provider), - ), + available(catalog, profile.tiers), ), ].map((profile) => option(catalog, profile)); } diff --git a/apps/web/src/pages/profiles.tsx b/apps/web/src/pages/profiles.tsx index 6b745221..1139688f 100644 --- a/apps/web/src/pages/profiles.tsx +++ b/apps/web/src/pages/profiles.tsx @@ -20,6 +20,7 @@ import { DialogTitle, } from "@corbits/react-ui/ui/dialog"; import { useEffect, useState } from "react"; +import { useQueryClient } from "@tanstack/react-query"; import type { CustomProfile, ModelCatalog, @@ -28,6 +29,7 @@ import type { Tier, } from "@corbits-builder/schemas/models"; import { Banner, Button, QuietSelect, QuietState } from "../components.jsx"; +import { keys } from "../queries/keys.ts"; import { productClient } from "../api/transport.ts"; import { workspaceTenantId } from "../api/auth.ts"; import { useModelCatalog } from "./providers.jsx"; @@ -66,6 +68,17 @@ export function connected(catalog: ModelCatalog, provider: string): boolean { ); } +export function available(catalog: ModelCatalog, tiers: Tiers): boolean { + return Object.values(tiers).every((binding) => + connected(catalog, binding.provider), + ); +} + +function profileSubtitle(profile: Profile, runnable: boolean) { + if (!runnable) return copy.unavailable; + return profile.custom ? copy.yours : profile.hint; +} + function complete(draft: Draft, tiers: readonly Tier[]): Tiers | null { const done: Partial> = {}; for (const tier of tiers) { @@ -76,20 +89,22 @@ function complete(draft: Draft, tiers: readonly Tier[]): Tiers | null { return done as Tiers; } -export function Profiles() { +export function Profiles({ initialField }: { initialField?: Field } = {}) { const catalog = useModelCatalog(); - const [field, setField] = useState(null); + const queryClient = useQueryClient(); + const [field, setField] = useState(initialField ?? null); const [error, setError] = useState(null); const [editing, setEditing] = useState(null); const [viewing, setViewing] = useState(null); useEffect(() => { + if (initialField) return; void workspaceTenantId() .then((tenantId) => productClient().personalSettings(tenantId)) .then((record) => setField(record.fields.models ?? { value: null, version: 0 }), ) .catch((cause: unknown) => setError(message(cause))); - }, []); + }, [initialField]); const save = async (choice: ModelChoice) => { if (!field) return false; @@ -103,6 +118,7 @@ export function Profiles() { }, ); setField(receipt.record.fields.models ?? field); + void queryClient.invalidateQueries({ queryKey: keys.modelChoice }); setError(receipt.conflicts.length ? wording.message.stepFailed : null); return receipt.conflicts.length === 0; } catch (cause) { @@ -128,8 +144,14 @@ export function Profiles() { ...custom.map((entry) => ({ ...entry, custom: true })), ]; const active = - profiles.find((entry) => entry.id === field.value?.profile) ?? - profiles.find((entry) => entry.id === data.automaticProfile); + profiles.find( + (entry) => + entry.id === field.value?.profile && available(data, entry.tiers), + ) ?? + profiles.find( + (entry) => + entry.id === data.automaticProfile && available(data, entry.tiers), + ); const viewed = profiles.find((entry) => entry.id === viewing) ?? active; const choice = ( profile: string | undefined, @@ -138,8 +160,6 @@ export function Profiles() { ...(profile ? { profile } : {}), ...(list.length ? { custom: list } : {}), }); - const available = (tiers: Tiers) => - Object.values(tiers).every((binding) => connected(data, binding.provider)); return ( <> @@ -156,72 +176,68 @@ export function Profiles() { {!active && data.unresolved ? ( ) : null} - {profiles.map((profile) => ( -
-
- {profile.label} - - {!available(profile.tiers) - ? copy.unavailable - : profile.custom - ? copy.yours - : profile.hint} - -
- - {profile.custom ? ( - <> + {profiles.map((profile) => { + const runnable = available(data, profile.tiers); + const activeRow = profile.id === active?.id; + return ( +
+
+ {profile.label} + {profileSubtitle(profile, runnable)} +
+ + {profile.custom ? ( + <> + + + + ) : null} + {profile.id === viewed?.id ? null : ( + )} + {activeRow ? ( + {copy.active} + ) : null} + {!activeRow && runnable ? ( - - ) : null} - {profile.id === viewed?.id ? null : ( - - )} - {profile.id === active?.id ? ( - {copy.active} - ) : ( - - )} - -
- ))} + ) : null} + +
+ ); + })} {error ? : null} @@ -264,7 +280,9 @@ export function Profiles() { {editing ? ( + available(data, profile.tiers), + )} editing={editing} onClose={() => setEditing(null)} onSave={async (profile) => { @@ -274,8 +292,7 @@ export function Profiles() { entry.id === profile.id ? profile : entry, ) : [...custom, profile]; - if (await save(choice(field.value?.profile, list))) - setEditing(null); + if (await save(choice(active?.id, list))) setEditing(null); }} /> ) : null} @@ -283,6 +300,12 @@ export function Profiles() { ); } +export function editorProviders(catalog: ModelCatalog, current?: string) { + return catalog.providers.filter( + (entry) => connected(catalog, entry.id) || entry.id === current, + ); +} + /** The four tiers of one profile; a new one starts blank or from any existing profile. */ function ProfileEditor({ catalog, @@ -307,7 +330,10 @@ function ProfileEditor({ draft, catalog.tiers.map((tier) => tier.id), ); - const ready = filled !== null && draft.label.trim().length > 0; + const ready = + filled !== null && + draft.label.trim().length > 0 && + available(catalog, filled); const setTier = (tier: Tier, binding: Partial) => setDraft((current) => ({ ...current, @@ -389,6 +415,7 @@ function ProfileEditor({ const provider = catalog.providers.find( (entry) => entry.id === binding.provider, ); + const providers = editorProviders(catalog, binding.provider); return (
{copy.choose} - {catalog.providers.map((entry) => ( - - ))} + {providers.map((entry) => { + const availableProvider = connected(catalog, entry.id); + return ( + + ); + })}