Skip to content

Commit 8da229f

Browse files
feat(check): parse vapi-checks.yml check configuration (#60)
## Value **V.A.L.U.E. tier:** project — PR 4 of 10 for inline simulation PR checks ([TEST-141](https://linear.app/vapi/issue/TEST-141/gitops-run-simulation-suites-against-pr-changes-inline-as-ci-checks)); this PR adds a config parser with no caller yet. - **Problem:** a customer needs one place to say which agents to test, with which suites, in which org, and how strictly. `promotion.yml` can't hold it: `promotionConfigParse` requires a pipeline of two or more orgs, and most customers have one. - **Who it affects:** gitops users setting up PR checks, single-org and CI-org alike. The PR check (PRs 5–8) and the promotion gate (PR 10) both read this file, so the gate runs the same tests as the PR check. - **What changes:** - `src/check-config.ts` parses a root `vapi-checks.yml` into typed check definitions: - `org`, plus an optional `runOrg` and `baseUrl`; - `targets` (`assistants/<id>` or `squads/<id>`, nested IDs allowed); - `suites` and/or `simulations`; - `bindings` (promotion's shape, via `promotionBindingsParse`); - extra `paths`; - run settings with repo-wide `defaults`: transport `chat`, 1 iteration, 20 min, `toolMocks: strict`, `stripWebhooks: true`. - `vapi-checks.example.yml` is the commented starting point: a single-org check, plus a commented CI-org check. - Every key is validated, and unknown keys fail, so a typo can't silently run fewer tests. A `mode:` key fails with a message explaining that checks always build from the branch, since a "run what's deployed" mode was deliberately left out. ## Evidence of value | Check | Result | |---|---| | Valid configs | Minimal check (defaults applied, `runOrg` = `org`, credentials `bind` / phones `omit`); default and per-check overrides; a CI-org check with EU `baseUrl` and bindings | | Rejections | 31 invalid shapes, each asserted against its exact message naming the field (table in `tests/check-config.test.ts`) | | Shipped example | Parses as-is, and the commented CI-org check parses once uncommented, so the docs can't drift from the parser | | `npm test` | 398 → 404 passing; `npm run build` clean | ## Testing plan - `tests/check-config.test.ts` covers: - parse results for valid configs; - one table row per rejection: version, empty or unknown keys at each level, `mode`, slugs, target shape, extension and `..` in IDs, duplicates, missing suites/simulations, each setting's range, `baseUrl`, `paths`, bindings; - the example file; - `checksConfigLoad`, which returns `null` when no `vapi-checks.yml` exists (checks are opt-in). - **Not tested:** nothing uses the config yet. Resolving targets and suites against real files is PR 5, and the `baseUrl` fallback order (`.env.<runOrg>`, `$VAPI_BASE_URL`, `api.vapi.ai`) is applied at run time in PR 7. Stacked on #59. Refs TEST-141 🤖 Generated with [Claude Code](https://claude.com/claude-code)
2 parents 93deaf3 + c9a6467 commit 8da229f

3 files changed

Lines changed: 620 additions & 0 deletions

File tree

‎src/check-config.ts‎

Lines changed: 295 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,295 @@
1+
// Parser for `vapi-checks.yml`, the PR check configuration.
2+
//
3+
// Config-free and pure (apart from checksConfigLoad's one read), so the
4+
// check CLI, the PR workflow and the promotion gate share it. It is a
5+
// separate file from promotion.yml because promotionConfigParse requires a
6+
// pipeline of two or more orgs, which single-org customers don't have.
7+
//
8+
// Every key is validated and unknown keys are rejected: a typo in a CI
9+
// config should fail loudly, not silently run fewer tests.
10+
11+
import { existsSync, readFileSync } from "node:fs";
12+
import { join } from "node:path";
13+
import { parse as parseYaml } from "yaml";
14+
import type { PromotionBindings } from "./promotion.ts";
15+
import { promotionBindingsParse, SLUG_RE } from "./promotion.ts";
16+
17+
export const CHECKS_CONFIG_FILE = "vapi-checks.yml";
18+
19+
// chat → vapi.webchat, voice → vapi.websocket.
20+
export type CheckTransport = "chat" | "voice";
21+
22+
// strict: every tool is mocked or the build fails (see check-mocks.ts).
23+
// off: tools are sent as written — only safe in a dedicated CI org.
24+
export type CheckToolMocks = "strict" | "off";
25+
26+
export type CheckTargetType = "assistants" | "squads";
27+
28+
export interface CheckTarget {
29+
type: CheckTargetType;
30+
id: string;
31+
}
32+
33+
export interface CheckRunSettings {
34+
transport: CheckTransport;
35+
iterations: number;
36+
timeoutMinutes: number;
37+
toolMocks: CheckToolMocks;
38+
stripWebhooks: boolean;
39+
}
40+
41+
export interface CheckDefinition extends CheckRunSettings {
42+
name: string;
43+
// Resources are read from resources/<org>/ at the checked-out commit.
44+
org: string;
45+
// The org the run executes in; its state supplies credential UUIDs.
46+
runOrg: string;
47+
baseUrl?: string;
48+
targets: CheckTarget[];
49+
suites: string[];
50+
simulations: string[];
51+
bindings: PromotionBindings;
52+
// Extra globs (repo-relative) that mark this check affected.
53+
paths: string[];
54+
}
55+
56+
export interface ChecksConfig {
57+
version: 1;
58+
checks: Record<string, CheckDefinition>;
59+
}
60+
61+
export const CHECK_DEFAULTS: CheckRunSettings = {
62+
transport: "chat",
63+
iterations: 1,
64+
timeoutMinutes: 20,
65+
toolMocks: "strict",
66+
stripWebhooks: true,
67+
};
68+
69+
const MAX_ITERATIONS = 10;
70+
const MAX_TIMEOUT_MINUTES = 120;
71+
const TOP_LEVEL_KEYS = ["version", "defaults", "checks"];
72+
const SETTINGS_KEYS = Object.keys(CHECK_DEFAULTS);
73+
const CHECK_KEYS = [
74+
"org",
75+
"runOrg",
76+
"baseUrl",
77+
"targets",
78+
"suites",
79+
"simulations",
80+
"bindings",
81+
"paths",
82+
...SETTINGS_KEYS,
83+
];
84+
const TARGET_TYPES: readonly CheckTargetType[] = ["assistants", "squads"];
85+
86+
function mapping(value: unknown, label: string): Record<string, unknown> {
87+
if (!value || typeof value !== "object" || Array.isArray(value))
88+
throw new Error(`${label} must be a mapping`);
89+
return Object.fromEntries(Object.entries(value));
90+
}
91+
92+
function keysAllowed(
93+
raw: Record<string, unknown>,
94+
allowed: readonly string[],
95+
label: string,
96+
): void {
97+
for (const key of Object.keys(raw)) {
98+
if (key === "mode")
99+
throw new Error(
100+
`${label}.mode is not supported: checks always build the target from the branch's files`,
101+
);
102+
if (!allowed.includes(key))
103+
throw new Error(
104+
`${label} has unknown key "${key}" (allowed: ${allowed.join(", ")})`,
105+
);
106+
}
107+
}
108+
109+
function slug(value: unknown, label: string): string {
110+
if (typeof value !== "string" || !SLUG_RE.test(value))
111+
throw new Error(`${label} must be a lowercase slug (a-z, 0-9, -)`);
112+
return value;
113+
}
114+
115+
// A resource ID as it appears in a file path under resources/<org>/<folder>/,
116+
// without the extension. Nested IDs (`team/intake`) are allowed.
117+
function resourceId(value: unknown, label: string): string {
118+
if (typeof value !== "string" || value.length === 0)
119+
throw new Error(`${label} must be a non-empty resource ID`);
120+
if (/\.(ya?ml|md|ts)$/.test(value))
121+
throw new Error(`${label} must not include a file extension: ${value}`);
122+
const parts = value.split("/");
123+
if (parts.some((part) => part === "" || part === "." || part === ".."))
124+
throw new Error(`${label} must be a relative resource ID: ${value}`);
125+
return value;
126+
}
127+
128+
function stringList(value: unknown, label: string): unknown[] {
129+
if (value === undefined) return [];
130+
if (!Array.isArray(value)) throw new Error(`${label} must be a list`);
131+
return value;
132+
}
133+
134+
function unique<T>(values: T[], key: (value: T) => string, label: string): T[] {
135+
const seen = new Set<string>();
136+
for (const value of values) {
137+
const id = key(value);
138+
if (seen.has(id)) throw new Error(`${label} lists ${id} twice`);
139+
seen.add(id);
140+
}
141+
return values;
142+
}
143+
144+
function target(value: unknown, label: string): CheckTarget {
145+
if (typeof value !== "string")
146+
throw new Error(`${label} must be "assistants/<id>" or "squads/<id>"`);
147+
const slash = value.indexOf("/");
148+
const type = value.slice(0, slash);
149+
if (slash < 0 || !TARGET_TYPES.includes(type as CheckTargetType))
150+
throw new Error(
151+
`${label} must be "assistants/<id>" or "squads/<id>", got "${value}"`,
152+
);
153+
return {
154+
type: type as CheckTargetType,
155+
id: resourceId(value.slice(slash + 1), label),
156+
};
157+
}
158+
159+
function settings(
160+
raw: Record<string, unknown>,
161+
base: CheckRunSettings,
162+
label: string,
163+
): CheckRunSettings {
164+
const result = { ...base };
165+
if (raw.transport !== undefined) {
166+
if (raw.transport !== "chat" && raw.transport !== "voice")
167+
throw new Error(`${label}.transport must be "chat" or "voice"`);
168+
result.transport = raw.transport;
169+
}
170+
if (raw.iterations !== undefined) {
171+
const iterations = raw.iterations;
172+
if (
173+
typeof iterations !== "number" ||
174+
!Number.isInteger(iterations) ||
175+
iterations < 1 ||
176+
iterations > MAX_ITERATIONS
177+
)
178+
throw new Error(
179+
`${label}.iterations must be an integer from 1 to ${MAX_ITERATIONS}`,
180+
);
181+
result.iterations = iterations;
182+
}
183+
if (raw.timeoutMinutes !== undefined) {
184+
const timeout = raw.timeoutMinutes;
185+
if (
186+
typeof timeout !== "number" ||
187+
!(timeout > 0) ||
188+
timeout > MAX_TIMEOUT_MINUTES
189+
)
190+
throw new Error(
191+
`${label}.timeoutMinutes must be a number above 0 and at most ${MAX_TIMEOUT_MINUTES}`,
192+
);
193+
result.timeoutMinutes = timeout;
194+
}
195+
if (raw.toolMocks !== undefined) {
196+
if (raw.toolMocks !== "strict" && raw.toolMocks !== "off")
197+
throw new Error(`${label}.toolMocks must be "strict" or "off"`);
198+
result.toolMocks = raw.toolMocks;
199+
}
200+
if (raw.stripWebhooks !== undefined) {
201+
if (typeof raw.stripWebhooks !== "boolean")
202+
throw new Error(`${label}.stripWebhooks must be true or false`);
203+
result.stripWebhooks = raw.stripWebhooks;
204+
}
205+
return result;
206+
}
207+
208+
function baseUrl(value: unknown, label: string): string | undefined {
209+
if (value === undefined) return undefined;
210+
if (typeof value !== "string" || !/^https?:\/\/[^\s/]+/.test(value))
211+
throw new Error(`${label}.baseUrl must be an http(s) URL`);
212+
return value.replace(/\/+$/, "");
213+
}
214+
215+
function check(
216+
name: string,
217+
value: unknown,
218+
defaults: CheckRunSettings,
219+
): CheckDefinition {
220+
const label = `checks.${name}`;
221+
const raw = mapping(value, label);
222+
keysAllowed(raw, CHECK_KEYS, label);
223+
const org = slug(raw.org, `${label}.org`);
224+
const runOrg =
225+
raw.runOrg === undefined ? org : slug(raw.runOrg, `${label}.runOrg`);
226+
const targets = unique(
227+
stringList(raw.targets, `${label}.targets`).map((item, index) =>
228+
target(item, `${label}.targets[${index}]`),
229+
),
230+
(item) => `${item.type}/${item.id}`,
231+
`${label}.targets`,
232+
);
233+
if (targets.length === 0)
234+
throw new Error(`${label}.targets must list at least one target`);
235+
const suites = unique(
236+
stringList(raw.suites, `${label}.suites`).map((item, index) =>
237+
resourceId(item, `${label}.suites[${index}]`),
238+
),
239+
(item) => item,
240+
`${label}.suites`,
241+
);
242+
const simulations = unique(
243+
stringList(raw.simulations, `${label}.simulations`).map((item, index) =>
244+
resourceId(item, `${label}.simulations[${index}]`),
245+
),
246+
(item) => item,
247+
`${label}.simulations`,
248+
);
249+
if (suites.length === 0 && simulations.length === 0)
250+
throw new Error(`${label} must list at least one suite or simulation`);
251+
const paths = stringList(raw.paths, `${label}.paths`).map((item, index) => {
252+
if (typeof item !== "string" || item.length === 0)
253+
throw new Error(`${label}.paths[${index}] must be a non-empty glob`);
254+
return item;
255+
});
256+
return {
257+
name,
258+
org,
259+
runOrg,
260+
baseUrl: baseUrl(raw.baseUrl, label),
261+
targets,
262+
suites,
263+
simulations,
264+
bindings: promotionBindingsParse(raw.bindings),
265+
paths,
266+
...settings(raw, defaults, label),
267+
};
268+
}
269+
270+
export function checksConfigParse(content: string): ChecksConfig {
271+
const raw = mapping(parseYaml(content), CHECKS_CONFIG_FILE);
272+
keysAllowed(raw, TOP_LEVEL_KEYS, CHECKS_CONFIG_FILE);
273+
if (raw.version !== 1)
274+
throw new Error(`${CHECKS_CONFIG_FILE} version must be 1`);
275+
const defaultsRaw =
276+
raw.defaults === undefined ? {} : mapping(raw.defaults, "defaults");
277+
keysAllowed(defaultsRaw, SETTINGS_KEYS, "defaults");
278+
const defaults = settings(defaultsRaw, CHECK_DEFAULTS, "defaults");
279+
const checksRaw = mapping(raw.checks, "checks");
280+
if (Object.keys(checksRaw).length === 0)
281+
throw new Error(`${CHECKS_CONFIG_FILE} must declare at least one check`);
282+
const checks: Record<string, CheckDefinition> = {};
283+
for (const [name, value] of Object.entries(checksRaw)) {
284+
slug(name, `check name "${name}"`);
285+
checks[name] = check(name, value, defaults);
286+
}
287+
return { version: 1, checks };
288+
}
289+
290+
// Returns null when the repo has no vapi-checks.yml: checks are opt-in.
291+
export function checksConfigLoad(rootDir: string): ChecksConfig | null {
292+
const path = join(rootDir, CHECKS_CONFIG_FILE);
293+
if (!existsSync(path)) return null;
294+
return checksConfigParse(readFileSync(path, "utf8"));
295+
}

0 commit comments

Comments
 (0)