diff --git a/cli/package.json b/cli/package.json index a630eaa9..d3e43ef8 100644 --- a/cli/package.json +++ b/cli/package.json @@ -39,8 +39,8 @@ "dependencies": { "@aws-sdk/client-s3": "^3.1041.0", "@aws-sdk/s3-request-presigner": "^3.1041.0", - "@flui-cloud/infra": "^0.5.1", - "@flui-cloud/spec": "^0.11.0", + "@flui-cloud/infra": "^0.6.4", + "@flui-cloud/spec": "^0.12.0", "@inquirer/prompts": "^7.10.1", "@kubernetes/client-node": "^1.4.0", "@nestjs/axios": "^4.0.0", diff --git a/cli/pnpm-lock.yaml b/cli/pnpm-lock.yaml index 1ad1cfaf..b2285b82 100644 --- a/cli/pnpm-lock.yaml +++ b/cli/pnpm-lock.yaml @@ -14,11 +14,11 @@ importers: specifier: ^3.1041.0 version: 3.1084.0 '@flui-cloud/infra': - specifier: ^0.5.1 - version: 0.5.1(@nestjs/common@11.1.28(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/config@4.0.4(@nestjs/common@11.1.28(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(rxjs@7.8.2))(@nestjs/swagger@11.4.5(@nestjs/common@11.1.28(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.28)(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2))(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2) + specifier: ^0.6.4 + version: 0.6.4(@nestjs/common@11.1.28(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/config@4.0.4(@nestjs/common@11.1.28(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(rxjs@7.8.2))(@nestjs/swagger@11.4.5(@nestjs/common@11.1.28(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.28)(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2))(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2) '@flui-cloud/spec': - specifier: ^0.11.0 - version: 0.11.0 + specifier: ^0.12.0 + version: 0.12.0 '@inquirer/prompts': specifier: ^7.10.1 version: 7.10.1(@types/node@20.19.25) @@ -419,10 +419,10 @@ packages: } engines: { node: '>=12' } - '@flui-cloud/infra@0.5.1': + '@flui-cloud/infra@0.6.4': resolution: { - integrity: sha512-5Y3D5kULxgX+J7MtlZ+9GHeb9R2CdGn8AcBfDVyV1D31L8jAy9q3rCwyYQA94zLfll4YcCRycR9JopAXmPBHbQ==, + integrity: sha512-93ObFCnOjdUiKDxOCBwf6SvioVkuxSVdMa/5uHAl4vkFnG+LNN7TdCsxmCWXna9mqPsGSLb7PC5vkg4QlQmI1A==, } peerDependencies: '@nestjs/common': '>=10' @@ -433,10 +433,10 @@ packages: reflect-metadata: '>=0.1' rxjs: '>=7' - '@flui-cloud/spec@0.11.0': + '@flui-cloud/spec@0.12.0': resolution: { - integrity: sha512-8ZDBn0IW+tPj3fqDBJQTmqOiczv9kEBlE1bI/Sq9mhad/5F4IhohjSoqbcdeaDNhCSQgHcKfuCL9xdTKHTjh2g==, + integrity: sha512-0OPU49trPl4Nf1ENO+PiT5NBDKqMx/iAfbFRm0UG4ib1YFoc7QpCtMLXgtzRyxRTP8DdaVmCAct42wO81S9n1g==, } hasBin: true @@ -5105,7 +5105,7 @@ snapshots: dependencies: '@jridgewell/trace-mapping': 0.3.9 - '@flui-cloud/infra@0.5.1(@nestjs/common@11.1.28(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/config@4.0.4(@nestjs/common@11.1.28(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(rxjs@7.8.2))(@nestjs/swagger@11.4.5(@nestjs/common@11.1.28(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.28)(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2))(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2)': + '@flui-cloud/infra@0.6.4(@nestjs/common@11.1.28(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/config@4.0.4(@nestjs/common@11.1.28(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(rxjs@7.8.2))(@nestjs/swagger@11.4.5(@nestjs/common@11.1.28(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.28)(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2))(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2)': dependencies: '@nestjs/common': 11.1.28(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2) '@nestjs/config': 4.0.4(@nestjs/common@11.1.28(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(rxjs@7.8.2) @@ -5119,7 +5119,7 @@ snapshots: - debug - supports-color - '@flui-cloud/spec@0.11.0': + '@flui-cloud/spec@0.12.0': dependencies: ajv: 8.20.0 ajv-formats: 3.0.1(ajv@8.20.0) diff --git a/cli/src/commands/cluster/create.ts b/cli/src/commands/cluster/create.ts index 72137ab9..e95c8bed 100644 --- a/cli/src/commands/cluster/create.ts +++ b/cli/src/commands/cluster/create.ts @@ -7,7 +7,70 @@ import { listClusters } from '../../lib/cluster-listing'; import { getRecommendedServerType } from '../../config/defaults'; const POLL_INTERVAL_MS = 5000; -const MAX_WAIT_MS = 1_800_000; // 30 min — matches the server-side estimate for master + N workers +// The API waits for the install to end too, and gives a creation up to an hour. +const MAX_WAIT_MS = 3_600_000; + +/** The sizes asked for, in order; the node size alone when none were. */ +function sizesOf(raw: string | undefined, nodeSize: string): string[] { + const sizes = (raw ?? '') + .split(',') + .map((s) => s.trim()) + .filter(Boolean); + return sizes.length ? sizes : [nodeSize]; +} + +function nodeLines(plan: HaPlan | null, workerCount: number): string[] { + if (!plan) return [` ${chalk.bold('Worker nodes:')} ${workerCount}`]; + const width = Math.max( + 6, + ...plan.servers.map((s) => s.zone.length), + ...plan.later.map((zone) => zone.length), + ); + return [ + ` ${chalk.bold('Availability:')} three locations (${plan.region})`, + ...plan.servers.map((s) => { + const role = s.role === 'data' ? 'copies of data' : 'votes'; + const first = s.first ? ', starts the cluster' : ''; + return ` ${s.zone.padEnd(width)} ${s.shape.padEnd(8)} ${role}${first}`; + }), + ...plan.later.map( + (zone) => + ` ${zone.padEnd(width)} ${chalk.yellow('none available now: add it later')}`, + ), + ...(plan.measured === false + ? [ + chalk.dim( + ' The provider declares these as close zones of one region; Flui has not measured them yet.', + ), + ] + : []), + ]; +} + +function haRequest( + ha: boolean, + sizes: string[], + completeLater: boolean, +): Record { + if (!ha) return {}; + return { + highAvailability: 'three-locations', + shapes: sizes, + ...(completeLater ? { completeLater: true } : {}), + }; +} + +interface HaPlan { + region: string; + measured?: boolean; + servers: Array<{ + zone: string; + shape: string; + role: 'data' | 'vote'; + first: boolean; + }>; + later: string[]; +} export default class ClusterCreate extends Command { static readonly description = @@ -23,6 +86,7 @@ export default class ClusterCreate extends Command { '<%= config.bin %> <%= command.id %> edge --provider hetzner --region hel1', '<%= config.bin %> <%= command.id %> edge --provider byos --flui-network', '<%= config.bin %> <%= command.id %> edge --vnet 3f2a… --subnet 9c1b…', + '<%= config.bin %> <%= command.id %> ha-hz --ha --region fsn1 --sizes cpx22,cx23', ]; static readonly args = { @@ -76,8 +140,102 @@ export default class ClusterCreate extends Command { description: 'Return immediately after queuing creation', default: false, }), + ha: Flags.boolean({ + description: + 'Highly available across three locations: one server in each location ' + + 'the provider validated for this region, bought together and joined one ' + + "at a time. --region is the first server's location, one for data.", + default: false, + }), + sizes: Flags.string({ + description: + 'With --ha: the sizes each location may take, comma separated, in order ' + + 'of preference. Each location gets the first one it has available. ' + + 'Defaults to --node-size.', + dependsOn: ['ha'], + }), + 'complete-later': Flags.boolean({ + description: + 'With --ha: when a location has none of the sizes available, create the ' + + 'cluster in the others and add that server later. Off, nothing is bought.', + default: false, + dependsOn: ['ha'], + }), }; + private refuseWorkersWithHa(ha: boolean, workerCount: number): void { + if (!ha || workerCount === 0) return; + this.error( + 'A cluster across three locations starts with its three servers; add workers once it is ready.', + { exit: 1 }, + ); + } + + private async ensureNameFree( + apiClient: ApiClient, + name: string, + provider: string, + ): Promise { + const spinner = ora(`Checking "${name}" is free on ${provider}...`).start(); + let taken: string | null = null; + try { + const availability = await apiClient.get<{ + available: boolean; + reason?: string; + }>( + `/infrastructure/clusters/name-availability?name=${encodeURIComponent(name)}&provider=${encodeURIComponent(provider)}`, + ); + if (availability.available) spinner.succeed('Name available'); + else taken = availability.reason ?? 'Name not available'; + } catch (error: any) { + // Best-effort — the create call re-validates server-side regardless. + spinner.warn( + `Could not pre-check name availability: ${error.message} — continuing`, + ); + } + if (taken) { + // Outside the try: oclif's exit is an exception the catch would swallow. + spinner.fail(taken); + this.exit(1); + } + } + + private async haPlan( + apiClient: ApiClient, + ha: boolean, + query: { + provider: string; + region: string; + shapes: string[]; + completeLater: boolean; + }, + ): Promise { + if (!ha) return null; + const params = new URLSearchParams({ + provider: query.provider, + region: query.region, + shapes: query.shapes.join(','), + completeLater: String(query.completeLater), + }); + const plan = await apiClient.get< + HaPlan & { + layoutAvailable: boolean; + possible: boolean; + refusal: string | null; + } + >(`/infrastructure/clusters/high-availability/plan?${params}`); + if (!plan.possible) { + this.error( + plan.refusal ?? + 'This cluster cannot be created across three locations.', + { + exit: 1, + }, + ); + } + return plan; + } + async run(): Promise { const { args, flags } = await this.parse(ClusterCreate); @@ -154,28 +312,17 @@ export default class ClusterCreate extends Command { if (flags.subnet && !flags.vnet) { this.error('--subnet needs the --vnet it belongs to.', { exit: 1 }); } + this.refuseWorkersWithHa(flags.ha, flags['worker-count']); + const sizes = sizesOf(flags.sizes, nodeSize); - const spinner = ora( - `Checking "${args.name}" is free on ${provider}...`, - ).start(); - try { - const availability = await apiClient.get<{ - available: boolean; - reason?: string; - }>( - `/infrastructure/clusters/name-availability?name=${encodeURIComponent(args.name)}&provider=${encodeURIComponent(provider)}`, - ); - if (!availability.available) { - spinner.fail(availability.reason ?? 'Name not available'); - this.exit(1); - } - spinner.succeed('Name available'); - } catch (error: any) { - // Best-effort — the create call re-validates server-side regardless. - spinner.warn( - `Could not pre-check name availability: ${error.message} — continuing`, - ); - } + await this.ensureNameFree(apiClient, args.name, provider); + + const haPlan = await this.haPlan(apiClient, flags.ha, { + provider, + region, + shapes: sizes, + completeLater: flags['complete-later'], + }); console.log(''); console.log(` ${chalk.bold('Name:')} ${args.name}`); @@ -183,8 +330,10 @@ export default class ClusterCreate extends Command { console.log( ` ${chalk.bold('Region:')} ${region || '(provider default)'}`, ); - console.log(` ${chalk.bold('Node size:')} ${nodeSize}`); - console.log(` ${chalk.bold('Worker nodes:')} ${flags['worker-count']}`); + console.log(` ${chalk.bold('Node size:')} ${sizes.join(', ')}`); + for (const line of nodeLines(haPlan, flags['worker-count'])) { + console.log(line); + } if (fluiNetwork) { console.log( ` ${chalk.bold('Network:')} built by Flui ` + @@ -209,8 +358,9 @@ export default class ClusterCreate extends Command { name: args.name, provider, region, - nodeSize, + nodeSize: sizes[0], workerCount: flags['worker-count'], + ...haRequest(flags.ha, sizes, flags['complete-later']), ...(fluiNetwork ? { fluiManagedNetwork: flags['network-cidr'] @@ -253,23 +403,24 @@ export default class ClusterCreate extends Command { return; } - await this.waitForCreation(apiClient, operationId, args.name); + await this.waitForCreation(apiClient, operationId, args.name, MAX_WAIT_MS); } private async waitForCreation( apiClient: ApiClient, operationId: string, clusterName: string, + maxWaitMs: number, ): Promise { console.log( chalk.dim( - ` Waiting for creation to complete (up to ${MAX_WAIT_MS / 60000} min)…`, + ` Waiting for creation to complete (up to ${maxWaitMs / 60000} min)…`, ), ); const waitSpinner = ora('Creating cluster…').start(); const started = Date.now(); - while (Date.now() - started < MAX_WAIT_MS) { + while (Date.now() - started < maxWaitMs) { await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS)); const result = await this.pollOperation( apiClient, @@ -319,6 +470,14 @@ export default class ClusterCreate extends Command { console.log(chalk.red(`\n Error: ${msg}\n`)); return 'failed'; } + if (op.status === 'CANCELLED') { + waitSpinner.warn('Creation stopped'); + const msg = + op.metadata?.message ?? + 'The cluster was deleted while it was created'; + console.log(chalk.yellow(`\n ${msg}\n`)); + return 'failed'; + } } catch { /* polling error — keep trying */ } diff --git a/cli/src/commands/cluster/disks.ts b/cli/src/commands/cluster/disks.ts new file mode 100644 index 00000000..356dbd59 --- /dev/null +++ b/cli/src/commands/cluster/disks.ts @@ -0,0 +1,74 @@ +import { Args, Command, Flags } from '@oclif/core'; +import chalk from 'chalk'; +import { ApiClient } from '../../lib/api-client'; +import { ConfigStorage } from '../../lib/config-storage'; +import { resolveClusterRef } from '../../lib/resolve-cluster'; + +interface DiskSpace { + totalGiB: number; + freeGiB: number; +} + +interface NodeDisksView { + measuredAt: string; + nodes: Array<{ + node: string; + zone: string | null; + disk: DiskSpace | null; + copies: DiskSpace | null; + }>; + warnings: string[]; +} + +const space = (s: DiskSpace | null) => + s ? `${s.freeGiB} of ${s.totalGiB} GiB free` : chalk.dim('—'); + +export default class ClusterDisks extends Command { + static readonly description = + 'How much room is left on each node: its own disk, where databases and\n' + + 'data kept on the node live, and the space set aside for copies of\n' + + 'replicated volumes. Names a node running short.'; + + static readonly examples = [ + '<%= config.bin %> <%= command.id %> workload-cluster-1', + ]; + + static readonly args = { + cluster: Args.string({ description: 'Cluster name or id', required: true }), + }; + + static readonly flags = { json: Flags.boolean({ default: false }) }; + + async run(): Promise { + const { args, flags } = await this.parse(ClusterDisks); + const cfg = new ConfigStorage(); + const api = new ApiClient({ + baseUrl: cfg.getApiUrlOrThrow(), + apiKey: cfg.getApiKeyOrThrow(), + }); + const cluster = await resolveClusterRef(args.cluster); + const view = await api.get( + `/infrastructure/clusters/${cluster.id}/storage/disks`, + ); + if (flags.json) { + this.log(JSON.stringify(view, null, 2)); + return; + } + this.log(''); + this.log(` ${chalk.bold(args.cluster)}`); + this.log(''); + this.log( + ` ${'NODE'.padEnd(28)} ${'LOCATION'.padEnd(10)} ${'DISK'.padEnd(24)} COPIES`, + ); + for (const n of view.nodes) { + this.log( + ` ${n.node.padEnd(28)} ${(n.zone ?? '—').padEnd(10)} ${space(n.disk).padEnd(24)} ${space(n.copies)}`, + ); + } + if (view.warnings.length) { + this.log(''); + for (const w of view.warnings) this.log(` ${chalk.yellow(w)}`); + } + this.log(''); + } +} diff --git a/cli/src/commands/cluster/ha.ts b/cli/src/commands/cluster/ha.ts new file mode 100644 index 00000000..4b7a5b6b --- /dev/null +++ b/cli/src/commands/cluster/ha.ts @@ -0,0 +1,233 @@ +import { Args, Command, Flags } from '@oclif/core'; +import chalk from 'chalk'; +import { ApiClient } from '../../lib/api-client'; +import { ConfigStorage } from '../../lib/config-storage'; +import { resolveClusterRef } from '../../lib/resolve-cluster'; + +interface ClusterHaView { + level: 'standard' | 'three-locations'; + region: string | null; + topology: { + validated: boolean; + zones: Array<{ + zone: string; + role: 'data' | 'vote'; + servers: string[]; + dataHosts: string[]; + }>; + } | null; + unsupported: string | null; + servers: { count: number; ready: number; zones: string[] }; + survives: string[]; + doesNotSurvive: string[]; + warnings: string[]; + missing: Array<{ role: 'server' | 'data-host'; zone: string; why: string }>; + observed?: { + at: string | null; + etcdLeader: { node: string | null; zone: string | null } | null; + roundTrips: Array<{ from: string; to: string; ms: number | null }>; + reachedThrough: string | null; + reachableServers: string[]; + }; +} + +const LEVEL = { + standard: 'standard', + 'three-locations': 'highly available across three locations', +}; + +const ROLE = { + server: 'a server', + 'data-host': 'a node for copies of data', +}; + +type Role = 'server' | 'data-host'; + +const plural = (n: number, word: string) => `${n} ${word}${n === 1 ? '' : 's'}`; + +function headerLine(name: string, view: ClusterHaView): string { + const region = view.region ? ` (${view.region})` : ''; + const servers = plural(view.servers.count, 'server'); + return ` ${chalk.bold(name)}: ${chalk.cyan(LEVEL[view.level])}, ${servers}${region}`; +} + +function zoneLines(view: ClusterHaView): string[] { + if (!view.topology) return []; + const width = Math.max(8, ...view.topology.zones.map((z) => z.zone.length)); + return [ + '', + ...view.topology.zones.map((z) => { + const servers = z.servers.length + ? z.servers.join(', ') + : chalk.red('no server'); + return ` ${z.zone.padEnd(width)} ${servers}${zoneData(z)}`; + }), + ]; +} + +function zoneData(z: { role: 'data' | 'vote'; dataHosts: string[] }): string { + if (z.role !== 'data') return ', votes only'; + const hosts = z.dataHosts.length + ? z.dataHosts.join(', ') + : chalk.red('no node yet'); + return `, copies of data on ${hosts}`; +} + +function observedLines(view: ClusterHaView): string[] { + const o = view.observed; + const lines: string[] = []; + if (o?.at) { + lines.push('', chalk.dim(` Measured ${new Date(o.at).toLocaleString()}`)); + if (o.etcdLeader?.node) { + const zone = o.etcdLeader.zone ?? 'no location'; + lines.push(` Leads the cluster: ${o.etcdLeader.node} (${zone})`); + } + for (const t of o.roundTrips) { + const ms = + t.ms === null ? chalk.red('no answer') : `${t.ms.toFixed(1)} ms`; + lines.push(` ${t.from} to ${t.to}: ${ms}`); + } + } + if (o?.reachedThrough) { + const others = o.reachableServers.filter((s) => s !== o.reachedThrough); + const move = others.length ? `, and can move to ${others.join(', ')}` : ''; + lines.push(` Flui reaches it through ${o.reachedThrough}${move}`); + } + return lines; +} + +function survivalLines(view: ClusterHaView): string[] { + return [ + '', + ' Survives', + ...view.survives.map((s) => ` ${chalk.green('✓')} ${s}`), + ' Does not survive', + ...view.doesNotSurvive.map((s) => ` ${chalk.red('✗')} ${s}`), + ]; +} + +function missingLines(name: string, view: ClusterHaView): string[] { + if (!view.missing.length) return []; + return [ + '', + ' Still missing', + ...view.missing.flatMap((m) => [ + ` + ${ROLE[m.role]} in ${m.zone}`, + chalk.dim(` ${m.why}`), + chalk.dim( + ` flui cluster ha ${name} --add ${m.role} --location ${m.zone}`, + ), + ]), + ]; +} + +export default class ClusterHa extends Command { + static readonly description = + 'Show what a cluster survives with the nodes it has now. Flui offers one\n' + + 'highly available layout: a server in each of three locations of a region\n' + + 'it has validated, copies of data in the two closest, the third voting\n' + + 'only. The command lists each location with its role and what is still\n' + + 'missing. With --add and --location, Flui buys the missing server or node\n' + + 'for data there and joins it with its role; scaling never removes it.'; + + static readonly examples = [ + '<%= config.bin %> <%= command.id %> workload-cluster-1', + '<%= config.bin %> <%= command.id %> workload-cluster-1 --add server --location hel1', + '<%= config.bin %> <%= command.id %> workload-cluster-1 --add data-host --location nbg1 --size cx33', + ]; + + static readonly args = { + cluster: Args.string({ description: 'Cluster name or id', required: true }), + }; + + static readonly flags = { + add: Flags.string({ + options: ['server', 'data-host'], + description: + 'Add what the cluster still needs: a server (also runs the control plane and holds a vote) or a node for data', + dependsOn: ['location'], + }), + location: Flags.string({ + description: + 'Where to add it, one of the locations listed under "Still missing"', + dependsOn: ['add'], + }), + size: Flags.string({ + description: "Machine size to buy (default: the cluster's own size)", + dependsOn: ['add'], + }), + force: Flags.boolean({ + default: false, + description: + 'Accept a region Flui has not validated for high availability; its three locations are treated as equal', + }), + json: Flags.boolean({ default: false }), + }; + + async run(): Promise { + const { args, flags } = await this.parse(ClusterHa); + const cfg = new ConfigStorage(); + const api = new ApiClient({ + baseUrl: cfg.getApiUrlOrThrow(), + apiKey: cfg.getApiKeyOrThrow(), + }); + const cluster = await resolveClusterRef(args.cluster); + if (flags.add) { + await this.addNode(api, cluster.id, args.cluster, flags); + return; + } + const view = await api.get( + `/infrastructure/clusters/${cluster.id}/high-availability`, + ); + if (flags.json) { + this.log(JSON.stringify(view, null, 2)); + return; + } + const lines = [ + '', + headerLine(args.cluster, view), + ...(view.unsupported ? [` ${chalk.yellow(view.unsupported)}`] : []), + ...view.warnings.map((w) => ` ${chalk.yellow(w)}`), + ...zoneLines(view), + ...observedLines(view), + ...survivalLines(view), + ...missingLines(args.cluster, view), + '', + ]; + for (const line of lines) this.log(line); + } + + private async addNode( + api: ApiClient, + clusterId: string, + name: string, + flags: { + add?: string; + location?: string; + size?: string; + force: boolean; + json: boolean; + }, + ): Promise { + const started = await api.post<{ operationId: string }>( + `/infrastructure/clusters/${clusterId}/high-availability/nodes`, + { + role: flags.add, + zone: flags.location, + ...(flags.size && { serverType: flags.size }), + ...(flags.force && { force: true }), + }, + ); + if (flags.json) { + this.log(JSON.stringify(started, null, 2)); + return; + } + this.log(''); + this.log( + ` ${chalk.bold(name)}: adding ${ROLE[flags.add as Role]} in ${flags.location}`, + ); + this.log(` Operation ${started.operationId}`); + this.log(chalk.dim(` Follow it with: flui cluster ha ${name}`)); + this.log(''); + } +} diff --git a/cli/src/commands/cluster/replicated-volumes.ts b/cli/src/commands/cluster/replicated-volumes.ts new file mode 100644 index 00000000..6fbf37a2 --- /dev/null +++ b/cli/src/commands/cluster/replicated-volumes.ts @@ -0,0 +1,168 @@ +import { Args, Command, Flags } from '@oclif/core'; +import chalk from 'chalk'; +import { ApiClient } from '../../lib/api-client'; +import { ConfigStorage } from '../../lib/config-storage'; +import { resolveClusterRef } from '../../lib/resolve-cluster'; + +interface ReplicatedVolumesView { + clusterId: string; + enabled: boolean; + storageClass: string; + storageNodes: string[]; + warnings: string[]; + plan: + | { + feasible: true; + zones: string[]; + storageNodes: string[]; + tieBreakers: string[]; + } + | { feasible: false; reason: string }; + operation: { id: string; status: string } | null; +} + +function listOf(items: string[]): string { + return items.length < 2 + ? items.join('') + : `${items.slice(0, -1).join(', ')} and ${items.at(-1)}`; +} + +function enablingWould(plan: { + zones: string[]; + storageNodes: string[]; + tieBreakers: string[]; +}): string { + const vote = plan.tieBreakers.length + ? `, with ${listOf(plan.tieBreakers)} for the tie-breaking vote` + : ', one in each location, with no vote needed'; + return `Enabling would keep copies on ${listOf(plan.storageNodes)} (${listOf(plan.zones)})${vote}.`; +} + +export default class ClusterReplicatedVolumes extends Command { + static readonly description = + 'Show whether a cluster offers replicated volumes, or turn them on.\n' + + 'A replicated volume keeps a synchronous copy in each location for data,\n' + + 'so an application with one writer comes back in another of them, with\n' + + 'its data, when its server or its location is lost. Where only two\n' + + 'locations hold data, the third only votes. The cluster must be highly\n' + + 'available across three locations (flui cluster ha).\n' + + 'With --enable, a node in each location for data sets aside disk space and\n' + + 'the volume engine is installed; existing volumes do not change.\n' + + 'Applications ask for it per volume with replicated: true in flui.yaml.'; + + static readonly examples = [ + '<%= config.bin %> <%= command.id %> workload-cluster-1', + '<%= config.bin %> <%= command.id %> workload-cluster-1 --enable', + '<%= config.bin %> <%= command.id %> workload-cluster-1 --enable --nodes workload-cluster-1-worker-1,workload-cluster-1-worker-2 --size 50', + ]; + + static readonly args = { + cluster: Args.string({ + description: 'Cluster name or id', + required: true, + }), + }; + + static readonly flags = { + enable: Flags.boolean({ + default: false, + description: 'Turn replicated volumes on', + }), + nodes: Flags.string({ + description: + 'The nodes that hold the copies, comma separated, one in each location for data (default: Flui picks them)', + }), + size: Flags.integer({ + min: 5, + max: 2000, + description: + 'GiB each of those nodes sets aside from its own disk (default 20)', + }), + force: Flags.boolean({ + default: false, + description: + 'Accept a region Flui has not validated for high availability; its three locations are treated as equal', + }), + json: Flags.boolean({ default: false }), + }; + + async run(): Promise { + const { args, flags } = await this.parse(ClusterReplicatedVolumes); + const cfg = new ConfigStorage(); + const api = new ApiClient({ + baseUrl: cfg.getApiUrlOrThrow(), + apiKey: cfg.getApiKeyOrThrow(), + }); + const cluster = await resolveClusterRef(args.cluster); + const path = `/infrastructure/clusters/${cluster.id}/replicated-volumes`; + + if (flags.enable) { + const nodes = flags.nodes + ?.split(',') + .map((n) => n.trim()) + .filter(Boolean); + const started = await api.post<{ + operationId: string; + storageNodes: string[]; + sizeGiB: number; + }>(path, { + ...(nodes?.length ? { nodes } : {}), + ...(flags.size ? { sizeGiB: flags.size } : {}), + ...(flags.force ? { force: true } : {}), + }); + if (flags.json) { + this.log(JSON.stringify(started, null, 2)); + return; + } + this.log(''); + this.log( + ` ${chalk.bold(args.cluster)}: enabling replicated volumes on ${listOf(started.storageNodes)} (${started.sizeGiB} GiB each)`, + ); + this.log(` Operation ${started.operationId}`); + this.log( + chalk.dim( + ` Follow it with: flui cluster replicated-volumes ${args.cluster}`, + ), + ); + this.log(''); + return; + } + + const view = await api.get(path); + if (flags.json) { + this.log(JSON.stringify(view, null, 2)); + return; + } + this.log(''); + this.log(` ${chalk.bold(args.cluster)}`); + if (view.operation) { + this.log( + ` ${chalk.yellow('Being enabled')}: operation ${view.operation.id} (${view.operation.status})`, + ); + } else if (view.enabled) { + this.log( + ` ${chalk.green('Replicated volumes on')}: copies on ${listOf(view.storageNodes)}`, + ); + this.log( + chalk.dim( + ' A volume asks for it with replicated: true in flui.yaml. An application moves to another copy only if every volume it mounts is replicated.', + ), + ); + } else if (view.plan.feasible) { + this.log(` Off. ${enablingWould(view.plan)}`); + this.log( + chalk.dim( + ` Turn it on: flui cluster replicated-volumes ${args.cluster} --enable`, + ), + ); + } else { + this.log( + ` Off, and not possible yet: ${(view.plan as { reason: string }).reason}`, + ); + } + for (const warning of view.warnings ?? []) { + this.log(` ${chalk.yellow('!')} ${warning}`); + } + this.log(''); + } +} diff --git a/cli/src/commands/db/ha.ts b/cli/src/commands/db/ha.ts new file mode 100644 index 00000000..c2d09896 --- /dev/null +++ b/cli/src/commands/db/ha.ts @@ -0,0 +1,177 @@ +import { Args, Command, Flags } from '@oclif/core'; +import chalk from 'chalk'; +import { ApiClient } from '../../lib/api-client'; +import { ConfigStorage } from '../../lib/config-storage'; +import { resolveAppOnAnyCluster } from '../../lib/resolve-app'; +import { resolveClusterRef } from '../../lib/resolve-cluster'; + +interface DatabaseHaView { + appId: string; + highlyAvailable: boolean; + state: { + cluster: string; + durability: 'required' | 'preferred'; + instances: number; + state: 'copying' | 'active' | 'failed'; + since: string; + detail?: string; + } | null; + placement?: { + primary: { instance: string; zone: string | null } | null; + copies: Array<{ + instance: string; + zone: string | null; + ready: boolean; + confirms: boolean; + flushLagMs: number | null; + }>; + } | null; +} + +const DURABILITY = { + required: + 'no confirmed write is ever lost; writes wait while no copy can confirm', + preferred: 'writes go on while a copy is away; it catches up later', +}; + +type Placement = NonNullable; + +function copyState(c: Placement['copies'][number]): string { + if (!c.ready) return chalk.red('not ready'); + return c.confirms ? 'confirms commits' : chalk.yellow('not streaming'); +} + +function placementLines(p: DatabaseHaView['placement']): string[] { + if (!p?.primary) return []; + return [ + ` Primary: ${p.primary.instance} (${p.primary.zone ?? 'no location'})`, + ...p.copies.map((c) => { + const lag = + c.flushLagMs === null + ? 'nothing written lately' + : `${c.flushLagMs} ms behind`; + return ` Copy: ${c.instance} (${c.zone ?? 'no location'}), ${copyState(c)}, ${lag}`; + }), + ]; +} + +function stateLines(view: DatabaseHaView, ref: string): string[] { + const s = view.state; + if (!s) { + return [ + ' Single instance: if its server is lost, the database is down until it returns or is restored.', + chalk.dim(` Make it highly available: flui db ha ${ref} --enable`), + ]; + } + if (s.state === 'active') { + const notReady = (view.placement?.copies ?? []).filter((c) => !c.ready); + const header = notReady.length + ? ` ${chalk.yellow('Highly available, not all copies ready')}: ${s.instances - notReady.length} of ${s.instances} instances serve; until the rest are back it survives less` + : ` ${chalk.green('Highly available')}: ${s.instances} instances, one per location; survives losing a server or a location`; + return [ + header, + ` Writes: ${DURABILITY[s.durability]}`, + ...placementLines(view.placement), + ]; + } + if (s.state === 'copying') { + return [ + ` ${chalk.yellow('Being made highly available')}: the copy is catching up; the single instance still serves`, + ]; + } + return [ + ` ${chalk.red('The last attempt stopped')}: ${s.detail ?? 'no detail recorded'}`, + ]; +} + +export default class DbHa extends Command { + static readonly description = + 'Show whether a managed PostgreSQL is highly available, or make it so.\n' + + "With --enable, Flui builds three instances, one in each of the cluster's\n" + + 'three locations, as live copies of the database and moves the application\n' + + 'to them without stopping it: writes pause for a few seconds. A commit waits\n' + + 'for whichever copy answers first. The single instance is kept, stopped, as\n' + + 'the way back. The cluster must be highly available across three locations\n' + + '(flui cluster ha).'; + + static readonly examples = [ + '<%= config.bin %> <%= command.id %> postgresql-051f58', + '<%= config.bin %> <%= command.id %> postgresql-051f58 --enable', + '<%= config.bin %> <%= command.id %> postgresql-051f58 --enable --durability preferred', + ]; + + static readonly args = { + app: Args.string({ + description: 'Database application name, slug, or id', + required: true, + }), + }; + + static readonly flags = { + cluster: Flags.string({ + char: 'c', + description: 'Cluster name or id (default: look the name up everywhere)', + }), + enable: Flags.boolean({ + default: false, + description: 'Make the database highly available', + }), + durability: Flags.string({ + options: ['required', 'preferred'], + default: 'required', + description: `required: ${DURABILITY.required}; preferred: ${DURABILITY.preferred}`, + }), + force: Flags.boolean({ + default: false, + description: + 'Accept a region Flui has not validated for high availability; its three locations are treated as equal', + }), + json: Flags.boolean({ default: false }), + }; + + async run(): Promise { + const { args, flags } = await this.parse(DbHa); + const cfg = new ConfigStorage(); + const api = new ApiClient({ + baseUrl: cfg.getApiUrlOrThrow(), + apiKey: cfg.getApiKeyOrThrow(), + }); + const clusterId = flags.cluster + ? (await resolveClusterRef(flags.cluster)).id + : undefined; + const app = await resolveAppOnAnyCluster(args.app, clusterId); + const path = `/applications/${app.id}/db/high-availability`; + + if (flags.enable) { + const started = await api.post<{ operationId: string }>(path, { + durability: flags.durability, + ...(flags.force && { force: true }), + }); + if (flags.json) { + this.log(JSON.stringify(started, null, 2)); + return; + } + this.log(''); + this.log( + ` ${chalk.bold(app.name)}: making it highly available (${flags.durability})`, + ); + this.log(` Operation ${started.operationId}`); + this.log(chalk.dim(` Follow it with: flui db ha ${args.app}`)); + this.log(''); + return; + } + + const view = await api.get(path); + if (flags.json) { + this.log(JSON.stringify(view, null, 2)); + return; + } + const lines = [ + '', + ` ${chalk.bold(app.name)}`, + ...stateLines(view, args.app), + '', + ]; + for (const line of lines) this.log(line); + } +} diff --git a/cli/src/commands/env/scale-master.ts b/cli/src/commands/env/scale-master.ts index 1659aedf..35cfa126 100644 --- a/cli/src/commands/env/scale-master.ts +++ b/cli/src/commands/env/scale-master.ts @@ -10,6 +10,22 @@ import { ClusterCapacityService } from 'src/modules/infrastructure/clusters/serv import { CliNodeRepository } from '../../lib/repositories/cli-node.repository'; import { NodeType } from 'src/modules/infrastructure/clusters/entities/cluster-node.entity'; import { confirmByTypingPrompt } from '../../lib/prompts'; +import { getRepositoryToken } from '@nestjs/typeorm'; +import type { INestApplication } from '@nestjs/common'; +import type { Repository } from 'typeorm'; +import { InfrastructureOperationEntity } from 'src/modules/infrastructure/servers/entities/infrastructure-operations.entity'; + +/** The run reports through the operation record, failures included. */ +async function operationOf( + app: INestApplication, + id: string, +): Promise { + return app + .get< + Repository + >(getRepositoryToken(InfrastructureOperationEntity)) + .findOne({ where: { id } }); +} export default class EnvScaleMaster extends Command { static readonly description = @@ -189,20 +205,10 @@ export default class EnvScaleMaster extends Command { } } - const run = ora({ - text: `Scaling master to ${targetType}...`, - color: 'yellow', - }).start(); - try { - const op = await scalingService.scaleNode(cluster.id, master.id, { - targetServerType: targetType, - upgradeDisk: flags.upgradeDisk, - }); - run.succeed(`Master scaled. Operation ${op.id} → ${op.status}`); - } catch (error) { - run.fail('Scale-master failed'); - throw error; - } + await this.scaleInProcess(app, cluster.id, master.id, { + targetServerType: targetType, + upgradeDisk: flags.upgradeDisk, + }); console.log(''); } catch (error) { console.log(chalk.red('\n❌ Error:\n')); @@ -214,4 +220,39 @@ export default class EnvScaleMaster extends Command { await closeNestApp(); } } + + /** The master hosts the API, so its resize runs here and not in the API's queue. */ + private async scaleInProcess( + app: INestApplication, + clusterId: string, + masterId: string, + request: { targetServerType: string; upgradeDisk: boolean }, + ): Promise { + const scalingService = app.get(ClusterNodeScalingService); + const run = ora({ + text: `Scaling master to ${request.targetServerType}...`, + color: 'yellow', + }).start(); + try { + const queued = await scalingService.scaleNode( + clusterId, + masterId, + request, + undefined, + { queue: false }, + ); + await scalingService.runScaleNode(queued.id); + const op = await operationOf(app, queued.id); + if (op?.status !== 'COMPLETED') { + run.fail( + `Scale-master failed: ${op?.errorMessage ?? 'no detail recorded'}`, + ); + this.exit(1); + } + run.succeed(`Master scaled. Operation ${queued.id} → ${op.status}`); + } catch (error) { + run.fail('Scale-master failed'); + throw error; + } + } } diff --git a/cli/src/commands/env/scale-node.ts b/cli/src/commands/env/scale-node.ts index a63ed6de..104803b9 100644 --- a/cli/src/commands/env/scale-node.ts +++ b/cli/src/commands/env/scale-node.ts @@ -1,32 +1,16 @@ import { Args, Command, Flags } from '@oclif/core'; -import chalk from 'chalk'; -import { archLabel, armWarningLine } from '../../lib/server-architecture-view'; -import ora from 'ora'; import { getNestApp, closeNestApp } from '../../lib/nest-app'; import { printContextBanner } from '../../lib/context-banner'; import { openControlPlane, printControlPlaneError, } from '../../lib/control-plane-api'; -import { findClusterNode } from '../../lib/cluster-nodes'; -import type { ClusterCapacityPlanDto } from 'src/modules/infrastructure/clusters/dto/cluster-capacity-plan.dto'; -import { confirmByTypingPrompt } from '../../lib/prompts'; - -interface ScaleNodePreview { - node: { currentServerType: string }; - affectedDedicatedApps: Array<{ slug: string }>; - expectedDowntimeMs: number; -} - -interface OperationSummary { - id: string; - status: string; -} +import { resizeNode } from '../../lib/node-resize'; export default class EnvScaleNode extends Command { static readonly description = - 'Vertically scale a worker node to a new server type. Planned-maintenance operation. ' + - 'For the master, use `flui env scale-master` instead.'; + 'Vertically scale a worker node of the control cluster to a new server type. Planned-maintenance operation: ' + + 'the node is off for 3-5 minutes. For the master, use `flui env scale-master`; for a node of a workload cluster, `flui node resize`.'; static readonly examples = [ '<%= config.bin %> <%= command.id %> worker-1 --type cx32', @@ -50,118 +34,36 @@ export default class EnvScaleNode extends Command { description: '[Hetzner] Also grow the local OS disk (one-way).', default: false, }), + copiesGib: Flags.integer({ + description: + 'Grow the space this node sets aside for copies of replicated volumes to this many GiB once it is back', + min: 1, + }), confirm: Flags.boolean({ description: 'Skip the typed confirmation prompt', default: false, }), + wait: Flags.boolean({ + description: + 'Follow the resize until it ends (--no-wait returns once it is queued)', + default: true, + allowNo: true, + }), }; async run(): Promise { const { args, flags } = await this.parse(EnvScaleNode); printContextBanner(); - const spinner = ora('Preparing scale-node plan...').start(); - try { const { cluster, api } = await openControlPlane(await getNestApp()); - const base = `/infrastructure/clusters/${cluster.id}`; - const target = await findClusterNode(api, cluster.id, args.name); - const [preview, plan] = await Promise.all([ - api.get(`${base}/nodes/${target.id}/scale/preview`), - api.get(`${base}/capacity-plan`), - ]); - spinner.succeed('Plan computed'); - - let targetType = flags.type; - let cand = targetType - ? plan.candidates.find((c) => c.name === targetType) - : undefined; - if (!targetType) { - cand = plan.candidates.find((c) => c.direction === 'upgrade'); - if (!cand) { - console.log( - chalk.red( - '\n❌ No upgrade candidate available. Pass --type explicitly.\n', - ), - ); - this.exit(1); - } - targetType = cand.name; - console.log( - chalk.dim( - ` (auto-selected next-bigger type: ${chalk.bold(targetType)})`, - ), - ); - } - console.log(chalk.cyan('\n🖥 Scale Node Plan\n')); - console.log(` ${chalk.bold('Node:')} ${target.serverName}`); - console.log( - ` ${chalk.bold('From:')} ${preview.node.currentServerType}`, - ); - console.log( - ` ${chalk.bold('To:')} ${targetType}${cand ? ` (${archLabel(cand.architecture)})` : ''}`, - ); - if (archLabel(cand?.architecture) === 'arm') { - console.log(` ${armWarningLine()}`); - } - if (cand) { - let deltaStr: string; - if (cand.monthlyDeltaEur === 'n/a') deltaStr = 'n/a'; - else { - const sign = Number.parseFloat(cand.monthlyDeltaEur) > 0 ? '+' : ''; - deltaStr = sign + cand.monthlyDeltaEur; - } - console.log( - ` ${chalk.bold('Cost/month:')} €${cand.monthlyCostEur} (${deltaStr} delta)`, - ); - } - console.log( - ` ${chalk.bold('Downtime:')} ~${Math.round(preview.expectedDowntimeMs / 60000)} min`, - ); - if (preview.affectedDedicatedApps.length > 0) { - console.log( - chalk.yellow( - `\n ⚠️ Hosts ${preview.affectedDedicatedApps.length} dedicated workload(s):`, - ), - ); - for (const app of preview.affectedDedicatedApps) { - console.log(chalk.yellow(` • ${app.slug}`)); - } - } - - if (!flags.confirm) { - console.log(''); - console.log( - chalk.yellow( - ` To confirm, type the node name exactly: ${chalk.bold(target.serverName)}`, - ), - ); - const ok = await confirmByTypingPrompt( - chalk.yellow('⚠️ Node name'), - target.serverName, - ); - if (!ok) { - console.log(chalk.green('\n✅ Cancelled\n')); - return; - } - } - - const run = ora({ - text: `Scaling ${target.serverName} to ${targetType}...`, - color: 'yellow', - }).start(); - try { - const op = await api.post( - `${base}/nodes/${target.id}/scale`, - { - targetServerType: targetType, - upgradeDisk: flags.upgradeDisk, - }, - ); - run.succeed(`Node scaled. Operation ${op.id} → ${op.status}`); - } catch (error) { - run.fail('Scale-node failed'); - throw error; - } + const ok = await resizeNode(api, cluster.id, args.name, { + type: flags.type, + upgradeDisk: flags.upgradeDisk, + copiesGib: flags.copiesGib, + confirm: flags.confirm, + wait: flags.wait, + }); + if (!ok) this.exit(1); console.log(''); } catch (error) { printControlPlaneError(error); diff --git a/cli/src/commands/env/storage.ts b/cli/src/commands/env/storage.ts index 9f0b2052..918ea937 100644 --- a/cli/src/commands/env/storage.ts +++ b/cli/src/commands/env/storage.ts @@ -101,6 +101,12 @@ export default class EnvStorage extends Command { console.log( ` ${chalk.bold('Client opts:')} ${chalk.dim(s.nfs.mountOptions)}`, ); + if (s.nfs.durability) { + const say = s.nfs.confirmsBeforeDisk ? chalk.yellow : chalk.dim; + console.log( + ` ${chalk.bold('Durability:')} ${say(s.nfs.durability)}`, + ); + } } if (s.pvcs) { diff --git a/cli/src/commands/node/resize.ts b/cli/src/commands/node/resize.ts new file mode 100644 index 00000000..fa9db3fd --- /dev/null +++ b/cli/src/commands/node/resize.ts @@ -0,0 +1,87 @@ +import { Args, Command, Flags } from '@oclif/core'; +import chalk from 'chalk'; +import { ApiClient } from '../../lib/api-client'; +import { ConfigStorage } from '../../lib/config-storage'; +import { resolveClusterRef } from '../../lib/resolve-cluster'; +import { resizeNode } from '../../lib/node-resize'; + +export default class NodeResize extends Command { + static readonly description = + 'Move a node of a workload cluster to another machine type: the node is\n' + + 'switched off for 3-5 minutes, its type changed by the provider, then it\n' + + 'comes back. On a cluster across three locations one node is resized at a\n' + + 'time, only while the rest is healthy, and a database primary on the node is\n' + + 'handed over first. For the control cluster use `flui env scale-node`.'; + + static readonly examples = [ + '<%= config.bin %> <%= command.id %> workload-1-worker-2 --cluster workload-1 --type cx33', + '<%= config.bin %> <%= command.id %> workload-1-worker-2 --cluster workload-1 --type cx43 --upgrade-disk --copies-gib 60', + ]; + + static readonly args = { + node: Args.string({ + description: 'Node name or id (from `flui node list`)', + required: true, + }), + }; + + static readonly flags = { + cluster: Flags.string({ + char: 'c', + description: 'Cluster name or id', + required: true, + }), + type: Flags.string({ + char: 't', + description: + 'Target machine type. Omitted: the next bigger type offered for this node', + }), + 'upgrade-disk': Flags.boolean({ + description: '[Hetzner] Also grow the local disk (one-way)', + default: false, + }), + 'copies-gib': Flags.integer({ + description: + 'Grow the space this node sets aside for copies of replicated volumes to this many GiB once it is back', + min: 1, + }), + confirm: Flags.boolean({ + description: 'Skip the typed confirmation prompt', + default: false, + }), + wait: Flags.boolean({ + description: + 'Follow the resize until it ends (--no-wait returns once it is queued)', + default: true, + allowNo: true, + }), + }; + + async run(): Promise { + const { args, flags } = await this.parse(NodeResize); + const cfg = new ConfigStorage(); + const api = new ApiClient({ + baseUrl: cfg.getApiUrlOrThrow(), + apiKey: cfg.getApiKeyOrThrow(), + }); + try { + const cluster = await resolveClusterRef(flags.cluster); + const ok = await resizeNode(api, cluster.id, args.node, { + type: flags.type, + upgradeDisk: flags['upgrade-disk'], + copiesGib: flags['copies-gib'], + confirm: flags.confirm, + wait: flags.wait, + }); + if (!ok) this.exit(1); + console.log(''); + } catch (error) { + console.log( + chalk.red( + `\n❌ ${error instanceof Error ? error.message : String(error)}\n`, + ), + ); + this.exit(1); + } + } +} diff --git a/cli/src/lib/node-resize.ts b/cli/src/lib/node-resize.ts new file mode 100644 index 00000000..9f12d0f0 --- /dev/null +++ b/cli/src/lib/node-resize.ts @@ -0,0 +1,193 @@ +import chalk from 'chalk'; +import ora from 'ora'; +import { ApiClient } from './api-client'; +import { findClusterNode } from './cluster-nodes'; +import { followOperation } from './follow-operation'; +import { confirmByTypingPrompt } from './prompts'; +import { archLabel, armWarningLine } from './server-architecture-view'; + +interface Candidate { + name: string; + direction: 'upgrade' | 'downgrade' | 'current'; + monthlyCostEur: string; + monthlyDeltaEur: string; + architecture?: string; + diskGb?: number; +} + +interface ResizePreview { + node: { currentServerType: string }; + affectedDedicatedApps: Array<{ slug: string }>; + expectedDowntimeMs: number; + candidates: Candidate[]; + highAvailability: { + refusal: string | null; + switchovers: Array<{ namespace: string; cluster: string; target: string }>; + } | null; +} + +export interface NodeResizeFlags { + type?: string; + upgradeDisk: boolean; + copiesGib?: number; + confirm: boolean; + wait: boolean; +} + +function deltaOf(cand: Candidate): string { + if (cand.monthlyDeltaEur === 'n/a') return 'n/a'; + const sign = Number.parseFloat(cand.monthlyDeltaEur) > 0 ? '+' : ''; + return `${sign}${cand.monthlyDeltaEur}`; +} + +function planLines( + serverName: string, + preview: ResizePreview, + targetType: string, + cand: Candidate | undefined, + flags: NodeResizeFlags, +): string[] { + const arch = cand ? ` (${archLabel(cand.architecture)})` : ''; + const downtime = Math.round(preview.expectedDowntimeMs / 60000); + const lines = [ + chalk.cyan('\n🖥 Resize plan\n'), + ` ${chalk.bold('Node:')} ${serverName}`, + ` ${chalk.bold('From:')} ${preview.node.currentServerType}`, + ` ${chalk.bold('To:')} ${targetType}${arch}`, + ]; + if (archLabel(cand?.architecture) === 'arm') + lines.push(` ${armWarningLine()}`); + if (cand) { + lines.push( + ` ${chalk.bold('Cost/month:')} €${cand.monthlyCostEur} (${deltaOf(cand)} delta)`, + ); + } + lines.push(` ${chalk.bold('Downtime:')} ~${downtime} min for this node`); + if (flags.copiesGib) { + lines.push( + ` ${chalk.bold('Copies:')} space for copies of replicated volumes grown to ${flags.copiesGib} GiB once the node is back`, + ); + } + const apps = preview.affectedDedicatedApps; + if (apps.length > 0) { + lines.push( + chalk.yellow(`\n ⚠️ Hosts ${apps.length} dedicated workload(s):`), + ...apps.map((app) => chalk.yellow(` • ${app.slug}`)), + ); + } + return lines; +} + +async function confirmNode(serverName: string): Promise { + console.log(''); + console.log( + chalk.yellow( + ` To confirm, type the node name exactly: ${chalk.bold(serverName)}`, + ), + ); + const ok = await confirmByTypingPrompt( + chalk.yellow('⚠️ Node name'), + serverName, + ); + if (!ok) console.log(chalk.green('\n✅ Cancelled\n')); + return ok; +} + +async function followResize( + api: ApiClient, + operationId: string, + serverName: string, + targetType: string, +): Promise { + const run = ora({ + text: `Resizing ${serverName} to ${targetType}...`, + color: 'yellow', + }).start(); + const done = await followOperation<{ message?: string }>(api, operationId, { + onUpdate: (o) => { + if (o.metadata?.message) run.text = String(o.metadata.message); + }, + }); + if (done?.status === 'COMPLETED') { + run.succeed(`${serverName} is now ${targetType}. Operation ${operationId}`); + return true; + } + const status = done?.status?.toLowerCase() ?? 'still running'; + const detail = + done?.errorMessage ?? `follow it with flui operation ${operationId}`; + run.fail(`Resize ${status}: ${detail}`); + return false; +} + +/** + * Shows what a resize of one node would do, asks for the node's name, queues + * it and follows the operation to its end. Returns false when nothing was + * done or the resize failed. + */ +export async function resizeNode( + api: ApiClient, + clusterId: string, + nodeRef: string, + flags: NodeResizeFlags, +): Promise { + const base = `/infrastructure/clusters/${clusterId}`; + const target = await findClusterNode(api, clusterId, nodeRef); + const preview = await api.get( + `${base}/nodes/${target.id}/scale/preview`, + ); + + const cand = flags.type + ? preview.candidates.find((c) => c.name === flags.type) + : preview.candidates.find((c) => c.direction === 'upgrade'); + const targetType = flags.type ?? cand?.name; + if (!targetType) { + console.log( + chalk.red('\n❌ No bigger type is offered for this node. Pass --type.\n'), + ); + return false; + } + if (!flags.type) { + console.log( + chalk.dim( + ` (next bigger type for this node: ${chalk.bold(targetType)})`, + ), + ); + } + for (const line of planLines( + target.serverName, + preview, + targetType, + cand, + flags, + )) { + console.log(line); + } + + const ha = preview.highAvailability; + if (ha?.refusal) { + console.log(chalk.red(`\n❌ ${ha.refusal}\n`)); + return false; + } + for (const s of ha?.switchovers ?? []) { + console.log( + chalk.dim( + ` The primary of ${s.namespace}/${s.cluster} moves to ${s.target} first; writes pause for a few seconds.`, + ), + ); + } + if (!flags.confirm && !(await confirmNode(target.serverName))) return false; + + const op = await api.post<{ id: string }>( + `${base}/nodes/${target.id}/scale`, + { + targetServerType: targetType, + upgradeDisk: flags.upgradeDisk, + ...(flags.copiesGib && { copiesGiB: flags.copiesGib }), + }, + ); + if (!flags.wait) { + console.log(`\n Resize queued. Operation ${op.id}\n`); + return true; + } + return followResize(api, op.id, target.serverName, targetType); +} diff --git a/images/postgres/flui-entrypoint.sh b/images/postgres/flui-entrypoint.sh index ff4fc1ef..4178b4b2 100644 --- a/images/postgres/flui-entrypoint.sh +++ b/images/postgres/flui-entrypoint.sh @@ -103,6 +103,30 @@ if [ "${FLUI_PG_RESTORE:-}" = "1" ] && [ "${FLUI_PG_STANDBY:-}" != "1" ] && [ ! else gosu postgres pgbackrest --config="$RESTORE_CONF" --stanza=main --delta restore fi + # A backup a highly available cluster took carries CloudNativePG's own + # configuration: certificates and a socket under /controller, and access rules + # for its operator. This instance has neither, so it starts from the image's + # configuration and keeps only the data. + if [ -f "$PGDATA/custom.conf" ] || [ -f "$PGDATA/override.conf" ]; then + echo "[flui-entrypoint] restore-bootstrap: the backup came from a highly available cluster; using this image's configuration" + mkdir -p "$PVC_ROOT/cnpg-configuration" + for f in postgresql.conf custom.conf override.conf pg_hba.conf pg_ident.conf; do + if [ -f "$PGDATA/$f" ]; then mv "$PGDATA/$f" "$PVC_ROOT/cnpg-configuration/$f"; fi + done + cp /usr/share/postgresql/postgresql.conf.sample "$PGDATA/postgresql.conf" + printf "\n# Flui managed\ninclude_if_exists = '/etc/flui/postgresql.flui.conf'\n" >> "$PGDATA/postgresql.conf" + # Recovery refuses to replay the log of a primary that ran with more of + # these than this instance: the cluster's own values come along. + for key in max_connections max_worker_processes max_wal_senders max_prepared_transactions max_locks_per_transaction; do + line=$(cat "$PVC_ROOT/cnpg-configuration/postgresql.conf" "$PVC_ROOT/cnpg-configuration/custom.conf" "$PVC_ROOT/cnpg-configuration/override.conf" 2>/dev/null \ + | grep -E "^[[:space:]]*${key}[[:space:]]*=" | tail -1 || true) + if [ -n "$line" ]; then printf '%s\n' "$line" >> "$PGDATA/postgresql.conf"; fi + done + printf 'local all all trust\nhost all all 127.0.0.1/32 trust\nhost all all ::1/128 trust\nhost all all all scram-sha-256\n' > "$PGDATA/pg_hba.conf" + : > "$PGDATA/pg_ident.conf" + chown postgres:postgres "$PGDATA/postgresql.conf" "$PGDATA/pg_hba.conf" "$PGDATA/pg_ident.conf" + chmod 0600 "$PGDATA/postgresql.conf" "$PGDATA/pg_hba.conf" "$PGDATA/pg_ident.conf" + fi # The restored postgresql.auto.conf carries the source's archive_command, # whose config file lives outside PGDATA and was not restored — neutralize it # or WAL recycling stalls. Enabling backup on this install rewrites it. diff --git a/package.json b/package.json index 10e2c884..1c989e43 100644 --- a/package.json +++ b/package.json @@ -98,10 +98,10 @@ "@aws-sdk/client-s3": "^3.1041.0", "@aws-sdk/s3-request-presigner": "^3.1041.0", "@flui-cloud/cartographer": "^0.7.0", - "@flui-cloud/infra": "^0.6.1", + "@flui-cloud/infra": "^0.6.4", "@flui-cloud/mail": "^0.1.0", "@flui-cloud/semantic-surface": "^0.1.0", - "@flui-cloud/spec": "^0.11.0", + "@flui-cloud/spec": "^0.12.0", "@kubernetes/client-node": "^1.4.0", "@modelcontextprotocol/node": "2.0.0", "@modelcontextprotocol/server": "2.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ca8f40cd..95e02a70 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,8 +20,8 @@ importers: specifier: ^0.7.0 version: 0.7.0 '@flui-cloud/infra': - specifier: ^0.6.1 - version: 0.6.1(@nestjs/common@11.1.6(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/config@4.0.2(@nestjs/common@11.1.6(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2))(rxjs@7.8.2))(@nestjs/swagger@11.2.6(@nestjs/common@11.1.6(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.6)(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2))(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2) + specifier: ^0.6.4 + version: 0.6.4(@nestjs/common@11.1.6(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/config@4.0.2(@nestjs/common@11.1.6(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2))(rxjs@7.8.2))(@nestjs/swagger@11.2.6(@nestjs/common@11.1.6(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.6)(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2))(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2) '@flui-cloud/mail': specifier: ^0.1.0 version: 0.1.0 @@ -29,8 +29,8 @@ importers: specifier: ^0.1.0 version: 0.1.0 '@flui-cloud/spec': - specifier: ^0.11.0 - version: 0.11.0 + specifier: ^0.12.0 + version: 0.12.0 '@kubernetes/client-node': specifier: ^1.4.0 version: 1.4.0(encoding@0.1.13) @@ -1037,10 +1037,10 @@ packages: } engines: { node: '>=20.0.0' } - '@flui-cloud/infra@0.6.1': + '@flui-cloud/infra@0.6.4': resolution: { - integrity: sha512-Bsoue7cZNe6SOueyam0Ap/mrwwYHXzdlKhYOGy8XXOzG9jEINrN1m/bgYrISUdwDajEpkxBxSfkna7GtTTXdLw==, + integrity: sha512-93ObFCnOjdUiKDxOCBwf6SvioVkuxSVdMa/5uHAl4vkFnG+LNN7TdCsxmCWXna9mqPsGSLb7PC5vkg4QlQmI1A==, } peerDependencies: '@nestjs/common': '>=10' @@ -1065,10 +1065,10 @@ packages: } engines: { node: '>=20.0.0' } - '@flui-cloud/spec@0.11.0': + '@flui-cloud/spec@0.12.0': resolution: { - integrity: sha512-8ZDBn0IW+tPj3fqDBJQTmqOiczv9kEBlE1bI/Sq9mhad/5F4IhohjSoqbcdeaDNhCSQgHcKfuCL9xdTKHTjh2g==, + integrity: sha512-0OPU49trPl4Nf1ENO+PiT5NBDKqMx/iAfbFRm0UG4ib1YFoc7QpCtMLXgtzRyxRTP8DdaVmCAct42wO81S9n1g==, } hasBin: true @@ -10399,7 +10399,7 @@ snapshots: '@flui-cloud/cartographer@0.7.0': {} - '@flui-cloud/infra@0.6.1(@nestjs/common@11.1.6(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/config@4.0.2(@nestjs/common@11.1.6(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2))(rxjs@7.8.2))(@nestjs/swagger@11.2.6(@nestjs/common@11.1.6(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.6)(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2))(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2)': + '@flui-cloud/infra@0.6.4(@nestjs/common@11.1.6(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/config@4.0.2(@nestjs/common@11.1.6(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2))(rxjs@7.8.2))(@nestjs/swagger@11.2.6(@nestjs/common@11.1.6(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.6)(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2))(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2)': dependencies: '@nestjs/common': 11.1.6(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2) '@nestjs/config': 4.0.2(@nestjs/common@11.1.6(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2))(rxjs@7.8.2) @@ -10418,7 +10418,7 @@ snapshots: dependencies: ajv: 8.20.0 - '@flui-cloud/spec@0.11.0': + '@flui-cloud/spec@0.12.0': dependencies: ajv: 8.20.0 ajv-formats: 3.0.1(ajv@8.20.0) diff --git a/src/app.module.ts b/src/app.module.ts index 7c52d9ff..72fbe4ec 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -54,6 +54,9 @@ import { AuthzModule } from './modules/authz/authz.module'; import { BackupsModule } from './modules/backups/backups.module'; import { FleetModule } from './modules/fleet/fleet.module'; import { DbLifecycleModule } from './modules/db-lifecycle/db-lifecycle.module'; +import { PostgresHaModule } from './modules/postgres-ha/postgres-ha.module'; +import { ReplicatedVolumesModule } from './modules/replicated-volumes/replicated-volumes.module'; +import { ClusterHaModule } from './modules/cluster-ha/cluster-ha.module'; import { StorageModule } from './modules/storage/storage.module'; import { AppMigrationModule } from './modules/app-migration/app-migration.module'; import { FullMigrationModule } from './modules/full-migration/full-migration.module'; @@ -162,6 +165,9 @@ import { QueueIntakeModule } from './common/queues/queue-intake.module'; BackupsModule, FleetModule, DbLifecycleModule, + PostgresHaModule, + ReplicatedVolumesModule, + ClusterHaModule, AppMigrationModule, FullMigrationModule, DemoModule, diff --git a/src/migrations/1790000000028-PhysicalDbMigration.spec.ts b/src/migrations/1790000000028-PhysicalDbMigration.spec.ts index e45beb13..c8a8fc4a 100644 --- a/src/migrations/1790000000028-PhysicalDbMigration.spec.ts +++ b/src/migrations/1790000000028-PhysicalDbMigration.spec.ts @@ -61,6 +61,8 @@ describe('PhysicalDbMigration1790000000028', () => { 'AppHistoryIndexes1790000000042', 'InferenceUsageIpHash1790000000043', 'InferenceUsageCachedTokens1790000000044', + 'DatabaseHa1790000000045', + 'ReplicatedVolumes1790000000046', ]); expect(migrations[at - 1].name).toMatch(/179000000002[67]$/); }); diff --git a/src/migrations/1790000000029-MigrationMeasures.spec.ts b/src/migrations/1790000000029-MigrationMeasures.spec.ts index 1db94253..5c150d80 100644 --- a/src/migrations/1790000000029-MigrationMeasures.spec.ts +++ b/src/migrations/1790000000029-MigrationMeasures.spec.ts @@ -48,6 +48,8 @@ describe('MigrationMeasures1790000000029', () => { 'AppHistoryIndexes1790000000042', 'InferenceUsageIpHash1790000000043', 'InferenceUsageCachedTokens1790000000044', + 'DatabaseHa1790000000045', + 'ReplicatedVolumes1790000000046', ]); }); }); diff --git a/src/migrations/1790000000044-InferenceUsageCachedTokens.spec.ts b/src/migrations/1790000000044-InferenceUsageCachedTokens.spec.ts index 366e98b3..f051960a 100644 --- a/src/migrations/1790000000044-InferenceUsageCachedTokens.spec.ts +++ b/src/migrations/1790000000044-InferenceUsageCachedTokens.spec.ts @@ -26,12 +26,9 @@ describe('InferenceUsageCachedTokens1790000000044', () => { ]); }); - it('runs last, after the address hash', () => { - expect(migrations[migrations.length - 1]).toBe( - InferenceUsageCachedTokens1790000000044, - ); - expect(migrations[migrations.length - 2].name).toBe( - 'InferenceUsageIpHash1790000000043', - ); + it('runs after the address hash, right before the database HA column', () => { + const at = migrations.indexOf(InferenceUsageCachedTokens1790000000044); + expect(migrations[at - 1].name).toBe('InferenceUsageIpHash1790000000043'); + expect(migrations[at + 1].name).toBe('DatabaseHa1790000000045'); }); }); diff --git a/src/migrations/1790000000045-DatabaseHa.spec.ts b/src/migrations/1790000000045-DatabaseHa.spec.ts new file mode 100644 index 00000000..227f56b9 --- /dev/null +++ b/src/migrations/1790000000045-DatabaseHa.spec.ts @@ -0,0 +1,24 @@ +import { migrations } from './index'; +import { DatabaseHa1790000000045 } from './1790000000045-DatabaseHa'; + +describe('DatabaseHa1790000000045', () => { + it('adds the highly available state as a nullable column, so every existing database reads as a single instance', async () => { + const sql: string[] = []; + await new DatabaseHa1790000000045().up({ + query: async (q: string) => { + sql.push(q); + }, + } as never); + expect(sql[0]).toBe( + 'ALTER TABLE "applications" ADD COLUMN IF NOT EXISTS "databaseHa" jsonb', + ); + expect(sql.join('\n')).not.toMatch(/NOT NULL|DEFAULT/); + }); + + it('runs after the cached-token column', () => { + const at = migrations.indexOf(DatabaseHa1790000000045); + expect(migrations[at - 1].name).toBe( + 'InferenceUsageCachedTokens1790000000044', + ); + }); +}); diff --git a/src/migrations/1790000000045-DatabaseHa.ts b/src/migrations/1790000000045-DatabaseHa.ts new file mode 100644 index 00000000..9e7ac52e --- /dev/null +++ b/src/migrations/1790000000045-DatabaseHa.ts @@ -0,0 +1,35 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +/** + * A managed Postgres that moved to a highly available cluster records it on + * its own application row. NULL is the single instance every install has. + */ +export class DatabaseHa1790000000045 implements MigrationInterface { + name = 'DatabaseHa1790000000045'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "applications" ADD COLUMN IF NOT EXISTS "databaseHa" jsonb`, + ); + await queryRunner.query( + `ALTER TYPE "public"."infrastructure_operations_operationtype_enum" ADD VALUE IF NOT EXISTS 'activate_database_ha'`, + ); + for (const step of [ + 'db_ha_prepare', + 'db_ha_copy', + 'db_ha_fence', + 'db_ha_promote', + 'db_ha_switch', + ]) { + await queryRunner.query( + `ALTER TYPE "public"."infrastructure_operations_currentstep_enum" ADD VALUE IF NOT EXISTS '${step}'`, + ); + } + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "applications" DROP COLUMN IF EXISTS "databaseHa"`, + ); + } +} diff --git a/src/migrations/1790000000046-ReplicatedVolumes.spec.ts b/src/migrations/1790000000046-ReplicatedVolumes.spec.ts new file mode 100644 index 00000000..763234a7 --- /dev/null +++ b/src/migrations/1790000000046-ReplicatedVolumes.spec.ts @@ -0,0 +1,21 @@ +import { migrations } from './index'; +import { ReplicatedVolumes1790000000046 } from './1790000000046-ReplicatedVolumes'; + +describe('ReplicatedVolumes1790000000046', () => { + it('only adds enum values, so no existing row changes', async () => { + const sql: string[] = []; + await new ReplicatedVolumes1790000000046().up({ + query: async (q: string) => { + sql.push(q); + }, + } as never); + expect(sql).toHaveLength(4); + expect(sql.every((q) => q.includes('ADD VALUE IF NOT EXISTS'))).toBe(true); + }); + + it('runs last', () => { + expect(migrations[migrations.length - 1]).toBe( + ReplicatedVolumes1790000000046, + ); + }); +}); diff --git a/src/migrations/1790000000046-ReplicatedVolumes.ts b/src/migrations/1790000000046-ReplicatedVolumes.ts new file mode 100644 index 00000000..f0e44a24 --- /dev/null +++ b/src/migrations/1790000000046-ReplicatedVolumes.ts @@ -0,0 +1,30 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +/** + * Enabling replicated volumes on a cluster is an operation of its own: the + * nodes that hold the copies are prepared, the volume engine is installed and + * the storage class appears. + */ +export class ReplicatedVolumes1790000000046 implements MigrationInterface { + name = 'ReplicatedVolumes1790000000046'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TYPE "public"."infrastructure_operations_operationtype_enum" ADD VALUE IF NOT EXISTS 'enable_replicated_volumes'`, + ); + for (const step of [ + 'replicated_prepare_nodes', + 'replicated_install_engine', + 'replicated_storage_class', + ]) { + await queryRunner.query( + `ALTER TYPE "public"."infrastructure_operations_currentstep_enum" ADD VALUE IF NOT EXISTS '${step}'`, + ); + } + } + + public async down(): Promise { + // Postgres cannot drop an enum value, and an operation recorded while it + // existed would be unreadable if it could. + } +} diff --git a/src/migrations/index.ts b/src/migrations/index.ts index 7443f126..32665189 100644 --- a/src/migrations/index.ts +++ b/src/migrations/index.ts @@ -98,6 +98,8 @@ import { ControlPeerPerNode1790000000041 } from './1790000000041-ControlPeerPerN import { AppHistoryIndexes1790000000042 } from './1790000000042-AppHistoryIndexes'; import { InferenceUsageIpHash1790000000043 } from './1790000000043-InferenceUsageIpHash'; import { InferenceUsageCachedTokens1790000000044 } from './1790000000044-InferenceUsageCachedTokens'; +import { DatabaseHa1790000000045 } from './1790000000045-DatabaseHa'; +import { ReplicatedVolumes1790000000046 } from './1790000000046-ReplicatedVolumes'; import { BackfillArtifactApplicationId1787500000000 } from './1787500000000-BackfillArtifactApplicationId'; import { WireGuardHubKeySealed1790000000000 } from './1790000000000-WireGuardHubKeySealed'; import { RecoverNodeAccessOperation1790000000001 } from './1790000000001-RecoverNodeAccessOperation'; @@ -209,4 +211,6 @@ export const migrations = [ AppHistoryIndexes1790000000042, InferenceUsageIpHash1790000000043, InferenceUsageCachedTokens1790000000044, + DatabaseHa1790000000045, + ReplicatedVolumes1790000000046, ]; diff --git a/src/modules/applications/applications.module.ts b/src/modules/applications/applications.module.ts index c7656652..ee3d309e 100644 --- a/src/modules/applications/applications.module.ts +++ b/src/modules/applications/applications.module.ts @@ -20,6 +20,7 @@ import { RepositoryCredentialEntity } from '../repositories/entities/repository- import { AppBuildEntity } from '../app-builds/entities/app-build.entity'; import { CatalogInstallEntity } from '../catalog/entities/catalog-install.entity'; import { SandboxTenantEntity } from '../sandbox/entities/sandbox-tenant.entity'; +import { ReplicatedVolumesModule } from '../replicated-volumes/replicated-volumes.module'; import { SharedInfrastructureModule } from '../infrastructure/shared/shared-infrastructure.module'; import { EgressModule } from '../infrastructure/egress/egress.module'; import { ClustersModule } from '../infrastructure/clusters/clusters.module'; @@ -156,6 +157,7 @@ import { FluiRegistryModule } from '../flui-registry/flui-registry.module'; BullModule.registerQueue({ name: GHCR_SECRET_REFRESH_QUEUE }), BullModule.registerQueue({ name: 'backup' }), SharedInfrastructureModule, + ReplicatedVolumesModule, EgressModule, EncryptionModule, ImagesModule, diff --git a/src/modules/applications/dto/app-management.dto.ts b/src/modules/applications/dto/app-management.dto.ts index 903f69e4..fab7bc6d 100644 --- a/src/modules/applications/dto/app-management.dto.ts +++ b/src/modules/applications/dto/app-management.dto.ts @@ -306,6 +306,15 @@ export class AppRuntimeResponseDto { 'Set when some replicas wait for a node with room: that is a wait, not a failure, and it lasts as long as scaling takes.', }) waitingForRoom?: RoomWaitDto | null; + + @Sensitivity(Sensitivity.PUBLIC) + @ApiPropertyOptional({ + type: String, + nullable: true, + description: + 'Set when the number of copies cannot be changed, nor the application stopped or started: a highly available database, whose instances Flui keeps across three locations.', + }) + replicasRefusal?: string | null; } export class ResourcesConsequenceDto { diff --git a/src/modules/applications/entities/application.entity.ts b/src/modules/applications/entities/application.entity.ts index 839f0b71..33f6a0c4 100644 --- a/src/modules/applications/entities/application.entity.ts +++ b/src/modules/applications/entities/application.entity.ts @@ -35,6 +35,7 @@ import { import type { AppMaintenance } from '../../infrastructure/maintenance/maintenance-window.core'; import type { AppBackupDecision } from '../../backups/utils/app-backup-decision.rules'; import type { PoolerSpec } from '../utils/db-pooler.core'; +import type { DatabaseHaSpec } from '../../postgres-ha/postgres-ha.types'; @Entity('applications') export class ApplicationEntity { @@ -172,6 +173,13 @@ export class ApplicationEntity { @Column({ type: 'jsonb', nullable: true }) pooler?: PoolerSpec | null; + /** + * The highly available cluster a managed PostgreSQL moved to. Null is the + * single instance, which is what every install made before it existed reads as. + */ + @Column({ type: 'jsonb', nullable: true }) + databaseHa?: DatabaseHaSpec | null; + @Column({ type: 'uuid', nullable: true }) currentRevisionId?: string; diff --git a/src/modules/applications/manifest-checks.core.spec.ts b/src/modules/applications/manifest-checks.core.spec.ts index 7cc0a5b4..4068fc61 100644 --- a/src/modules/applications/manifest-checks.core.spec.ts +++ b/src/modules/applications/manifest-checks.core.spec.ts @@ -150,3 +150,74 @@ describe('what only the installation can say about a manifest', () => { expect(byId(checksFor(facts()), 'placement').status).toBe('pass'); }); }); + +describe('replicated volumes in a manifest', () => { + const replicated = ( + over: Partial> = {}, + ) => ({ declared: true, enabled: true, mismatches: [], ...over }); + + it('adds no line for a manifest without replicated volumes', () => { + expect(byId(checksFor(facts()), 'replicated-volumes')).toBeUndefined(); + }); + + it('passes when the cluster keeps replicated volumes', () => { + const c = byId( + checksFor(facts({ replicatedVolumes: replicated() })), + 'replicated-volumes', + ); + expect(c.status).toBe('pass'); + }); + + it('refuses a cluster without them and says how to enable them', () => { + const checks = checksFor( + facts({ replicatedVolumes: replicated({ enabled: false }) }), + ); + expect(byId(checks, 'replicated-volumes').status).toBe('fail'); + expect(byId(checks, 'replicated-volumes').detail).toContain( + 'flui cluster replicated-volumes control-cluster --enable', + ); + expect(wouldDeploy(checks)).toBe(false); + }); + + it('does not refuse a cluster it could not read', () => { + const checks = checksFor( + facts({ replicatedVolumes: replicated({ enabled: null }) }), + ); + expect(byId(checks, 'replicated-volumes').status).toBe('unknown'); + expect(wouldDeploy(checks)).toBe(true); + }); + + it('refuses replicated on a volume that already has one copy', () => { + const c = byId( + checksFor( + facts({ + replicatedVolumes: replicated({ + mismatches: [{ name: 'data', declared: true }], + }), + }), + ), + 'replicated-volumes', + ); + expect(c.status).toBe('fail'); + expect(c.detail).toContain( + 'Volume "data" already exists with a single copy', + ); + }); + + it('refuses dropping replicated from a replicated volume', () => { + const c = byId( + checksFor( + facts({ + replicatedVolumes: replicated({ + declared: false, + enabled: null, + mismatches: [{ name: 'data', declared: false }], + }), + }), + ), + 'replicated-volumes', + ); + expect(c.status).toBe('fail'); + expect(c.detail).toContain('Keep replicated: true'); + }); +}); diff --git a/src/modules/applications/manifest-checks.core.ts b/src/modules/applications/manifest-checks.core.ts index d04196d7..f9bb32c1 100644 --- a/src/modules/applications/manifest-checks.core.ts +++ b/src/modules/applications/manifest-checks.core.ts @@ -54,6 +54,17 @@ export interface ManifestFacts { targetIsControlCluster: boolean; /** Whether this installation has any workload cluster at all. */ hasWorkloadCluster: boolean | null; + /** Absent unless the manifest or the application it updates has a replicated volume. */ + replicatedVolumes?: ReplicatedVolumesFact | null; +} + +export interface ReplicatedVolumesFact { + /** The manifest asks for at least one replicated volume. */ + declared: boolean; + /** Null when the cluster could not be asked. */ + enabled: boolean | null; + /** Volumes the application already has whose storage the manifest now contradicts. */ + mismatches: Array<{ name: string; declared: boolean }>; } const check = ( @@ -72,6 +83,7 @@ export function checksFor(facts: ManifestFacts): ManifestCheck[] { placementCheck(facts), identityCheck(facts), exposureCheck(facts), + ...(facts.replicatedVolumes ? [replicatedVolumesCheck(facts)] : []), ]; } @@ -284,3 +296,45 @@ function exposureCheck(facts: ManifestFacts): ManifestCheck { 'Public, but no DNS zone is bound to this cluster — it will answer on the default host rather than a name of yours.', ); } + +const quoted = (names: string[]) => names.map((n) => `"${n}"`).join(', '); + +/** The sentence a deploy is refused with, shared by the preview and the deploy itself. */ +export function replicatedVolumesRefusal( + fact: ReplicatedVolumesFact, + clusterName: string | null, +): string | null { + const added = fact.mismatches.filter((m) => m.declared).map((m) => m.name); + const dropped = fact.mismatches.filter((m) => !m.declared).map((m) => m.name); + if (fact.declared && fact.enabled === false) { + return `${clusterName ?? 'The cluster'} does not keep replicated volumes. Enable them first with \`flui cluster replicated-volumes ${clusterName ?? ''} --enable\`, or remove replicated from the manifest.`; + } + if (added.length) { + return `Volume ${quoted(added)} already exists with a single copy, and replicated is read only when a volume is created, so the application would run without the copy the manifest declares. Remove replicated from it, or deploy under a new name with replicated volumes and restore a backup into the new application (\`flui app backup restore --to\`).`; + } + if (dropped.length) { + return `Volume ${quoted(dropped)} is replicated, and removing replicated from the manifest does not move its data to a single copy. Keep replicated: true on it.`; + } + return null; +} + +function replicatedVolumesCheck(facts: ManifestFacts): ManifestCheck { + const fact = facts.replicatedVolumes!; + const title = 'Replicated volumes'; + const refusal = replicatedVolumesRefusal(fact, facts.clusterName); + if (refusal) return check('replicated-volumes', 'fail', title, refusal); + if (fact.declared && fact.enabled === null) { + return check( + 'replicated-volumes', + 'unknown', + title, + `Whether ${facts.clusterName ?? 'the cluster'} keeps replicated volumes could not be read just now, so this is unanswered, not refused.`, + ); + } + return check( + 'replicated-volumes', + 'pass', + title, + `${facts.clusterName ?? 'The cluster'} keeps a second copy of these volumes on another server of the same location.`, + ); +} diff --git a/src/modules/applications/services/app-availability.service.spec.ts b/src/modules/applications/services/app-availability.service.spec.ts index be3372d4..78c0187c 100644 --- a/src/modules/applications/services/app-availability.service.spec.ts +++ b/src/modules/applications/services/app-availability.service.spec.ts @@ -103,3 +103,51 @@ describe('the availability of one application, read from its cluster', () => { expect(result).toEqual({ highlyAvailable: true, reasons: [] }); }); }); + +describe('the availability of a database that moved to a highly available cluster', () => { + const ha = { + engine: 'cnpg', + cluster: 'orders-db-ha', + durability: 'required', + instances: 2, + state: 'active', + since: 'x', + }; + + it('survives losing a worker once its instances run on two nodes', async () => { + const result = await serviceWith({ + pods: [pod('worker-1'), pod('worker-2')], + bindings: [ + { claimName: 'orders-db-ha-1', node: 'worker-1' }, + { claimName: 'orders-db-ha-2', node: 'worker-2' }, + { claimName: 'data-orders-db-0', node: 'worker-2' }, + ], + app: { replicas: 1, persistenceScope: 'dedicated', databaseHa: ha }, + }).forApplication('a1'); + expect(result).toEqual({ highlyAvailable: true, reasons: [] }); + }); + + it('says so while only one instance is ready', async () => { + const result = await serviceWith({ + pods: [pod('worker-1'), pod('worker-2', false)], + app: { replicas: 1, persistenceScope: 'dedicated', databaseHa: ha }, + }).forApplication('a1'); + expect(result.highlyAvailable).toBe(false); + expect(result.reasons.map((r) => r.code)).toEqual(['copies_on_one_node']); + }); + + it('is still a single database while the copy is being built', async () => { + const result = await serviceWith({ + pods: [pod('worker-2')], + app: { + replicas: 1, + persistenceScope: 'dedicated', + databaseHa: { ...ha, state: 'copying' }, + }, + }).forApplication('a1'); + expect(result.reasons.map((r) => r.code)).toEqual([ + 'single_copy', + 'dedicated_placement', + ]); + }); +}); diff --git a/src/modules/applications/services/app-availability.service.ts b/src/modules/applications/services/app-availability.service.ts index 3f835f45..8c825d56 100644 --- a/src/modules/applications/services/app-availability.service.ts +++ b/src/modules/applications/services/app-availability.service.ts @@ -14,6 +14,7 @@ import { AppAvailability, appAvailability, } from '../utils/app-availability.core'; +import { databaseHaServes } from '../../postgres-ha/postgres-ha.types'; /** * Gathers what the availability rule needs — where the copies run, which @@ -65,9 +66,13 @@ export class AppAvailabilityService { const boundTo = new Map(bindings.map((b) => [b.claimName, b.node])); const scaling = app.scaling; - const desiredCopies = scaling?.enabled + // A highly available database is its instances, each on its own disk by + // design; the stopped single instance and its volume no longer count. + const ha = databaseHaServes(app.databaseHa) ? app.databaseHa : null; + const scaledCopies = scaling?.enabled ? (scaling.horizontal?.min ?? scaling.minReplicas ?? app.replicas ?? 1) : (app.replicas ?? 1); + const desiredCopies = ha ? ha.instances : scaledCopies; return appAvailability({ desiredCopies, @@ -82,8 +87,8 @@ export class AppAvailabilityService { .map((pod) => pod.spec?.nodeName ?? '') .filter(Boolean), ingressNodeCount: clusterIngressValues(cluster).length, - dedicated: app.persistenceScope === 'dedicated', - volumes: claims.map((claim) => ({ + dedicated: !ha && app.persistenceScope === 'dedicated', + volumes: (ha ? [] : claims).map((claim) => ({ name: claim.name, boundToNode: boundTo.get(claim.name) ?? null, })), diff --git a/src/modules/applications/services/app-management.database-ha.spec.ts b/src/modules/applications/services/app-management.database-ha.spec.ts new file mode 100644 index 00000000..77f1c7dd --- /dev/null +++ b/src/modules/applications/services/app-management.database-ha.spec.ts @@ -0,0 +1,175 @@ +jest.mock('@kubernetes/client-node', () => ({})); + +import { ConflictException } from '@nestjs/common'; +import { AppManagementService } from './app-management.service'; + +type Internals = Record; + +const active = { + engine: 'cnpg', + cluster: 'orders-db-ha', + durability: 'quorum', + instances: 3, + state: 'active', + since: '2026-11-07T18:00:00Z', +}; + +function build(databaseHa: object | null) { + const kubernetes = { + readObject: jest.fn().mockResolvedValue({ + spec: { + resources: { + requests: { cpu: '500m', memory: '1024Mi' }, + limits: { memory: '1024Mi' }, + }, + }, + }), + mergePatchObject: jest.fn().mockResolvedValue(undefined), + patchWorkloadContainerResources: jest.fn(), + scaleWorkload: jest.fn(), + restartWorkload: jest.fn(), + }; + const applications = { + update: jest.fn(), + findById: jest.fn().mockResolvedValue({ id: 'a1', replicas: 1 }), + updateStatus: jest.fn(), + }; + const revisions = { createAuditEvent: jest.fn() }; + const service = new AppManagementService( + {} as never, + applications as never, + revisions as never, + {} as never, + kubernetes as never, + {} as never, + {} as never, + {} as never, + ); + const internals = service as unknown as Internals; + internals.resolveAppAndKubeconfig = jest.fn().mockResolvedValue({ + app: { + id: 'a1', + slug: 'orders-db', + k8sNamespace: 'ns', + status: 'running', + replicas: 1, + resources: {}, + databaseHa, + }, + kubeconfig: 'kc', + }); + internals.getWorkloadOrThrow = jest.fn().mockResolvedValue({ + kind: 'StatefulSet', + name: 'orders-db', + resource: { + spec: { + template: { + spec: { + containers: [ + { + name: 'postgres', + resources: { + requests: { cpu: '250m', memory: '256Mi' }, + limits: { cpu: '2', memory: '2Gi' }, + }, + }, + ], + }, + }, + }, + }, + }); + internals.watchRollout = jest.fn(); + internals.buildRuntimeResponse = jest.fn().mockResolvedValue({}); + return { service, kubernetes, applications, revisions, internals }; +} + +describe('AppManagementService on a highly available database', () => { + it('gives the new resources to its cluster, keeping what was not asked to change', async () => { + const { service, kubernetes, revisions } = build(active); + + await service.updateResources('a1', { + requests: { memory: '2Gi' }, + limits: { memory: '2Gi' }, + }); + + expect(kubernetes.mergePatchObject).toHaveBeenCalledWith('kc', { + apiVersion: 'postgresql.cnpg.io/v1', + kind: 'Cluster', + metadata: { name: 'orders-db-ha', namespace: 'ns' }, + spec: { + resources: { + requests: { cpu: '500m', memory: '2Gi' }, + limits: { memory: '2Gi' }, + }, + primaryUpdateMethod: 'switchover', + }, + }); + expect(kubernetes.patchWorkloadContainerResources).toHaveBeenCalledWith( + 'kc', + 'StatefulSet', + 'ns', + 'orders-db', + 'postgres', + { + requests: { cpu: '500m', memory: '2Gi' }, + limits: { memory: '2Gi' }, + }, + ); + expect(revisions.createAuditEvent).toHaveBeenCalledWith( + expect.objectContaining({ eventType: expect.anything() }), + ); + }); + + it('starts from the requests and limits of the single instance when its cluster was made without any', async () => { + const { service, kubernetes } = build(active); + kubernetes.readObject.mockResolvedValue({ spec: {} }); + + await service.updateResources('a1', { requests: { memory: '384Mi' } }); + + expect(kubernetes.mergePatchObject.mock.calls[0][1].spec.resources).toEqual( + { + requests: { cpu: '250m', memory: '384Mi' }, + limits: { cpu: '2', memory: '2Gi' }, + }, + ); + }); + + it('restarts its instances through the cluster, one at a time', async () => { + const { service, kubernetes } = build(active); + + await service.restartDeployment('a1'); + + const patch = kubernetes.mergePatchObject.mock.calls[0][1]; + expect(patch.metadata.name).toBe('orders-db-ha'); + expect( + patch.metadata.annotations['kubectl.kubernetes.io/restartedAt'], + ).toEqual(expect.any(String)); + expect(patch.spec.primaryUpdateMethod).toBe('switchover'); + expect(kubernetes.restartWorkload).not.toHaveBeenCalled(); + }); + + it('refuses to change its copies, or to stop or start it', async () => { + const { service, kubernetes } = build(active); + + await expect(service.updateReplicas('a1', { replicas: 2 })).rejects.toThrow( + ConflictException, + ); + await expect(service.stop('a1')).rejects.toThrow( + 'Flui keeps its 3 instances across three locations', + ); + expect(kubernetes.scaleWorkload).not.toHaveBeenCalled(); + }); + + it('waits for the move to end before any change', async () => { + const { service, kubernetes } = build({ ...active, state: 'copying' }); + + await expect( + service.updateResources('a1', { requests: { cpu: '1' } }), + ).rejects.toThrow('being made highly available'); + await expect(service.restartDeployment('a1')).rejects.toThrow( + ConflictException, + ); + expect(kubernetes.mergePatchObject).not.toHaveBeenCalled(); + }); +}); diff --git a/src/modules/applications/services/app-management.service.ts b/src/modules/applications/services/app-management.service.ts index e182a14f..7d65e181 100644 --- a/src/modules/applications/services/app-management.service.ts +++ b/src/modules/applications/services/app-management.service.ts @@ -65,6 +65,18 @@ import { StatefulSetVolumeSwapService, } from './statefulset-volume-swap.service'; import { systemReplicasRefusal } from '../schedulers/platform-api-sizing'; +import { databaseHaServes } from '../../postgres-ha/postgres-ha.types'; +import { + CNPG_API_VERSION, + CNPG_PRIMARY_UPDATE_METHOD, +} from '../../postgres-ha/postgres-ha.constants'; +import { + CNPG_RESTART_ANNOTATION, + databaseHaReplicasRefusal, + databaseHaResizeRefusal, + haResourceBase, + mergedResources, +} from '../utils/database-ha-edits.core'; /** Rollout poll interval (ms) */ const ROLLOUT_POLL_INTERVAL_MS = 3000; @@ -109,6 +121,16 @@ export class AppManagementService { }, ): Promise { const { app, kubeconfig } = await this.resolveAppAndKubeconfig(appId); + const resizeRefused = databaseHaResizeRefusal(app.databaseHa); + if (resizeRefused) throw new ConflictException(resizeRefused); + if (databaseHaServes(app.databaseHa)) { + return this.updateHaResources( + app, + kubeconfig, + this.normalizedResources(dto), + context, + ); + } const { kind, name: deploymentName, @@ -174,36 +196,108 @@ export class AppManagementService { `Updated resources for container "${container.name}" in ${kind} "${deploymentName}"`, ); - // Sync DB entity so the stored resources reflect the live state - const merged = this.buildEntityResources(dto, app.resources); - await this.applicationsRepository.update(appId, { resources: merged }); + await this.recordResources(app, dto, beforeResources, context); - // Audit event + // Fire-and-forget: track the rollout until all pods are ready + this.watchRollout( + app, + kubeconfig, + 'update-resources', + RolloutSection.RESOURCES, + true, + ); + + return this.buildRuntimeResponse(app, kubeconfig); + } + + /** + * A highly available database takes new resources on its cluster, which + * restarts the instances one at a time with the primary last. The single + * instance at zero takes them too, so the way back runs with them. + */ + private async updateHaResources( + app: ApplicationEntity, + kubeconfig: string, + dto: UpdateResourcesDto, + context?: { + actor?: { id?: string; name?: string }; + reason?: string; + proposal?: string[]; + }, + ): Promise { + const name = app.databaseHa!.cluster; + const cluster = await this.kubernetesService.readObject( + kubeconfig, + CNPG_API_VERSION, + 'Cluster', + name, + app.k8sNamespace, + ); + if (!cluster) { + throw new NotFoundException( + `The highly available cluster ${name} of this database is not there`, + ); + } + const single = await this.getWorkloadOrThrow(app, kubeconfig); + const container = single.resource.spec?.template?.spec?.containers?.[0]; + const merged = mergedResources( + haResourceBase(cluster.spec?.resources, container?.resources), + dto, + ); + const healed = healedResources(merged); + const resources = healed ? { ...merged, ...healed } : merged; + const belowRequest = limitBelowRequest(healed ?? {}); + if (belowRequest) throw new BadRequestException(belowRequest); + await this.kubernetesService.mergePatchObject(kubeconfig, { + apiVersion: CNPG_API_VERSION, + kind: 'Cluster', + metadata: { name, namespace: app.k8sNamespace }, + spec: { resources, primaryUpdateMethod: CNPG_PRIMARY_UPDATE_METHOD }, + }); + if (container) { + await this.kubernetesService.patchWorkloadContainerResources( + kubeconfig, + single.kind, + app.k8sNamespace, + single.name, + container.name, + resources, + ); + } + this.logger.log( + `Updated resources of the highly available cluster ${name} in ${app.k8sNamespace}`, + ); + await this.recordResources(app, dto, app.resources, context); + return this.buildRuntimeResponse(app, kubeconfig); + } + + /** Keeps the stored resources on the live ones, and says who changed them. */ + private async recordResources( + app: ApplicationEntity, + dto: UpdateResourcesDto, + before: ApplicationResources | undefined, + context?: { + actor?: { id?: string; name?: string }; + reason?: string; + proposal?: string[]; + }, + ): Promise { + const merged = this.buildEntityResources(dto, app.resources); + await this.applicationsRepository.update(app.id, { resources: merged }); await this.appRevisionsRepository.createAuditEvent({ - applicationId: appId, + applicationId: app.id, eventType: AppEventType.RESOURCE_UPDATE, actor: context?.actor ? { type: AppEventActorType.USER, ...context.actor } : { type: AppEventActorType.API }, changeMetadata: { - before: beforeResources ?? {}, + before: before ?? {}, after: merged, containerName: dto.containerName ?? null, ...(context?.reason ? { reason: context.reason } : {}), ...(context?.proposal ? { proposal: context.proposal } : {}), }, }); - - // Fire-and-forget: track the rollout until all pods are ready - this.watchRollout( - app, - kubeconfig, - 'update-resources', - RolloutSection.RESOURCES, - true, - ); - - return this.buildRuntimeResponse(app, kubeconfig); } async applyReplicas( @@ -211,6 +305,8 @@ export class AppManagementService { replicas: number, ): Promise<{ app: ApplicationEntity; kubeconfig: string; previous: number }> { const { app, kubeconfig } = await this.resolveAppAndKubeconfig(appId); + const haRefused = databaseHaReplicasRefusal(app.databaseHa); + if (haRefused) throw new ConflictException(haRefused); const refused = systemReplicasRefusal(app, replicas); if (refused) throw new BadRequestException(refused); const { kind, name: deploymentName } = await this.getWorkloadOrThrow( @@ -272,12 +368,34 @@ export class AppManagementService { 'restart', ); if (refused) throw new ConflictException(refused); + const moving = databaseHaResizeRefusal(app.databaseHa); + if (moving) throw new ConflictException(moving); + const restartedAt = new Date().toISOString(); + if (databaseHaServes(app.databaseHa)) { + await this.kubernetesService.mergePatchObject(kubeconfig, { + apiVersion: CNPG_API_VERSION, + kind: 'Cluster', + metadata: { + name: app.databaseHa.cluster, + namespace: app.k8sNamespace, + annotations: { [CNPG_RESTART_ANNOTATION]: restartedAt }, + }, + spec: { primaryUpdateMethod: CNPG_PRIMARY_UPDATE_METHOD }, + }); + await this.appRevisionsRepository.createAuditEvent({ + applicationId: appId, + eventType: AppEventType.RESTART, + actor: { type: AppEventActorType.API }, + changeMetadata: { triggeredAt: restartedAt, highlyAvailable: true }, + envSnapshot: app.env ?? [], + }); + return this.buildRuntimeResponse(app, kubeconfig); + } const { kind, name: deploymentName } = await this.getWorkloadOrThrow( app, kubeconfig, ); - const restartedAt = new Date().toISOString(); await this.kubernetesService.restartWorkload( kubeconfig, kind, @@ -750,6 +868,33 @@ export class AppManagementService { return { kind, name, resource }; } + /** + * A highly available database serves from its cluster's instances; the + * single instance it keeps at zero copies would read as none running. + */ + private async haReplicas( + app: ApplicationEntity, + kubeconfig: string, + fallback: T, + ): Promise { + const cluster = await this.kubernetesService + .readObject( + kubeconfig, + CNPG_API_VERSION, + 'Cluster', + app.databaseHa!.cluster, + app.k8sNamespace, + ) + .catch(() => null); + if (!cluster) return fallback; + const ready = Number(cluster.status?.readyInstances ?? 0); + return { + desired: Number(cluster.spec?.instances ?? app.databaseHa!.instances), + ready, + available: ready, + }; + } + private async buildRuntimeResponse( app: ApplicationEntity, kubeconfig: string, @@ -811,7 +956,9 @@ export class AppManagementService { }; }); - const replicas = detail?.replicas ?? {}; + const replicas = databaseHaServes(app.databaseHa) + ? await this.haReplicas(app, kubeconfig, detail?.replicas ?? {}) + : (detail?.replicas ?? {}); const pods = await this.kubernetesService .listPodsByLabel(kubeconfig, app.k8sNamespace, `flui-app-id=${app.id}`) .catch(() => [] as k8s.V1Pod[]); @@ -835,6 +982,7 @@ export class AppManagementService { replicas.ready ?? 0, pods, ), + replicasRefusal: databaseHaReplicasRefusal(app.databaseHa), }; } diff --git a/src/modules/applications/services/application-manifest-generator.database-ha.spec.ts b/src/modules/applications/services/application-manifest-generator.database-ha.spec.ts new file mode 100644 index 00000000..0d83f15f --- /dev/null +++ b/src/modules/applications/services/application-manifest-generator.database-ha.spec.ts @@ -0,0 +1,130 @@ +import { load } from 'js-yaml'; +import { ApplicationManifestGeneratorService } from './application-manifest-generator.service'; +import { EncryptionService } from '../../shared/encryption/services/encryption.service'; +import { ApplicationEntity } from '../entities/application.entity'; +import { PoolerSpec } from '../utils/db-pooler.core'; +import { DatabaseHaSpec } from '../../postgres-ha/postgres-ha.types'; + +const generator = new ApplicationManifestGeneratorService({ + encrypt: (v: string) => `enc:${v}`, + decrypt: (v: string) => (v.startsWith('enc:') ? v.slice(4) : v), +} as unknown as EncryptionService); + +const pooler: PoolerSpec = { + enabled: true, + mode: 'transaction', + replicas: 1, + image: 'ghcr.io/cloudnative-pg/pgbouncer:1.26.0', + authPasswordEnc: 'enc:authpw', + adminPasswordEnc: 'enc:adminpw', + upstream: { mode: 'local' }, + enabledAt: '2026-10-12T00:00:00.000Z', +}; + +const ha = (state: DatabaseHaSpec['state']): DatabaseHaSpec => ({ + engine: 'cnpg', + cluster: 'postgresql-ab12-ha', + durability: 'required', + instances: 2, + state, + since: '2026-10-28T17:25:00.000Z', +}); + +function database(over: Partial = {}): ApplicationEntity { + return { + id: 'db-1', + slug: 'postgresql-ab12', + name: 'postgresql-ab12', + kind: 'DATABASE', + k8sNamespace: 'user-x', + replicas: 1, + port: 5432, + workloadKind: 'StatefulSet', + persistenceScope: 'dedicated', + dedicatedNodeName: 'worker-1', + labels: { 'flui.cloud/db-engine': 'postgres' }, + volumes: [{ name: 'data', mountPath: '/var/lib/postgresql', size: '10Gi' }], + env: [ + { name: 'POSTGRES_DB', value: 'postgresql-ab12' }, + { name: 'POSTGRES_USER', value: 'postgresql-ab12' }, + { name: 'POSTGRES_PASSWORD', value: 'enc:pw', secret: true }, + ], + sourceConfig: { + type: 'docker_image', + imageRef: 'ghcr.io/flui-cloud/flui-postgres:17', + }, + ...over, + } as unknown as ApplicationEntity; +} + +type Doc = { kind: string; metadata: { name: string }; spec?: any }; + +const render = (app: ApplicationEntity) => + generator.generateForDockerImage(app).map((m) => ({ + ...m, + doc: load(m.yaml) as Doc, + })); + +const find = (app: ApplicationEntity, kind: string, name: string) => + render(app).find((m) => m.doc.kind === kind && m.doc.metadata.name === name); + +describe('a managed Postgres that moved to a highly available cluster', () => { + /** Byte for byte, so no existing database is redeployed because this exists. */ + it('renders exactly as before while it is a single instance or still copying', () => { + const before = render(database()).map((m) => m.yaml); + expect(render(database({ databaseHa: null })).map((m) => m.yaml)).toEqual( + before, + ); + expect( + render(database({ databaseHa: ha('copying') })).map((m) => m.yaml), + ).toEqual(before); + expect( + render(database({ databaseHa: ha('failed') })).map((m) => m.yaml), + ).toEqual(before); + }); + + it("points the application's address at the cluster's primary once it serves", () => { + expect( + find(database(), 'Service', 'postgresql-ab12-svc')!.doc.spec.selector, + ).toEqual({ app: 'postgresql-ab12' }); + expect( + find( + database({ databaseHa: ha('active') }), + 'Service', + 'postgresql-ab12-svc', + )!.doc.spec.selector, + ).toEqual({ + 'cnpg.io/cluster': 'postgresql-ab12-ha', + 'cnpg.io/instanceRole': 'primary', + }); + }); + + /** Its volume is the way back if the move has to be undone. */ + it('keeps the single instance at zero copies instead of removing it', () => { + const sts = find( + database({ databaseHa: ha('active') }), + 'StatefulSet', + 'postgresql-ab12', + )!; + expect(sts.doc.spec.replicas).toBe(0); + expect( + find(database(), 'StatefulSet', 'postgresql-ab12')!.doc.spec.replicas, + ).toBe(1); + }); + + it('sends the pooler to the primary and lets it run on any node', () => { + const docs = generator + .generatePooler(database({ pooler, databaseHa: ha('active') })) + .map((m) => load(m.yaml) as Doc); + const upstream = docs.find( + (d) => d.metadata.name === 'postgresql-ab12-pooler-upstream', + )!; + expect(upstream.spec.selector).toEqual({ + 'cnpg.io/cluster': 'postgresql-ab12-ha', + 'cnpg.io/instanceRole': 'primary', + }); + const deployment = docs.find((d) => d.kind === 'Deployment')!; + expect(deployment.spec.template.spec.affinity).toBeUndefined(); + expect(deployment.spec.template.spec.nodeSelector).toBeUndefined(); + }); +}); diff --git a/src/modules/applications/services/application-manifest-generator.replicated-volume.spec.ts b/src/modules/applications/services/application-manifest-generator.replicated-volume.spec.ts new file mode 100644 index 00000000..4d92bece --- /dev/null +++ b/src/modules/applications/services/application-manifest-generator.replicated-volume.spec.ts @@ -0,0 +1,70 @@ +import { load } from 'js-yaml'; +import { ApplicationManifestGeneratorService } from './application-manifest-generator.service'; +import { EncryptionService } from '../../shared/encryption/services/encryption.service'; +import { ApplicationEntity } from '../entities/application.entity'; + +const generator = new ApplicationManifestGeneratorService({ + encrypt: (v: string) => `enc:${v}`, + decrypt: (v: string) => (v.startsWith('enc:') ? v.slice(4) : v), +} as unknown as EncryptionService); + +type Doc = { + kind: string; + spec: { strategy?: { type: string }; replicas: number }; +}; + +function app(volumes: ApplicationEntity['volumes']): ApplicationEntity { + return { + id: 'app-1', + slug: 'gitea-ab12', + name: 'gitea-ab12', + k8sNamespace: 'user-x', + replicas: 1, + port: 3000, + volumes, + sourceConfig: { type: 'docker_image', imageRef: 'gitea/gitea:1.24' }, + } as unknown as ApplicationEntity; +} + +const deployment = (a: ApplicationEntity) => + generator + .generateForDockerImage(a) + .map((m) => ({ yaml: m.yaml, doc: load(m.yaml) as Doc })) + .find((m) => m.doc.kind === 'Deployment')!; + +describe('ApplicationManifestGenerator, replicated volumes', () => { + it('replaces the pod instead of rolling it when a volume is replicated', () => { + const d = deployment( + app([ + { + name: 'data', + mountPath: '/data', + size: '10Gi', + storageClass: 'flui-replicated', + }, + ] as ApplicationEntity['volumes']), + ); + expect(d.doc.spec.strategy).toEqual({ type: 'Recreate' }); + expect(d.doc.spec.replicas).toBe(1); + }); + + it('leaves every other Deployment as it was, without a blank line where the strategy would go', () => { + for (const volumes of [ + [], + [{ name: 'data', mountPath: '/data', size: '10Gi' }], + [ + { + name: 'data', + mountPath: '/data', + size: '10Gi', + storageClass: 'flui-local', + }, + ], + ] as ApplicationEntity['volumes'][]) { + const d = deployment(app(volumes)); + expect(d.doc.spec.strategy).toBeUndefined(); + expect(d.yaml).not.toContain('STRATEGY_BLOCK'); + expect(d.yaml).toMatch(/ {2}replicas: 1\n {2}selector:/); + } + }); +}); diff --git a/src/modules/applications/services/application-manifest-generator.service.ts b/src/modules/applications/services/application-manifest-generator.service.ts index 96597c46..31c9beb4 100644 --- a/src/modules/applications/services/application-manifest-generator.service.ts +++ b/src/modules/applications/services/application-manifest-generator.service.ts @@ -40,6 +40,11 @@ import { renderUserlist, } from '../utils/db-pooler.core'; import { isPlatformApi } from '../schedulers/platform-api-sizing'; +import { + databaseHaPrimarySelector, + databaseHaServes, +} from '../../postgres-ha/postgres-ha.types'; +import { REPLICATED_STORAGE_CLASS } from '../../replicated-volumes/replicated-volumes.constants'; export interface GeneratedManifest { kind: ApplicationResourceKind; @@ -482,6 +487,9 @@ export class ApplicationManifestGeneratorService { * wherever the database could. */ private poolerPlacement(app: ApplicationEntity): Record { + // The primary of a highly available cluster changes node, so the pooler is + // not tied to one. + if (databaseHaServes(app.databaseHa)) return {}; return { affinity: { podAffinity: { @@ -559,6 +567,7 @@ export class ApplicationManifestGeneratorService { .replaceAll('{{POD_LABELS_BLOCK}}', this.renderLabelsBlock(labels, 8)) .replaceAll('{{ANNOTATIONS_BLOCK}}', this.renderLabelsBlock(annotations)) .replaceAll('{{REPLICAS}}', String(app.replicas ?? 1)) + .replaceAll('{{STRATEGY_BLOCK}}\n', this.renderStrategyBlock(app)) .replaceAll('{{IMAGE}}', imageRef) .replaceAll('{{PULL_POLICY}}', config.pullPolicy || 'IfNotPresent') .replaceAll( @@ -637,6 +646,9 @@ export class ApplicationManifestGeneratorService { const ephemeralRequest = app.resources?.ephemeralStorage?.request; const ephemeralLimit = app.resources?.ephemeralStorage?.limit; + // Once a highly available cluster serves, the single instance stays at zero + // copies: its volume is the way back if the move has to be undone. + const replicas = databaseHaServes(app.databaseHa) ? 0 : (app.replicas ?? 1); const template = this.loadTemplate('statefulset.yaml'); const yaml = template .replaceAll('{{SLUG}}', app.slug) @@ -644,7 +656,7 @@ export class ApplicationManifestGeneratorService { .replaceAll('{{LABELS_BLOCK}}', this.renderLabelsBlock(labels)) .replaceAll('{{POD_LABELS_BLOCK}}', this.renderLabelsBlock(labels, 8)) .replaceAll('{{ANNOTATIONS_BLOCK}}', this.renderLabelsBlock(annotations)) - .replaceAll('{{REPLICAS}}', String(app.replicas ?? 1)) + .replaceAll('{{REPLICAS}}', String(replicas)) .replaceAll('{{IMAGE}}', imageRef) .replaceAll('{{PULL_POLICY}}', config.pullPolicy || 'IfNotPresent') .replaceAll( @@ -828,11 +840,20 @@ export class ApplicationManifestGeneratorService { const labels = this.buildLabels(app); const template = this.loadTemplate('service.yaml'); - const yaml = template + let yaml = template .replaceAll('{{SLUG}}', app.slug) .replaceAll('{{NAMESPACE}}', app.k8sNamespace) .replaceAll('{{LABELS_BLOCK}}', this.renderLabelsBlock(labels)) .replaceAll('{{PORT}}', String(app.port)); + if (databaseHaServes(app.databaseHa)) { + const selector = Object.entries(databaseHaPrimarySelector(app.databaseHa)) + .map(([k, v]) => ` ${k}: ${v}`) + .join('\n'); + yaml = yaml.replace( + ` selector:\n app: ${app.slug}\n`, + ` selector:\n${selector}\n`, + ); + } return { kind: ApplicationResourceKind.SERVICE, @@ -1028,6 +1049,17 @@ export class ApplicationManifestGeneratorService { return undefined; } + /** + * A replicated volume is attached to one node at a time, so the new pod of a + * rolling update would wait for a volume the old pod never lets go. + */ + private renderStrategyBlock(app: ApplicationEntity): string { + const replicated = (app.volumes ?? []).some( + (v) => this.resolveStorageClass(app, v) === REPLICATED_STORAGE_CLASS, + ); + return replicated ? ' strategy:\n type: Recreate\n' : ''; + } + private nodePlacement(app: ApplicationEntity): { nodeSelector?: Record; tolerations?: Array<{ key: string; operator: string; effect: string }>; diff --git a/src/modules/applications/services/application-reconciliation.service.ts b/src/modules/applications/services/application-reconciliation.service.ts index 292faf99..9a192312 100644 --- a/src/modules/applications/services/application-reconciliation.service.ts +++ b/src/modules/applications/services/application-reconciliation.service.ts @@ -15,6 +15,11 @@ import { AppResourceEntity } from '../entities/app-resource.entity'; import { ApplicationStatus } from '../enums/application-status.enum'; import { ApplicationCategory } from '../enums/application-category.enum'; import { ApplicationResourceStatus } from '../enums/application-resource-status.enum'; +import { + haClusterStatus, + servesFromHaCluster, +} from '../utils/database-ha-status.core'; +import { CNPG_API_VERSION } from '../../postgres-ha/postgres-ha.constants'; import { readEndpointFailure } from '../utils/endpoint-failure.util'; type K8sResource = { @@ -434,6 +439,31 @@ export class ApplicationReconciliationService { return { allReady, anyFailed, anyDrift, transientFailures }; } + private async readinessOf( + app: ApplicationEntity, + resource: AppResourceEntity, + actual: K8sResource, + kubeconfig: string, + ): Promise { + if (!servesFromHaCluster(app.databaseHa, resource.kind, actual)) { + return this.computeResourceStatus(resource.kind, actual); + } + const cluster = app.databaseHa!.cluster; + return haClusterStatus( + await this.withK8sRetry( + () => + this.kubernetesService.readObject( + kubeconfig, + CNPG_API_VERSION, + 'Cluster', + cluster, + resource.namespace, + ), + `Cluster/${cluster}`, + ), + ); + } + private async reconcileResource( app: ApplicationEntity, resource: AppResourceEntity, @@ -506,7 +536,12 @@ export class ApplicationReconciliationService { const resourceVersion = actual.metadata?.resourceVersion || ''; // 4. Determine readiness - const resourceStatus = this.computeResourceStatus(resource.kind, actual); + const resourceStatus = await this.readinessOf( + app, + resource, + actual, + kubeconfig, + ); // 5. Detect drift: compare actualHash vs desiredHash (unless ignored). // When actualHash is null (annotation missing) we cannot verify drift diff --git a/src/modules/applications/services/application-source-deploy.service.ts b/src/modules/applications/services/application-source-deploy.service.ts index 4b819c82..8396b8a4 100644 --- a/src/modules/applications/services/application-source-deploy.service.ts +++ b/src/modules/applications/services/application-source-deploy.service.ts @@ -82,11 +82,19 @@ import { normalizeClusterType, } from '../../infrastructure/clusters/entities/cluster.entity'; import { + replicatedVolumesRefusal, wouldDeploy, type CapacityFact, type ManifestCheck, + type ReplicatedVolumesFact, } from '../manifest-checks.core'; import { allChecksFor, type RepoFacts } from '../manifest-repo-checks.core'; +import { + appVolumesFromManifest, + declaresReplicatedVolume, + volumeReplicationMismatches, +} from '../utils/manifest-volumes.util'; +import { ReplicatedVolumesService } from '../../replicated-volumes/services/replicated-volumes.service'; import { manifestSelfFacts } from '../manifest-self-facts.core'; import { RepoFactsReaderService } from './repo-facts-reader.service'; import { EndpointType } from '../../dns/enums/endpoint-type.enum'; @@ -202,6 +210,8 @@ export class ApplicationSourceDeployService { private readonly attachedServices?: AttachedServicesPort, @Optional() private readonly registry?: FluiRegistryPublisherService, + @Optional() + private readonly replicatedVolumes?: ReplicatedVolumesService, ) {} /** Builds push to the instance's own registry, so no GHCR token is needed. */ @@ -384,6 +394,7 @@ export class ApplicationSourceDeployService { // a refusal here, not an application that comes up green without its // database. Same call the `--validate-only` preview makes. await this.validateAttachedServices(manifest); + await this.assertReplicatedVolumes(manifest, dto.clusterId, app); // Resolve the imageRef to use when skipping the build: // 1. dto.imageRef (explicit) — wins @@ -457,7 +468,7 @@ export class ApplicationSourceDeployService { ...scalingFromManifest(manifest, null), healthProbe: healthProbe as any, startCommand: manifest.deploy.startCommand, - volumes: (manifest.deploy.volumes as any) ?? [], + volumes: appVolumesFromManifest(manifest.deploy.volumes), autoDeploy: false, metadata: manifestMetadata, projectId: dto.projectId, @@ -678,6 +689,11 @@ export class ApplicationSourceDeployService { hasWorkloadCluster: cluster.isControl ? await this.hasWorkloadCluster() : null, + replicatedVolumes: await this.replicatedVolumesFact( + manifest, + dto.clusterId, + existingApp, + ), }, manifestSelfFacts(manifest), repo, @@ -721,6 +737,39 @@ export class ApplicationSourceDeployService { return this.repoFactsReader.factsFor(userId, { owner, repo, ref: branch }); } + private async replicatedVolumesFact( + manifest: ApplicationManifest, + clusterId: string, + app: ApplicationEntity | null, + ): Promise { + const declared = declaresReplicatedVolume(manifest.deploy?.volumes); + const mismatches = volumeReplicationMismatches( + manifest.deploy?.volumes, + app?.volumes, + ); + if (!declared && !mismatches.length) return null; + const enabled = + declared && this.replicatedVolumes + ? await this.replicatedVolumes.isEnabled(clusterId) + : null; + return { declared, enabled, mismatches }; + } + + /** The application would otherwise run without the copy its manifest declares. */ + private async assertReplicatedVolumes( + manifest: ApplicationManifest, + clusterId: string, + app: ApplicationEntity | null, + ): Promise { + const fact = await this.replicatedVolumesFact(manifest, clusterId, app); + if (!fact) return; + const refusal = replicatedVolumesRefusal( + fact, + (await this.readCluster(clusterId)).name, + ); + if (refusal) throw new BadRequestException(refusal); + } + private async readCluster(clusterId: string): Promise<{ found: boolean; ready: boolean | null; @@ -979,6 +1028,7 @@ export class ApplicationSourceDeployService { existingEnv, ); this.warnEnvShadows(existingEnv, manifestEnv); + await this.assertReplicatedVolumes(manifest, app.clusterId, app); const endpointSpecJson = manifest.deploy.domain ? JSON.stringify(manifest.deploy.domain) diff --git a/src/modules/applications/templates/k8s/deployment.yaml b/src/modules/applications/templates/k8s/deployment.yaml index 51e4bd51..e939f614 100644 --- a/src/modules/applications/templates/k8s/deployment.yaml +++ b/src/modules/applications/templates/k8s/deployment.yaml @@ -9,6 +9,7 @@ metadata: {{ANNOTATIONS_BLOCK}} spec: replicas: {{REPLICAS}} +{{STRATEGY_BLOCK}} selector: matchLabels: app: {{SLUG}} diff --git a/src/modules/applications/utils/database-ha-edits.core.ts b/src/modules/applications/utils/database-ha-edits.core.ts new file mode 100644 index 00000000..9457bd0f --- /dev/null +++ b/src/modules/applications/utils/database-ha-edits.core.ts @@ -0,0 +1,67 @@ +import { DatabaseHaSpec } from '../../postgres-ha/postgres-ha.types'; +import { ResourcePair } from '../../shared/utils/resource-quantity.util'; + +/** CloudNativePG restarts a cluster's instances one at a time when this annotation changes. */ +export const CNPG_RESTART_ANNOTATION = 'kubectl.kubernetes.io/restartedAt'; + +const MOVING = + 'This database is being made highly available; wait until the move ends.'; + +/** + * Why a change of copies, a stop or a start is refused: a highly available + * database keeps its instances across three locations, and its single instance + * stays at zero as the way back. Scaling that one would start a second + * Postgres beside the cluster that serves. + */ +export function databaseHaReplicasRefusal( + spec: DatabaseHaSpec | null | undefined, +): string | null { + if (spec?.state === 'active') { + return `This database is highly available: Flui keeps its ${spec.instances} instances across three locations, so the number of copies is not changed here, and it is not stopped or started.`; + } + return spec?.state === 'copying' ? MOVING : null; +} + +/** While the copy is made the single instance still serves, and the cluster is being built from it. */ +export function databaseHaResizeRefusal( + spec: DatabaseHaSpec | null | undefined, +): string | null { + return spec?.state === 'copying' ? MOVING : null; +} + +/** + * What a change applies over: the cluster's own resources, or, when it was + * made without any, those the single instance ran with. + */ +export function haResourceBase( + cluster: ResourcePair | undefined, + single: ResourcePair | undefined, +): ResourcePair { + return cluster && (cluster.requests || cluster.limits) + ? cluster + : (single ?? {}); +} + +/** The resources after a change: the values given over the current ones. */ +export function mergedResources( + current: ResourcePair, + change: ResourcePair, +): ResourcePair { + const side = ( + base?: { cpu?: string; memory?: string }, + next?: { cpu?: string; memory?: string }, + ) => + next + ? { + ...base, + ...(next.cpu !== undefined && { cpu: next.cpu }), + ...(next.memory !== undefined && { memory: next.memory }), + } + : base; + const requests = side(current.requests, change.requests); + const limits = side(current.limits, change.limits); + return { + ...(requests && { requests }), + ...(limits && { limits }), + }; +} diff --git a/src/modules/applications/utils/database-ha-status.core.spec.ts b/src/modules/applications/utils/database-ha-status.core.spec.ts new file mode 100644 index 00000000..33defcde --- /dev/null +++ b/src/modules/applications/utils/database-ha-status.core.spec.ts @@ -0,0 +1,50 @@ +import { ApplicationResourceStatus } from '../enums/application-resource-status.enum'; +import { + haClusterStatus, + servesFromHaCluster, +} from './database-ha-status.core'; + +const active = { + engine: 'cnpg', + cluster: 'db-ha', + durability: 'required', + instances: 3, + state: 'active', + since: 'x', +} as const; + +describe('a database that serves from its highly available cluster', () => { + it('reads the single instance kept at zero copies as the way back, not as a failure', () => { + expect( + servesFromHaCluster(active, 'StatefulSet', { spec: { replicas: 0 } }), + ).toBe(true); + expect( + servesFromHaCluster(active, 'StatefulSet', { spec: { replicas: 1 } }), + ).toBe(false); + expect( + servesFromHaCluster({ ...active, state: 'copying' }, 'StatefulSet', { + spec: { replicas: 0 }, + }), + ).toBe(false); + expect( + servesFromHaCluster(null, 'StatefulSet', { spec: { replicas: 0 } }), + ).toBe(false); + }); + + it('is ready with every instance up, degraded with one missing or during a failover', () => { + const cluster = (phase: string, ready: number) => ({ + spec: { instances: 3 }, + status: { phase, readyInstances: ready, currentPrimary: 'db-ha-1' }, + }); + expect(haClusterStatus(cluster('Cluster in healthy state', 3))).toBe( + ApplicationResourceStatus.READY, + ); + expect(haClusterStatus(cluster('Cluster in healthy state', 2))).toBe( + ApplicationResourceStatus.DEGRADED, + ); + expect(haClusterStatus(cluster('Failing over', 2))).toBe( + ApplicationResourceStatus.DEGRADED, + ); + expect(haClusterStatus(null)).toBe(ApplicationResourceStatus.DEGRADED); + }); +}); diff --git a/src/modules/applications/utils/database-ha-status.core.ts b/src/modules/applications/utils/database-ha-status.core.ts new file mode 100644 index 00000000..93443370 --- /dev/null +++ b/src/modules/applications/utils/database-ha-status.core.ts @@ -0,0 +1,41 @@ +import { ApplicationResourceStatus } from '../enums/application-resource-status.enum'; +import { DatabaseHaSpec } from '../../postgres-ha/postgres-ha.types'; + +/** + * A database made highly available keeps its single instance at zero copies + * as the way back. Read as a workload that wants one copy and has none, it + * made a serving database look broken; the instances that do serve are the + * highly available cluster's. + */ +export function servesFromHaCluster( + spec: DatabaseHaSpec | null | undefined, + kind: string, + actual: { spec?: { replicas?: number } }, +): boolean { + return ( + spec?.state === 'active' && + kind === 'StatefulSet' && + actual.spec?.replicas === 0 + ); +} + +/** Ready with every instance up; degraded while one is missing or a failover runs. */ +export function haClusterStatus( + cluster: { + spec?: { instances?: number }; + status?: { + phase?: string; + readyInstances?: number; + currentPrimary?: string; + }; + } | null, +): ApplicationResourceStatus { + if (!cluster?.status?.currentPrimary) + return ApplicationResourceStatus.DEGRADED; + const instances = cluster.spec?.instances ?? 0; + const ready = cluster.status.readyInstances ?? 0; + return cluster.status.phase === 'Cluster in healthy state' && + ready >= instances + ? ApplicationResourceStatus.READY + : ApplicationResourceStatus.DEGRADED; +} diff --git a/src/modules/applications/utils/db-pooler.core.ts b/src/modules/applications/utils/db-pooler.core.ts index f81c8655..2f3c5045 100644 --- a/src/modules/applications/utils/db-pooler.core.ts +++ b/src/modules/applications/utils/db-pooler.core.ts @@ -1,4 +1,9 @@ import { createHash, createHmac, pbkdf2Sync, randomBytes } from 'node:crypto'; +import { + DatabaseHaSpec, + databaseHaPrimarySelector, + databaseHaServes, +} from '../../postgres-ha/postgres-ha.types'; export type PoolerMode = 'transaction' | 'session'; @@ -110,7 +115,12 @@ export function poolerLabels(app: { * separate app, so this and the role exec on `flui-app-id` reach the same pod. */ export function poolerUpstreamObjects( - app: { id: string; slug: string; k8sNamespace: string }, + app: { + id: string; + slug: string; + k8sNamespace: string; + databaseHa?: DatabaseHaSpec | null; + }, spec: Pick, ): { service: Record; @@ -126,7 +136,13 @@ export function poolerUpstreamObjects( spec: { type: 'ClusterIP', ...(spec.upstreamClusterIP ? { clusterIP: spec.upstreamClusterIP } : {}), - ...(forward ? {} : { selector: { app: app.slug } }), + ...(forward + ? {} + : { + selector: databaseHaServes(app.databaseHa) + ? databaseHaPrimarySelector(app.databaseHa) + : { app: app.slug }, + }), ports: [ { name: 'postgres', diff --git a/src/modules/applications/utils/manifest-volumes.util.spec.ts b/src/modules/applications/utils/manifest-volumes.util.spec.ts new file mode 100644 index 00000000..c3030fc6 --- /dev/null +++ b/src/modules/applications/utils/manifest-volumes.util.spec.ts @@ -0,0 +1,57 @@ +import { + appVolumesFromManifest, + declaresReplicatedVolume, + volumeReplicationMismatches, +} from './manifest-volumes.util'; + +describe('manifest volumes', () => { + it('turns replicated into the replicated storage class and nothing else', () => { + expect( + appVolumesFromManifest([ + { name: 'data', mountPath: '/data', size: '2Gi', replicated: true }, + { name: 'cache', mountPath: '/cache', replicated: false }, + ]), + ).toEqual([ + { + name: 'data', + mountPath: '/data', + size: '2Gi', + storageClass: 'flui-replicated', + }, + { name: 'cache', mountPath: '/cache' }, + ]); + expect(appVolumesFromManifest(undefined)).toEqual([]); + }); + + it('says whether any volume asks for a copy', () => { + expect( + declaresReplicatedVolume([ + { name: 'd', mountPath: '/d', replicated: true }, + ]), + ).toBe(true); + expect(declaresReplicatedVolume([{ name: 'd', mountPath: '/d' }])).toBe( + false, + ); + }); + + it('names existing volumes whose storage the manifest now contradicts', () => { + expect( + volumeReplicationMismatches( + [ + { name: 'data', mountPath: '/data', replicated: true }, + { name: 'logs', mountPath: '/logs' }, + { name: 'new', mountPath: '/new', replicated: true }, + { name: 'kept', mountPath: '/kept', replicated: true }, + ], + [ + { name: 'data', mountPath: '/data', storageClass: 'flui-local' }, + { name: 'logs', mountPath: '/logs', storageClass: 'flui-replicated' }, + { name: 'kept', mountPath: '/kept', storageClass: 'flui-replicated' }, + ], + ), + ).toEqual([ + { name: 'data', declared: true }, + { name: 'logs', declared: false }, + ]); + }); +}); diff --git a/src/modules/applications/utils/manifest-volumes.util.ts b/src/modules/applications/utils/manifest-volumes.util.ts new file mode 100644 index 00000000..cd9d1db7 --- /dev/null +++ b/src/modules/applications/utils/manifest-volumes.util.ts @@ -0,0 +1,55 @@ +import { REPLICATED_STORAGE_CLASS } from '../../replicated-volumes/replicated-volumes.constants'; +import { ApplicationVolume } from '../interfaces/source-config.interface'; + +export interface ManifestVolumeInput { + name: string; + mountPath: string; + size?: string; + replicated?: boolean; +} + +export interface VolumeReplicationMismatch { + name: string; + declared: boolean; +} + +/** `replicated: true` is the replicated storage class; every other volume keeps the cluster's default. */ +export function appVolumesFromManifest( + volumes: readonly ManifestVolumeInput[] | undefined, +): ApplicationVolume[] { + return (volumes ?? []).map(({ name, mountPath, size, replicated }) => ({ + name, + mountPath, + ...(size ? { size } : {}), + ...(replicated ? { storageClass: REPLICATED_STORAGE_CLASS } : {}), + })); +} + +export function declaresReplicatedVolume( + volumes: readonly ManifestVolumeInput[] | undefined, +): boolean { + return (volumes ?? []).some((v) => v.replicated === true); +} + +/** + * A volume's storage is chosen when the volume is created and a redeploy never + * moves its data, so a manifest that now says otherwise about a volume the + * application already has would describe storage that is not the one running. + */ +export function volumeReplicationMismatches( + declared: readonly ManifestVolumeInput[] | undefined, + existing: readonly ApplicationVolume[] | undefined, +): VolumeReplicationMismatch[] { + const actual = new Map( + (existing ?? []).map((v) => [ + v.name, + v.storageClass === REPLICATED_STORAGE_CLASS, + ]), + ); + return (declared ?? []) + .filter( + (v) => + actual.has(v.name) && actual.get(v.name) !== (v.replicated === true), + ) + .map((v) => ({ name: v.name, declared: v.replicated === true })); +} diff --git a/src/modules/auth/constants/api-key-groups.spec.ts b/src/modules/auth/constants/api-key-groups.spec.ts index 58d773e9..27614ba8 100644 --- a/src/modules/auth/constants/api-key-groups.spec.ts +++ b/src/modules/auth/constants/api-key-groups.spec.ts @@ -123,6 +123,8 @@ describe('permission groups — the taxonomy', () => { 'catalog_search', 'cluster_list', 'cluster_resources', + 'db_ha_enable', + 'db_ha_get', 'db_pooler_client_close', 'db_pooler_get', 'db_session_cancel', diff --git a/src/modules/auth/constants/api-key-groups.ts b/src/modules/auth/constants/api-key-groups.ts index 6a570744..140248fe 100644 --- a/src/modules/auth/constants/api-key-groups.ts +++ b/src/modules/auth/constants/api-key-groups.ts @@ -154,7 +154,7 @@ export const PERMISSION_GROUPS: PermissionGroupDef[] = [ depth: PERMISSION_DEPTH.CHANGE, label: 'Deploy and operate applications', summary: - 'Deploy, install, scale, restart, stop and roll back applications, change how much CPU and memory they may use, set their non-secret configuration, set up the routes, schedules, wildcard DNS and repository links they need, choose whether a backup is taken before each deploy, read a connected repository and commit the manifests Flui renders for it onto a branch of Flui’s own — never yours — starting builds that spend that repository’s own GitHub Actions minutes, read the logs, metrics, traffic and alerts of what it is running, and ask you in person for a sensitive value it must never hold itself, on top of everything See applications reads — but never delete one.', + 'Deploy, install, scale, restart, stop and roll back applications, make a managed PostgreSQL highly available once a person approves it, change how much CPU and memory they may use, set their non-secret configuration, set up the routes, schedules, wildcard DNS and repository links they need, choose whether a backup is taken before each deploy, read a connected repository and commit the manifests Flui renders for it onto a branch of Flui’s own — never yours — starting builds that spend that repository’s own GitHub Actions minutes, read the logs, metrics, traffic and alerts of what it is running, and ask you in person for a sensitive value it must never hold itself, on top of everything See applications reads — but never delete one.', scopes: APPS_CHANGE, }, { diff --git a/src/modules/backups/backups.module.ts b/src/modules/backups/backups.module.ts index c86179c0..246102ff 100644 --- a/src/modules/backups/backups.module.ts +++ b/src/modules/backups/backups.module.ts @@ -100,6 +100,9 @@ import { AppVolumeBackupProcessor } from './processors/app-volume-backup.process import { VeleroUninstallService } from './services/velero-uninstall.service'; import { UninstallVeleroProcessor } from './processors/uninstall-velero.processor'; import { VeleroUninstallController } from './controllers/velero-uninstall.controller'; +import { PgBackrestHaService } from './services/pgbackrest-ha.service'; +import { DatabaseHaBackupHandoverService } from './services/database-ha-backup-handover.service'; +import { PgbackrestPluginInstallerService } from '../postgres-ha/services/pgbackrest-plugin-installer.service'; @Module({ imports: [ @@ -180,6 +183,9 @@ import { VeleroUninstallController } from './controllers/velero-uninstall.contro DeclaredEngineResolver, BackupStatusService, PgBackrestService, + PgBackrestHaService, + PgbackrestPluginInstallerService, + DatabaseHaBackupHandoverService, PgLegacyRepoRetirer, PlaintextRetirementService, DestinationPlacementValidator, diff --git a/src/modules/backups/processors/run-db-backup.processor.ts b/src/modules/backups/processors/run-db-backup.processor.ts index f533d5a1..8e94aa7f 100644 --- a/src/modules/backups/processors/run-db-backup.processor.ts +++ b/src/modules/backups/processors/run-db-backup.processor.ts @@ -1,5 +1,5 @@ import { Processor, Process } from '@nestjs/bull'; -import { Logger } from '@nestjs/common'; +import { Logger, Optional } from '@nestjs/common'; import { Job } from 'bull'; import { InjectRepository } from '@nestjs/typeorm'; import { Repository } from 'typeorm'; @@ -31,6 +31,7 @@ import { ArtifactLocationState } from '../enums/artifact-location-state.enum'; import { BackupArtifactLocationEntity } from '../entities/backup-artifact-location.entity'; import { BACKUP_QUEUE, BACKUP_JOB_TYPES } from '../backups.constants'; import { EncryptionMode } from '../enums/destination-health.enum'; +import { DatabaseHaBackupHandoverService } from '../services/database-ha-backup-handover.service'; @Processor(BACKUP_QUEUE) export class RunDbBackupProcessor { @@ -44,6 +45,7 @@ export class RunDbBackupProcessor { private readonly destRepo: BackupDestinationRepository, private readonly artifactRepo: BackupArtifactRepository, private readonly engines: ContinuousBackupEngineRegistry, + @Optional() private readonly handover?: DatabaseHaBackupHandoverService, ) {} @Process(BACKUP_JOB_TYPES.RUN_DB_BACKUP) @@ -89,6 +91,11 @@ export class RunDbBackupProcessor { // the workload: this job can run when that workload is mid-redeploy, and // driving one database's tool against another's is the failure the // registry exists to make impossible. + // A database made highly available whose backup did not follow it yet + // ships from its cluster from this run on. + if (await this.handover?.handOver(appId, { takeBase: false })) { + Object.assign(policy, await this.policiesService.findById(policy.id)); + } const engine = this.engines.forEngine(policy.engine); // Idempotent + self-healing: (re)write conf, ensure the stanza exists, flip diff --git a/src/modules/backups/services/continuous-backup-engine.registry.ts b/src/modules/backups/services/continuous-backup-engine.registry.ts index e12c22ab..2867838a 100644 --- a/src/modules/backups/services/continuous-backup-engine.registry.ts +++ b/src/modules/backups/services/continuous-backup-engine.registry.ts @@ -1,6 +1,7 @@ import { BadRequestException, Injectable } from '@nestjs/common'; import { ContinuousBackupEngine } from './continuous-backup-engine.interface'; import { PgBackrestService } from './pgbackrest.service'; +import { PgBackrestHaService } from './pgbackrest-ha.service'; import { MariadbPitrService } from './mariadb-pitr.service'; import { MariadbDumpService, @@ -20,23 +21,27 @@ import { export class ContinuousBackupEngineRegistry { private readonly engines: ReadonlyMap; - private readonly dumpFallback: ReadonlyMap; + private readonly fallbacks: ReadonlyMap; constructor( pgbackrest: PgBackrestService, mariadb: MariadbPitrService, postgresDump: PostgresDumpService, mariadbDump: MariadbDumpService, + pgbackrestHa: PgBackrestHaService, ) { this.engines = new Map([ [pgbackrest.engine, pgbackrest], [mariadb.engine, mariadb], [postgresDump.engine, postgresDump], [mariadbDump.engine, mariadbDump], + [pgbackrestHa.engine, pgbackrestHa], ]); - this.dumpFallback = new Map([ - [pgbackrest.engine, postgresDump], - [mariadb.engine, mariadbDump], + // A highly available Postgres keeps continuous backup through its own + // instances before anything falls back to a dump. + this.fallbacks = new Map([ + [pgbackrest.engine, [pgbackrestHa, postgresDump]], + [mariadb.engine, [mariadbDump]], ]); } @@ -55,14 +60,16 @@ export class ContinuousBackupEngineRegistry { await continuous.requireTooling(appId); return continuous; } catch (err) { - const dump = this.dumpFallback.get(continuous.engine); - if (!dump || !(err instanceof BadRequestException)) throw err; - try { - await dump.requireTooling(appId); - } catch { - throw err; + if (!(err instanceof BadRequestException)) throw err; + for (const next of this.fallbacks.get(continuous.engine) ?? []) { + try { + await next.requireTooling(appId); + return next; + } catch { + continue; + } } - return dump; + throw err; } } diff --git a/src/modules/backups/services/database-ha-backup-handover.service.spec.ts b/src/modules/backups/services/database-ha-backup-handover.service.spec.ts new file mode 100644 index 00000000..f7904fd5 --- /dev/null +++ b/src/modules/backups/services/database-ha-backup-handover.service.spec.ts @@ -0,0 +1,73 @@ +jest.mock('@kubernetes/client-node', () => ({})); +jest.mock('jose', () => ({})); +jest.mock('ip-cidr', () => ({})); + +import { DatabaseHaBackupHandoverService } from './database-ha-backup-handover.service'; + +describe('continuous backup following a database to high availability', () => { + function build(policy: Record | null) { + const policyRepo = { + findDbPolicyForApp: jest.fn().mockResolvedValue(policy), + update: jest.fn(), + }; + const jobs = { createOnDemand: jest.fn() }; + const ha = { + engine: 'postgres-ha', + mintGeneration: () => '20261108120000', + enable: jest.fn(), + requireTooling: jest.fn().mockResolvedValue(undefined), + }; + const service = new DatabaseHaBackupHandoverService( + policyRepo as never, + { findById: jest.fn().mockResolvedValue({ id: 'd1' }) } as never, + { primaryDestinationOf: () => ({ destinationId: 'd1' }) } as never, + jobs as never, + ha as never, + ); + return { service, policyRepo, jobs, ha }; + } + + it('moves the policy to a repository of its own and takes a full right away', async () => { + const { service, policyRepo, jobs, ha } = build({ + id: 'p1', + engine: 'postgres', + userId: 'u1', + retentionMaxCopies: 3, + metadata: { generation: 'old' }, + }); + + await expect(service.handOver('a1')).resolves.toContain( + 'first full backup', + ); + + expect(ha.enable).toHaveBeenCalledWith( + 'a1', + { id: 'd1' }, + { + retentionFull: 3, + generation: '20261108120000', + }, + ); + expect(policyRepo.update).toHaveBeenCalledWith('p1', { + engine: 'postgres-ha', + metadata: { generation: '20261108120000' }, + }); + expect(jobs.createOnDemand).toHaveBeenCalledWith( + 'u1', + expect.objectContaining({ policyId: 'p1' }), + 'scheduled', + ); + }); + + it('leaves alone a database with no policy, one already moved, and one backed up by dumps', async () => { + for (const policy of [ + null, + { engine: 'postgres-ha' }, + { engine: 'postgres-dump' }, + ]) { + const { service, ha } = build(policy); + await expect(service.handOver('a1')).resolves.toBeNull(); + expect(ha.enable).not.toHaveBeenCalled(); + } + }); +}); diff --git a/src/modules/backups/services/database-ha-backup-handover.service.ts b/src/modules/backups/services/database-ha-backup-handover.service.ts new file mode 100644 index 00000000..2e22dbea --- /dev/null +++ b/src/modules/backups/services/database-ha-backup-handover.service.ts @@ -0,0 +1,73 @@ +import { BadRequestException, Injectable, Logger } from '@nestjs/common'; +import { BackupPolicyRepository } from '../repositories/backup-policy.repository'; +import { BackupDestinationRepository } from '../repositories/backup-destination.repository'; +import { BackupPoliciesService } from './backup-policies.service'; +import { BackupJobsService } from './backup-jobs.service'; +import { BackupJobTriggerType } from '../enums/backup-job.enum'; +import { PgBackrestHaService } from './pgbackrest-ha.service'; + +/** Rows written before the engine column existed are pgBackRest's, like the registry reads them. */ +const SINGLE_INSTANCE_ENGINE = 'postgres'; + +/** + * Moves a database's continuous backup to its highly available cluster once + * that cluster serves: a repository of its own, a full base right away, and + * the policy pointing at the new engine. The single instance's repository is + * kept as it is, and its backups restore as before. + */ +@Injectable() +export class DatabaseHaBackupHandoverService { + private readonly logger = new Logger(DatabaseHaBackupHandoverService.name); + + constructor( + private readonly policyRepo: BackupPolicyRepository, + private readonly destRepo: BackupDestinationRepository, + private readonly policies: BackupPoliciesService, + private readonly jobs: BackupJobsService, + private readonly ha: PgBackrestHaService, + ) {} + + /** What happened to the backup, in words for the operation; null when the database had none. */ + async handOver( + appId: string, + { userId, takeBase = true }: { userId?: string; takeBase?: boolean } = {}, + ): Promise { + const policy = await this.policyRepo.findDbPolicyForApp(appId); + if (!policy) return null; + if (policy.engine === this.ha.engine) return null; + if ((policy.engine ?? SINGLE_INSTANCE_ENGINE) !== SINGLE_INSTANCE_ENGINE) { + return null; + } + const highlyAvailable = await this.ha.requireTooling(appId).then( + () => true, + (err) => { + if (err instanceof BadRequestException) return false; + throw err; + }, + ); + if (!highlyAvailable) return null; + const primary = this.policies.primaryDestinationOf(policy); + const destination = await this.destRepo.findById(primary.destinationId); + if (!destination) throw new Error('primary destination not found'); + const generation = this.ha.mintGeneration(); + await this.ha.enable(appId, destination, { + retentionFull: policy.retentionMaxCopies ?? 2, + generation, + }); + await this.policyRepo.update(policy.id, { + engine: this.ha.engine, + metadata: { ...policy.metadata, generation }, + }); + this.logger.log( + `[backup] ${appId}: continuous backup moved to the highly available cluster (${generation})`, + ); + if (!takeBase) + return 'Continuous backup now ships from the highly available cluster.'; + await this.jobs.createOnDemand( + userId ?? policy.userId, + { policyId: policy.id, metadata: { reason: 'database-ha-handover' } }, + BackupJobTriggerType.SCHEDULED, + ); + return 'Continuous backup now ships from the highly available cluster; its first full backup is running.'; + } +} diff --git a/src/modules/backups/services/database-ha-backup.spec.ts b/src/modules/backups/services/database-ha-backup.spec.ts new file mode 100644 index 00000000..892a421a --- /dev/null +++ b/src/modules/backups/services/database-ha-backup.spec.ts @@ -0,0 +1,100 @@ +jest.mock('@kubernetes/client-node', () => ({})); +jest.mock('jose', () => ({})); +jest.mock('ip-cidr', () => ({})); + +import { BadRequestException } from '@nestjs/common'; +import { PostgresDumpService } from './logical-dump.service'; +import { PgBackrestService } from './pgbackrest.service'; + +const app = { + id: 'a1', + slug: 'orders-db', + clusterId: 'c1', + k8sNamespace: 'ns', + databaseHa: { cluster: 'orders-db-ha', state: 'active', instances: 3 }, +}; + +const credentials = [ + { + name: 'POSTGRES_PASSWORD', + valueFrom: { secretKeyRef: { name: 'orders-db-secret', key: 'password' } }, + }, + { name: 'POSTGRES_USER', value: 'orders' }, +]; + +describe('backing up a highly available Postgres', () => { + it("dumps from the primary, through the application's address, with the credentials its single instance kept", async () => { + const k8s = { + listResourcesByLabel: jest.fn(async (_k: string, kind: string) => + kind === 'Pod' + ? [ + { + metadata: { labels: { 'cnpg.io/instanceRole': 'replica' } }, + status: { phase: 'Running' }, + spec: { containers: [{ name: 'postgres', image: 'cnpg' }] }, + }, + { + metadata: { labels: { 'cnpg.io/instanceRole': 'primary' } }, + status: { phase: 'Running', podIP: '10.42.0.9' }, + spec: { + containers: [{ name: 'postgres', image: 'cnpg@sha256:1' }], + }, + }, + ] + : [ + { + metadata: { name: 'orders-db-ha-r' }, + spec: { ports: [{ port: 5432 }] }, + }, + { + metadata: { name: 'orders-db-svc' }, + spec: { ports: [{ port: 5432 }] }, + }, + ], + ), + readObject: jest.fn().mockResolvedValue({ + spec: { + template: { + spec: { containers: [{ name: 'orders-db', env: credentials }] }, + }, + }, + }), + }; + const service = new PostgresDumpService( + k8s as never, + { decrypt: () => 'kc' } as never, + { findOne: jest.fn().mockResolvedValue(app) } as never, + { + findOne: jest.fn().mockResolvedValue({ kubeconfigEncrypted: 'x' }), + } as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + ); + + const target = await service.resolveTarget('a1'); + + expect(target).toMatchObject({ + container: 'postgres', + image: 'cnpg@sha256:1', + host: 'orders-db-svc.ns.svc.cluster.local', + env: credentials, + }); + }); + + it('leaves log shipping to pgBackRest only on a single instance', async () => { + const service = new PgBackrestService( + {} as never, + {} as never, + { findOne: jest.fn().mockResolvedValue(app) } as never, + {} as never, + {} as never, + {} as never, + ); + await expect(service.requireTooling('a1')).rejects.toThrow( + BadRequestException, + ); + }); +}); diff --git a/src/modules/backups/services/logical-dump.service.ts b/src/modules/backups/services/logical-dump.service.ts index 3495bd7d..9b5c1162 100644 --- a/src/modules/backups/services/logical-dump.service.ts +++ b/src/modules/backups/services/logical-dump.service.ts @@ -46,6 +46,8 @@ import { remoteName, restorePasswords, } from '../utils/rclone-crypt.util'; +import { databaseHaServes } from '../../postgres-ha/postgres-ha.types'; +import { runningPodForDatabase } from '../../infrastructure/shared/utils/database-pod.util'; const JOB_TIMEOUT_SECONDS = 2 * 60 * 60; const JOB_POLL_INTERVAL_MS = 5_000; @@ -123,7 +125,10 @@ abstract class LogicalDumpEngine implements ContinuousBackupEngine { app.k8sNamespace, labelSelector, ); - const pod = pods.find((p: any) => p?.status?.phase === 'Running'); + const ha = databaseHaServes(app.databaseHa); + const pod = ha + ? runningPodForDatabase(pods) + : pods.find((p: any) => p?.status?.phase === 'Running'); if (!pod) { throw new BadRequestException( 'This database is not running, so it cannot be backed up. Start it and try again.', @@ -138,8 +143,14 @@ abstract class LogicalDumpEngine implements ContinuousBackupEngine { app.k8sNamespace, labelSelector, ); - const service = services[0]; + const service = ha + ? (services.find((s: any) => s.metadata?.name === `${app.slug}-svc`) ?? + services[0]) + : services[0]; const port = service?.spec?.ports?.[0]?.port ?? this.family.defaultPort; + const credentials = ha + ? await this.singleInstanceCredentials(kubeconfig, app) + : null; return { kubeconfig, namespace: app.k8sNamespace, @@ -147,8 +158,8 @@ abstract class LogicalDumpEngine implements ContinuousBackupEngine { labelSelector, container: container.name, image: container.image, - env: container.env ?? [], - envFrom: container.envFrom ?? [], + env: credentials?.env ?? container.env ?? [], + envFrom: credentials?.envFrom ?? container.envFrom ?? [], imagePullSecrets: pod.spec?.imagePullSecrets ?? [], tolerations: pod.spec?.tolerations ?? [], host: service @@ -158,6 +169,32 @@ abstract class LogicalDumpEngine implements ContinuousBackupEngine { }; } + /** + * A highly available database's instances carry no application credentials + * in their env; the single instance kept at zero still does, and its users + * and passwords are the ones the copy kept. + */ + private async singleInstanceCredentials( + kubeconfig: string, + app: ApplicationEntity, + ): Promise<{ env: unknown[]; envFrom: unknown[] } | null> { + const sts = await this.k8s + .readObject( + kubeconfig, + 'apps/v1', + 'StatefulSet', + app.slug, + app.k8sNamespace, + ) + .catch(() => null); + const containers: any[] = sts?.spec?.template?.spec?.containers ?? []; + const container = + containers.find((c) => c.name === app.slug) ?? containers[0]; + return container + ? { env: container.env ?? [], envFrom: container.envFrom ?? [] } + : null; + } + private async exec(target: DumpTarget, script: string): Promise { const b64 = Buffer.from(script, 'utf-8').toString('base64'); return this.k8s.execInPod( diff --git a/src/modules/backups/services/logical-dump.spec.ts b/src/modules/backups/services/logical-dump.spec.ts index ce23ada5..1f8322ab 100644 --- a/src/modules/backups/services/logical-dump.spec.ts +++ b/src/modules/backups/services/logical-dump.spec.ts @@ -23,14 +23,34 @@ describe('choosing the engine that protects a database', () => { }; const ok = async () => undefined; - const registry = (pg: any, pgDump: any) => + const registry = ( + pg: any, + pgDump: any, + pgHa = engine('postgres-ha', refuse('not highly available')), + ) => new ContinuousBackupEngineRegistry( pg, engine('mariadb', ok), pgDump, engine('mariadb-dump', ok), + pgHa, ); + it('keeps continuous backup through the instances of a highly available database, before any dump', async () => { + const pg = engine( + 'postgres', + refuse('ships its log through its own instances'), + ); + const dump = engine('postgres-dump', ok); + const chosen = await registry( + pg, + dump, + engine('postgres-ha', ok), + ).chooseFor('postgres', 'app-1'); + expect(chosen.engine).toBe('postgres-ha'); + expect(dump.requireTooling).not.toHaveBeenCalled(); + }); + it('keeps continuous backup when the image can ship its log', async () => { const pg = engine('postgres', ok); const chosen = await registry(pg, engine('postgres-dump', ok)).chooseFor( diff --git a/src/modules/backups/services/pgbackrest-ha.service.spec.ts b/src/modules/backups/services/pgbackrest-ha.service.spec.ts new file mode 100644 index 00000000..2de08413 --- /dev/null +++ b/src/modules/backups/services/pgbackrest-ha.service.spec.ts @@ -0,0 +1,230 @@ +jest.mock('@kubernetes/client-node', () => ({ + dumpYaml: (o: unknown) => JSON.stringify(o), +})); +jest.mock('jose', () => ({})); +jest.mock('ip-cidr', () => ({})); + +import { BadRequestException } from '@nestjs/common'; +import { PgBackrestHaService, describeFailure } from './pgbackrest-ha.service'; + +const HEALTHY = 'Cluster in healthy state'; + +const ha = { + engine: 'cnpg', + cluster: 'orders-db-ha', + durability: 'quorum', + instances: 3, + state: 'active', + since: '2026-11-08T12:00:00Z', +}; + +function build(databaseHa: object | null = ha) { + const k8s = { + serverSideApply: jest.fn().mockResolvedValue(1), + mergePatchObject: jest.fn().mockResolvedValue(undefined), + execInPod: jest.fn().mockResolvedValue(''), + listPodsByLabel: jest + .fn() + .mockResolvedValue([ + { metadata: { name: 'orders-db-ha-1' } }, + { metadata: { name: 'orders-db-ha-2' } }, + ]), + readObject: jest.fn().mockResolvedValue({ status: { phase: HEALTHY } }), + deleteObject: jest.fn(), + }; + const plugin = { ensureInstalled: jest.fn() }; + const service = new PgBackrestHaService( + k8s as never, + { decrypt: (v: string) => `plain-${v}` } as never, + { + findOne: jest.fn().mockResolvedValue({ + id: 'a1', + clusterId: 'c1', + k8sNamespace: 'ns', + databaseHa, + }), + } as never, + { + findOne: jest.fn().mockResolvedValue({ kubeconfigEncrypted: 'kc' }), + } as never, + { passphraseFor: jest.fn().mockResolvedValue('pass') } as never, + {} as never, + plugin as never, + ); + Object.assign(service, { logger: { log: jest.fn() } }); + return { service, k8s, plugin }; +} + +const destination = { + bucket: 'b', + endpoint: 'https://fsn1.your-objectstorage.com', + region: 'fsn1', + pathPrefix: 'eu', + forcePathStyle: false, + accessKeyEncrypted: 'ak', + secretKeyEncrypted: 'sk', +}; + +describe('continuous backup of a highly available Postgres', () => { + it('ships its log to a repository of its own, proven by a segment that reaches it', async () => { + const { service, k8s, plugin } = build(); + k8s.execInPod.mockImplementation(async (...args: unknown[]) => { + const script = (args[4] as string[])[2]; + if (script.includes('pg_switch_wal')) return '000000020000000000000011\n'; + if (script.includes('pg_stat_archiver')) return 't|\n'; + return ''; + }); + + await service.enable('a1', destination as never, { + generation: '20261108120000', + }); + + expect(plugin.ensureInstalled).toHaveBeenCalledWith('plain-kc'); + const applied = k8s.serverSideApply.mock.calls[0][1] as string; + expect(applied).toContain( + '"repoPath":"/eu/pgbackrest/a1/ha-20261108120000"', + ); + expect(applied).toContain( + '"flui.cloud/repository-prefix":"pgbackrest/a1/ha-20261108120000/"', + ); + expect(applied).toContain('"ENCRYPTION_PASS":"pass"'); + expect( + k8s.mergePatchObject.mock.calls.at(-1)[1].spec.plugins[0].isWALArchiver, + ).toBe(true); + const cleared = k8s.execInPod.mock.calls.filter((c) => c[5]); + expect(cleared.map((c) => c[5])).toEqual([ + 'orders-db-ha-1', + 'orders-db-ha-2', + ]); + expect(cleared[0][4][2]).toContain('postgresql.auto.conf'); + expect( + k8s.execInPod.mock.calls.some((c) => + (c[4] as string[])[2].includes( + "last_archived_wal >= '000000020000000000000011'", + ), + ), + ).toBe(true); + }); + + it('touches no instance while one restarts, and retries the segment switch while the plugin is added', async () => { + jest.useFakeTimers(); + try { + const { service, k8s } = build(); + k8s.readObject + .mockResolvedValueOnce({ spec: {} }) + .mockResolvedValueOnce({ status: { phase: 'Switchover in progress' } }) + .mockResolvedValue({ status: { phase: HEALTHY } }); + let switches = 0; + k8s.execInPod.mockImplementation(async (...args: unknown[]) => { + const script = (args[4] as string[])[2]; + if (script.includes('pg_switch_wal') && switches++ === 0) { + throw { status: 'Failure', message: 'no such file or directory' }; + } + if (script.includes('pg_switch_wal')) return '000000020000000000000012'; + if (script.includes('pg_stat_archiver')) return 't|'; + return ''; + }); + + const enabled = service.enable('a1', destination as never, {}); + await jest.advanceTimersByTimeAsync(5_000); + expect(k8s.execInPod).not.toHaveBeenCalled(); + await jest.advanceTimersByTimeAsync(30_000); + await enabled; + + expect(switches).toBe(2); + } finally { + jest.useRealTimers(); + } + }); + + it('returns the label of the base the plugin took', async () => { + const { service, k8s, plugin } = build(); + k8s.readObject.mockResolvedValue({ + status: { phase: 'completed', backupId: '20261108-120500F' }, + }); + + await expect(service.baseBackup('a1', 'full')).resolves.toBe( + '20261108-120500F', + ); + expect(plugin.ensureInstalled).toHaveBeenCalledWith('plain-kc'); + expect(k8s.serverSideApply.mock.calls[0][1]).toContain( + '"backupType":"full"', + ); + }); + + it('forgets the source of a promoted copy, and removes the plugin again when shipping never starts', async () => { + const { service, k8s } = build(); + k8s.readObject.mockResolvedValue({ + spec: { replica: { enabled: false, source: 'flui-single' } }, + status: { phase: HEALTHY }, + }); + Object.assign(service, { + untilArchiving: jest + .fn() + .mockRejectedValue(new Error('not shipping: stanza missing')), + }); + + await expect( + service.enable('a1', destination as never, {}), + ).rejects.toThrow('stanza missing'); + + const patches = k8s.mergePatchObject.mock.calls.map((c) => c[1].spec); + expect(patches).toEqual([ + { replica: null }, + expect.objectContaining({ plugins: expect.any(Array) }), + { plugins: null }, + ]); + }); + + it('reads the message of a failed exec, which is not an Error', () => { + expect( + describeFailure({ + status: 'Failure', + message: 'command terminated with exit code 28', + }), + ).toBe('command terminated with exit code 28'); + }); + + it('says why when the database is not highly available', async () => { + const { service } = build(null); + await expect(service.requireTooling('a1')).rejects.toThrow( + BadRequestException, + ); + }); +}); + +describe('the bases of a highly available repository, as its Stanza records them', () => { + const { infoFromStanza, parseArchivedSince } = jest.requireActual( + './pgbackrest-ha.util', + ); + + it('reads the window and the counts the plugin wrote, the way they looked live', () => { + const base = { + label: '20261108-192628F', + type: 'full', + timestamp: { start: 1794165988, stop: 1794165991 }, + }; + expect( + infoFromStanza({ + backupsCount: { Diff: 0, Full: 1, Incr: 0 }, + recoveryWindow: { firstBackup: base, lastBackup: base }, + }), + ).toEqual({ + latestLabel: '20261108-192628F', + oldestRecoverable: new Date(1794165991000).toISOString(), + newestRecoverable: new Date(1794165991000).toISOString(), + lastFullAt: new Date(1794165991000).toISOString(), + backupCount: 1, + latestSizeBytes: null, + }); + expect(infoFromStanza(undefined).backupCount).toBe(0); + }); + + it('reads whether a segment reached the repository, and the last that failed', () => { + expect(parseArchivedSince('t|\n')).toEqual({ archived: true, failed: '' }); + expect(parseArchivedSince('f|000000020000000000000011')).toEqual({ + archived: false, + failed: '000000020000000000000011', + }); + }); +}); diff --git a/src/modules/backups/services/pgbackrest-ha.service.ts b/src/modules/backups/services/pgbackrest-ha.service.ts new file mode 100644 index 00000000..814c993d --- /dev/null +++ b/src/modules/backups/services/pgbackrest-ha.service.ts @@ -0,0 +1,508 @@ +import { + BadRequestException, + Injectable, + Logger, + NotFoundException, +} from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import * as k8s from '@kubernetes/client-node'; +import { KubernetesService } from '../../infrastructure/shared/services/kubernetes.service'; +import { EncryptionService } from '../../shared/encryption/services/encryption.service'; +import { ApplicationEntity } from '../../applications/entities/application.entity'; +import { ClusterEntity } from '../../infrastructure/clusters/entities/cluster.entity'; +import { BackupDestinationEntity } from '../entities/backup-destination.entity'; +import { RestoreStrategy } from '../enums/restore-job.enum'; +import { BackupDestinationsService } from './backup-destinations.service'; +import { PgBackrestService } from './pgbackrest.service'; +import { + ArtifactEngineFacts, + ContinuousBackupEngine, + EngineEnableOptions, + RestorePointMark, +} from './continuous-backup-engine.interface'; +import { + PGBACKREST_STANZA, + PgBackupInfo, + s3EndpointHost, + withArchivedEdge, +} from './pgbackrest-config.util'; +import { repoPathIn } from './pgbackrest-repo.util'; +import { parseRestorePoint } from '../utils/restore-point.util'; +import { + databaseHaServes, + DatabaseHaSpec, +} from '../../postgres-ha/postgres-ha.types'; +import { + CNPG_API_VERSION, + CNPG_HEALTHY_PHASE, +} from '../../postgres-ha/postgres-ha.constants'; +import { PgbackrestPluginInstallerService } from '../../postgres-ha/services/pgbackrest-plugin-installer.service'; +import { + archiveThroughPluginPatch, + PGBACKREST_PLUGIN_API_VERSION, + PGBACKREST_SIDECAR, + pgbackrestStanzaName, + pluginBackupState, + promotedReplicaPatch, + renderPgbackrestSecret, + renderPgbackrestStanza, + renderPluginBackup, + stopArchivingPatch, +} from '../../postgres-ha/utils/pgbackrest-plugin.core'; +import { + archivedSince, + CLEAR_COPIED_ARCHIVE_SETTINGS, + infoFromStanza, + parseArchivedSince, + SWITCH_SEGMENT, + haArchivedEdgeScript, + haRepositoryPrefix, + haRestorePointScript, + REPOSITORY_ANNOTATION, +} from './pgbackrest-ha.util'; + +const ARCHIVING_TIMEOUT_MS = 15 * 60_000; +const BACKUP_TIMEOUT_MS = 2 * 60 * 60_000; +const POLL_MS = 10_000; + +/** A failed exec rejects with a Kubernetes status, not an Error: its message is what says why. */ +export function describeFailure(err: unknown): string { + if (err instanceof Error) return err.message; + const status = err as { message?: string; body?: { message?: string } }; + return ( + status?.message ?? + status?.body?.message ?? + JSON.stringify(err)?.slice(0, 300) ?? + String(err) + ); +} + +interface HaTarget { + app: ApplicationEntity; + ha: DatabaseHaSpec; + kubeconfig: string; + namespace: string; + labelSelector: string; +} + +/** + * Continuous backup of a highly available Postgres, through the pgBackRest + * plugin of CloudNativePG: its instances ship their log to an encrypted + * pgBackRest repository of their own, one per move to high availability, and + * base backups are taken on the primary. + * + * The repository has the layout, stanza and cipher of the single instance's, + * so a restore is the single instance's restore: a new flui-postgres booted + * from it, to any moment the repository covers. + */ +@Injectable() +export class PgBackrestHaService implements ContinuousBackupEngine { + readonly engine = 'postgres-ha'; + readonly catalogSlug = 'postgresql'; + readonly restoreEnvPrefix = 'FLUI_PG_'; + readonly restoreStrategy = RestoreStrategy.PG_PITR; + readonly selfPrunesRepository = true; + readonly replaysToEndWithoutTarget = true; + + private readonly logger = new Logger(PgBackrestHaService.name); + + constructor( + private readonly k8s: KubernetesService, + private readonly encryption: EncryptionService, + @InjectRepository(ApplicationEntity) + private readonly appRepo: Repository, + @InjectRepository(ClusterEntity) + private readonly clusterRepo: Repository, + private readonly destinations: BackupDestinationsService, + private readonly single: PgBackrestService, + private readonly plugin: PgbackrestPluginInstallerService, + ) {} + + private async target(appId: string): Promise { + const app = await this.appRepo.findOne({ where: { id: appId } }); + if (!app) throw new NotFoundException(`Application ${appId} not found`); + if (!databaseHaServes(app.databaseHa)) { + throw new BadRequestException( + 'This database is not highly available, so its own instance ships its log.', + ); + } + const cluster = await this.clusterRepo.findOne({ + where: { id: app.clusterId }, + }); + if (!cluster?.kubeconfigEncrypted) { + throw new NotFoundException(`Cluster ${app.clusterId} missing`); + } + return { + app, + ha: app.databaseHa, + kubeconfig: this.encryption.decrypt(cluster.kubeconfigEncrypted), + namespace: app.k8sNamespace, + labelSelector: `cnpg.io/cluster=${app.databaseHa.cluster}`, + }; + } + + private sidecar(t: HaTarget, command: string): Promise { + return this.k8s.execInPod( + t.kubeconfig, + t.namespace, + t.labelSelector, + PGBACKREST_SIDECAR, + ['sh', '-c', command], + ); + } + + private postgres(t: HaTarget, script: string): Promise { + return this.k8s.execInPod( + t.kubeconfig, + t.namespace, + t.labelSelector, + 'postgres', + ['sh', '-c', script], + ); + } + + async requireTooling(appId: string): Promise { + const t = await this.target(appId); + const cluster = await this.k8s + .readObject( + t.kubeconfig, + CNPG_API_VERSION, + 'Cluster', + t.ha.cluster, + t.namespace, + ) + .catch(() => null); + if (!cluster) { + throw new BadRequestException( + `The highly available cluster ${t.ha.cluster} of this database is not there.`, + ); + } + } + + /** A repository per move to high availability: the instance it was copied from may have shipped to the old one until the last moment. */ + mintGeneration(): string { + return new Date().toISOString().replace(/\D/g, '').slice(0, 14); + } + + artifactObjectPrefix(appId: string, generation?: string): string { + return haRepositoryPrefix(appId, generation); + } + + async enable( + appId: string, + dest: BackupDestinationEntity, + opts?: EngineEnableOptions, + ): Promise { + const t = await this.target(appId); + await this.plugin.ensureInstalled(t.kubeconfig); + const prefix = this.artifactObjectPrefix(appId, opts?.generation); + const secret = renderPgbackrestSecret({ + cluster: t.ha.cluster, + namespace: t.namespace, + accessKey: this.encryption.decrypt(dest.accessKeyEncrypted), + secretKey: this.encryption.decrypt(dest.secretKeyEncrypted), + cipherPass: await this.destinations.passphraseFor(dest), + }); + const stanza = renderPgbackrestStanza({ + cluster: t.ha.cluster, + namespace: t.namespace, + stanza: PGBACKREST_STANZA, + repository: { + bucket: dest.bucket, + endpoint: s3EndpointHost(dest.endpoint), + region: dest.region, + repoPath: repoPathIn(dest.pathPrefix, prefix), + pathStyle: !!dest.forcePathStyle, + retentionFull: opts?.retentionFull ?? 2, + }, + }) as { metadata: { annotations?: Record } }; + stanza.metadata.annotations = { [REPOSITORY_ANNOTATION]: prefix }; + await this.k8s.serverSideApply( + t.kubeconfig, + [secret, stanza].map((o) => k8s.dumpYaml(o)).join('---\n'), + ); + await this.forgetPromotedSource(t); + await this.untilHealthy(t); + await this.clearCopiedArchiveSettings(t); + await this.k8s.mergePatchObject( + t.kubeconfig, + archiveThroughPluginPatch({ + cluster: t.ha.cluster, + namespace: t.namespace, + }), + ); + try { + await this.untilArchiving(t); + } catch (err) { + // A plugin left on a cluster that does not ship stops CloudNativePG + // from reconciling it, failover included. + await this.k8s + .mergePatchObject( + t.kubeconfig, + stopArchivingPatch({ cluster: t.ha.cluster, namespace: t.namespace }), + ) + .catch(() => undefined); + throw err; + } + this.logger.log( + `[pgbackrest-ha] ${t.ha.cluster}: shipping its log to ${prefix}`, + ); + } + + private async forgetPromotedSource(t: HaTarget): Promise { + const cluster = await this.k8s.readObject( + t.kubeconfig, + CNPG_API_VERSION, + 'Cluster', + t.ha.cluster, + t.namespace, + ); + const replica = cluster?.spec?.replica; + if (replica && replica.enabled !== true) { + await this.k8s.mergePatchObject( + t.kubeconfig, + promotedReplicaPatch({ cluster: t.ha.cluster, namespace: t.namespace }), + ); + } + } + + /** A promotion or a move of the primary restarts an instance; nothing is run in one while it does. */ + private async untilHealthy(t: HaTarget): Promise { + const deadline = Date.now() + ARCHIVING_TIMEOUT_MS; + let phase = ''; + while (Date.now() < deadline) { + const cluster = await this.k8s + .readObject( + t.kubeconfig, + CNPG_API_VERSION, + 'Cluster', + t.ha.cluster, + t.namespace, + ) + .catch(() => null); + phase = cluster?.status?.phase ?? ''; + if (phase === CNPG_HEALTHY_PHASE) return; + await new Promise((r) => setTimeout(r, POLL_MS)); + } + throw new Error( + `The highly available cluster did not settle before its backup was set up: ${phase || 'no status'}`, + ); + } + + /** + * Adding the plugin restarts the instances one at a time, so the segment + * switch is retried until the primary answers; shipping is proven once that + * segment is in the repository. + */ + private async untilArchiving(t: HaTarget): Promise { + const deadline = Date.now() + ARCHIVING_TIMEOUT_MS; + let segment = ''; + let last = ''; + while (Date.now() < deadline) { + try { + segment ||= (await this.postgres(t, SWITCH_SEGMENT)) + .trim() + .split('\n') + .pop()!; + const state = parseArchivedSince( + await this.postgres(t, archivedSince(segment)), + ); + if (state.archived) return; + last = state.failed + ? `segment ${state.failed} failed to reach the repository` + : `segment ${segment} has not reached the repository yet`; + } catch (err) { + last = describeFailure(err); + } + await new Promise((r) => setTimeout(r, POLL_MS)); + } + throw new Error( + `The highly available cluster did not start shipping its log to the backup: ${last}`, + ); + } + + /** The archive settings the copy brought along leave every instance, primary and copies alike. */ + private async clearCopiedArchiveSettings(t: HaTarget): Promise { + const pods = await this.k8s.listPodsByLabel( + t.kubeconfig, + t.namespace, + t.labelSelector, + ); + for (const pod of pods) { + const name = pod?.metadata?.name; + if (!name) continue; + await this.k8s.execInPod( + t.kubeconfig, + t.namespace, + t.labelSelector, + 'postgres', + ['sh', '-c', CLEAR_COPIED_ARCHIVE_SETTINGS], + name, + ); + } + } + + async disable(appId: string): Promise { + const t = await this.target(appId); + await this.k8s.mergePatchObject( + t.kubeconfig, + stopArchivingPatch({ cluster: t.ha.cluster, namespace: t.namespace }), + ); + const name = pgbackrestStanzaName(t.ha.cluster); + await this.k8s + .deleteObject( + t.kubeconfig, + PGBACKREST_PLUGIN_API_VERSION, + 'Stanza', + name, + t.namespace, + ) + .catch(() => undefined); + await this.k8s + .deleteObject(t.kubeconfig, 'v1', 'Secret', name, t.namespace) + .catch(() => undefined); + this.logger.log(`[pgbackrest-ha] ${t.ha.cluster}: stopped shipping`); + } + + async baseBackup( + appId: string, + type: 'full' | 'incr' | 'diff' = 'full', + ): Promise { + const t = await this.target(appId); + await this.plugin.ensureInstalled(t.kubeconfig); + const name = `${t.ha.cluster}-${this.mintGeneration()}`; + await this.k8s.serverSideApply( + t.kubeconfig, + k8s.dumpYaml( + renderPluginBackup({ + name, + cluster: t.ha.cluster, + namespace: t.namespace, + type, + }), + ), + ); + const deadline = Date.now() + BACKUP_TIMEOUT_MS; + while (Date.now() < deadline) { + const backup = await this.k8s + .readObject(t.kubeconfig, CNPG_API_VERSION, 'Backup', name, t.namespace) + .catch(() => null); + const state = pluginBackupState(backup ?? {}); + if (state.phase === 'completed') { + this.logger.log(`[pgbackrest-ha] base backup ${state.label}`); + return state.label; + } + if (state.phase === 'failed') throw new Error(state.error); + await new Promise((r) => setTimeout(r, POLL_MS)); + } + throw new Error(`The base backup ${name} did not finish in time`); + } + + async chooseBackupType( + appId: string, + fullEveryDays: number, + ): Promise<'full' | 'incr'> { + const info = await this.info(appId); + if (info.backupCount === 0 || !info.lastFullAt) return 'full'; + const ageMs = Date.now() - new Date(info.lastFullAt).getTime(); + return ageMs >= fullEveryDays * 86_400_000 ? 'full' : 'incr'; + } + + async info(appId: string): Promise { + const t = await this.target(appId); + const stanza = await this.stanzaOf(t); + const edge = await this.postgres(t, haArchivedEdgeScript()).catch(() => ''); + return withArchivedEdge(infoFromStanza(stanza?.status), edge); + } + + private stanzaOf(t: HaTarget): Promise { + return this.k8s + .readObject( + t.kubeconfig, + PGBACKREST_PLUGIN_API_VERSION, + 'Stanza', + pgbackrestStanzaName(t.ha.cluster), + t.namespace, + ) + .catch(() => null); + } + + async describeForArtifact(appId: string): Promise { + const t = await this.target(appId); + const single = await this.single.resolveTarget(appId); + const read = (run: Promise) => + run.then( + (out) => out.trim().split('\n').pop(), + () => undefined, + ); + const stanza = await this.stanzaOf(t); + return { + engine: this.engine, + engineVersion: await read( + this.postgres(t, `psql -U postgres -Atc 'SHOW server_version'`), + ), + tool: 'pgbackrest', + toolVersion: await read( + this.sidecar(t, `pgbackrest version | awk '{print $2}'`), + ), + catalogSlug: this.catalogSlug, + identities: { user: single.pgUser, database: single.pgDb }, + repository: { + objectKeyPrefix: + stanza?.metadata?.annotations?.[REPOSITORY_ANNOTATION] ?? + this.artifactObjectPrefix(appId), + cipher: + stanza?.spec?.stanzaConfiguration?.s3Repositories?.[0]?.cipherConfig + ?.type ?? 'unknown', + }, + }; + } + + async markRestorePoint( + appId: string, + label: string, + ): Promise { + const t = await this.target(appId); + return parseRestorePoint( + await this.postgres(t, haRestorePointScript(label)), + ); + } + + buildRestoreEnv( + sourceAppId: string, + destination: BackupDestinationEntity, + recoveryTargetTime?: Date, + restoreSet?: string, + generation?: string, + artifactSummary?: Record, + ): Record { + return this.single.buildRestoreEnv( + sourceAppId, + destination, + recoveryTargetTime, + restoreSet, + generation, + artifactSummary, + ); + } + + identityEnv(identities: { + user: string; + database: string; + }): Record { + return this.single.identityEnv(identities); + } + + endedBeforeTarget(restoredAppId: string): Promise { + return this.single.endedBeforeTarget(restoredAppId); + } + + restoreFailure(restoredAppId: string): Promise { + return this.single.restoreFailure(restoredAppId); + } + + reconcileAfterRestore(newAppId: string): Promise { + return this.single.reconcileAfterRestore(newAppId); + } +} diff --git a/src/modules/backups/services/pgbackrest-ha.util.ts b/src/modules/backups/services/pgbackrest-ha.util.ts new file mode 100644 index 00000000..1d79da9d --- /dev/null +++ b/src/modules/backups/services/pgbackrest-ha.util.ts @@ -0,0 +1,115 @@ +import { + RESTORE_POINT_MARKER, + restorePointLabel, +} from '../utils/restore-point.util'; + +/** Where the Stanza records the repository it ships to, relative to the destination. */ +export const REPOSITORY_ANNOTATION = 'flui.cloud/repository-prefix'; + +/** + * A highly available database's repository, beside the single instance's + * under the same application: one per move to high availability, because + * the instance it was copied from shipped to its own until the move. + */ +export function haRepositoryPrefix(appId: string, generation?: string): string { + return `pgbackrest/${appId}/ha-${generation ?? 'current'}/`; +} + +const psql = `psql -U postgres -v ON_ERROR_STOP=1 -tA`; + +/** The moment the last segment reached the repository, in the marker the single instance's info reads. */ +export function haArchivedEdgeScript(): string { + return `echo "FLUI_LAST_ARCHIVED=$(${psql} -c "SELECT coalesce(floor(extract(epoch from last_archived_time))::bigint, 0) FROM pg_stat_archiver")"`; +} + +/** A named restore point on the primary, then a segment switch so it ships now. */ +export function haRestorePointScript(label: string): string { + return [ + 'set -e', + `MARK="$(${psql} -F ' ' -c "SELECT floor(extract(epoch from clock_timestamp()) * 1000)::bigint, pg_create_restore_point('${restorePointLabel(label)}')")"`, + `${psql} -c "SELECT pg_switch_wal()" >/dev/null`, + `echo "${RESTORE_POINT_MARKER}$MARK"`, + ].join('\n'); +} + +/** + * The copy carries the single instance's `postgresql.auto.conf`, where Flui + * pointed `archive_command` at a pgBackRest configuration the cluster's image + * does not have; read last, it would win over the operator's own command. + */ +export const CLEAR_COPIED_ARCHIVE_SETTINGS = [ + 'sed -i -E "/^(archive_command|archive_timeout)[[:space:]]*=/d" "$PGDATA/postgresql.auto.conf"', + `${psql} -c "SELECT pg_reload_conf()" >/dev/null`, +].join('\n'); + +export const SWITCH_SEGMENT = `${psql} -c "SELECT pg_walfile_name(pg_switch_wal())"`; + +/** `archived` once the segment reached the repository; the last failure otherwise. */ +export function archivedSince(segment: string): string { + return `${psql} -F '|' -c "SELECT last_archived_wal >= '${segment}', coalesce(last_failed_wal, '') FROM pg_stat_archiver"`; +} + +export function parseArchivedSince(out: string): { + archived: boolean; + failed: string; +} { + const [archived, failed] = out.trim().split('\n').pop()!.split('|'); + return { archived: archived === 't', failed: failed ?? '' }; +} + +interface StanzaBackup { + label?: string; + type?: string; + timestamp?: { start?: number; stop?: number }; +} + +/** + * The repository's bases as the plugin records them on its Stanza: the first + * and the last, and how many of each type. The newest full is the last backup + * when it is one, otherwise no later than the first, which is how a cadence + * errs towards a full too early rather than too late. + */ +export function infoFromStanza( + status: + | { + backupsCount?: { Full?: number; Incr?: number; Diff?: number }; + recoveryWindow?: { + firstBackup?: StanzaBackup; + lastBackup?: StanzaBackup; + }; + } + | null + | undefined, +): { + latestLabel: string | null; + oldestRecoverable: string | null; + newestRecoverable: string | null; + lastFullAt: string | null; + backupCount: number; + latestSizeBytes: null; +} { + const count = status?.backupsCount ?? {}; + const backupCount = (count.Full ?? 0) + (count.Incr ?? 0) + (count.Diff ?? 0); + const first = status?.recoveryWindow?.firstBackup; + const last = status?.recoveryWindow?.lastBackup; + const at = (b?: StanzaBackup) => + b?.timestamp?.stop ? new Date(b.timestamp.stop * 1000).toISOString() : null; + if (!backupCount || !last?.label) { + return { + latestLabel: null, + oldestRecoverable: null, + newestRecoverable: null, + lastFullAt: null, + backupCount: 0, + latestSizeBytes: null, + }; + } + return { + latestLabel: last.label, + oldestRecoverable: at(first), + newestRecoverable: at(last), + lastFullAt: at(last.type === 'full' ? last : first), + backupCount, + latestSizeBytes: null, + }; +} diff --git a/src/modules/backups/services/pgbackrest.service.ts b/src/modules/backups/services/pgbackrest.service.ts index 8b498d66..69ce3a9f 100644 --- a/src/modules/backups/services/pgbackrest.service.ts +++ b/src/modules/backups/services/pgbackrest.service.ts @@ -62,6 +62,7 @@ import { restoreFailureFrom, withArchivedEdge, } from './pgbackrest-config.util'; +import { databaseHaServes } from '../../postgres-ha/postgres-ha.types'; /** * Drives pgBackRest inside a managed Postgres pod over `kubectl exec`. Config + @@ -92,7 +93,18 @@ export class PgBackrestService implements ContinuousBackupEngine { private readonly retirer: PgLegacyRepoRetirer, ) {} + /** + * A highly available database serves from CloudNativePG's instances, whose + * image has no pgBackRest and whose log shipping the operator owns; refused + * the way a missing tool is, so the scheduled dump takes over. + */ async requireTooling(appId: string): Promise { + const app = await this.appRepo.findOne({ where: { id: appId } }); + if (databaseHaServes(app?.databaseHa)) { + throw new BadRequestException( + 'A highly available database ships its log through its own instances, not the single one.', + ); + } await this.requirePgBackrest(await this.resolveTarget(appId)); } diff --git a/src/modules/cluster-ha/cluster-ha.module.ts b/src/modules/cluster-ha/cluster-ha.module.ts new file mode 100644 index 00000000..8db01b79 --- /dev/null +++ b/src/modules/cluster-ha/cluster-ha.module.ts @@ -0,0 +1,29 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { ClusterEntity } from '../infrastructure/clusters/entities/cluster.entity'; +import { SharedInfrastructureModule } from '../infrastructure/shared/shared-infrastructure.module'; +import { EncryptionModule } from '../shared/encryption/encryption.module'; +import { ClusterHaService } from './services/cluster-ha.service'; +import { EtcdLeaderPlacementService } from './services/etcd-leader-placement.service'; +import { SiteRoundTripService } from './services/site-round-trip.service'; +import { ClusterHaSweepService } from './services/cluster-ha-sweep.service'; +import { ClusterHaSweepScheduler } from './schedulers/cluster-ha-sweep.scheduler'; +import { ClusterHaController } from './controllers/cluster-ha.controller'; + +@Module({ + imports: [ + TypeOrmModule.forFeature([ClusterEntity]), + SharedInfrastructureModule, + EncryptionModule, + ], + controllers: [ClusterHaController], + providers: [ + ClusterHaService, + EtcdLeaderPlacementService, + SiteRoundTripService, + ClusterHaSweepService, + ClusterHaSweepScheduler, + ], + exports: [ClusterHaService], +}) +export class ClusterHaModule {} diff --git a/src/modules/cluster-ha/controllers/cluster-ha.controller.ts b/src/modules/cluster-ha/controllers/cluster-ha.controller.ts new file mode 100644 index 00000000..5a1627e5 --- /dev/null +++ b/src/modules/cluster-ha/controllers/cluster-ha.controller.ts @@ -0,0 +1,32 @@ +import { Controller, Get, Param } from '@nestjs/common'; +import { + ApiBearerAuth, + ApiOperation, + ApiParam, + ApiTags, +} from '@nestjs/swagger'; +import { RequireSection } from '../../iam/decorators/require-section.decorator'; +import { ClusterHaService } from '../services/cluster-ha.service'; + +@ApiTags('Infrastructure - Clusters') +@ApiBearerAuth() +@Controller('infrastructure/clusters/:id/high-availability') +export class ClusterHaController { + constructor(private readonly service: ClusterHaService) {} + + @Get() + @RequireSection('clusters') + @ApiOperation({ + summary: + 'What a cluster survives now, and what it needs to survive the loss of a location', + description: + 'Flui offers one highly available layout: a server in each of three locations of a region it has validated, copies of data in the two closest, and the third location voting only. ' + + '`level` is `standard` or `three-locations`; `topology` names each location with its role (`data` or `vote`), its servers and its hosts for data, or is null with the reason in `unsupported`. ' + + '`survives` and `doesNotSurvive` are plain sentences; `missing` lists the roles to add (`server`, `data-host`) with the location and why. Read from the nodes as they are now. ' + + '`observed` is what Flui last measured, as of `observed.at`, on a cluster across three locations: the location of the etcd leader, which Flui brings back to a location for data; the round trip in milliseconds between each pair of locations (null when one did not answer); the server Flui reaches the cluster through and the ones it can move to.', + }) + @ApiParam({ name: 'id', description: 'Cluster ID' }) + view(@Param('id') id: string) { + return this.service.view(id); + } +} diff --git a/src/modules/cluster-ha/schedulers/cluster-ha-sweep.scheduler.ts b/src/modules/cluster-ha/schedulers/cluster-ha-sweep.scheduler.ts new file mode 100644 index 00000000..39b5429c --- /dev/null +++ b/src/modules/cluster-ha/schedulers/cluster-ha-sweep.scheduler.ts @@ -0,0 +1,26 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { Cron, CronExpression } from '@nestjs/schedule'; +import { ClusterHaSweepService } from '../services/cluster-ha-sweep.service'; + +@Injectable() +export class ClusterHaSweepScheduler { + private readonly logger = new Logger(ClusterHaSweepScheduler.name); + private running = false; + + constructor(private readonly sweeper: ClusterHaSweepService) {} + + @Cron( + process.env.FLUI_CLUSTER_HA_SWEEP_CRON || CronExpression.EVERY_10_MINUTES, + ) + async tick(): Promise { + if (this.running) return; + this.running = true; + try { + await this.sweeper.sweep(); + } catch (err: any) { + this.logger.error(`[ha] sweep failed: ${err?.message ?? err}`); + } finally { + this.running = false; + } + } +} diff --git a/src/modules/cluster-ha/services/cluster-ha-sweep.service.spec.ts b/src/modules/cluster-ha/services/cluster-ha-sweep.service.spec.ts new file mode 100644 index 00000000..845f0ebb --- /dev/null +++ b/src/modules/cluster-ha/services/cluster-ha-sweep.service.spec.ts @@ -0,0 +1,103 @@ +jest.mock('@kubernetes/client-node', () => ({})); +jest.mock( + '../../infrastructure/clusters/utils/cluster-metadata-merge.util', + () => ({ mergeClusterMetadata: jest.fn() }), +); + +import { Repository } from 'typeorm'; +import { ClusterEntity } from '../../infrastructure/clusters/entities/cluster.entity'; +import { mergeClusterMetadata } from '../../infrastructure/clusters/utils/cluster-metadata-merge.util'; +import { KubernetesService } from '../../infrastructure/shared/services/kubernetes.service'; +import { ClusterHaService } from './cluster-ha.service'; +import { ClusterHaSweepService } from './cluster-ha-sweep.service'; +import { EtcdLeaderPlacementService } from './etcd-leader-placement.service'; +import { SiteRoundTripService } from './site-round-trip.service'; + +const server = (name: string, zone: string, address: string) => ({ + metadata: { + name, + labels: { + 'topology.kubernetes.io/zone': zone, + 'node-role.kubernetes.io/control-plane': 'true', + }, + }, + status: { + conditions: [{ type: 'Ready', status: 'True' }], + addresses: [{ type: 'InternalIP', address }], + }, +}); + +function setup(level: 'three-locations' | 'standard' = 'three-locations') { + const runImageOnNode = jest.fn( + async (_k: string, _node: string, run: { command: string[] }) => + run.command[2] + .split('\n') + .map((line) => /-q (\S+)/.exec(line)![1]) + .map( + (a) => + `${a} round-trip min/avg/max = 1.0/${a.endsWith('.4') ? '35.0' : '3.0'}/4.0 ms`, + ) + .join('\n'), + ); + const kubernetes = { runImageOnNode } as unknown as KubernetesService; + const etcd = { + check: jest + .fn() + .mockResolvedValue({ node: 'w1', zone: 'nbg1', moved: false }), + } as unknown as EtcdLeaderPlacementService; + const clusterHa = { + inspect: jest.fn().mockResolvedValue({ + cluster: { name: 'ha-lab' }, + kubeconfig: 'kc', + nodes: [ + server('master', 'fsn1', '10.0.0.2'), + server('w1', 'nbg1', '10.0.0.3'), + server('w3', 'hel1', '10.0.0.4'), + ], + state: { level }, + topology: { + zones: ['fsn1', 'nbg1', 'hel1'], + dataZones: ['fsn1', 'nbg1'], + validated: true, + measured: true, + }, + }), + } as unknown as ClusterHaService; + const service = new ClusterHaSweepService( + {} as Repository, + clusterHa, + etcd, + new SiteRoundTripService(kubernetes), + ); + return { service, etcd }; +} + +const cluster = { id: 'c1', name: 'ha-lab', metadata: {} } as ClusterEntity; + +describe('ClusterHaSweepService', () => { + beforeEach(() => jest.mocked(mergeClusterMetadata).mockClear()); + + it('records the etcd leader and the round trip between each pair of locations', async () => { + const { service } = setup(); + await service.check(cluster); + expect(mergeClusterMetadata).toHaveBeenCalledWith(expect.anything(), 'c1', { + haObserved: { + at: expect.any(String), + etcdLeader: { node: 'w1', zone: 'nbg1' }, + roundTrips: [ + { from: 'fsn1', to: 'hel1', ms: 35 }, + { from: 'fsn1', to: 'nbg1', ms: 3 }, + { from: 'hel1', to: 'nbg1', ms: 3 }, + ], + etcdLeaderMovedAt: undefined, + }, + }); + }); + + it('leaves a cluster that does not span three locations alone', async () => { + const { service, etcd } = setup('standard'); + await service.check(cluster); + expect(etcd.check).not.toHaveBeenCalled(); + expect(mergeClusterMetadata).not.toHaveBeenCalled(); + }); +}); diff --git a/src/modules/cluster-ha/services/cluster-ha-sweep.service.ts b/src/modules/cluster-ha/services/cluster-ha-sweep.service.ts new file mode 100644 index 00000000..3aa1ab53 --- /dev/null +++ b/src/modules/cluster-ha/services/cluster-ha-sweep.service.ts @@ -0,0 +1,70 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { IsNull, Not, Repository } from 'typeorm'; +import { ClusterEntity } from '../../infrastructure/clusters/entities/cluster.entity'; +import { mergeClusterMetadata } from '../../infrastructure/clusters/utils/cluster-metadata-merge.util'; +import { ClusterHaService } from './cluster-ha.service'; +import { EtcdLeaderPlacementService } from './etcd-leader-placement.service'; +import { SiteRoundTripService } from './site-round-trip.service'; +import { HaObserved } from '../utils/ha-observed'; + +/** + * On each cluster that spans three locations: puts the etcd leader back in a + * location for data when it is not, and records where it is and how far the + * locations are from one another, for the cluster's high availability view. + */ +@Injectable() +export class ClusterHaSweepService { + private readonly logger = new Logger(ClusterHaSweepService.name); + + constructor( + @InjectRepository(ClusterEntity) + private readonly clusters: Repository, + private readonly clusterHa: ClusterHaService, + private readonly etcdLeader: EtcdLeaderPlacementService, + private readonly roundTrips: SiteRoundTripService, + ) {} + + async sweep(): Promise { + const clusters = await this.clusters.find({ + where: { deletedAt: IsNull(), kubeconfigEncrypted: Not(IsNull()) }, + }); + for (const cluster of clusters) { + await this.check(cluster).catch((err: any) => + this.logger.warn(`[ha] ${cluster.name}: ${err?.message ?? err}`), + ); + } + } + + async check(cluster: ClusterEntity): Promise { + const ha = await this.clusterHa.inspect(cluster.id); + if (ha.state.level !== 'three-locations' || !ha.topology) return; + const three = { ...ha, topology: ha.topology }; + const previous = (cluster.metadata as { haObserved?: HaObserved } | null) + ?.haObserved; + const leader = await this.etcdLeader + .check(three, previous?.etcdLeaderMovedAt) + .catch((err: any) => { + this.logger.warn( + `[ha] ${cluster.name}: etcd leader not read (${err?.message ?? err})`, + ); + return null; + }); + const trips = await this.roundTrips.measure(three).catch((err: any) => { + this.logger.warn( + `[ha] ${cluster.name}: round trips not measured (${err?.message ?? err})`, + ); + return null; + }); + const at = new Date().toISOString(); + const observed: HaObserved = { + at, + etcdLeader: leader ? { node: leader.node, zone: leader.zone } : null, + roundTrips: trips ?? [], + etcdLeaderMovedAt: leader?.moved ? at : previous?.etcdLeaderMovedAt, + }; + await mergeClusterMetadata(this.clusters, cluster.id, { + haObserved: observed, + }); + } +} diff --git a/src/modules/cluster-ha/services/cluster-ha.service.ts b/src/modules/cluster-ha/services/cluster-ha.service.ts new file mode 100644 index 00000000..bcc50569 --- /dev/null +++ b/src/modules/cluster-ha/services/cluster-ha.service.ts @@ -0,0 +1,158 @@ +import { + BadRequestException, + Injectable, + NotFoundException, + ServiceUnavailableException, +} from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import * as k8s from '@kubernetes/client-node'; +import { ClusterEntity } from '../../infrastructure/clusters/entities/cluster.entity'; +import { KubernetesService } from '../../infrastructure/shared/services/kubernetes.service'; +import { EncryptionService } from '../../shared/encryption/services/encryption.service'; +import { CapabilitiesProviderFactory } from '../../providers/core/factories/capabilities-provider.factory'; +import { CloudProvider } from '../../providers/enums/cloud-provider.enum'; +import { + HaTopology, + resolveHaTopology, +} from '../../providers/core/ha-topology'; +import { + ClusterHaState, + clusterHaNodesFrom, + clusterHaRegion, + clusterHaState, + zonesWithServers, +} from '../utils/cluster-ha.core'; +import { HaObserved } from '../utils/ha-observed'; +import { serverHostOf } from '../../infrastructure/networking/kubeconfig-demotion'; +import { KnownApiServer } from '../../infrastructure/clusters/services/api-server-failover.core'; + +export interface ClusterHaObserved { + at: string | null; + etcdLeader: HaObserved['etcdLeader']; + roundTrips: HaObserved['roundTrips']; + /** The server Flui talks to the cluster through now. */ + reachedThrough: string | null; + /** The servers Flui can move to if that one is lost. */ + reachableServers: string[]; +} + +export interface ThreeLocationCluster { + cluster: ClusterEntity; + kubeconfig: string; + nodes: k8s.V1Node[]; + state: ClusterHaState; + topology: HaTopology; +} + +@Injectable() +export class ClusterHaService { + constructor( + @InjectRepository(ClusterEntity) + private readonly clusters: Repository, + private readonly kubernetes: KubernetesService, + private readonly encryption: EncryptionService, + private readonly capabilities: CapabilitiesProviderFactory, + ) {} + + /** + * Read from the nodes as they are now, not from what was asked for. What + * the sweep measured (etcd leader, round trips) is as of `observed.at`. + */ + async view( + clusterId: string, + force = false, + ): Promise< + ClusterHaState & { clusterId: string; observed: ClusterHaObserved } + > { + const { cluster, kubeconfig, state } = await this.inspect(clusterId, force); + const metadata = (cluster.metadata ?? {}) as { + haObserved?: HaObserved; + apiServers?: KnownApiServer[]; + }; + const host = serverHostOf(kubeconfig); + return { + clusterId, + ...state, + observed: { + at: metadata.haObserved?.at ?? null, + etcdLeader: metadata.haObserved?.etcdLeader ?? null, + roundTrips: metadata.haObserved?.roundTrips ?? [], + reachedThrough: + metadata.apiServers?.find((s) => s.address === host)?.name ?? null, + reachableServers: (metadata.apiServers ?? []).map((s) => s.name), + }, + }; + } + + /** + * Databases and volumes are made highly available only on a cluster that + * already survives the loss of a location: copies of data are no use + * if the one server that would move the application to them is lost. + */ + async requireThreeLocations( + clusterId: string, + force = false, + ): Promise { + const read = await this.inspect(clusterId, force); + if (read.state.unsupported) { + throw new BadRequestException(read.state.unsupported); + } + if (read.state.level !== 'three-locations' || !read.topology) { + throw new BadRequestException( + notThreeLocations(read.cluster.name, read.state), + ); + } + return { ...read, topology: read.topology }; + } + + /** The cluster's nodes, the layout that applies to it and where it stands against it. */ + async inspect( + clusterId: string, + force = false, + ): Promise< + Omit & { topology: HaTopology | null } + > { + const cluster = await this.clusters.findOne({ where: { id: clusterId } }); + if (!cluster) throw new NotFoundException(`Cluster ${clusterId} not found`); + if (!cluster.kubeconfigEncrypted) { + throw new BadRequestException('This cluster cannot be reached.'); + } + const kubeconfig = this.encryption.decrypt(cluster.kubeconfigEncrypted); + const nodes = await this.kubernetes.listNodes(kubeconfig).catch(() => { + throw new ServiceUnavailableException( + `Flui could not reach ${cluster.name} to read its nodes. When the cluster has more servers, Flui moves to one that answers within a minute.`, + ); + }); + const haNodes = clusterHaNodesFrom(nodes); + const provider = cluster.provider as CloudProvider; + const declared = this.capabilities.isProviderSupported(provider) + ? this.capabilities.getCapabilitiesService(provider).haTopologies() + : []; + const resolution = resolveHaTopology( + declared, + clusterHaRegion(haNodes), + zonesWithServers(haNodes), + force, + ); + return { + cluster, + kubeconfig, + nodes, + state: clusterHaState(haNodes, resolution), + topology: resolution.topology, + }; + } +} + +/** Why a cluster is not ready for highly available data, in one sentence. */ +export function notThreeLocations(name: string, state: ClusterHaState): string { + if (state.unsupported) return state.unsupported; + const roles = state.missing + .map( + (m) => + `a ${m.role === 'server' ? 'server' : 'node for copies of data'} in ${m.zone}`, + ) + .join(', '); + return `${name} is not highly available across three locations yet; it still needs ${roles}. \`flui cluster ha ${name}\` shows where it stands.`; +} diff --git a/src/modules/cluster-ha/services/etcd-leader-placement.service.spec.ts b/src/modules/cluster-ha/services/etcd-leader-placement.service.spec.ts new file mode 100644 index 00000000..f9b08952 --- /dev/null +++ b/src/modules/cluster-ha/services/etcd-leader-placement.service.spec.ts @@ -0,0 +1,105 @@ +jest.mock('@kubernetes/client-node', () => ({})); + +import { KubernetesService } from '../../infrastructure/shared/services/kubernetes.service'; +import { ThreeLocationCluster } from './cluster-ha.service'; +import { + ETCDCTL_IMAGE, + EtcdLeaderPlacementService, +} from './etcd-leader-placement.service'; + +const IDS = { + master: '10276657743932975437', + w1: '9372538179322589801', + w3: '17855658209131950253', +}; + +const server = (name: string, zone: string) => ({ + metadata: { + name, + labels: { + 'topology.kubernetes.io/zone': zone, + 'node-role.kubernetes.io/control-plane': 'true', + }, + }, + status: { conditions: [{ type: 'Ready', status: 'True' }] }, +}); + +function setup(leader: string, validated = true) { + const members = `{"members":[{"ID":${IDS.master},"name":"lab-master-1a2b3c4d"},{"ID":${IDS.w1},"name":"lab-worker-1-0f9e8d7c"},{"ID":${IDS.w3},"name":"lab-worker-3-aabbccdd"}]}`; + const runImageOnNode = jest.fn( + async (_k: string, _node: string, run: { command: string[] }) => { + if (run.command.includes('status')) { + return `[{"Endpoint":"https://127.0.0.1:2379","Status":{"leader":${leader}}}]`; + } + if (run.command.includes('list')) return members; + return 'Leadership transferred'; + }, + ); + const service = new EtcdLeaderPlacementService({ + runImageOnNode, + } as unknown as KubernetesService); + const ha = { + cluster: { name: 'ha-lab' }, + kubeconfig: 'kc', + nodes: [ + server('lab-master', 'fsn1'), + server('lab-worker-1', 'nbg1'), + server('lab-worker-3', 'hel1'), + ], + state: { level: 'three-locations' }, + topology: { + zones: ['fsn1', 'nbg1', 'hel1'], + dataZones: validated ? ['fsn1', 'nbg1'] : ['fsn1', 'nbg1', 'hel1'], + validated, + }, + } as unknown as ThreeLocationCluster; + return { service, runImageOnNode, ha }; +} + +describe('EtcdLeaderPlacementService', () => { + it('asks the leader in the voting location to hand over to a member in a location for data', async () => { + const { service, runImageOnNode, ha } = setup(IDS.w3); + await expect(service.check(ha)).resolves.toEqual({ + node: 'lab-master', + zone: 'fsn1', + moved: true, + }); + expect(runImageOnNode).toHaveBeenLastCalledWith( + 'kc', + 'lab-worker-3', + expect.objectContaining({ + image: ETCDCTL_IMAGE, + hostNetwork: true, + command: expect.arrayContaining([ + 'move-leader', + BigInt(IDS.master).toString(16), + ]), + }), + ); + }); + + it('only reads when the leader is already in a location for data', async () => { + const { service, runImageOnNode, ha } = setup(IDS.w1); + await expect(service.check(ha)).resolves.toEqual({ + node: 'lab-worker-1', + zone: 'nbg1', + moved: false, + }); + expect(runImageOnNode).toHaveBeenCalledTimes(2); + }); + + it('reports the leader but never moves it on a forced region, where every location is one for data', async () => { + const { service, runImageOnNode, ha } = setup(IDS.w3, false); + await expect(service.check(ha)).resolves.toMatchObject({ + node: 'lab-worker-3', + moved: false, + }); + expect(runImageOnNode).toHaveBeenCalledTimes(2); + }); + + it('waits after a move', async () => { + const { service, runImageOnNode, ha } = setup(IDS.w3); + await service.check(ha, new Date().toISOString()); + expect(runImageOnNode).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/modules/cluster-ha/services/etcd-leader-placement.service.ts b/src/modules/cluster-ha/services/etcd-leader-placement.service.ts new file mode 100644 index 00000000..0add9add --- /dev/null +++ b/src/modules/cluster-ha/services/etcd-leader-placement.service.ts @@ -0,0 +1,129 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { KubernetesService } from '../../infrastructure/shared/services/kubernetes.service'; +import { ThreeLocationCluster } from './cluster-ha.service'; +import { clusterHaNodesFrom } from '../utils/cluster-ha.core'; +import { + ETCD_MEMBER_ANNOTATION, + EtcdServer, + etcdLeaderOf, + etcdLeaderPlacement, + etcdMembersOf, + memberOfServer, +} from '../utils/etcd-leader.core'; + +/** The etcd release K3s embeds is 3.6; its etcdctl speaks to it. */ +export const ETCDCTL_IMAGE = 'quay.io/coreos/etcd:v3.6.16'; +const ETCD_TLS_DIR = '/var/lib/rancher/k3s/server/tls/etcd'; + +const etcdctl = (...args: string[]) => [ + '/usr/local/bin/etcdctl', + '--endpoints=https://127.0.0.1:2379', + '--cacert=/etcd/server-ca.crt', + '--cert=/etcd/client.crt', + '--key=/etcd/client.key', + ...args, +]; + +export interface EtcdLeaderObservation { + node: string | null; + zone: string | null; + /** This check handed leadership to the member on `node`. */ + moved: boolean; +} + +/** + * Brings the etcd leader back to a location for data after a loss handed it + * to the voting one, where every change to the cluster would wait for the far + * link. Leadership moves between live members without stopping anything. + */ +@Injectable() +export class EtcdLeaderPlacementService { + private readonly logger = new Logger(EtcdLeaderPlacementService.name); + + constructor(private readonly kubernetes: KubernetesService) {} + + async check( + ha: ThreeLocationCluster, + lastMovedAt?: string, + ): Promise { + const preferredZones = ha.topology.dataZones; + const annotations = new Map( + ha.nodes.map((n) => [ + n.metadata?.name ?? '', + n.metadata?.annotations?.[ETCD_MEMBER_ANNOTATION], + ]), + ); + const servers: EtcdServer[] = clusterHaNodesFrom(ha.nodes) + .filter((n) => n.server) + .map((n) => ({ + node: n.name, + zone: n.zone, + ready: n.ready, + member: annotations.get(n.name), + })); + const reader = servers.find( + (s) => s.ready && s.zone && preferredZones.includes(s.zone), + ); + if (!reader) return null; + const status = await this.etcdctl( + ha.kubeconfig, + reader.node, + 'endpoint', + 'status', + '-w', + 'json', + ); + const memberList = await this.etcdctl( + ha.kubeconfig, + reader.node, + 'member', + 'list', + '-w', + 'json', + ); + const leader = etcdLeaderOf(status); + const members = etcdMembersOf(memberList); + const leaderServer = servers.find( + (s) => memberOfServer(s, members)?.id === leader, + ); + const observed: EtcdLeaderObservation = { + node: leaderServer?.node ?? null, + zone: leaderServer?.zone ?? null, + moved: false, + }; + if (preferredZones.length === ha.topology.zones.length) return observed; + const decision = etcdLeaderPlacement({ + leader, + members, + servers, + preferredZones, + lastMovedAt, + now: Date.now(), + }); + if (!decision.move) return observed; + await this.etcdctl( + ha.kubeconfig, + decision.leaderNode, + 'move-leader', + decision.target, + ); + this.logger.log( + `[etcd] ${ha.cluster.name}: leader on ${decision.leaderNode} sat in the voting location; handed it to ${decision.targetNode}`, + ); + return { + node: decision.targetNode, + zone: servers.find((s) => s.node === decision.targetNode)?.zone ?? null, + moved: true, + }; + } + + private etcdctl(kubeconfig: string, node: string, ...args: string[]) { + return this.kubernetes.runImageOnNode(kubeconfig, node, { + image: ETCDCTL_IMAGE, + command: etcdctl(...args), + hostDir: ETCD_TLS_DIR, + mountPath: '/etcd', + hostNetwork: true, + }); + } +} diff --git a/src/modules/cluster-ha/services/site-round-trip.service.ts b/src/modules/cluster-ha/services/site-round-trip.service.ts new file mode 100644 index 00000000..350d6392 --- /dev/null +++ b/src/modules/cluster-ha/services/site-round-trip.service.ts @@ -0,0 +1,63 @@ +import { Injectable } from '@nestjs/common'; +import { + KubernetesService, + NODE_PROBE_IMAGE, +} from '../../infrastructure/shared/services/kubernetes.service'; +import { ThreeLocationCluster } from './cluster-ha.service'; +import { clusterHaNodesFrom } from '../utils/cluster-ha.core'; +import { + SiteRoundTrip, + SiteServer, + parsePingOutput, + pingScript, + roundTripPlan, +} from '../utils/site-round-trip.core'; + +/** + * How long a message takes between the cluster's locations, measured from + * the servers themselves on the addresses their database and copies use: the + * floor under every write that waits for another location. + */ +@Injectable() +export class SiteRoundTripService { + constructor(private readonly kubernetes: KubernetesService) {} + + async measure(ha: ThreeLocationCluster): Promise { + const ready = new Set( + clusterHaNodesFrom(ha.nodes) + .filter((n) => n.server && n.ready) + .map((n) => n.name), + ); + const servers: SiteServer[] = ha.nodes + .filter((n) => ready.has(n.metadata?.name ?? '')) + .map((n) => ({ + node: n.metadata!.name!, + zone: n.metadata?.labels?.['topology.kubernetes.io/zone'] ?? '', + address: + n.status?.addresses?.find((a) => a.type === 'InternalIP')?.address ?? + '', + })) + .filter((s) => s.zone && s.address); + const trips: SiteRoundTrip[] = []; + for (const { from, to } of roundTripPlan(servers)) { + const output = await this.kubernetes.runImageOnNode( + ha.kubeconfig, + from.node, + { + image: NODE_PROBE_IMAGE, + command: ['sh', '-c', pingScript(to.map((t) => t.address))], + hostNetwork: true, + }, + ); + const averages = parsePingOutput(output); + for (const t of to) { + trips.push({ + from: from.zone, + to: t.zone, + ms: averages.get(t.address) ?? null, + }); + } + } + return trips; + } +} diff --git a/src/modules/cluster-ha/utils/cluster-ha.core.spec.ts b/src/modules/cluster-ha/utils/cluster-ha.core.spec.ts new file mode 100644 index 00000000..13831541 --- /dev/null +++ b/src/modules/cluster-ha/utils/cluster-ha.core.spec.ts @@ -0,0 +1,199 @@ +import { + ClusterHaNode, + clusterHaNodesFrom, + clusterHaRegion, + clusterHaState, +} from './cluster-ha.core'; +import { HaTopology } from '../../providers/core/ha-topology'; + +const hetzner: HaTopology = { + region: 'eu-central', + zones: ['fsn1', 'nbg1', 'hel1'], + dataZones: ['fsn1', 'nbg1'], + validated: true, + measured: true, +}; +const layout = { topology: hetzner }; + +const n = ( + name: string, + zone: string | undefined, + server = false, + schedulable = true, +): ClusterHaNode => ({ + name, + zone, + region: zone ? 'eu-central' : undefined, + server, + ready: true, + schedulable, +}); + +describe('cluster HA state', () => { + it('a master and a worker are standard and need a server in each other location', () => { + const s = clusterHaState( + [n('master', 'fsn1', true, false), n('w1', 'fsn1')], + layout, + ); + expect(s.level).toBe('standard'); + expect(s.missing).toEqual([ + expect.objectContaining({ role: 'server', zone: 'nbg1' }), + expect.objectContaining({ role: 'server', zone: 'hel1' }), + expect.objectContaining({ role: 'data-host', zone: 'nbg1' }), + ]); + expect(s.doesNotSurvive.join(' ')).toContain('the loss of the master'); + }); + + it('three servers in Falkenstein, Nuremberg and Helsinki that take applications reach three locations', () => { + const s = clusterHaState( + [n('s1', 'fsn1', true), n('s2', 'nbg1', true), n('s3', 'hel1', true)], + layout, + ); + expect(s.level).toBe('three-locations'); + expect(s.missing).toEqual([]); + expect(s.topology?.zones.map((z) => [z.zone, z.role])).toEqual([ + ['fsn1', 'data'], + ['nbg1', 'data'], + ['hel1', 'vote'], + ]); + expect(s.survives.join(' ')).toContain('the loss of a whole location'); + }); + + it('a data location needs a node that takes applications, the voting one does not', () => { + const s = clusterHaState( + [ + n('s1', 'fsn1', true), + n('s2', 'nbg1', true, false), + n('s3', 'hel1', true, false), + ], + layout, + ); + expect(s.level).toBe('standard'); + expect(s.missing).toEqual([ + expect.objectContaining({ role: 'data-host', zone: 'nbg1' }), + ]); + }); + + it('where every location holds data, each needs a node that takes applications', () => { + const s = clusterHaState( + [n('a', 'z1', true), n('b', 'z2', true), n('c', 'z3', true, false)], + { + topology: { + region: 'r', + zones: ['z1', 'z2', 'z3'], + dataZones: ['z1', 'z2', 'z3'], + validated: true, + measured: true, + }, + }, + ); + expect(s.level).toBe('standard'); + expect(s.missing).toEqual([ + { + role: 'data-host', + zone: 'z3', + why: 'Copies of data live in z1, z2 and z3, one in each, on a node that takes applications.', + }, + ]); + }); + + it("says when Flui took the layout on the provider's word", () => { + const s = clusterHaState( + [n('a', 'z1', true), n('b', 'z2', true), n('c', 'z3', true)], + { + topology: { + region: 'r', + zones: ['z1', 'z2', 'z3'], + dataZones: ['z1', 'z2', 'z3'], + validated: true, + measured: false, + }, + }, + ); + expect(s.level).toBe('three-locations'); + expect(s.warnings).toContain( + 'The provider declares z1, z2 and z3 as zones of one region, close together; Flui has not measured them yet.', + ); + }); + + it('warns that two servers are more fragile than one', () => { + const s = clusterHaState( + [n('s1', 'fsn1', true), n('s2', 'nbg1', true), n('w1', 'fsn1')], + layout, + ); + expect(s.warnings.join(' ')).toContain('more fragile than one'); + }); + + it('a region with no validated layout says why, and stays standard', () => { + const s = clusterHaState([n('s1', 'fsn1', true)], { + topology: null, + reason: + 'Flui has not validated a highly available layout for us-east yet.', + }); + expect(s.level).toBe('standard'); + expect(s.topology).toBeNull(); + expect(s.unsupported).toContain('us-east'); + }); + + it('a forced layout says it was not measured', () => { + const s = clusterHaState( + [n('a', 'z1', true), n('b', 'z2', true), n('c', 'z3', true)], + { + topology: { + region: 'r', + zones: ['z1', 'z2', 'z3'], + dataZones: ['z1', 'z2', 'z3'], + validated: false, + measured: false, + }, + }, + ); + expect(s.level).toBe('three-locations'); + expect(s.warnings.join(' ')).toContain('was forced'); + }); + + it('nodes without a known location are reported', () => { + const s = clusterHaState([n('master', undefined, true, false)], layout); + expect(s.warnings.join(' ')).toContain('No node reports its location'); + }); + + it('reads roles, locations, region and readiness from the nodes', () => { + const nodes = clusterHaNodesFrom([ + { + metadata: { + name: 'm', + labels: { + 'node-role.kubernetes.io/control-plane': 'true', + 'topology.kubernetes.io/zone': 'fsn1', + 'topology.kubernetes.io/region': 'eu-central', + }, + }, + status: { conditions: [{ type: 'Ready', status: 'True' }] }, + spec: { taints: [{ key: 'x', effect: 'NoSchedule' }] }, + }, + { + metadata: { name: 'w', labels: {} }, + status: { conditions: [{ type: 'Ready', status: 'False' }] }, + }, + ]); + expect(nodes).toEqual([ + { + name: 'm', + zone: 'fsn1', + region: 'eu-central', + server: true, + ready: true, + schedulable: false, + }, + { + name: 'w', + zone: undefined, + region: undefined, + server: false, + ready: false, + schedulable: false, + }, + ]); + expect(clusterHaRegion(nodes)).toBe('eu-central'); + }); +}); diff --git a/src/modules/cluster-ha/utils/cluster-ha.core.ts b/src/modules/cluster-ha/utils/cluster-ha.core.ts new file mode 100644 index 00000000..8379b9fb --- /dev/null +++ b/src/modules/cluster-ha/utils/cluster-ha.core.ts @@ -0,0 +1,222 @@ +import { + TOPOLOGY_REGION_LABEL, + TOPOLOGY_ZONE_LABEL, +} from '../../providers/core/node-topology'; +import { + HaTopologyResolution, + locationList, +} from '../../providers/core/ha-topology'; + +const CONTROL_PLANE_LABEL = 'node-role.kubernetes.io/control-plane'; + +export type ClusterHaLevel = 'standard' | 'three-locations'; + +export interface ClusterHaNode { + name: string; + zone?: string; + region?: string; + server: boolean; + ready: boolean; + /** Takes ordinary workloads: ready, not cordoned, no reserving taint. */ + schedulable: boolean; +} + +export interface MissingRole { + role: 'server' | 'data-host'; + zone: string; + why: string; +} + +export interface ClusterHaZone { + zone: string; + /** `data`: copies of data live here. `vote`: it only votes, a write never waits for it. */ + role: 'data' | 'vote'; + servers: string[]; + dataHosts: string[]; +} + +export interface ClusterHaState { + level: ClusterHaLevel; + region: string | null; + /** The layout this cluster grows into; null when none applies, with the reason. */ + topology: { validated: boolean; zones: ClusterHaZone[] } | null; + unsupported: string | null; + servers: { count: number; ready: number; zones: string[] }; + survives: string[]; + doesNotSurvive: string[]; + warnings: string[]; + missing: MissingRole[]; +} + +interface NodeLike { + metadata?: { name?: string; labels?: Record }; + spec?: { + unschedulable?: boolean; + taints?: Array<{ key: string; effect: string }>; + }; + status?: { conditions?: Array<{ type: string; status: string }> }; +} + +export function clusterHaNodesFrom(nodes: NodeLike[]): ClusterHaNode[] { + return nodes + .filter((n) => n.metadata?.name) + .map((n) => { + const ready = (n.status?.conditions ?? []).some( + (c) => c.type === 'Ready' && c.status === 'True', + ); + const reserved = (n.spec?.taints ?? []).some( + (t) => t.effect === 'NoSchedule' || t.effect === 'NoExecute', + ); + return { + name: n.metadata!.name!, + zone: n.metadata?.labels?.[TOPOLOGY_ZONE_LABEL], + region: n.metadata?.labels?.[TOPOLOGY_REGION_LABEL], + server: n.metadata?.labels?.[CONTROL_PLANE_LABEL] === 'true', + ready, + schedulable: ready && !n.spec?.unschedulable && !reserved, + }; + }); +} + +/** The region most of the cluster's nodes report. */ +export function clusterHaRegion(nodes: ClusterHaNode[]): string | undefined { + const count = new Map(); + for (const n of nodes.filter((n) => n.region)) { + count.set(n.region!, (count.get(n.region!) ?? 0) + 1); + } + return [...count.entries()].sort((a, b) => b[1] - a[1])[0]?.[0]; +} + +export function zonesWithServers(nodes: ClusterHaNode[]): string[] { + return [ + ...new Set(nodes.filter((n) => n.server && n.zone).map((n) => n.zone!)), + ].sort((a, b) => a.localeCompare(b)); +} + +/** + * What a cluster survives with the nodes it has now, against the one layout + * Flui offers: a server in each of three locations, copies of data in the + * locations close enough to confirm a write together. Only what is there + * counts; a role on its way does not raise the level until its node arrives. + */ +export function clusterHaState( + nodes: ClusterHaNode[], + resolution: HaTopologyResolution, +): ClusterHaState { + const servers = nodes.filter((n) => n.server); + const warnings: string[] = []; + if (!nodes.some((n) => n.zone)) { + warnings.push( + 'No node reports its location yet, so Flui cannot tell which locations the cluster spans.', + ); + } + if (servers.length === 2) { + warnings.push( + 'Two servers are more fragile than one: losing either of them stops the control plane. A third server is needed.', + ); + } + const notReady = servers.filter((n) => !n.ready).map((n) => n.name); + if (notReady.length) warnings.push(`Not ready: ${notReady.join(', ')}.`); + + const base = { + region: clusterHaRegion(nodes) ?? null, + servers: { + count: servers.length, + ready: servers.filter((n) => n.ready).length, + zones: zonesWithServers(nodes), + }, + warnings, + }; + + if (!resolution.topology) { + return { + ...base, + level: 'standard', + topology: null, + unsupported: (resolution as { reason: string }).reason, + ...standardSentences(servers.length), + missing: [], + }; + } + + const topology = resolution.topology; + if (!topology.validated) { + warnings.push( + 'This layout was forced: Flui has not measured how far apart these locations are.', + ); + } else if (!topology.measured) { + warnings.push( + `The provider declares ${locationList(topology.zones)} as zones of one region, close together; Flui has not measured them yet.`, + ); + } + const zones: ClusterHaZone[] = topology.zones.map((zone) => ({ + zone, + role: topology.dataZones.includes(zone) ? 'data' : 'vote', + servers: servers + .filter((n) => n.ready && n.zone === zone) + .map((n) => n.name), + dataHosts: topology.dataZones.includes(zone) + ? nodes.filter((n) => n.schedulable && n.zone === zone).map((n) => n.name) + : [], + })); + + const missing: MissingRole[] = zones + .filter((z) => z.servers.length === 0) + .map((z) => ({ + role: 'server' as const, + zone: z.zone, + why: 'One server in each of the three locations keeps the cluster manageable when a location is lost.', + })); + const dataZonesWithHosts = zones.filter( + (z) => z.role === 'data' && z.dataHosts.length > 0, + ); + if (dataZonesWithHosts.length < topology.dataZones.length) { + for (const z of zones.filter( + (z) => z.role === 'data' && z.dataHosts.length === 0, + )) { + missing.push({ + role: 'data-host', + zone: z.zone, + why: `Copies of data live in ${locationList(topology.dataZones)}, one in each, on a node that takes applications.`, + }); + } + } + + const ready = missing.length === 0; + return { + ...base, + level: ready ? 'three-locations' : 'standard', + topology: { validated: topology.validated, zones }, + unsupported: null, + ...(ready + ? { + survives: [ + 'the loss of a server', + 'the loss of a whole location: the control plane, highly available databases and replicated volumes carry on in the other two', + ], + doesNotSurvive: ['the loss of two locations at once'], + } + : standardSentences(servers.length)), + missing, + }; +} + +function standardSentences(serverCount: number): { + survives: string[]; + doesNotSurvive: string[]; +} { + return { + survives: [ + 'the loss of a worker, for applications with two or more copies', + ], + doesNotSurvive: [ + ...(serverCount >= 3 + ? [] + : [ + 'the loss of the master: applications keep answering, but nothing can change and no copy can take over until it returns', + ]), + 'the loss of the node a database or a volume lives on: it waits for the node or a restore', + 'the loss of a whole location', + ], + }; +} diff --git a/src/modules/cluster-ha/utils/etcd-leader.core.spec.ts b/src/modules/cluster-ha/utils/etcd-leader.core.spec.ts new file mode 100644 index 00000000..550f8462 --- /dev/null +++ b/src/modules/cluster-ha/utils/etcd-leader.core.spec.ts @@ -0,0 +1,92 @@ +import { + EtcdServer, + etcdLeaderOf, + etcdLeaderPlacement, + etcdMembersOf, +} from './etcd-leader.core'; + +const IDS = { + master: '10276657743932975437', + w1: '9372538179322589801', + w3: '17855658209131950253', +}; +const hex = (id: string) => BigInt(id).toString(16); + +const memberList = JSON.stringify({ header: {}, members: [] }).replace( + '[]', + `[{"ID":${IDS.master},"name":"lab-master-1a2b3c4d"},{"ID":${IDS.w1},"name":"lab-worker-1-0f9e8d7c"},{"ID":${IDS.w3},"name":"lab-worker-3-aabbccdd"}]`, +); +const status = (leader: string) => + `[{"Endpoint":"https://127.0.0.1:2379","Status":{"header":{"member_id":${IDS.master}},"version":"3.6.7","leader":${leader}}}]`; + +const servers: EtcdServer[] = [ + { node: 'lab-master', zone: 'fsn1', ready: true }, + { node: 'lab-worker-1', zone: 'nbg1', ready: true }, + { node: 'lab-worker-3', zone: 'hel1', ready: true }, +]; + +const decide = ( + leader: string, + over: Partial[0]> = {}, +) => + etcdLeaderPlacement({ + leader: etcdLeaderOf(status(leader)), + members: etcdMembersOf(memberList), + servers, + preferredZones: ['fsn1', 'nbg1'], + now: Date.parse('2026-10-29T10:00:00Z'), + ...over, + }); + +describe('etcd leader placement', () => { + it('keeps the 64-bit member ids exact', () => { + expect(etcdLeaderOf(status(IDS.w3))).toBe(hex(IDS.w3)); + expect(etcdMembersOf(memberList).map((m) => m.id)).toEqual([ + hex(IDS.master), + hex(IDS.w1), + hex(IDS.w3), + ]); + }); + + it('hands a leader in the voting location to a member in a location for data', () => { + expect(decide(IDS.w3)).toEqual({ + move: true, + leaderNode: 'lab-worker-3', + target: hex(IDS.master), + targetNode: 'lab-master', + }); + }); + + it('leaves a leader already in a location for data alone', () => { + expect(decide(IDS.w1)).toMatchObject({ move: false }); + }); + + it('never moves while a server is down or not yet a member', () => { + expect( + decide(IDS.w3, { + servers: servers.map((s, i) => (i === 0 ? { ...s, ready: false } : s)), + }), + ).toMatchObject({ move: false, reason: 'a server is not ready' }); + expect( + decide(IDS.w3, { + servers: [ + ...servers, + { node: 'lab-worker-4', zone: 'fsn1', ready: true }, + ], + }), + ).toMatchObject({ move: false }); + }); + + it('prefers the name K3s recorded on the node, and waits after a move', () => { + expect( + decide(IDS.w3, { + servers: servers.map((s) => + s.node === 'lab-master' ? { ...s, member: 'lab-master-1a2b3c4d' } : s, + ), + }), + ).toMatchObject({ move: true, targetNode: 'lab-master' }); + expect( + decide(IDS.w3, { lastMovedAt: '2026-10-29T09:50:00Z' }), + ).toMatchObject({ move: false, reason: 'moved recently' }); + }); +}); diff --git a/src/modules/cluster-ha/utils/etcd-leader.core.ts b/src/modules/cluster-ha/utils/etcd-leader.core.ts new file mode 100644 index 00000000..145e5b77 --- /dev/null +++ b/src/modules/cluster-ha/utils/etcd-leader.core.ts @@ -0,0 +1,102 @@ +/** K3s records on each server node the name of its etcd member. */ +export const ETCD_MEMBER_ANNOTATION = 'etcd.k3s.cattle.io/node-name'; + +export const ETCD_LEADER_MOVE_COOLDOWN_MS = 30 * 60 * 1000; + +export interface EtcdMember { + /** Hex, the form `etcdctl move-leader` takes. */ + id: string; + name: string; +} + +export interface EtcdServer { + node: string; + zone?: string; + ready: boolean; + member?: string; +} + +export type EtcdLeaderPlacement = + | { move: false; reason: string } + | { move: true; leaderNode: string; target: string; targetNode: string }; + +/** etcd prints member ids as 64-bit integers, more than a JavaScript number holds. */ +function parseEtcdJson(text: string): any { + return JSON.parse(text.replace(/:(\s*)(\d{16,})/g, ':$1"$2"')); +} + +const hex = (id: string | number) => BigInt(id).toString(16); + +export function etcdLeaderOf(statusJson: string): string | undefined { + const parsed = parseEtcdJson(statusJson); + const leader = (Array.isArray(parsed) ? parsed[0] : parsed)?.Status?.leader; + return leader === undefined ? undefined : hex(leader); +} + +export function etcdMembersOf(memberListJson: string): EtcdMember[] { + return (parseEtcdJson(memberListJson)?.members ?? []).map( + (m: { ID: string; name: string }) => ({ id: hex(m.ID), name: m.name }), + ); +} + +/** The member K3s named after the node: the node's name and eight hex characters. */ +export function memberOfServer( + server: EtcdServer, + members: EtcdMember[], +): EtcdMember | undefined { + return members.find((m) => + server.member + ? m.name === server.member + : /^(.+)-[0-9a-f]{8}$/.exec(m.name)?.[1] === server.node, + ); +} + +/** + * Whether the etcd leader sits outside the locations a write is confirmed + * fastest from, and which member to hand it to. Only while every server is + * ready and an etcd member, so leadership never moves in the middle of a loss. + */ +export function etcdLeaderPlacement(input: { + leader: string | undefined; + members: EtcdMember[]; + servers: EtcdServer[]; + preferredZones: string[]; + lastMovedAt?: string; + now: number; +}): EtcdLeaderPlacement { + if (input.servers.some((s) => !s.ready)) { + return { move: false, reason: 'a server is not ready' }; + } + const placed = input.servers.map((s) => ({ + ...s, + id: memberOfServer(s, input.members)?.id, + })); + if ( + placed.some((s) => !s.id) || + input.members.length !== input.servers.length + ) { + return { move: false, reason: 'servers and members do not match' }; + } + const leader = placed.find((s) => s.id === input.leader); + if (!leader?.zone) return { move: false, reason: 'leader not placed' }; + if (input.preferredZones.includes(leader.zone)) { + return { move: false, reason: 'already in place' }; + } + if ( + input.lastMovedAt && + input.now - Date.parse(input.lastMovedAt) < ETCD_LEADER_MOVE_COOLDOWN_MS + ) { + return { move: false, reason: 'moved recently' }; + } + const target = placed + .filter((s) => s.zone && input.preferredZones.includes(s.zone)) + .sort((a, b) => a.node.localeCompare(b.node))[0]; + if (!target) + return { move: false, reason: 'no member in a preferred location' }; + return { + move: true, + leaderNode: leader.node, + target: target.id!, + targetNode: target.node, + }; +} diff --git a/src/modules/cluster-ha/utils/ha-observed.ts b/src/modules/cluster-ha/utils/ha-observed.ts new file mode 100644 index 00000000..7c5a042e --- /dev/null +++ b/src/modules/cluster-ha/utils/ha-observed.ts @@ -0,0 +1,9 @@ +import { SiteRoundTrip } from './site-round-trip.core'; + +/** What the high availability sweep last saw on a cluster, kept in its metadata. */ +export interface HaObserved { + at: string; + etcdLeader: { node: string | null; zone: string | null } | null; + roundTrips: SiteRoundTrip[]; + etcdLeaderMovedAt?: string; +} diff --git a/src/modules/cluster-ha/utils/site-round-trip.core.spec.ts b/src/modules/cluster-ha/utils/site-round-trip.core.spec.ts new file mode 100644 index 00000000..9921db92 --- /dev/null +++ b/src/modules/cluster-ha/utils/site-round-trip.core.spec.ts @@ -0,0 +1,35 @@ +import { + parsePingOutput, + pingScript, + roundTripPlan, +} from './site-round-trip.core'; + +describe('round trips between locations', () => { + const servers = [ + { zone: 'hel1', node: 'w3', address: '10.0.0.4' }, + { zone: 'fsn1', node: 'master', address: '10.0.0.2' }, + { zone: 'fsn1', node: 'w2', address: '10.0.0.5' }, + { zone: 'nbg1', node: 'w1', address: '10.0.0.3' }, + ]; + + it('measures each pair of locations once, from one server in each', () => { + expect( + roundTripPlan(servers).map((p) => [p.from.node, p.to.map((t) => t.node)]), + ).toEqual([ + ['master', ['w3', 'w1']], + ['w3', ['w1']], + ]); + }); + + it('reads the average of busybox ping, and null when nothing came back', () => { + const out = [ + '10.0.0.4 round-trip min/avg/max = 34.120/35.004/36.510 ms', + '10.0.0.3 ', + ].join('\n'); + expect([...parsePingOutput(out)]).toEqual([ + ['10.0.0.4', 35.004], + ['10.0.0.3', null], + ]); + expect(pingScript(['10.0.0.4'])).toContain('ping -c 3 -W 2 -q 10.0.0.4'); + }); +}); diff --git a/src/modules/cluster-ha/utils/site-round-trip.core.ts b/src/modules/cluster-ha/utils/site-round-trip.core.ts new file mode 100644 index 00000000..c3b8e008 --- /dev/null +++ b/src/modules/cluster-ha/utils/site-round-trip.core.ts @@ -0,0 +1,49 @@ +export interface SiteRoundTrip { + from: string; + to: string; + /** Average of three pings; null when the other location did not answer. */ + ms: number | null; +} + +export interface SiteServer { + zone: string; + node: string; + address: string; +} + +/** One server per location, the first by name, so the pairs stay the same between sweeps. */ +export function serverPerZone(servers: SiteServer[]): SiteServer[] { + const byZone = new Map(); + for (const s of [...servers].sort((a, b) => a.node.localeCompare(b.node))) { + if (!byZone.has(s.zone)) byZone.set(s.zone, s); + } + return [...byZone.values()].sort((a, b) => a.zone.localeCompare(b.zone)); +} + +/** Each location measures the ones after it: three locations, three pairs, two probes. */ +export function roundTripPlan( + servers: SiteServer[], +): { from: SiteServer; to: SiteServer[] }[] { + const zones = serverPerZone(servers); + return zones + .map((from, i) => ({ from, to: zones.slice(i + 1) })) + .filter((p) => p.to.length > 0); +} + +export function pingScript(addresses: string[]): string { + return addresses + .map((a) => `echo "${a} $(ping -c 3 -W 2 -q ${a} 2>/dev/null | tail -1)"`) + .join('\n'); +} + +/** Reads busybox's `round-trip min/avg/max = 0.4/0.5/0.6 ms`, one line per address. */ +export function parsePingOutput(output: string): Map { + const result = new Map(); + for (const line of output.split('\n')) { + const [address, ...rest] = line.trim().split(' '); + if (!address) continue; + const avg = /=\s*[\d.]+\/([\d.]+)\//.exec(rest.join(' '))?.[1]; + result.set(address, avg === undefined ? null : Number(avg)); + } + return result; +} diff --git a/src/modules/database-console/services/db-disk.service.ts b/src/modules/database-console/services/db-disk.service.ts index eb2cf4a3..c0db4ff8 100644 --- a/src/modules/database-console/services/db-disk.service.ts +++ b/src/modules/database-console/services/db-disk.service.ts @@ -7,6 +7,7 @@ import { DbConnectionResolveInput, DbConnectionResolver, } from '../interfaces/db-connection'; +import { runningPodForDatabase } from '../../infrastructure/shared/utils/database-pod.util'; export interface DbDiskInfo { available: boolean; @@ -67,7 +68,7 @@ export class DbDiskService { r.target.namespace, r.target.podLabelSelector, ); - const pod = pods.find((p) => p.status?.phase === 'Running') ?? pods[0]; + const pod = runningPodForDatabase(pods) ?? pods[0]; if (!pod) { return { ...EMPTY, available: false, reason: 'No running pod found.' }; } diff --git a/src/modules/database-console/services/kube-port-forward.service.ts b/src/modules/database-console/services/kube-port-forward.service.ts index 53a1aff8..497ff950 100644 --- a/src/modules/database-console/services/kube-port-forward.service.ts +++ b/src/modules/database-console/services/kube-port-forward.service.ts @@ -12,6 +12,7 @@ import { CONSOLE_TARGET_ABSENT, platformFoundationAtTarget, } from '../constants/platform-foundations'; +import { runningPodForDatabase } from '../../infrastructure/shared/utils/database-pod.util'; export interface PortForwardTunnel { /** Loopback port a TCP client (e.g. a pg.Pool) connects to. */ @@ -200,7 +201,7 @@ export class KubePortForwardService implements OnModuleDestroy { namespace: entry.namespace, labelSelector: selector, }); - const pod = (pods.items ?? []).find((p) => p.status?.phase === 'Running'); + const pod = runningPodForDatabase(pods.items ?? []); if (!pod?.metadata?.name) { // An absence, not a fault. `redis` answered 500 here because discovery // labels a bootstrap-created pod with nothing Flui coined: the selector diff --git a/src/modules/db-lifecycle/db-lifecycle.module.ts b/src/modules/db-lifecycle/db-lifecycle.module.ts index df963eaf..bc187fe0 100644 --- a/src/modules/db-lifecycle/db-lifecycle.module.ts +++ b/src/modules/db-lifecycle/db-lifecycle.module.ts @@ -79,6 +79,8 @@ import { FullMigrationEntity } from '../full-migration/entities/full-migration.e DbMigrationService, DbMigrationViewService, DbPoolerCutService, + DbPodExecService, + DbWriteFenceService, ], }) export class DbLifecycleModule {} diff --git a/src/modules/infrastructure/clusters/clusters.controller.fence.spec.ts b/src/modules/infrastructure/clusters/clusters.controller.fence.spec.ts index 88b177d4..d88baf9a 100644 --- a/src/modules/infrastructure/clusters/clusters.controller.fence.spec.ts +++ b/src/modules/infrastructure/clusters/clusters.controller.fence.spec.ts @@ -25,6 +25,7 @@ import { ClusterVNetService } from './services/cluster-vnet.service'; import { ClusterScalingService } from './services/cluster-scaling.service'; import { ClusterStorageService } from './services/cluster-storage.service'; import { ClusterStorageUsageService } from './services/cluster-storage-usage.service'; +import { NodeDisksService } from './services/node-disks.service'; import { ClusterCapacityService } from './services/cluster-capacity.service'; import { ClusterNodeScalingService } from './services/cluster-node-scaling.service'; import { OrphanVolumesService } from './services/orphan-volumes.service'; @@ -223,6 +224,7 @@ describe('clusters controller — the fence around the cluster key', () => { { provide: ClusterScalingService, useValue: {} }, { provide: ClusterStorageService, useValue: {} }, { provide: ClusterStorageUsageService, useValue: {} }, + { provide: NodeDisksService, useValue: {} }, { provide: ClusterCapacityService, useValue: { getPlan: async () => ({ candidates: [] }) }, diff --git a/src/modules/infrastructure/clusters/clusters.controller.ts b/src/modules/infrastructure/clusters/clusters.controller.ts index 7db63b14..8c435301 100644 --- a/src/modules/infrastructure/clusters/clusters.controller.ts +++ b/src/modules/infrastructure/clusters/clusters.controller.ts @@ -77,6 +77,7 @@ import { import { NodeAccessRecoveryService } from './services/node-access-recovery.service'; import { byOf } from '../scaling/scaling-actor'; import { ClusterStorageService } from './services/cluster-storage.service'; +import { NodeDisksService } from './services/node-disks.service'; import { ClusterStorageUsageService } from './services/cluster-storage-usage.service'; import { ControlRestoreDto, @@ -125,6 +126,7 @@ import { assertWritableClusterMetadata, assertWritableNodeMetadata, } from './utils/writable-metadata.util'; +import { HaPlanResponseDto } from './dto/ha-creation-plan.dto'; export const CLUSTER_ACCESS_NEEDS_DATA_ACCESS = 'Your own SSH keys or OS image on a new cluster give a shell on its nodes, ' + @@ -180,6 +182,7 @@ export class ClustersController { private readonly clusterScalingService: ClusterScalingService, private readonly clusterStorageService: ClusterStorageService, private readonly clusterStorageUsageService: ClusterStorageUsageService, + private readonly nodeDisksService: NodeDisksService, private readonly clusterCapacityService: ClusterCapacityService, private readonly clusterNodeScalingService: ClusterNodeScalingService, private readonly orphanVolumesService: OrphanVolumesService, @@ -233,6 +236,41 @@ export class ClustersController { return this.clusterCreationService.workloadProviderVerdict(provider); } + @Get('high-availability/plan') + @RequireSection('infrastructure') + @ApiOperation({ + summary: 'What a cluster across three locations would be bought as', + description: + "Buys nothing. For a provider and the first server's location: the three " + + 'servers with their size and role, each location taking the first of ' + + '`shapes` it has available now, or why it cannot be created, with the ' + + 'sizes every location has. Creating the cluster gives the same answer.', + }) + @ApiQuery({ name: 'provider', enum: CloudProvider }) + @ApiQuery({ name: 'region', description: "The first server's location." }) + @ApiQuery({ + name: 'shapes', + description: 'Sizes in order of preference, comma separated.', + }) + @ApiQuery({ name: 'completeLater', required: false, type: Boolean }) + @ApiResponse({ status: 200, type: HaPlanResponseDto }) + async highAvailabilityPlan( + @Query('provider') provider: CloudProvider, + @Query('region') region: string, + @Query('shapes') shapes: string, + @Query('completeLater') completeLater?: string, + ): Promise { + return this.clusterCreationService.previewHighAvailability({ + provider, + region, + shapes: (shapes ?? '') + .split(',') + .map((s) => s.trim()) + .filter(Boolean), + completeLater: completeLater === 'true', + }); + } + @Get('orphan-volumes') @RequireSection('infrastructure') @ApiOperation({ @@ -975,9 +1013,12 @@ export class ClustersController { @ApiOperation({ summary: 'Preview impact of a scale-node operation', description: - 'Returns the current server type, affected dedicated workloads and ' + - 'expected downtime for a vertical scale of the named node. Use this ' + - 'before invoking POST .../scale to confirm the maintenance window.', + 'Returns the current server type, affected dedicated workloads, ' + + 'expected downtime and the machine types the node could move to ' + + '(`candidates`, against its own type) for a vertical scale of the named node. ' + + 'On a cluster across three locations `highAvailability` says why the resize would be refused now ' + + '(another node being resized, a server down, a copy of a replicated volume behind) and which ' + + 'Postgres primaries are handed over before the node goes down. Use this before POST .../scale.', }) async previewScaleNode( @Param('id') id: string, @@ -1025,15 +1066,18 @@ export class ClustersController { summary: 'Vertically scale a cluster node (power-off → change_type → power-on)', description: - 'Maintenance-window operation. Cordons the node, powers it off, asks the ' + + 'Maintenance-window operation, queued: returns the operation to follow. Cordons the node, powers it off, asks the ' + 'provider to change its server type, powers it back on and waits for ' + 'k3s to report Ready. Total downtime ~3–5 min. All pods scheduled on ' + - 'the node — including dedicated databases — are unavailable during ' + - 'this window. Snapshot DBs first.', + 'the node, dedicated databases included, are unavailable during ' + + 'this window. Snapshot DBs first. Each step checks where the machine already is, so the job resumes after an API restart; ' + + 'a failure once the node is off tries to power it back on. On a cluster across three locations one node is resized at a time, ' + + 'only while every other server is ready and every copy of a replicated volume is up to date, and a Postgres primary on the node is ' + + 'handed over first. `copiesGiB` grows the space for copies of replicated volumes once the node is back.', }) @ApiResponse({ - status: 200, - description: 'Operation completed successfully', + status: 201, + description: 'Operation queued', }) async scaleNode( @Param('id') id: string, @@ -1088,6 +1132,20 @@ export class ClustersController { return this.clusterStorageService.getStatus(id); } + @Get(':id/storage/disks') + @RequireSection('clusters') + @ApiOperation({ + summary: 'How much room is left on each node', + description: + 'For each node, its own disk (`disk`), where databases and data kept on the node live, and the space set aside for copies of replicated volumes (`copies`, null on a node that holds none), each as total and free GiB. ' + + '`warnings` names a node running short, under 15% free or a few GiB: a full disk stops what is kept on it from growing, and a full space for copies refuses new or larger replicated volumes. Read from the nodes as they are now.', + }) + @ApiParam({ name: 'id', description: 'Cluster ID' }) + @ApiResponse({ status: 404, description: 'Cluster not found' }) + async getNodeDisks(@Param('id') id: string) { + return this.nodeDisksService.view(id); + } + @Get(':id/storage/usage') @RequireSection('clusters') @ApiOperation({ diff --git a/src/modules/infrastructure/clusters/clusters.module.ts b/src/modules/infrastructure/clusters/clusters.module.ts index 8e503138..ea9e9c39 100644 --- a/src/modules/infrastructure/clusters/clusters.module.ts +++ b/src/modules/infrastructure/clusters/clusters.module.ts @@ -43,6 +43,8 @@ import { ClustersController } from './clusters.controller'; import { ClustersService } from './clusters.service'; import { ClusterValidationService } from './services/cluster-validation.service'; import { ClusterCreationService } from './services/cluster-creation.service'; +import { HaCreationPlanService } from './services/ha-creation-plan.service'; +import { HaServerJoinService } from './services/ha-server-join.service'; import { ClusterDeletionService } from './services/cluster-deletion.service'; import { ClusterMapperService } from './services/cluster-mapper.service'; import { ClusterOperationsService } from './services/cluster-operations.service'; @@ -56,8 +58,14 @@ import { SharedStorageExportScheduler } from './schedulers/shared-storage-export import { ClusterDnsSpreadScheduler } from './schedulers/cluster-dns-spread.scheduler'; import { SharedVolumeModeScheduler } from './schedulers/shared-volume-mode.scheduler'; import { SharedVolumeModeReconciler } from './services/shared-volume-mode.reconciler'; +import { NodeTopologyScheduler } from './schedulers/node-topology.scheduler'; +import { NodeTopologyReconciler } from './services/node-topology.reconciler'; +import { FluiLocalDirectIoScheduler } from './schedulers/flui-local-direct-io.scheduler'; +import { FluiLocalDirectIoReconciler } from './services/flui-local-direct-io.reconciler'; import { ClusterDnsSpreadReconciler } from './services/cluster-dns-spread.reconciler'; import { KubeconfigEndpointPromoter } from './services/kubeconfig-endpoint.promoter'; +import { ApiServerFailoverService } from './services/api-server-failover.service'; +import { ApiServerFailoverScheduler } from './schedulers/api-server-failover.scheduler'; import { KubeconfigEndpointScheduler } from './schedulers/kubeconfig-endpoint.scheduler'; import { ClusterPowerManagementService } from './services/cluster-power-management.service'; import { ClusterSshCleanupService } from './services/cluster-ssh-cleanup.service'; @@ -79,6 +87,7 @@ import { OrphanedClaimsService } from './services/orphaned-claims.service'; import { ClusterOrphanedClaimsController } from './cluster-orphaned-claims.controller'; import { ClusterCapacityService } from './services/cluster-capacity.service'; import { ClusterNodeScalingService } from './services/cluster-node-scaling.service'; +import { NodeResizeHaService } from './services/node-resize-ha.service'; import { OrphanVolumesService } from './services/orphan-volumes.service'; import { ApplicationEntity } from '../../applications/entities/application.entity'; import { ApplicationServiceEntity } from '../../attached-services/entities/application-service.entity'; @@ -107,8 +116,17 @@ import { UserEntity } from '../../auth/entities/user.entity'; import { UserEventsModule } from '../../auth/gateway/user-events.module'; import { IamModule } from '../../iam/iam.module'; +import { HaNodesController } from './ha-nodes.controller'; +import { HaNodesService } from './services/ha-nodes.service'; +import { NodeDisksService } from './services/node-disks.service'; +import { NodeScaleProcessor } from './processors/node-scale.processor'; +import { ClusterHaModule } from '../../cluster-ha/cluster-ha.module'; +import { ReplicatedVolumesModule } from '../../replicated-volumes/replicated-volumes.module'; + @Module({ imports: [ + ClusterHaModule, + ReplicatedVolumesModule, ConfigModule, // Both behind forwardRef: applications and backups already reach back // here, and a rebuild needs the deploy path and the artifacts at once. @@ -174,8 +192,12 @@ import { IamModule } from '../../iam/iam.module'; ClustersController, ClusterOrphanedClaimsController, CostsController, + HaNodesController, ], providers: [ + HaNodesService, + NodeScaleProcessor, + NodeDisksService, NodeLifeEventsService, LostNodeOperationsService, MasterEvacuationService, @@ -189,6 +211,8 @@ import { IamModule } from '../../iam/iam.module'; // Modular services ClusterValidationService, ClusterCreationService, + HaCreationPlanService, + HaServerJoinService, ClusterDeletionService, ClusterOperationsService, ClusterMapperService, @@ -202,7 +226,13 @@ import { IamModule } from '../../iam/iam.module'; ClusterDnsSpreadScheduler, SharedVolumeModeReconciler, SharedVolumeModeScheduler, + NodeTopologyReconciler, + NodeTopologyScheduler, + FluiLocalDirectIoReconciler, + FluiLocalDirectIoScheduler, KubeconfigEndpointPromoter, + ApiServerFailoverService, + ApiServerFailoverScheduler, KubeconfigEndpointScheduler, ClusterPowerManagementService, ClusterSshCleanupService, @@ -228,6 +258,7 @@ import { IamModule } from '../../iam/iam.module'; OrphanedClaimsService, ClusterCapacityService, ClusterNodeScalingService, + NodeResizeHaService, OrphanVolumesService, K3sScriptService, ByosNodeJoinService, diff --git a/src/modules/infrastructure/clusters/constants.ts b/src/modules/infrastructure/clusters/constants.ts index 4fbc0742..99362f94 100644 --- a/src/modules/infrastructure/clusters/constants.ts +++ b/src/modules/infrastructure/clusters/constants.ts @@ -7,3 +7,6 @@ export const FLUI_CONTROL_NAMESPACE = 'flui-control'; /** Legacy namespace used by control clusters provisioned before the control-cluster rename. */ export const FLUI_LEGACY_CONTROL_NAMESPACE = 'flui-observability'; + +/** The K3s pod network every Flui cluster uses. */ +export const POD_NETWORK = '10.42.0.0/16'; diff --git a/src/modules/infrastructure/clusters/constants/storage-conventions.spec.ts b/src/modules/infrastructure/clusters/constants/storage-conventions.spec.ts new file mode 100644 index 00000000..00d813b4 --- /dev/null +++ b/src/modules/infrastructure/clusters/constants/storage-conventions.spec.ts @@ -0,0 +1,26 @@ +import { NFS_EXPORT_OPTIONS, nfsDurability } from './storage-conventions'; + +describe('what the shared storage promises about a confirmed write', () => { + /** The master answers before the disk does, so a crash can take the newest writes with it. */ + it('says recent writes can be lost while the export confirms before the disk', () => { + expect(nfsDurability(NFS_EXPORT_OPTIONS)).toEqual({ + confirmsBeforeDisk: true, + durability: + "Writes are confirmed before they reach the master's disk: if the master stops abruptly, the most recent ones can be lost. Keep databases on dedicated storage.", + }); + }); + + it('says writes are safe once the export waits for the disk', () => { + expect(nfsDurability('rw,sync,no_subtree_check,no_root_squash')).toEqual({ + confirmsBeforeDisk: false, + durability: "Writes are confirmed once they are on the master's disk.", + }); + }); + + it('reads the option itself, not a substring of another one', () => { + expect(nfsDurability('rw, async ,no_root_squash').confirmsBeforeDisk).toBe( + true, + ); + expect(nfsDurability('rw,no_async_hint').confirmsBeforeDisk).toBe(false); + }); +}); diff --git a/src/modules/infrastructure/clusters/constants/storage-conventions.ts b/src/modules/infrastructure/clusters/constants/storage-conventions.ts index 1a7397f1..bb63e3ee 100644 --- a/src/modules/infrastructure/clusters/constants/storage-conventions.ts +++ b/src/modules/infrastructure/clusters/constants/storage-conventions.ts @@ -57,6 +57,26 @@ export const FLUI_FSCACHE_DEFAULT_SIZE_GB = 5; */ export const NFS_EXPORT_OPTIONS = 'rw,async,no_subtree_check,no_root_squash'; +/** + * What the shared storage promises about a write it has confirmed, read off + * the export options so the answer follows what the master actually runs. + */ +export function nfsDurability(exportOptions: string): { + confirmsBeforeDisk: boolean; + durability: string; +} { + const confirmsBeforeDisk = exportOptions + .split(',') + .map((option) => option.trim()) + .includes('async'); + return { + confirmsBeforeDisk, + durability: confirmsBeforeDisk + ? "Writes are confirmed before they reach the master's disk: if the master stops abruptly, the most recent ones can be lost. Keep databases on dedicated storage." + : "Writes are confirmed once they are on the master's disk.", + }; +} + /** * NFS mount options used on workers. * - vers=4.2: full POSIX locking and modern features diff --git a/src/modules/infrastructure/clusters/dto/add-ha-node.dto.ts b/src/modules/infrastructure/clusters/dto/add-ha-node.dto.ts new file mode 100644 index 00000000..a23e0c67 --- /dev/null +++ b/src/modules/infrastructure/clusters/dto/add-ha-node.dto.ts @@ -0,0 +1,36 @@ +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { IsBoolean, IsIn, IsOptional, IsString } from 'class-validator'; +import { HA_NODE_ROLES, HaNodeRole } from '../utils/ha-node-role'; + +export class AddHaNodeDto { + @ApiProperty({ + enum: HA_NODE_ROLES, + description: + '`server`: a node that also runs the control plane and holds a vote, one per location. `data-host`: a node that takes applications in a location for data, where copies of databases and volumes live.', + }) + @IsIn(HA_NODE_ROLES as unknown as string[]) + role!: HaNodeRole; + + @ApiProperty({ + description: + 'The location to add it in, one the cluster view lists under `missing` for that role (for example `hel1`).', + }) + @IsString() + zone!: string; + + @ApiPropertyOptional({ + description: "The machine size to buy. Omitted: the cluster's own size.", + }) + @IsOptional() + @IsString() + serverType?: string; + + @ApiPropertyOptional({ + default: false, + description: + 'Accept a region Flui has not validated for high availability; its three locations are treated as equal.', + }) + @IsOptional() + @IsBoolean() + force?: boolean; +} diff --git a/src/modules/infrastructure/clusters/dto/cluster-storage.dto.ts b/src/modules/infrastructure/clusters/dto/cluster-storage.dto.ts index c9805b7e..d17a2691 100644 --- a/src/modules/infrastructure/clusters/dto/cluster-storage.dto.ts +++ b/src/modules/infrastructure/clusters/dto/cluster-storage.dto.ts @@ -40,6 +40,18 @@ export class ClusterStorageNfsDto { @ApiProperty({ description: 'NFS mount options applied on the workers' }) mountOptions: string; + + @ApiProperty({ + description: + 'Whether the master confirms a write before it reaches its disk. When true, the most recent confirmed writes can be lost if the master stops abruptly.', + }) + confirmsBeforeDisk: boolean; + + @ApiProperty({ + description: + 'What the shared storage promises about confirmed writes, in one sentence', + }) + durability: string; } export class ClusterStoragePvcSummaryDto { diff --git a/src/modules/infrastructure/clusters/dto/create-cluster.dto.ts b/src/modules/infrastructure/clusters/dto/create-cluster.dto.ts index 2291fdd7..4787572c 100644 --- a/src/modules/infrastructure/clusters/dto/create-cluster.dto.ts +++ b/src/modules/infrastructure/clusters/dto/create-cluster.dto.ts @@ -14,6 +14,7 @@ import { IsUUID, IsPositive, Matches, + IsIn, } from 'class-validator'; import { Type } from 'class-transformer'; import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; @@ -22,6 +23,7 @@ import { RELEASE } from 'src/config/release.config'; import { AsyncOperationResponseDto } from 'src/modules/common/dto'; import { FirewallRuleDto } from 'src/modules/providers/dto/firewall.dto'; import { HostnameMode } from 'src/modules/dns/enums/hostname-mode.enum'; +import type { ClusterHaLevel } from 'src/modules/cluster-ha/utils/cluster-ha.core'; /** * DTO for VNet configuration during cluster creation @@ -114,6 +116,36 @@ export class CreateClusterDto { @Max(19) workerCount: number; + @ApiPropertyOptional({ + enum: ['standard', 'three-locations'], + default: 'standard', + description: + "`three-locations`: one server in each of the three locations the provider validated for this region, bought together and joined one at a time. Only offered where the provider declares such a layout; `region` is the first server's location and must be one for data. Needs `workerCount: 0`.", + }) + @IsOptional() + @IsIn(['standard', 'three-locations']) + highAvailability?: ClusterHaLevel; + + @ApiPropertyOptional({ + type: [String], + example: ['cpx22', 'cx23'], + description: + 'With `three-locations`: the sizes each location may take, in order of preference, all of one architecture. Each location gets the first one it has available. Omitted: `nodeSize` only.', + }) + @IsOptional() + @IsArray() + @IsString({ each: true }) + shapes?: string[]; + + @ApiPropertyOptional({ + default: false, + description: + 'With `three-locations`: when a location has none of `shapes` available, create the cluster in the others and leave that server to add later. Off: nothing is bought and the answer lists sizes every location has.', + }) + @IsOptional() + @IsBoolean() + completeLater?: boolean; + @ApiPropertyOptional({ example: false, description: 'Enable autoscaling', diff --git a/src/modules/infrastructure/clusters/dto/ha-creation-plan.dto.ts b/src/modules/infrastructure/clusters/dto/ha-creation-plan.dto.ts new file mode 100644 index 00000000..dd949a3f --- /dev/null +++ b/src/modules/infrastructure/clusters/dto/ha-creation-plan.dto.ts @@ -0,0 +1,84 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { Sensitivity } from '../../../mask/decorators/sensitivity.decorator'; + +export class HaPlanServerDto { + @Sensitivity(Sensitivity.PUBLIC) + @ApiProperty({ description: 'The location, for example `nbg1`.' }) + zone: string; + + @Sensitivity(Sensitivity.PUBLIC) + @ApiProperty({ description: 'The size it would be bought as.' }) + shape: string; + + @Sensitivity(Sensitivity.PUBLIC) + @ApiProperty({ + enum: ['data', 'vote'], + description: + '`data`: copies of databases and volumes live here. `vote`: only confirms writes and keeps the cluster deciding.', + }) + role: 'data' | 'vote'; + + @Sensitivity(Sensitivity.PUBLIC) + @ApiProperty({ description: 'The server that starts the cluster.' }) + first: boolean; +} + +export class HaPlanResponseDto { + @Sensitivity(Sensitivity.PUBLIC) + @ApiProperty({ + description: + 'Whether the provider declares a layout across three locations for this region at all; false, the option is not offered.', + }) + layoutAvailable: boolean; + + @Sensitivity(Sensitivity.PUBLIC) + @ApiProperty({ + description: + 'Whether creating the cluster across three locations would go ahead now, the same answer creating it would give.', + }) + possible: boolean; + + @Sensitivity(Sensitivity.ARBITRARY_TEXT) + @ApiProperty({ + nullable: true, + type: String, + description: + 'Why not, in words to show, with the sizes every location has when that is what is missing.', + }) + refusal: string | null; + + @Sensitivity(Sensitivity.PUBLIC) + @ApiProperty({ + description: + 'True when the refusal is a location with none of the sizes while the first one has one: asking again with `completeLater` would go ahead.', + }) + completeLaterHelps: boolean; + + @Sensitivity(Sensitivity.PUBLIC) + @ApiProperty({ + nullable: true, + type: String, + description: 'The region the three locations belong to.', + }) + region: string | null; + + @Sensitivity(Sensitivity.PUBLIC) + @ApiProperty({ + description: + "False when Flui took the three locations on the provider's word that they are close zones of one region and has not measured them yet.", + }) + measured: boolean; + + @ApiProperty({ + type: [HaPlanServerDto], + description: 'The servers it would buy, in the order they join.', + }) + servers: HaPlanServerDto[]; + + @Sensitivity(Sensitivity.PUBLIC) + @ApiProperty({ + type: [String], + description: 'Locations left to add later, with `completeLater`.', + }) + later: string[]; +} diff --git a/src/modules/infrastructure/clusters/dto/scale-node.dto.ts b/src/modules/infrastructure/clusters/dto/scale-node.dto.ts index e1272ab9..e831d67f 100644 --- a/src/modules/infrastructure/clusters/dto/scale-node.dto.ts +++ b/src/modules/infrastructure/clusters/dto/scale-node.dto.ts @@ -17,6 +17,16 @@ export class ScaleNodeDto { @IsOptional() @IsBoolean() upgradeDisk?: boolean; + + @ApiPropertyOptional({ + description: + 'New size, in GiB, of the space this node sets aside for copies of replicated volumes, grown once the node is back. Only grows; ignored on a node that holds no copies, and kept when the disk has not the room.', + minimum: 1, + }) + @IsOptional() + @IsInt() + @Min(1) + copiesGiB?: number; } export class ExpandSharedVolumeDto { diff --git a/src/modules/infrastructure/clusters/ha-nodes.controller.ts b/src/modules/infrastructure/clusters/ha-nodes.controller.ts new file mode 100644 index 00000000..fbe08eb8 --- /dev/null +++ b/src/modules/infrastructure/clusters/ha-nodes.controller.ts @@ -0,0 +1,57 @@ +import { + Body, + Controller, + HttpCode, + HttpStatus, + Param, + Post, + Req, +} from '@nestjs/common'; +import { + ApiBearerAuth, + ApiOperation, + ApiParam, + ApiTags, +} from '@nestjs/swagger'; +import { RequirePermission } from '../../iam/decorators/require-permission.decorator'; +import { IAM_PERMISSION } from '../../iam/constants/iam-permissions'; +import { ActionCycle } from '../../action-cycle/action-cycle.decorator'; +import { byOf } from '../scaling/scaling-actor'; +import { AddHaNodeDto } from './dto/add-ha-node.dto'; +import { HaNodesService } from './services/ha-nodes.service'; + +@ApiTags('Infrastructure - Clusters') +@ApiBearerAuth() +@Controller('infrastructure/clusters/:id/high-availability/nodes') +export class HaNodesController { + constructor(private readonly service: HaNodesService) {} + + @Post() + @HttpCode(HttpStatus.ACCEPTED) + @RequirePermission(IAM_PERMISSION.CLUSTER_MANAGE) + @ActionCycle({ + action: 'POST /infrastructure/clusters/:id/high-availability/nodes', + bind: ['id'], + sentence: 'add a node to cluster {id} for high availability', + consequence: + 'Flui buys a machine in the chosen location and joins it to the cluster with its role: a server also runs the control plane and holds a vote, a node for data takes applications where copies of data live. The machine is billed from then on.', + }) + @ApiOperation({ + summary: 'Add the server or the node for data a cluster still needs', + description: + 'Adds what `GET /infrastructure/clusters/:id/high-availability` lists under `missing`, one node per call, with its role (`server` or `data-host`) and its location. Anything not missing is refused with the reason: a fourth server, or a second one in a location, changes the votes the control plane counts without making the cluster survive more. ' + + 'The node is added outside the scaling group, which never removes it. Returns the operation to follow. With two servers the cluster is more fragile than with one until the third joins, so add the second and the third one after the other.', + }) + @ApiParam({ name: 'id', description: 'Cluster ID' }) + add( + @Param('id') id: string, + @Body() dto: AddHaNodeDto, + @Req() + req: { user?: { email?: string; displayName?: string } } & Record< + string, + unknown + >, + ) { + return this.service.add(id, dto, byOf(req)); + } +} diff --git a/src/modules/infrastructure/clusters/processors/cluster-queue.processor.spec.ts b/src/modules/infrastructure/clusters/processors/cluster-queue.processor.spec.ts index 39bc9621..54a11a2f 100644 --- a/src/modules/infrastructure/clusters/processors/cluster-queue.processor.spec.ts +++ b/src/modules/infrastructure/clusters/processors/cluster-queue.processor.spec.ts @@ -1,6 +1,7 @@ jest.mock('@kubernetes/client-node', () => ({})); jest.mock('ip-cidr', () => ({})); +import { haJoinOutcome } from '../services/ha-server-join.service'; import { ClusterQueueProcessor } from './cluster-queue.processor'; import { ClusterStatus, ClusterType } from '../entities/cluster.entity'; import { @@ -70,6 +71,15 @@ describe('ClusterQueueProcessor.handleCreateCluster', () => { .mockResolvedValue({ id: 'node-master', ipAddress: '1.2.3.4' }), createWorkerNodes: jest.fn().mockResolvedValue([]), }; + const haServerJoin = { + startBuying: jest.fn(async (_c: unknown, plan: unknown) => + plan ? { purchases: Promise.resolve([]) } : null, + ), + joinBought: jest.fn(), + waitForPlatform: jest + .fn() + .mockResolvedValue({ at: '2026-11-08T12:00:00Z', problem: null }), + }; const capabilitiesFactory = { getCapabilitiesService: jest.fn().mockReturnValue({ getStaticCapabilities: jest @@ -111,6 +121,7 @@ describe('ClusterQueueProcessor.handleCreateCluster', () => { clusterRepository, operationRepository, orchestrationService, + haServerJoin, capabilitiesFactory, clusterDnsZoneService, grafanaConfigService, @@ -131,8 +142,10 @@ describe('ClusterQueueProcessor.handleCreateCluster', () => { cluster, operation, orchestrationService, + haServerJoin, infraGateway, clusterFirewallIntegrationService, + clusterRepository, }; } @@ -170,6 +183,99 @@ describe('ClusterQueueProcessor.handleCreateCluster', () => { ]); }); + it('buys nothing again when a retried creation finds its cluster destroyed', async () => { + const { run, cluster, operation, orchestrationService } = build(0); + cluster.status = ClusterStatus.DELETED; + + await run(); + + expect(orchestrationService.createMasterNode).not.toHaveBeenCalled(); + expect(cluster.status).toBe(ClusterStatus.DELETED); + expect(operation.status).toBe(OperationStatus.CANCELLED); + }); + + const PLAN = { + region: 'eu-central', + first: { zone: 'fsn1', shape: 'cpx22' }, + servers: [ + { zone: 'nbg1', shape: 'cpx22' }, + { zone: 'hel1', shape: 'cpx22' }, + ], + later: [] as string[], + }; + + it('across three locations: buys the servers with the first, joins them after it, then waits for the install', async () => { + const { run, cluster, operation, orchestrationService, haServerJoin } = + build(0); + operation.metadata.haPlan = PLAN; + operation.metadata.operationSteps = getOperationSteps( + OperationType.CREATE_CLUSTER, + { workerCount: 0, haServers: ['nbg1', 'hel1'] }, + ); + const order: string[] = []; + haServerJoin.startBuying.mockImplementation(async () => { + order.push('buy'); + return { purchases: Promise.resolve([]) }; + }); + orchestrationService.createMasterNode.mockImplementation(async () => { + order.push('first'); + return { id: 'node-master', ipAddress: '1.2.3.4' }; + }); + haServerJoin.joinBought.mockImplementation(async () => { + order.push('join'); + return { + joined: 2, + joinedAt: 'then', + missing: [], + ...haJoinOutcome(PLAN, ['nbg1', 'hel1'], []), + }; + }); + haServerJoin.waitForPlatform.mockImplementation(async () => { + order.push('install'); + return { at: 'later', problem: null }; + }); + + await run(); + + expect(order).toEqual(['buy', 'first', 'join', 'install']); + expect(orchestrationService.createWorkerNodes).not.toHaveBeenCalled(); + expect(cluster.nodeCount).toBe(3); + expect(operation.status).toBe(OperationStatus.COMPLETED); + expect(operation.metadata.message).toBe( + 'Cluster ready across fsn1, nbg1, hel1', + ); + expect(operation.metadata.platformInstalledAt).toBe('later'); + }); + + it('says when the install on the first server stopped', async () => { + const { run, operation, haServerJoin } = build(0); + const plan = { ...PLAN, servers: [PLAN.servers[0]], later: ['hel1'] }; + operation.metadata.haPlan = plan; + operation.metadata.operationSteps = getOperationSteps( + OperationType.CREATE_CLUSTER, + { workerCount: 0, haServers: ['nbg1'] }, + ); + haServerJoin.joinBought.mockResolvedValue({ + joined: 1, + joinedAt: 'then', + missing: ['hel1'], + ...haJoinOutcome(plan, ['nbg1'], ['hel1']), + }); + haServerJoin.waitForPlatform.mockResolvedValue({ + at: null, + problem: + 'The install on the first server stopped before the end (its log is /var/log/k3s-master-init.log)', + }); + + await run(); + + expect(operation.metadata.message).toBe( + 'Cluster ready in fsn1, nbg1; a server is still missing in hel1: add it from the High availability tab. The install on the first server stopped before the end (its log is /var/log/k3s-master-init.log)', + ); + expect(operation.metadata.missingLocations).toEqual(['hel1']); + expect(operation.metadata.platformInstalledAt).toBeNull(); + }); + it('takes the single-node path and never calls createWorkerNodes when workerCount is 0', async () => { const { run, cluster, operation, orchestrationService, infraGateway } = build(0); @@ -188,6 +294,43 @@ describe('ClusterQueueProcessor.handleCreateCluster', () => { expect(infraGateway.emitCompleted).toHaveBeenCalledTimes(1); expect(infraGateway.emitFailed).not.toHaveBeenCalled(); }); + + it('stops, without bringing the cluster back, when it is deleted while its servers are being made', async () => { + const { run, cluster, operation, orchestrationService, clusterRepository } = + build(2); + clusterRepository.findOne + .mockResolvedValueOnce(cluster) + .mockResolvedValue({ ...cluster, status: ClusterStatus.DELETING }); + + await expect(run()).resolves.toBeUndefined(); + + expect(orchestrationService.createWorkerNodes).not.toHaveBeenCalled(); + expect(cluster.status).toBe(ClusterStatus.CREATING); + expect(clusterRepository.save).not.toHaveBeenCalled(); + expect(operation.status).toBe(OperationStatus.CANCELLED); + }); + + it('waits for Flui to finish installing on a standard cluster too, and says when it did not', async () => { + const { run, cluster, operation, haServerJoin } = build(0); + haServerJoin.waitForPlatform.mockResolvedValue({ + at: null, + problem: 'Flui had not finished installing on the first server in time', + }); + + await run(); + + expect(haServerJoin.waitForPlatform).toHaveBeenCalledWith( + cluster, + expect.any(Function), + 3, + expect.any(Function), + ); + expect(cluster.status).toBe(ClusterStatus.READY); + expect(operation.metadata.message).toBe( + 'Single-node cluster ready. Flui had not finished installing on the first server in time', + ); + expect(operation.metadata.platformInstalledAt).toBeNull(); + }); }); describe('ClusterQueueProcessor.handleRemoveWorker', () => { @@ -576,6 +719,9 @@ describe('ClusterQueueProcessor.handleDeleteCluster', () => { remove: jest.fn().mockResolvedValue(undefined), delete: jest.fn().mockResolvedValue(undefined), }; + const operationRepository = { + findOne: jest.fn().mockResolvedValue(null), + }; const billingIntervals = { closeNodeIntervals: jest.fn().mockResolvedValue(undefined), closeVolumeIntervals: jest.fn().mockResolvedValue(undefined), @@ -592,6 +738,7 @@ describe('ClusterQueueProcessor.handleDeleteCluster', () => { debug: jest.fn(), }, clusterRepository, + operationRepository, nodeRepository, billingIntervals, updateOperationStatus: jest.fn().mockResolvedValue(undefined), @@ -621,9 +768,38 @@ describe('ClusterQueueProcessor.handleDeleteCluster', () => { id: 'job-1', data: { operationId: 'op-1', clusterId: 'cluster-1', force }, } as never); - return { run, clusterRepository, nodeRepository, billingIntervals }; + return { + run, + clusterRepository, + operationRepository, + nodeRepository, + billingIntervals, + }; } + it('reads the nodes only once a creation still running has stopped', async () => { + jest.useFakeTimers(); + try { + const t = build(false, [ + { id: 'n1', serverName: 'master', providerResourceId: '123' }, + ]); + t.operationRepository.findOne + .mockResolvedValueOnce({ id: 'op-create', status: 'in_progress' }) + .mockResolvedValue(null); + + const done = t.run(); + await jest.advanceTimersByTimeAsync(0); + expect(t.clusterRepository.findOne).not.toHaveBeenCalled(); + await jest.advanceTimersByTimeAsync(10_000); + await done; + + expect(t.operationRepository.findOne).toHaveBeenCalledTimes(2); + expect(t.clusterRepository.findOne).toHaveBeenCalled(); + } finally { + jest.useRealTimers(); + } + }); + it.each([false, true])( 'ends every billed lifetime and every node record at the deletion time (force=%s)', async (force) => { diff --git a/src/modules/infrastructure/clusters/processors/cluster-queue.processor.ts b/src/modules/infrastructure/clusters/processors/cluster-queue.processor.ts index 9aca8a01..55e32236 100644 --- a/src/modules/infrastructure/clusters/processors/cluster-queue.processor.ts +++ b/src/modules/infrastructure/clusters/processors/cluster-queue.processor.ts @@ -8,7 +8,7 @@ import { RecoverNodeAccessJob, } from '../services/node-access-recovery.service'; import { InjectRepository } from '@nestjs/typeorm'; -import { Repository } from 'typeorm'; +import { In, Repository } from 'typeorm'; import { CreateClusterJobData, DeleteClusterJobData, @@ -76,6 +76,20 @@ import { removalPreflight } from './removal-preflight'; import { MasterEvacuationService } from '../services/master-evacuation.service'; import { movesSentence } from '../../scaling/engine/drain.core'; import { retryHeldDelivery } from '../../../common/leadership/held-delivery'; +import { HaCreationPlan } from '../utils/ha-creation-plan.core'; + +import { + HaJoined, + HaPurchase, + HaServerJoinService, +} from '../services/ha-server-join.service'; + +function isSingleNodeCreation( + workerCount: number, + haPlan: HaCreationPlan | undefined, +): boolean { + return workerCount === 0 && !haPlan; +} /** cert-manager's webhooks register after the cluster reports ready. */ const ZONE_RECONCILE_ATTEMPTS = 5; @@ -83,6 +97,22 @@ const ZONE_RECONCILE_DELAY_MS = 30_000; const REMOVAL_HAND_ON_LIMIT = 5; +const CREATION_STOP_WAIT_MS = 30 * 60_000; +const CREATION_STOP_POLL_MS = 10_000; + +class CreationCancelled extends Error {} + +function beingDestroyed(cluster: ClusterEntity): boolean { + return ( + !!cluster.deletedAt || + [ + ClusterStatus.DELETING, + ClusterStatus.DELETION_FAILED, + ClusterStatus.DELETED, + ].includes(cluster.status) + ); +} + @Processor('infrastructure') export class ClusterQueueProcessor { private readonly logger = new Logger(ClusterQueueProcessor.name); @@ -96,6 +126,7 @@ export class ClusterQueueProcessor { private readonly operationRepository: Repository, @InjectQueue('infrastructure') private readonly infrastructureQueue: Queue, private readonly orchestrationService: ClusterOrchestrationService, + private readonly haServerJoin: HaServerJoinService, private readonly serversService: ServersService, private readonly clusterDeletionService: ClusterDeletionService, private readonly clusterPowerManagementService: ClusterPowerManagementService, @@ -293,7 +324,9 @@ export class ClusterQueueProcessor { const workerCount = (operation.metadata as CreateClusterOperationMetadata)?.clusterConfig ?.workerCount || 0; - const isSingleNode = workerCount === 0; + const haPlan = (operation.metadata as CreateClusterOperationMetadata) + ?.haPlan; + const isSingleNode = isSingleNodeCreation(workerCount, haPlan); // Updated: Use providerFirewallId (string) instead of providerFirewallIds (array) const providerFirewallId = ( @@ -320,11 +353,9 @@ export class ClusterQueueProcessor { throw new Error(`Cluster ${clusterId} not found`); } - // Reset status to CREATING in case this is a Bull retry after a previous failed attempt - if (cluster.status === ClusterStatus.ERROR) { - cluster.status = ClusterStatus.CREATING; - await this.clusterRepository.save(cluster); - } + // A retry of a creation whose cluster was destroyed meanwhile would buy + // its servers again for a cluster nobody has. + if (await this.resumeOrStop(cluster, operationId)) return; const totalSteps = isSingleNode ? 4 : 5; const startedAt = Date.now(); @@ -363,11 +394,19 @@ export class ClusterQueueProcessor { timestamp: new Date(), } as InfrastructureOperationProgressDto); + const bought = await this.haServerJoin.startBuying( + cluster, + haPlan, + operationId, + providerFirewallIds, + ); const masterNode = await this.orchestrationService.createMasterNode( cluster, operationId, providerFirewallIds, + haPlan?.first.zone, ); + const k3sReadyAt = new Date().toISOString(); await this.updateOperationStep(operationId, 1, 100, { message: 'Master node created and K3s ready', @@ -386,6 +425,7 @@ export class ClusterQueueProcessor { timestamp: new Date(), } as InfrastructureOperationProgressDto); + await this.stopIfDestroyed(clusterId); cluster.nodeCount = 1; await this.clusterRepository.save(cluster); @@ -461,10 +501,18 @@ export class ClusterQueueProcessor { if (isSingleNode) { // === SINGLE-NODE: STEP 3 = FINALIZE === + const completion = await this.completionOf( + cluster, + operationId, + null, + k3sReadyAt, + { stepIndex: 3, ready: 'Single-node cluster ready' }, + ); await this.updateOperationStep(operationId, 3, 50, { message: 'Finalizing single-node cluster', }); + await this.stopIfDestroyed(clusterId); cluster.status = ClusterStatus.READY; await this.clusterRepository.save(cluster); @@ -483,7 +531,7 @@ export class ClusterQueueProcessor { await this.updateOperationStep(operationId, 3, 100, { status: OperationStatus.COMPLETED, - message: 'Single-node cluster ready', + ...completion, clusterName: cluster.name, masterIp: cluster.masterIpAddress, nodeCount: 1, @@ -519,16 +567,14 @@ export class ClusterQueueProcessor { timestamp: new Date(), } as InfrastructureOperationProgressDto); - await this.orchestrationService.createWorkerNodes( + const initial = await this.createInitialNodes( cluster, workerCount, operationId, providerFirewallIds, + { plan: haPlan, bought, k3sReadyAt }, ); - - await this.updateOperationStep(operationId, 3, 100, { - message: `All ${workerCount} worker nodes created`, - }); + await this.updateOperationStep(operationId, 3, 100, initial.stepResult); this.infraGateway.emitProgress(operationId, clusterId, { operationId, resourceId: clusterId, @@ -541,15 +587,23 @@ export class ClusterQueueProcessor { timestamp: new Date(), } as InfrastructureOperationProgressDto); - cluster.nodeCount = 1 + workerCount; + await this.stopIfDestroyed(clusterId); + cluster.nodeCount = initial.nodeCount; await this.clusterRepository.save(cluster); await this.clusterFirewallIntegrationService.syncHostLayer(clusterId); + const completion = await this.completionOf( + cluster, + operationId, + initial.ha, + k3sReadyAt, + ); // === MULTI-NODE: STEP 4 = FINALIZE === await this.updateOperationStep(operationId, 4, 50, { message: 'Finalizing multi-node cluster', }); + await this.stopIfDestroyed(clusterId); cluster.status = ClusterStatus.READY; await this.clusterRepository.save(cluster); @@ -571,7 +625,7 @@ export class ClusterQueueProcessor { await this.updateOperationStep(operationId, 4, 100, { status: OperationStatus.COMPLETED, - message: 'Multi-node cluster ready', + ...completion, clusterName: cluster.name, masterIp: cluster.masterIpAddress, nodeCount: cluster.nodeCount, @@ -592,53 +646,70 @@ export class ClusterQueueProcessor { ); } } catch (error) { - this.logger.error( - `❌ Cluster creation failed for ${clusterId}: ${error.message}`, - error.stack, - ); - - // Update operation as failed - const failedOp = await this.operationRepository.findOne({ - where: { id: operationId }, - }); - if (failedOp) { - failedOp.status = OperationStatus.FAILED; - failedOp.errorMessage = error.message; - failedOp.completedAt = new Date(); - failedOp.metadata = { - ...failedOp.metadata, - error: error.message, - stack: error.stack, - clusterId, - failedAt: new Date().toISOString(), - }; - await this.operationRepository.save(failedOp); + if (error instanceof CreationCancelled) { + this.logger.warn(`Creation of ${clusterId} stopped: ${error.message}`); + await this.updateOperationStatus( + operationId, + OperationStatus.CANCELLED, + { message: error.message }, + ); + return; } + await this.recordCreationFailure(operationId, clusterId, error); + throw error; // Re-throw for Bull to handle retries + } + } - this.infraGateway.emitFailed(operationId, clusterId, { - operationId, - resourceId: clusterId, - operationType: OperationType.CREATE_CLUSTER, - resourceType: 'cluster', + /** The operation and the cluster say why the creation failed, before Bull retries it. */ + private async recordCreationFailure( + operationId: string, + clusterId: string, + error: Error, + ): Promise { + this.logger.error( + `❌ Cluster creation failed for ${clusterId}: ${error.message}`, + error.stack, + ); + + // Update operation as failed + const failedOp = await this.operationRepository.findOne({ + where: { id: operationId }, + }); + if (failedOp) { + failedOp.status = OperationStatus.FAILED; + failedOp.errorMessage = error.message; + failedOp.completedAt = new Date(); + failedOp.metadata = { + ...failedOp.metadata, error: error.message, - timestamp: new Date(), - } as InfrastructureOperationFailedDto); + stack: error.stack, + clusterId, + failedAt: new Date().toISOString(), + }; + await this.operationRepository.save(failedOp); + } - // Update cluster status to error - const errorCluster = await this.clusterRepository.findOne({ - where: { id: clusterId }, - }); - if (errorCluster) { - errorCluster.status = ClusterStatus.ERROR; - errorCluster.metadata = { - ...errorCluster.metadata, - error: error.message, - failedAt: new Date().toISOString(), - }; - await this.clusterRepository.save(errorCluster); - } + this.infraGateway.emitFailed(operationId, clusterId, { + operationId, + resourceId: clusterId, + operationType: OperationType.CREATE_CLUSTER, + resourceType: 'cluster', + error: error.message, + timestamp: new Date(), + } as InfrastructureOperationFailedDto); - throw error; // Re-throw for Bull to handle retries + // Update cluster status to error + const errorCluster = await this.clusterRepository.findOne({ + where: { id: clusterId }, + }); + if (errorCluster) { + errorCluster.status = ClusterStatus.ERROR; + errorCluster.metadata = { + ...errorCluster.metadata, + error: error.message, + failedAt: new Date().toISOString(), + }; + await this.clusterRepository.save(errorCluster); } } @@ -677,6 +748,8 @@ export class ClusterQueueProcessor { timestamp: new Date(), } as InfrastructureOperationProgressDto); + await this.waitForCreationToStop(clusterId); + // Load cluster with nodes const cluster = await this.clusterRepository.findOne({ where: { id: clusterId }, @@ -1899,6 +1972,7 @@ export class ClusterQueueProcessor { providerFirewallIds, serverType, region, + haRole, } = job.data; const startedAt = Date.now(); this.logger.log( @@ -1969,6 +2043,7 @@ export class ClusterQueueProcessor { providerFirewallIds, serverType, region, + haRole, ); await this.updateOperationStep(operationId, 1, 100, { @@ -2686,6 +2761,154 @@ export class ClusterQueueProcessor { } } + /** + * Whether a run of the creation must stop before doing anything; a Bull + * retry after a failed attempt starts over from CREATING. + */ + /** + * The creation saves the cluster it read at its start; once a deletion has + * begun that would bring the cluster back, and the deletion, which reads the + * nodes once, would leave every server bought after it running. + */ + /** + * The nodes are read once, so a creation still buying servers must stop + * first; it does at its next check, between servers or while it waits. + */ + private async waitForCreationToStop(clusterId: string): Promise { + const deadline = Date.now() + CREATION_STOP_WAIT_MS; + while (Date.now() < deadline) { + const running = await this.operationRepository.findOne({ + where: { + operationType: OperationType.CREATE_CLUSTER, + resourceId: clusterId, + status: In([OperationStatus.PENDING, OperationStatus.IN_PROGRESS]), + }, + }); + if (!running) return; + this.logger.log( + `Deletion of ${clusterId} waits for its creation ${running.id} to stop`, + ); + await new Promise((r) => setTimeout(r, CREATION_STOP_POLL_MS)); + } + this.logger.warn( + `Creation of ${clusterId} did not stop within ${CREATION_STOP_WAIT_MS / 60_000} minutes; deleting what exists now`, + ); + } + + private async stopIfDestroyed(clusterId: string): Promise { + const fresh = await this.clusterRepository.findOne({ + where: { id: clusterId }, + }); + if (!fresh || beingDestroyed(fresh)) { + throw new CreationCancelled( + 'The cluster is being deleted, so its creation stopped', + ); + } + } + + private async resumeOrStop( + cluster: ClusterEntity, + operationId: string, + ): Promise { + if (!beingDestroyed(cluster)) { + if (cluster.status === ClusterStatus.ERROR) { + cluster.status = ClusterStatus.CREATING; + await this.clusterRepository.save(cluster); + } + return false; + } + this.logger.warn( + `Cluster ${cluster.name} was destroyed; its creation is not run again`, + ); + await this.updateOperationStep(operationId, 0, 100, { + status: OperationStatus.CANCELLED, + message: 'The cluster was destroyed before its creation finished', + }); + return true; + } + + /** The workers a standard cluster asked for, or the servers bought to join later. */ + private async createInitialNodes( + cluster: ClusterEntity, + workerCount: number, + operationId: string, + providerFirewallIds: string[], + ha: { + plan: HaCreationPlan | undefined; + bought: { purchases: Promise } | null; + k3sReadyAt: string; + }, + ): Promise<{ + nodeCount: number; + stepResult: Record; + ha: HaJoined | null; + }> { + if (!ha.plan || !ha.bought) { + await this.orchestrationService.createWorkerNodes( + cluster, + workerCount, + operationId, + providerFirewallIds, + ); + return { + nodeCount: 1 + workerCount, + stepResult: { message: `All ${workerCount} worker nodes created` }, + ha: null, + }; + } + const joined = await this.haServerJoin.joinBought( + cluster, + ha.plan, + ha.bought, + operationId, + (step, progress, data) => + this.updateOperationStep(operationId, step, progress, data), + () => this.stopIfDestroyed(cluster.id), + ); + return { + nodeCount: 1 + joined.joined, + stepResult: { + message: joined.joinedMessage, + k3sReadyAt: ha.k3sReadyAt, + serversJoinedAt: joined.joinedAt, + }, + ha: joined, + }; + } + + /** How the creation ends: a cluster across three locations waits for the install and says what it reached. */ + /** + * Ready once Flui finished installing on the master, not when K3s answered: + * the components it installs come minutes later. A cluster whose install + * stopped or ran late still exists, so that is said, not undone. + */ + private async completionOf( + cluster: ClusterEntity, + operationId: string, + ha: HaJoined | null, + k3sReadyAt: string, + { stepIndex = 4, ready = 'Multi-node cluster ready' } = {}, + ): Promise> { + const platform = await this.haServerJoin.waitForPlatform( + cluster, + (step, progress, data) => + this.updateOperationStep(operationId, step, progress, data), + stepIndex, + () => this.stopIfDestroyed(cluster.id), + ); + return { + message: [ha?.readyMessage ?? ready, platform.problem] + .filter(Boolean) + .join('. '), + k3sReadyAt, + platformInstalledAt: platform.at, + ...(ha && { + serversJoinedAt: ha.joinedAt, + missingLocations: ha.missing, + }), + }; + } + /** Taints the master when a control cluster crosses single-node → multi-node. */ private async maybeProtectMasterOnScaleOut( cluster: ClusterEntity, diff --git a/src/modules/infrastructure/clusters/processors/node-scale.processor.ts b/src/modules/infrastructure/clusters/processors/node-scale.processor.ts new file mode 100644 index 00000000..71763067 --- /dev/null +++ b/src/modules/infrastructure/clusters/processors/node-scale.processor.ts @@ -0,0 +1,21 @@ +import { Process, Processor } from '@nestjs/bull'; +import { Job } from 'bull'; +import { + ClusterNodeScalingService, + ScaleNodeJobData, +} from '../services/cluster-node-scaling.service'; + +/** + * A resize is minutes of a node switched off, so it runs as a job: an API + * that restarts meanwhile picks it up again instead of leaving the node + * cordoned or off. + */ +@Processor('infrastructure') +export class NodeScaleProcessor { + constructor(private readonly scaling: ClusterNodeScalingService) {} + + @Process('scale-node') + async handle(job: Job): Promise { + await this.scaling.runScaleNode(job.data.operationId); + } +} diff --git a/src/modules/infrastructure/clusters/schedulers/api-server-failover.scheduler.ts b/src/modules/infrastructure/clusters/schedulers/api-server-failover.scheduler.ts new file mode 100644 index 00000000..3c4c3576 --- /dev/null +++ b/src/modules/infrastructure/clusters/schedulers/api-server-failover.scheduler.ts @@ -0,0 +1,26 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { Cron, CronExpression } from '@nestjs/schedule'; +import { ApiServerFailoverService } from '../services/api-server-failover.service'; + +@Injectable() +export class ApiServerFailoverScheduler { + private readonly logger = new Logger(ApiServerFailoverScheduler.name); + private running = false; + + constructor(private readonly failover: ApiServerFailoverService) {} + + @Cron( + process.env.FLUI_API_SERVER_FAILOVER_CRON || CronExpression.EVERY_MINUTE, + ) + async tick(): Promise { + if (this.running) return; + this.running = true; + try { + await this.failover.sweep(); + } catch (err: any) { + this.logger.error(`[api-server] sweep failed: ${err?.message ?? err}`); + } finally { + this.running = false; + } + } +} diff --git a/src/modules/infrastructure/clusters/schedulers/flui-local-direct-io.scheduler.ts b/src/modules/infrastructure/clusters/schedulers/flui-local-direct-io.scheduler.ts new file mode 100644 index 00000000..b7f44412 --- /dev/null +++ b/src/modules/infrastructure/clusters/schedulers/flui-local-direct-io.scheduler.ts @@ -0,0 +1,45 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Cron, CronExpression } from '@nestjs/schedule'; +import { Repository } from 'typeorm'; +import { ClusterEntity, ClusterStatus } from '../entities/cluster.entity'; +import { FluiLocalDirectIoReconciler } from '../services/flui-local-direct-io.reconciler'; + +/** + * Off unless `FLUI_LOCAL_DIRECT_IO_RECONCILE=true`: it changes the storage of + * nodes that already hold data, so it stays opt-in until proven on a live + * cluster. New nodes get direct I/O from their bootstrap script regardless. + */ +@Injectable() +export class FluiLocalDirectIoScheduler { + private readonly logger = new Logger(FluiLocalDirectIoScheduler.name); + private running = false; + + constructor( + @InjectRepository(ClusterEntity) + private readonly clusters: Repository, + private readonly reconciler: FluiLocalDirectIoReconciler, + ) {} + + @Cron(process.env.FLUI_LOCAL_DIRECT_IO_CRON || CronExpression.EVERY_HOUR) + async tick(): Promise { + if (process.env.FLUI_LOCAL_DIRECT_IO_RECONCILE !== 'true') return; + if (this.running) return; + this.running = true; + try { + const clusters = await this.clusters.find({ + where: { status: ClusterStatus.READY }, + select: { id: true, name: true }, + }); + for (const cluster of clusters) { + await this.reconciler.reconcile(cluster.id).catch((err: unknown) => { + this.logger.warn( + `[local-direct-io] ${cluster.name}: ${err instanceof Error ? err.message : JSON.stringify(err)}`, + ); + }); + } + } finally { + this.running = false; + } + } +} diff --git a/src/modules/infrastructure/clusters/schedulers/node-topology.scheduler.ts b/src/modules/infrastructure/clusters/schedulers/node-topology.scheduler.ts new file mode 100644 index 00000000..177a6192 --- /dev/null +++ b/src/modules/infrastructure/clusters/schedulers/node-topology.scheduler.ts @@ -0,0 +1,41 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Cron, CronExpression } from '@nestjs/schedule'; +import { Repository } from 'typeorm'; +import { ClusterEntity, ClusterStatus } from '../entities/cluster.entity'; +import { NodeTopologyReconciler } from '../services/node-topology.reconciler'; + +/** Every half hour, labels any node that still lacks the zone and region its provider declares. */ +@Injectable() +export class NodeTopologyScheduler { + private readonly logger = new Logger(NodeTopologyScheduler.name); + private running = false; + + constructor( + @InjectRepository(ClusterEntity) + private readonly clusters: Repository, + private readonly reconciler: NodeTopologyReconciler, + ) {} + + @Cron(process.env.FLUI_NODE_TOPOLOGY_CRON || CronExpression.EVERY_30_MINUTES) + async tick(): Promise { + if (process.env.FLUI_NODE_TOPOLOGY_RECONCILE === 'false') return; + if (this.running) return; + this.running = true; + try { + const clusters = await this.clusters.find({ + where: { status: ClusterStatus.READY }, + select: { id: true, name: true }, + }); + for (const cluster of clusters) { + await this.reconciler.reconcile(cluster.id).catch((err: unknown) => { + this.logger.warn( + `[node-topology] ${cluster.name}: ${err instanceof Error ? err.message : JSON.stringify(err)}`, + ); + }); + } + } finally { + this.running = false; + } + } +} diff --git a/src/modules/infrastructure/clusters/services/api-server-failover.core.ts b/src/modules/infrastructure/clusters/services/api-server-failover.core.ts new file mode 100644 index 00000000..4e1b14fc --- /dev/null +++ b/src/modules/infrastructure/clusters/services/api-server-failover.core.ts @@ -0,0 +1,65 @@ +const CONTROL_PLANE_LABEL = 'node-role.kubernetes.io/control-plane'; + +export interface KnownApiServer { + nodeId: string; + name: string; + address: string; +} + +interface NodeLike { + metadata?: { name?: string; labels?: Record }; + status?: { conditions?: Array<{ type: string; status: string }> }; +} + +interface RecordedNode { + id: string; + serverName?: string; + nodeType?: string; +} + +/** + * The servers the cluster reports, joined to Flui's records and to the overlay + * address each one answers on. The master comes first, the order the sweep + * tries them in. + */ +export function knownApiServers( + nodes: NodeLike[], + recorded: RecordedNode[], + overlayOf: Map, +): KnownApiServer[] { + const servers = new Set( + nodes + .filter((n) => n.metadata?.labels?.[CONTROL_PLANE_LABEL] === 'true') + .map((n) => n.metadata!.name!), + ); + return recorded + .filter((r) => r.serverName && servers.has(r.serverName)) + .filter((r) => overlayOf.has(r.id)) + .sort( + (a, b) => + Number(b.nodeType === 'master') - Number(a.nodeType === 'master') || + a.serverName!.localeCompare(b.serverName!), + ) + .map((r) => ({ + nodeId: r.id, + name: r.serverName!, + address: overlayOf.get(r.id)!, + })); +} + +export function readyServerCount(nodes: NodeLike[]): number { + return nodes.filter( + (n) => + n.metadata?.labels?.[CONTROL_PLANE_LABEL] === 'true' && + (n.status?.conditions ?? []).some( + (c) => c.type === 'Ready' && c.status === 'True', + ), + ).length; +} + +export function sameServers(a: KnownApiServer[], b: KnownApiServer[]): boolean { + return ( + a.length === b.length && + a.every((s, i) => s.nodeId === b[i].nodeId && s.address === b[i].address) + ); +} diff --git a/src/modules/infrastructure/clusters/services/api-server-failover.service.ts b/src/modules/infrastructure/clusters/services/api-server-failover.service.ts new file mode 100644 index 00000000..3a03c3bb --- /dev/null +++ b/src/modules/infrastructure/clusters/services/api-server-failover.service.ts @@ -0,0 +1,156 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { IsNull, Repository } from 'typeorm'; +import { ClusterEntity, ClusterType } from '../entities/cluster.entity'; +import { EncryptionService } from '../../../shared/encryption/services/encryption.service'; +import { KubernetesService } from '../../shared/services/kubernetes.service'; +import { WireGuardPeerService } from '../../networking/services/wireguard-peer.service'; +import { ApiServerSanService } from '../../networking/services/api-server-san.service'; +import { managementNetworkOn } from '../../networking/management-network.state'; +import { apiServerAnswers } from '../../networking/api-server-probe'; +import { + SERVER_LINE, + overlayDemotionOf, + withServerHost, +} from '../../networking/kubeconfig-demotion'; +import { + KnownApiServer, + knownApiServers, + readyServerCount, + sameServers, +} from './api-server-failover.core'; +import { mergeClusterMetadata } from '../utils/cluster-metadata-merge.util'; + +const CA_DATA = /certificate-authority-data:\s*(\S+)/; + +/** Below this, restarting a server to change its certificate could stop the control plane. */ +const SERVERS_FOR_A_RESTART = 3; + +/** + * Keeps a cluster with several servers reachable when the one Flui talks to + * is lost. While that server answers, the sweep learns the others and makes + * sure each one's certificate names the overlay address Flui would use, one + * server per sweep. When it stops answering, the stored address moves to the + * first other server that answers with a certificate the cluster's CA + * vouches for. The promoter moves it back to the master once the master is + * reachable again. + */ +@Injectable() +export class ApiServerFailoverService { + private readonly logger = new Logger(ApiServerFailoverService.name); + + constructor( + @InjectRepository(ClusterEntity) + private readonly clusters: Repository, + private readonly encryption: EncryptionService, + private readonly kubernetes: KubernetesService, + private readonly wgPeers: WireGuardPeerService, + private readonly apiServerSan: ApiServerSanService, + ) {} + + async sweep(): Promise { + if (!managementNetworkOn()) return; + const clusters = await this.clusters.find({ + where: { clusterType: ClusterType.WORKLOAD, deletedAt: IsNull() }, + relations: ['nodes'], + }); + for (const cluster of clusters) { + try { + await this.check(cluster); + } catch (err: any) { + this.logger.warn( + `[api-server] ${cluster.name}: ${err?.message ?? err}`, + ); + } + } + } + + async check(cluster: ClusterEntity): Promise { + if (!cluster.kubeconfigEncrypted || overlayDemotionOf(cluster)) return; + const kubeconfig = this.encryption.decrypt(cluster.kubeconfigEncrypted); + const line = SERVER_LINE.exec(kubeconfig); + const ca = CA_DATA.exec(kubeconfig)?.[1]; + if (!line || !ca) return; + const current = line[2]; + const port = (line[3] ?? ':6443').slice(1); + + if (await apiServerAnswers(current, port, ca)) { + await this.learn(cluster, kubeconfig, current, port, ca); + return; + } + const known = knownOf(cluster); + for (const server of known.filter((s) => s.address !== current)) { + if (await apiServerAnswers(server.address, port, ca)) { + await this.clusters.update(cluster.id, { + kubeconfigEncrypted: this.encryption.encrypt( + withServerHost(kubeconfig, server.address), + ), + }); + this.logger.warn( + `[api-server] ${cluster.name}: ${current} does not answer; Flui now reaches the cluster through ${server.name}`, + ); + return; + } + } + if (known.length > 1) { + this.logger.warn( + `[api-server] ${cluster.name}: none of its ${known.length} servers answers`, + ); + } + } + + private async learn( + cluster: ClusterEntity, + kubeconfig: string, + current: string, + port: string, + ca: string, + ): Promise { + const nodes = await this.kubernetes.listNodes(kubeconfig); + const overlay = new Map(); + for (const node of cluster.nodes ?? []) { + const reserved = await this.wgPeers.nodeOverlayFor(node.id); + if (reserved?.enrolled) overlay.set(node.id, reserved.nodeAddress); + } + const servers = knownApiServers(nodes, cluster.nodes ?? [], overlay); + const reachable: KnownApiServer[] = []; + let restarted = false; + for (const server of servers) { + if ( + server.address === current || + (await apiServerAnswers(server.address, port, ca)) + ) { + reachable.push(server); + continue; + } + if (restarted || readyServerCount(nodes) < SERVERS_FOR_A_RESTART) + continue; + restarted = true; + const result = await this.apiServerSan + .enrolOverlayAddress(cluster.id, server.nodeId) + .catch((err: any) => { + this.logger.warn( + `[api-server] ${cluster.name}: could not name ${server.address} in ${server.name}'s certificate (${err?.message ?? err})`, + ); + return null; + }); + if (result && (await apiServerAnswers(server.address, port, ca))) { + reachable.push(server); + } + } + if (sameServers(reachable, knownOf(cluster))) return; + await mergeClusterMetadata(this.clusters, cluster.id, { + apiServers: reachable, + }); + this.logger.log( + `[api-server] ${cluster.name}: reachable through ${reachable.map((s) => s.name).join(', ') || 'no server yet'}`, + ); + } +} + +function knownOf(cluster: ClusterEntity): KnownApiServer[] { + return ( + (cluster.metadata as { apiServers?: KnownApiServer[] } | null) + ?.apiServers ?? [] + ); +} diff --git a/src/modules/infrastructure/clusters/services/api-server-failover.spec.ts b/src/modules/infrastructure/clusters/services/api-server-failover.spec.ts new file mode 100644 index 00000000..42e52257 --- /dev/null +++ b/src/modules/infrastructure/clusters/services/api-server-failover.spec.ts @@ -0,0 +1,137 @@ +jest.mock('@kubernetes/client-node', () => ({})); +jest.mock('../../networking/api-server-probe', () => ({ + apiServerAnswers: jest.fn(), +})); +jest.mock('../../networking/management-network.state', () => ({ + managementNetworkOn: () => true, +})); +jest.mock('../utils/cluster-metadata-merge.util', () => ({ + mergeClusterMetadata: jest.fn(), +})); + +import { apiServerAnswers } from '../../networking/api-server-probe'; +import { mergeClusterMetadata } from '../utils/cluster-metadata-merge.util'; +import { ApiServerFailoverService } from './api-server-failover.service'; +import { knownApiServers, readyServerCount } from './api-server-failover.core'; + +const answers = apiServerAnswers as jest.MockedFunction< + typeof apiServerAnswers +>; + +const kubeconfig = (host: string) => + `clusters:\n- cluster:\n certificate-authority-data: Q0E=\n server: https://${host}:6443\n`; + +const k8sNode = (name: string, server: boolean, ready = true) => ({ + metadata: { + name, + labels: server ? { 'node-role.kubernetes.io/control-plane': 'true' } : {}, + }, + status: { conditions: [{ type: 'Ready', status: ready ? 'True' : 'False' }] }, +}); + +const recorded = [ + { id: 'n-m', serverName: 'lab-master', nodeType: 'master' }, + { id: 'n-1', serverName: 'lab-worker-1', nodeType: 'worker' }, + { id: 'n-2', serverName: 'lab-worker-2', nodeType: 'worker' }, + { id: 'n-3', serverName: 'lab-worker-3', nodeType: 'worker' }, +]; +const overlay: Record = { + 'n-m': '10.9.0.1', + 'n-1': '10.9.0.2', + 'n-2': '10.9.0.3', + 'n-3': '10.9.0.4', +}; +const k8sNodes = [ + k8sNode('lab-master', true), + k8sNode('lab-worker-1', true), + k8sNode('lab-worker-2', false), + k8sNode('lab-worker-3', true), +]; + +function setup(metadata: Record = {}, host = '10.9.0.1') { + const cluster: any = { + id: 'c1', + name: 'lab', + kubeconfigEncrypted: kubeconfig(host), + metadata, + nodes: recorded, + }; + const saved: any[] = []; + jest + .mocked(mergeClusterMetadata) + .mockImplementation(async (_repo, _id, keys) => { + saved.push({ metadata: keys }); + }); + const enrol = jest.fn(async () => ({ outcome: 'applied' })); + const service = new ApiServerFailoverService( + { + update: jest.fn(async (_id, c) => saved.push({ ...c })), + find: jest.fn(), + } as any, + { decrypt: (v: string) => v, encrypt: (v: string) => v } as any, + { listNodes: jest.fn(async () => k8sNodes) } as any, + { + nodeOverlayFor: jest.fn(async (id: string) => + overlay[id] ? { nodeAddress: overlay[id], enrolled: true } : undefined, + ), + } as any, + { enrolOverlayAddress: enrol } as any, + ); + return { service, cluster, saved, enrol }; +} + +describe('reaching a cluster through any of its servers', () => { + beforeEach(() => answers.mockReset()); + + it('lists the servers, the master first, with the address each answers on', () => { + expect( + knownApiServers(k8sNodes, recorded, new Map(Object.entries(overlay))), + ).toEqual([ + { nodeId: 'n-m', name: 'lab-master', address: '10.9.0.1' }, + { nodeId: 'n-1', name: 'lab-worker-1', address: '10.9.0.2' }, + { nodeId: 'n-3', name: 'lab-worker-3', address: '10.9.0.4' }, + ]); + expect(readyServerCount(k8sNodes)).toBe(3); + }); + + it('while the current server answers, it learns the others and names one missing address per sweep', async () => { + answers.mockImplementation( + async (host) => host !== '10.9.0.2' && host !== '10.9.0.4', + ); + const { service, cluster, saved, enrol } = setup(); + await service.check(cluster); + expect(enrol).toHaveBeenCalledTimes(1); + expect(enrol).toHaveBeenCalledWith('c1', 'n-1'); + expect(saved[0].metadata.apiServers.map((s: any) => s.name)).toEqual([ + 'lab-master', + ]); + }); + + it('moves to the first other server that answers when the current one does not', async () => { + answers.mockImplementation(async (host) => host === '10.9.0.4'); + const { service, cluster, saved } = setup({ + apiServers: [ + { nodeId: 'n-m', name: 'lab-master', address: '10.9.0.1' }, + { nodeId: 'n-1', name: 'lab-worker-1', address: '10.9.0.2' }, + { nodeId: 'n-3', name: 'lab-worker-3', address: '10.9.0.4' }, + ], + }); + await service.check(cluster); + expect(saved[0].kubeconfigEncrypted).toContain( + 'server: https://10.9.0.4:6443', + ); + }); + + it('leaves alone a cluster a person moved back to its public address', async () => { + answers.mockResolvedValue(false); + const { service, cluster, saved } = setup({ + overlayDemotion: { at: 'x', by: 'p', from: 'a', to: 'b' }, + apiServers: [ + { nodeId: 'n-1', name: 'lab-worker-1', address: '10.9.0.2' }, + ], + }); + await service.check(cluster); + expect(saved).toEqual([]); + expect(answers).not.toHaveBeenCalled(); + }); +}); diff --git a/src/modules/infrastructure/clusters/services/cluster-capacity.service.ts b/src/modules/infrastructure/clusters/services/cluster-capacity.service.ts index 5ddc83c0..2f3252ea 100644 --- a/src/modules/infrastructure/clusters/services/cluster-capacity.service.ts +++ b/src/modules/infrastructure/clusters/services/cluster-capacity.service.ts @@ -150,6 +150,26 @@ export class ClusterCapacityService { }; } + /** Machine types a node of this cluster could move to, against its own current type. */ + async candidatesFor( + provider: string, + serverType: string | null | undefined, + ): Promise { + let sizes: NodeSizeDto[] = []; + try { + sizes = + (await this.providerFactory + .getProvider(provider as CloudProvider) + .getNodeSizes?.(false)) ?? []; + } catch { + return []; + } + return this.buildCandidates( + sizes, + serverType ? sizes.find((s) => s.name === serverType) : undefined, + ); + } + private buildCandidates( sizes: NodeSizeDto[], currentSize: NodeSizeDto | undefined, diff --git a/src/modules/infrastructure/clusters/services/cluster-creation.ha.spec.ts b/src/modules/infrastructure/clusters/services/cluster-creation.ha.spec.ts new file mode 100644 index 00000000..0aa179fe --- /dev/null +++ b/src/modules/infrastructure/clusters/services/cluster-creation.ha.spec.ts @@ -0,0 +1,217 @@ +import { BadRequestException } from '@nestjs/common'; +import { ManagementAddressResolver } from '../../shared/services/management-address.resolver'; +import { ClusterCreationService } from './cluster-creation.service'; +import { HaCreationPlanService } from './ha-creation-plan.service'; +import { CloudProvider } from '../../../providers/enums/cloud-provider.enum'; + +describe('ClusterCreationService.createCluster — across three locations', () => { + const sizes = [ + { + id: '109', + name: 'cpx22', + architecture: 'x86', + availability: ['fsn1', 'nbg1', 'hel1'].map((location) => ({ + location, + available: true, + })), + }, + { + id: '45', + name: 'cax11', + architecture: 'arm', + availability: ['fsn1', 'nbg1', 'hel1'].map((location) => ({ + location, + available: false, + })), + }, + ]; + + function build() { + const clusterRepo = { + findOne: jest.fn().mockResolvedValue(null), + create: jest.fn((x: object) => x), + save: jest.fn(async (x: object) => ({ id: 'cluster-1', ...x })), + delete: jest.fn(), + }; + const operationRepo = { + create: jest.fn((x: Record) => x), + save: jest.fn(async (x: object) => ({ id: 'op-1', ...x })), + }; + const envSubnet = { + id: 'subnet-1', + vnetId: 'vnet-1', + ipRange: '10.10.1.0/24', + vnet: { + provider: CloudProvider.HETZNER, + implementation: 'provider-native', + }, + }; + const queue = { add: jest.fn().mockResolvedValue(undefined) }; + const getNodeSizes = jest.fn().mockResolvedValue(sizes); + const providers = { getProvider: () => ({ getNodeSizes }) }; + const capabilitiesFactory = { + getCapabilitiesService: () => ({ + getStaticCapabilities: () => ({ + firewall: { backend: 'managed-api' }, + vnetRequired: true, + crossClusterAllowed: true, + }), + nodeTopology: (location: string) => ({ + zone: location, + region: 'eu-central', + }), + haTopologies: () => [ + { + region: 'eu-central', + zones: ['fsn1', 'nbg1', 'hel1'], + dataZones: ['fsn1', 'nbg1'], + validated: true, + measured: true, + }, + ], + }), + }; + const service = new ClusterCreationService( + clusterRepo as never, + operationRepo as never, + { + findOne: jest.fn().mockResolvedValue(envSubnet), + find: jest.fn().mockResolvedValue([envSubnet]), + } as never, + queue as never, + { + generateK3sToken: () => 'token', + encrypt: () => 'encrypted', + } as never, + { + createAndReconcileFirewall: jest.fn().mockResolvedValue('fw-1'), + } as never, + capabilitiesFactory as never, + { resolveControlEgressIps: jest.fn().mockResolvedValue([]) } as never, + new ManagementAddressResolver(), + {} as never, + providers as never, + new HaCreationPlanService( + capabilitiesFactory as never, + providers as never, + ), + ); + return { service, clusterRepo, operationRepo, queue, getNodeSizes }; + } + + const dto = { + name: 'ha-hz', + provider: CloudProvider.HETZNER, + region: 'fsn1', + nodeSize: 'cpx22', + workerCount: 0, + highAvailability: 'three-locations' as const, + }; + + it('records the servers it will buy, with live availability, before queueing the creation', async () => { + const { service, clusterRepo, operationRepo, queue, getNodeSizes } = + build(); + + await service.createCluster(dto as never); + + expect(getNodeSizes).toHaveBeenCalledWith(true); + expect(clusterRepo.create).toHaveBeenCalledWith( + expect.objectContaining({ + nodeSize: 'cpx22', + metadata: expect.objectContaining({ + highAvailability: 'three-locations', + }), + }), + ); + const operation = operationRepo.create.mock.calls[0][0]; + expect(operation.metadata.haPlan).toEqual({ + region: 'eu-central', + first: { zone: 'fsn1', shape: 'cpx22' }, + servers: [ + { zone: 'nbg1', shape: 'cpx22' }, + { zone: 'hel1', shape: 'cpx22' }, + ], + later: [], + }); + expect(operation.metadata.operationSteps[3].description).toBe( + 'Joining the servers in nbg1 and hel1, one at a time', + ); + expect(queue.add).toHaveBeenCalledWith( + 'create-cluster', + expect.anything(), + expect.anything(), + ); + }); + + it('buys nothing and saves nothing when a location has none of the sizes', async () => { + const { service, clusterRepo, queue } = build(); + + await expect( + service.createCluster({ ...dto, nodeSize: 'cax11' } as never), + ).rejects.toThrow(BadRequestException); + expect(clusterRepo.save).not.toHaveBeenCalled(); + expect(queue.add).not.toHaveBeenCalled(); + }); + + it('previews the servers with their role, and buys nothing', async () => { + const { service, clusterRepo, queue } = build(); + + const plan = await service.previewHighAvailability({ + provider: CloudProvider.HETZNER, + region: 'fsn1', + shapes: ['cpx22'], + completeLater: false, + }); + + expect(plan).toEqual({ + layoutAvailable: true, + possible: true, + refusal: null, + completeLaterHelps: false, + region: 'eu-central', + measured: true, + servers: [ + { zone: 'fsn1', shape: 'cpx22', role: 'data', first: true }, + { zone: 'nbg1', shape: 'cpx22', role: 'data', first: false }, + { zone: 'hel1', shape: 'cpx22', role: 'vote', first: false }, + ], + later: [], + }); + expect(clusterRepo.save).not.toHaveBeenCalled(); + expect(queue.add).not.toHaveBeenCalled(); + }); + + it('previews a refusal with the layout still on offer', async () => { + const { service } = build(); + + const plan = await service.previewHighAvailability({ + provider: CloudProvider.HETZNER, + region: 'fsn1', + shapes: ['cax11'], + completeLater: false, + }); + + expect(plan).toMatchObject({ + layoutAvailable: true, + possible: false, + completeLaterHelps: false, + }); + expect(plan.refusal).toBe( + 'fsn1, nbg1 and hel1 have none of cax11 available now. No arm size is available in all three locations right now.', + ); + }); + + it('leaves a standard cluster as it always was', async () => { + const { service, operationRepo, getNodeSizes } = build(); + + await service.createCluster({ + ...dto, + highAvailability: undefined, + } as never); + + expect(getNodeSizes).not.toHaveBeenCalledWith(true); + expect( + operationRepo.create.mock.calls[0][0].metadata.haPlan, + ).toBeUndefined(); + }); +}); diff --git a/src/modules/infrastructure/clusters/services/cluster-creation.service.ts b/src/modules/infrastructure/clusters/services/cluster-creation.service.ts index aaa2a9fd..5f730ce9 100644 --- a/src/modules/infrastructure/clusters/services/cluster-creation.service.ts +++ b/src/modules/infrastructure/clusters/services/cluster-creation.service.ts @@ -25,12 +25,17 @@ import { OperationType, } from '../../servers/entities/infrastructure-operations.entity'; import { CreateClusterDto } from '../dto/create-cluster.dto'; +import { HaCreationPlan } from '../utils/ha-creation-plan.core'; +import { HaPlanResponseDto } from '../dto/ha-creation-plan.dto'; +import { + HaCreationPlanService, + shapeNameVia, +} from './ha-creation-plan.service'; import { EncryptionService } from '../../../shared/encryption/services/encryption.service'; import { ClusterFirewallIntegrationService } from './cluster-firewall-integration.service'; import { CapabilitiesProviderFactory } from '../../../providers/core/factories/capabilities-provider.factory'; import { CloudProvider } from '../../../providers/enums/cloud-provider.enum'; import { ProviderFactory } from '../../../providers/core/factories/provider.factory'; -import { shapeNameOf } from '../../../providers/core/catalogue/shape-name'; import { getFirewallRulesForClusterType, sanitizeApiServerFirewallRules, @@ -54,6 +59,30 @@ import { HostnameMode } from '../../../dns/enums/hostname-mode.enum'; import { managementNetworkOn } from '../../networking/management-network.state'; import type { OperationWarning } from '../processors/control-hub-hooks'; +/** What a cluster across three locations changes in its creation; nothing for a standard one. */ +// Bull does not stop a job past its timeout, it runs a second one beside it; +// every creation also waits up to twenty minutes for the install to end. +const CREATE_JOB_TIMEOUT_MS = 3_600_000; + +function haCreationFacts(plan: HaCreationPlan | null) { + if (!plan) { + return { + firstShape: null, + clusterMetadata: {}, + operationMetadata: {}, + serverZones: [] as string[], + jobTimeoutMs: CREATE_JOB_TIMEOUT_MS, + }; + } + return { + firstShape: plan.first.shape, + clusterMetadata: { highAvailability: 'three-locations' }, + operationMetadata: { haPlan: plan }, + serverZones: plan.servers.map((s) => s.zone), + jobTimeoutMs: CREATE_JOB_TIMEOUT_MS, + }; +} + /** * Service responsible for cluster creation logic */ @@ -76,6 +105,7 @@ export class ClusterCreationService { private readonly managementAddress: ManagementAddressResolver, private readonly byosVNetService: ByosVNetService, @Optional() private readonly providers?: ProviderFactory, + @Optional() private readonly haPlans?: HaCreationPlanService, ) {} /** @@ -143,6 +173,8 @@ export class ClusterCreationService { } await this.enforceProviderPolicies(dto, clusterType, hasUsableSubnet); + const haPlan = (await this.haPlans?.plan(dto)) ?? null; + const ha = haCreationFacts(haPlan); // Resolve nip.io hostname token: when running in IP mode, every cluster gets // a unique token segment so the LE domain set differs between recreations, @@ -170,7 +202,7 @@ export class ClusterCreationService { name: dto.name, provider: dto.provider, region: dto.region, - nodeSize: await this.shapeNameFor(dto.provider, dto.nodeSize), + nodeSize: await this.clusterNodeSize(dto, ha.firstShape), nodeCount: 0, // Will be updated as nodes are created minNodes: dto.minNodes, maxNodes: dto.maxNodes, @@ -191,7 +223,7 @@ export class ClusterCreationService { // observability gets its own dedicated topology. sharedStorageEnabled: dto.sharedStorageEnabled !== false, sharedStorageVolumeSizeGb: dto.sharedStorageVolumeSizeGb ?? 20, - metadata, + metadata: { ...metadata, ...ha.clusterMetadata }, }); const savedCluster = await this.clusterRepository.save(cluster); @@ -274,6 +306,7 @@ export class ClusterCreationService { const operationSteps = getOperationSteps(OperationType.CREATE_CLUSTER, { workerCount, withFirewall, + haServers: ha.serverZones, }); // Create operation for tracking @@ -289,10 +322,13 @@ export class ClusterCreationService { currentStepProgress: 0, metadata: { clusterConfig: dto, - estimatedDurationInSeconds: estimateCreateDurationSeconds(workerCount), + estimatedDurationInSeconds: estimateCreateDurationSeconds( + workerCount + ha.serverZones.length, + ), providerFirewallId, // Single firewall ID (not array) — null when deferred operationSteps: operationSteps, // Fixed: was 'steps', now 'operationSteps' desiredFirewallRules: firewallDeferred ? desiredRules : undefined, + ...ha.operationMetadata, ...(warnings.length ? { warnings } : {}), }, }); @@ -311,7 +347,7 @@ export class ClusterCreationService { type: 'exponential', delay: 2000, }, - timeout: 1800000, // 30 minutes + timeout: ha.jobTimeoutMs, }); this.logger.log( @@ -352,25 +388,26 @@ export class ClusterCreationService { ); } - /** - * The size as the provider's catalogue names it: a client may send the id - * (`109` on Hetzner), and every later reader — the scaling group, the node - * list, billing — expects the name. Unchanged when the catalogue is unread. - */ + /** The first server's size; a cluster across three locations names it in its plan. */ + private async clusterNodeSize( + dto: CreateClusterDto, + planned: string | null, + ): Promise { + return planned ?? this.shapeNameFor(dto.provider, dto.nodeSize); + } + private async shapeNameFor( provider: string, nodeSize: string, ): Promise { - if (!nodeSize) return nodeSize; - try { - const service = this.providers?.getProvider(provider as CloudProvider); - return shapeNameOf( - nodeSize, - (await service?.getNodeSizes?.(false)) ?? [], - ); - } catch { - return nodeSize; - } + return shapeNameVia(this.providers, provider, nodeSize); + } + + /** What creating a cluster across three locations would buy now, without buying it. */ + async previewHighAvailability( + query: Parameters[0], + ): Promise { + return this.haPlans!.preview(query); } /** Whether this provider leaves Flui to build the private network. */ diff --git a/src/modules/infrastructure/clusters/services/cluster-deletion.service.ts b/src/modules/infrastructure/clusters/services/cluster-deletion.service.ts index a79ce052..1fa93c0e 100644 --- a/src/modules/infrastructure/clusters/services/cluster-deletion.service.ts +++ b/src/modules/infrastructure/clusters/services/cluster-deletion.service.ts @@ -134,7 +134,9 @@ export class ClusterDeletionService { type: 'exponential', delay: 2000, }, - timeout: 600000, // 10 minutes + // Bull runs a second job beside one past its timeout; the deletion may + // first wait up to thirty minutes for a creation still running to stop. + timeout: 2_700_000, }); this.logger.log( diff --git a/src/modules/infrastructure/clusters/services/cluster-dns-spread.reconciler.spec.ts b/src/modules/infrastructure/clusters/services/cluster-dns-spread.reconciler.spec.ts index c4bad7fb..d5b2f6fe 100644 --- a/src/modules/infrastructure/clusters/services/cluster-dns-spread.reconciler.spec.ts +++ b/src/modules/infrastructure/clusters/services/cluster-dns-spread.reconciler.spec.ts @@ -1,10 +1,19 @@ jest.mock('@kubernetes/client-node', () => ({})); -import { ClusterDnsSpreadReconciler } from './cluster-dns-spread.reconciler'; +import { + ClusterDnsSpreadReconciler, + dnsNeedsRespread, +} from './cluster-dns-spread.reconciler'; function setup(opts: { readyNodes: number; - dns?: { replicas: number; spreadAcrossNodes: boolean } | null; + dns?: { + replicas: number; + spreadAcrossNodes: boolean; + spreadAcrossZones?: boolean; + podZones?: string[]; + readyZones?: string[]; + } | null; fail?: Error; }) { const kubernetes = { @@ -18,14 +27,18 @@ function setup(opts: { externalIps: [], })), ), - readClusterDns: jest - .fn() - .mockResolvedValue( - opts.dns === undefined - ? { replicas: 1, spreadAcrossNodes: false } - : opts.dns, - ), + readClusterDns: jest.fn().mockResolvedValue( + opts.dns === undefined + ? { + replicas: 1, + spreadAcrossNodes: false, + podZones: [], + readyZones: [], + } + : opts.dns && { podZones: [], readyZones: [], ...opts.dns }, + ), spreadClusterDns: jest.fn().mockResolvedValue(undefined), + respreadClusterDns: jest.fn().mockResolvedValue(undefined), }; const reconciler = new ClusterDnsSpreadReconciler( { @@ -60,13 +73,23 @@ describe('keeping the cluster DNS on two nodes', () => { it('does nothing when it is already right', async () => { const { reconciler, kubernetes } = setup({ readyNodes: 2, - dns: { replicas: 2, spreadAcrossNodes: true }, + dns: { replicas: 2, spreadAcrossNodes: true, spreadAcrossZones: true }, }); await expect(reconciler.reconcile('c1')).resolves.toBe('already-spread'); expect(kubernetes.spreadClusterDns).not.toHaveBeenCalled(); }); + it('adds locations to a DNS spread only across nodes', async () => { + const { reconciler, kubernetes } = setup({ + readyNodes: 3, + dns: { replicas: 2, spreadAcrossNodes: true, spreadAcrossZones: false }, + }); + + await expect(reconciler.reconcile('c1')).resolves.toBe('spread'); + expect(kubernetes.spreadClusterDns).toHaveBeenCalledWith('k', 2); + }); + it('leaves a single-node cluster alone', async () => { const { reconciler, kubernetes } = setup({ readyNodes: 1 }); @@ -93,3 +116,47 @@ describe('keeping the cluster DNS on two nodes', () => { await expect(reconciler.reconcile('c1')).resolves.toBe('unreachable'); }); }); + +describe('moving the cluster DNS out of a single location', () => { + const now = Date.parse('2026-10-29T08:00:00Z'); + + it('restarts it when every copy sits in one location and another has room', async () => { + const { reconciler, kubernetes } = setup({ + readyNodes: 3, + dns: { + replicas: 2, + spreadAcrossNodes: true, + spreadAcrossZones: true, + podZones: ['fsn1', 'fsn1'], + readyZones: ['fsn1', 'nbg1', 'hel1'], + }, + }); + await expect(reconciler.reconcile('c1')).resolves.toBe('respread'); + expect(kubernetes.respreadClusterDns).toHaveBeenCalled(); + }); + + it('leaves copies already in two locations, or a cluster with one location', () => { + expect( + dnsNeedsRespread( + { podZones: ['fsn1', 'nbg1'], readyZones: ['fsn1', 'nbg1'] }, + now, + ), + ).toBe(false); + expect( + dnsNeedsRespread( + { podZones: ['fsn1', 'fsn1'], readyZones: ['fsn1'] }, + now, + ), + ).toBe(false); + }); + + it('waits an hour before trying again', () => { + const dns = { podZones: ['fsn1', 'fsn1'], readyZones: ['fsn1', 'nbg1'] }; + expect( + dnsNeedsRespread({ ...dns, respreadAt: '2026-10-29T07:30:00Z' }, now), + ).toBe(false); + expect( + dnsNeedsRespread({ ...dns, respreadAt: '2026-10-29T06:30:00Z' }, now), + ).toBe(true); + }); +}); diff --git a/src/modules/infrastructure/clusters/services/cluster-dns-spread.reconciler.ts b/src/modules/infrastructure/clusters/services/cluster-dns-spread.reconciler.ts index a87ce79c..135e5857 100644 --- a/src/modules/infrastructure/clusters/services/cluster-dns-spread.reconciler.ts +++ b/src/modules/infrastructure/clusters/services/cluster-dns-spread.reconciler.ts @@ -6,17 +6,36 @@ import { KubernetesService } from '../../shared/services/kubernetes.service'; import { EncryptionService } from '../../../shared/encryption/services/encryption.service'; export const CLUSTER_DNS_REPLICAS = 2; +const RESPREAD_COOLDOWN_MS = 60 * 60 * 1000; export type ClusterDnsSpreadOutcome = | 'spread' + | 'respread' | 'already-spread' | 'single-node' | 'no-cluster-dns' | 'unreachable'; +/** + * Every copy in one location while another location has room: losing that + * location takes name resolution with it, and with it every volume that has + * to move. Not more than once an hour, in case the scheduler + * keeps choosing the same location for its own reasons. + */ +export function dnsNeedsRespread( + dns: { podZones: string[]; readyZones: string[]; respreadAt?: string }, + now: number, +): boolean { + if (dns.podZones.length < 2 || new Set(dns.podZones).size > 1) return false; + if (dns.readyZones.filter((z) => z !== dns.podZones[0]).length === 0) + return false; + const last = dns.respreadAt ? Date.parse(dns.respreadAt) : Number.NaN; + return Number.isNaN(last) || now - last > RESPREAD_COOLDOWN_MS; +} + /** * Keeps the cluster DNS on two nodes once the cluster has two that are - * ready. + * ready, in two locations when it has them. * * K3s owns the `coredns` manifest and writes it back when it restarts, which * returns it to one copy. Changing that from K3s' side means restarting K3s on @@ -44,15 +63,27 @@ export class ClusterDnsSpreadReconciler { if (nodes.filter((n) => n.ready).length < 2) return 'single-node'; const dns = await this.kubernetes.readClusterDns(kubeconfig); if (!dns) return 'no-cluster-dns'; - if (dns.replicas >= CLUSTER_DNS_REPLICAS && dns.spreadAcrossNodes) { - return 'already-spread'; + if ( + dns.replicas >= CLUSTER_DNS_REPLICAS && + dns.spreadAcrossNodes && + dns.spreadAcrossZones + ) { + if (!dnsNeedsRespread(dns, Date.now())) return 'already-spread'; + await this.kubernetes.respreadClusterDns( + kubeconfig, + new Date().toISOString(), + ); + this.logger.log( + `[cluster-dns] ${cluster.name}: every copy was in ${dns.podZones[0]}; restarted so they spread over ${dns.readyZones.join(', ')}`, + ); + return 'respread'; } await this.kubernetes.spreadClusterDns( kubeconfig, Math.max(dns.replicas, CLUSTER_DNS_REPLICAS), ); this.logger.log( - `[cluster-dns] ${cluster.name}: ${dns.replicas} → ${Math.max(dns.replicas, CLUSTER_DNS_REPLICAS)} copies, spread across nodes`, + `[cluster-dns] ${cluster.name}: ${dns.replicas} → ${Math.max(dns.replicas, CLUSTER_DNS_REPLICAS)} copies, spread across nodes and locations`, ); return 'spread'; } catch (err) { diff --git a/src/modules/infrastructure/clusters/services/cluster-node-scaling.service.spec.ts b/src/modules/infrastructure/clusters/services/cluster-node-scaling.service.spec.ts new file mode 100644 index 00000000..1f416a93 --- /dev/null +++ b/src/modules/infrastructure/clusters/services/cluster-node-scaling.service.spec.ts @@ -0,0 +1,226 @@ +jest.mock('@kubernetes/client-node', () => ({})); + +import { BadRequestException } from '@nestjs/common'; +import { ClusterNodeScalingService } from './cluster-node-scaling.service'; +import { NodeResizeHaService } from './node-resize-ha.service'; +import { NodeType } from '../entities/cluster-node.entity'; +import { + OperationStatus, + OperationType, +} from '../../servers/entities/infrastructure-operations.entity'; + +type Fake = Record; + +function setup( + opts: { + ha?: 'three-locations' | 'standard'; + serverDown?: string; + failChangeType?: boolean; + primaryOnNode?: boolean; + } = {}, +) { + const server = { status: 'running', server_type: 'cx22' }; + const provider: Fake = { + changeServerType: jest.fn(async (_id, r: { targetServerType: string }) => { + if (opts.failChangeType) throw new Error('provider refused the type'); + server.server_type = r.targetServerType; + }), + powerOffServer: jest.fn(async () => { + server.status = 'off'; + }), + powerOnServer: jest.fn(async () => { + server.status = 'running'; + }), + getServerDetailsAsDto: jest.fn(async () => ({ ...server })), + }; + const operations = new Map(); + const operationRepository: Fake = { + create: jest.fn((o) => o), + save: jest.fn(async (o) => { + const saved = { id: o.id ?? 'op-1', ...o }; + operations.set(saved.id, saved); + return saved; + }), + findOne: jest.fn(async ({ where }) => operations.get(where.id) ?? null), + findOneOrFail: jest.fn(async ({ where }) => operations.get(where.id)), + update: jest.fn(async (id, patch) => { + operations.set(id, { ...operations.get(id), ...patch }); + }), + createQueryBuilder: jest.fn(() => { + const qb: any = { + where: () => qb, + andWhere: () => qb, + getOne: async () => null, + }; + return qb; + }), + }; + const node = { + id: 'n-1', + clusterId: 'c1', + serverName: 'lab-worker-1', + nodeType: NodeType.WORKER, + providerResourceId: 'srv-1', + region: 'nbg1', + }; + const kubernetes: Fake = { + cordonNode: jest.fn(), + uncordonNode: jest.fn(), + waitForNodeReady: jest.fn(), + execInPod: jest.fn(async () => '[[]]'), + listObjectsEverywhere: jest.fn(async () => + opts.primaryOnNode + ? [ + { + metadata: { namespace: 'p', name: 'db-ha' }, + status: { currentPrimary: 'db-ha-1' }, + }, + ] + : [], + ), + listPodsByLabel: jest.fn(async () => [ + { + metadata: { name: 'db-ha-1' }, + spec: { nodeName: 'lab-worker-1' }, + status: { conditions: [{ type: 'Ready', status: 'True' }] }, + }, + { + metadata: { name: 'db-ha-2' }, + spec: { nodeName: 'lab-master' }, + status: { conditions: [{ type: 'Ready', status: 'True' }] }, + }, + ]), + jsonPatchObjectStatus: jest.fn(), + readObject: jest.fn(async () => ({ + status: { currentPrimary: 'db-ha-2', phase: 'Cluster in healthy state' }, + })), + }; + const queue = { add: jest.fn() }; + const serverNode = (name: string, zone: string, ready = true) => ({ + metadata: { + name, + labels: { + 'topology.kubernetes.io/zone': zone, + 'node-role.kubernetes.io/control-plane': 'true', + }, + }, + status: { + conditions: [{ type: 'Ready', status: ready ? 'True' : 'False' }], + }, + }); + const clusterHa = { + inspect: jest.fn(async () => ({ + kubeconfig: 'kc', + nodes: [ + serverNode('lab-master', 'fsn1'), + serverNode('lab-worker-1', 'nbg1'), + serverNode('lab-worker-3', 'hel1', opts.serverDown !== 'lab-worker-3'), + ], + state: { level: opts.ha ?? 'three-locations' }, + topology: { + zones: ['fsn1', 'nbg1', 'hel1'], + dataZones: ['fsn1', 'nbg1'], + }, + })), + }; + const service = new ClusterNodeScalingService( + { + findOne: jest.fn(async () => ({ + id: 'c1', + name: 'lab', + provider: 'hetzner', + region: 'fsn1', + kubeconfigEncrypted: 'e', + })), + } as never, + { findOne: jest.fn(async () => node), save: jest.fn() } as never, + operationRepository as never, + { find: jest.fn(async () => []) } as never, + { getProvider: () => provider } as never, + kubernetes as never, + { decrypt: () => 'kc' } as never, + { openNodeInterval: jest.fn() } as never, + { resolveHourlyEur: jest.fn(async () => 0.01) } as never, + queue as never, + { candidatesFor: jest.fn(async () => []) } as never, + new NodeResizeHaService( + operationRepository as never, + kubernetes as never, + clusterHa as never, + { runHostScript: jest.fn(async () => 'GROWN 40') } as never, + ), + ); + return { service, provider, queue, operations, kubernetes, server }; +} + +describe('resizing a node', () => { + it('queues the work and answers at once', async () => { + const { service, queue, provider } = setup({ ha: 'standard' }); + const op = await service.scaleNode('c1', 'n-1', { + targetServerType: 'cx32', + }); + expect(op.status).toBe(OperationStatus.PENDING); + expect(op.operationType).toBe(OperationType.SCALE_NODE); + expect(queue.add).toHaveBeenCalledWith( + 'scale-node', + { operationId: 'op-1' }, + expect.any(Object), + ); + expect(provider.powerOffServer).not.toHaveBeenCalled(); + }); + + it('refuses on a cluster across three locations while another server is down', async () => { + const { service, queue } = setup({ serverDown: 'lab-worker-3' }); + await expect( + service.scaleNode('c1', 'n-1', { targetServerType: 'cx32' }), + ).rejects.toThrow(BadRequestException); + expect(queue.add).not.toHaveBeenCalled(); + }); + + it('runs the steps, hands a primary on the node over first, and grows the space for copies', async () => { + const { service, provider, operations, kubernetes, server } = setup({ + primaryOnNode: true, + }); + await service.scaleNode('c1', 'n-1', { + targetServerType: 'cx32', + copiesGiB: 40, + }); + await service.runScaleNode('op-1'); + expect(kubernetes.jsonPatchObjectStatus).toHaveBeenCalledWith( + 'kc', + expect.objectContaining({ name: 'db-ha', namespace: 'p' }), + expect.arrayContaining([ + { op: 'replace', path: '/status/targetPrimary', value: 'db-ha-2' }, + ]), + ); + expect(provider.powerOffServer).toHaveBeenCalled(); + expect(server).toEqual({ status: 'running', server_type: 'cx32' }); + expect(operations.get('op-1').status).toBe(OperationStatus.COMPLETED); + }); + + it('carries on where it was after a restart instead of starting over', async () => { + const { service, provider, server, operations } = setup({ ha: 'standard' }); + await service.scaleNode('c1', 'n-1', { targetServerType: 'cx32' }); + server.status = 'off'; + server.server_type = 'cx32'; + await service.runScaleNode('op-1'); + expect(provider.powerOffServer).not.toHaveBeenCalled(); + expect(provider.changeServerType).not.toHaveBeenCalled(); + expect(provider.powerOnServer).toHaveBeenCalled(); + expect(operations.get('op-1').status).toBe(OperationStatus.COMPLETED); + }); + + it('powers the node back on when a step fails once it is off', async () => { + const { service, provider, server, operations } = setup({ + ha: 'standard', + failChangeType: true, + }); + await service.scaleNode('c1', 'n-1', { targetServerType: 'cx32' }); + await service.runScaleNode('op-1'); + expect(provider.powerOnServer).toHaveBeenCalled(); + expect(server.status).toBe('running'); + const op = operations.get('op-1'); + expect(op.status).toBe(OperationStatus.FAILED); + expect(op.errorMessage).toContain('The node powered back on'); + }); +}); diff --git a/src/modules/infrastructure/clusters/services/cluster-node-scaling.service.ts b/src/modules/infrastructure/clusters/services/cluster-node-scaling.service.ts index 008188fd..5aa78283 100644 --- a/src/modules/infrastructure/clusters/services/cluster-node-scaling.service.ts +++ b/src/modules/infrastructure/clusters/services/cluster-node-scaling.service.ts @@ -3,8 +3,11 @@ import { Injectable, Logger, NotFoundException, + Optional, } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; +import { InjectQueue } from '@nestjs/bull'; +import { Queue } from 'bull'; import { IsNull, Repository } from 'typeorm'; import { ClusterEntity } from '../entities/cluster.entity'; import { ClusterNodeEntity, NodeType } from '../entities/cluster-node.entity'; @@ -26,6 +29,12 @@ import { ARM_SERVER_WARNING_CODE, providerServerTypeWarning, } from '../../../providers/core/catalogue/server-architecture'; +import { ClusterCapacityService } from './cluster-capacity.service'; +import { CapacityCandidateDto } from '../dto/cluster-capacity-plan.dto'; +import { + HaResizeReadiness, + NodeResizeHaService, +} from './node-resize-ha.service'; export interface ScaleNodeImpactDto { clusterId: string; @@ -40,11 +49,45 @@ export interface ScaleNodeImpactDto { affectedDedicatedApps: Array<{ id: string; slug: string }>; expectedDowntimeMs: number; warning: string; + /** Machine types this node could move to, against its own current type. */ + candidates: CapacityCandidateDto[]; + /** + * On a cluster across three locations: why the resize would be refused now, + * and the primaries handed over before the node goes down. Null elsewhere. + */ + highAvailability: { + refusal: string | null; + switchovers: Array<{ namespace: string; cluster: string; target: string }>; + } | null; } export interface ScaleNodeRequest { targetServerType: string; upgradeDisk?: boolean; + /** New size of the space for copies of replicated volumes on this node, grown once it is back. */ + copiesGiB?: number; +} + +export interface ScaleNodeJobData { + operationId: string; +} + +interface ScaleRun { + operationId: string; + meta: { + clusterId: string; + nodeId: string; + targetServerType: string; + upgradeDisk?: boolean; + copiesGiB?: number; + }; + cluster: ClusterEntity; + node: ClusterNodeEntity; + provider: ReturnType; + kubeconfig: string | null; + isMaster: boolean; + /** Set while the machine is off because of this run, so a failure brings it back. */ + poweredOff: boolean; } export interface ExpandSharedVolumeRequest { @@ -69,8 +112,20 @@ export class ClusterNodeScalingService { private readonly encryptionService: EncryptionService, private readonly billingIntervals: BillingIntervalsService, private readonly nodePriceService: NodePriceService, + @InjectQueue('infrastructure') private readonly infrastructureQueue: Queue, + private readonly capacity: ClusterCapacityService, + // Absent in the CLI's own container, which resizes the control cluster's + // master in process: a resize there works as it always has. + @Optional() private readonly resizeHa?: NodeResizeHaService, ) {} + private haReadiness( + cluster: ClusterEntity, + node: ClusterNodeEntity, + ): Promise { + return this.resizeHa?.readiness(cluster, node) ?? Promise.resolve(null); + } + // ─── Impact preview ──────────────────────────────────────────────────────── async previewScaleNode( @@ -104,6 +159,11 @@ export class ClusterNodeScalingService { slug: a.slug, })), expectedDowntimeMs: 240000, + candidates: await this.capacity.candidatesFor( + cluster.provider, + details?.server_type ?? node.serverType, + ), + highAvailability: await this.haReadiness(cluster, node), warning: affected.length > 0 ? `Scaling this node will stop ${affected.length} dedicated workload(s) (~3–5 min of downtime). Snapshot them with \`flui app snapshot create\` before proceeding.` @@ -118,6 +178,7 @@ export class ClusterNodeScalingService { nodeId: string, request: ScaleNodeRequest, userId?: string, + options: { queue?: boolean } = {}, ): Promise { const { cluster, node } = await this.resolveClusterAndNode( clusterId, @@ -156,6 +217,9 @@ export class ClusterNodeScalingService { ); } + const ha = await this.haReadiness(cluster, node); + if (ha?.refusal) throw new BadRequestException(ha.refusal); + const steps = getOperationSteps(OperationType.SCALE_NODE); const operation = this.operationRepository.create({ operationType: OperationType.SCALE_NODE, @@ -175,6 +239,8 @@ export class ClusterNodeScalingService { currentServerType: currentDetails?.server_type ?? null, targetServerType: request.targetServerType, upgradeDisk: request.upgradeDisk ?? false, + ...(request.copiesGiB && { copiesGiB: request.copiesGiB }), + ...(ha && { highAvailability: true }), operationSteps: steps, ...(warning ? { @@ -184,151 +250,249 @@ export class ClusterNodeScalingService { }, }); const saved = await this.operationRepository.save(operation); + // The control cluster's master runs the API: its resize is run by the + // CLI that asked for it, which runScaleNode then does in process. + if (options.queue === false) return saved; + await this.infrastructureQueue.add( + 'scale-node', + { operationId: saved.id } satisfies ScaleNodeJobData, + { attempts: 1, timeout: 30 * 60 * 1000 }, + ); + this.logger.log( + `Queued scale-node ${node.serverName} → ${request.targetServerType} (operation ${saved.id})`, + ); + return saved; + } + /** + * The steps of a resize, each checking where the machine already is, so a + * job that runs again after the API restarted carries on instead of + * starting over. A failure once the machine is off tries to bring it back, + * so a resize that could not finish does not leave a node down. + */ + async runScaleNode(operationId: string): Promise { + const operation = await this.operationRepository.findOneOrFail({ + where: { id: operationId }, + }); + const meta = operation.metadata as ScaleRun['meta']; + const { cluster, node } = await this.resolveClusterAndNode( + meta.clusterId, + meta.nodeId, + ); + const run: ScaleRun = { + operationId, + meta, + cluster, + node, + provider: this.providerFactory.getProvider( + cluster.provider as CloudProvider, + ), + kubeconfig: cluster.kubeconfigEncrypted + ? this.encryptionService.decrypt(cluster.kubeconfigEncrypted) + : null, + isMaster: node.nodeType === NodeType.MASTER, + poweredOff: false, + }; + + await this.operationRepository.update(operationId, { + status: OperationStatus.IN_PROGRESS, + startedAt: operation.startedAt ?? new Date(), + }); try { - saved.status = OperationStatus.IN_PROGRESS; - saved.startedAt = new Date(); - await this.operationRepository.save(saved); + await this.prepareForResize(run); + await this.changeMachine(run); + await this.waitHealthy(run); + await this.recordNewType(run); - const isMaster = node.nodeType === NodeType.MASTER; + // Step 5: finalize + await this.advanceStep(operationId, 5, 100, { + status: OperationStatus.COMPLETED, + message: `Node ${node.serverName} scaled to ${meta.targetServerType}`, + completedAt: new Date().toISOString(), + }); + await this.operationRepository.update(operationId, { + status: OperationStatus.COMPLETED, + completedAt: new Date(), + }); + } catch (error) { + await this.failScaleNode(run, error as Error); + } + } - // Step 0: precheck → cordon (workers only; for master, the k8s API - // goes down with the master so cordoning is pointless and we cannot - // call it once power_off starts). - if (!isMaster && cluster.kubeconfigEncrypted) { - await this.advanceStep(saved.id, 0, 50, { - message: 'Cordoning worker node', - }); - const kubeconfig = this.encryptionService.decrypt( - cluster.kubeconfigEncrypted, + /** Step 0: primaries handed over, then cordon. The master is not cordoned: its API goes down with it. */ + private async prepareForResize(run: ScaleRun): Promise { + const { operationId, kubeconfig, node } = run; + const ha = kubeconfig ? await this.haReadiness(run.cluster, node) : null; + for (const s of ha?.switchovers ?? []) { + await this.advanceStep(operationId, 0, 20, { + message: `Handing the primary of ${s.namespace}/${s.cluster} over to ${s.target}`, + }); + await this.resizeHa!.switchover(kubeconfig!, s); + } + if (!run.isMaster && kubeconfig) { + await this.advanceStep(operationId, 0, 50, { + message: 'Cordoning worker node', + }); + try { + await this.kubernetesService.cordonNode(kubeconfig, node.serverName); + } catch (err) { + this.logger.warn( + `Cordon failed for ${node.serverName}: ${(err as Error).message} — continuing`, ); - try { - await this.kubernetesService.cordonNode(kubeconfig, node.serverName); - } catch (err) { - this.logger.warn( - `Cordon failed for ${node.serverName}: ${(err as Error).message} — continuing`, - ); - } - } else { - await this.advanceStep(saved.id, 0, 50, { - message: 'Master node: skipping cordon (k8s API co-located)', - }); } - await this.advanceStep(saved.id, 0, 100); - - // Step 1: power off - await this.advanceStep(saved.id, 1, 0, { - message: 'Powering off node', + } else { + await this.advanceStep(operationId, 0, 50, { + message: 'Master node: skipping cordon (k8s API co-located)', }); - await provider.powerOffServer(node.providerResourceId); + } + await this.advanceStep(operationId, 0, 100); + } + + /** Steps 1-3, each skipped when the machine is already past it. */ + private async changeMachine(run: ScaleRun): Promise { + const { operationId, provider, node, meta } = run; + const details = () => + provider.getServerDetailsAsDto?.(node.providerResourceId); + const before = await details(); + + // Step 1: power off + await this.advanceStep(operationId, 1, 0, { + message: 'Powering off node', + }); + if (before?.server_type !== meta.targetServerType) { + if (before?.status !== 'off') { + await provider.powerOffServer!(node.providerResourceId); + } + run.poweredOff = true; await this.waitForServerStatus(provider, node.providerResourceId, 'off'); - await this.advanceStep(saved.id, 1, 100); + } + await this.advanceStep(operationId, 1, 100); - // Step 2: change_type - await this.advanceStep(saved.id, 2, 0, { - message: `Changing server type → ${request.targetServerType}`, - }); - await provider.changeServerType(node.providerResourceId, { - targetServerType: request.targetServerType, - upgradeDisk: request.upgradeDisk ?? false, + // Step 2: change_type + await this.advanceStep(operationId, 2, 0, { + message: `Changing server type → ${meta.targetServerType}`, + }); + if ((await details())?.server_type !== meta.targetServerType) { + await provider.changeServerType!(node.providerResourceId, { + targetServerType: meta.targetServerType, + upgradeDisk: meta.upgradeDisk ?? false, }); - await this.advanceStep(saved.id, 2, 100); + } + await this.advanceStep(operationId, 2, 100); - // Step 3: power on - await this.advanceStep(saved.id, 3, 0, { - message: 'Powering on node', - }); - await provider.powerOnServer(node.providerResourceId); - await this.waitForServerStatus( - provider, - node.providerResourceId, - 'running', - ); - await this.advanceStep(saved.id, 3, 100); - - // Step 4: wait healthy. - // For master: TCP probe the public IP on port 80 (Traefik). When - // Traefik answers, k3s + ingress controller + master are all up. - // For worker: poll k8s API on the (still-alive) master for the - // worker's Node Ready=True condition. - await this.advanceStep(saved.id, 4, 0, { - message: isMaster - ? 'Waiting for master ingress (TCP :80) to respond' - : 'Waiting for worker node Ready via master k8s API', - }); - if (isMaster) { - const masterIp = cluster.masterIpAddress; - if (!masterIp) { - throw new Error( - 'Cluster has no masterIpAddress recorded — cannot wait for health', - ); - } - await this.waitForTcpOpen(masterIp, 80, 480000, 5000); - } else if (cluster.kubeconfigEncrypted) { - const kubeconfig = this.encryptionService.decrypt( - cluster.kubeconfigEncrypted, + // Step 3: power on + await this.advanceStep(operationId, 3, 0, { + message: 'Powering on node', + }); + if ((await details())?.status !== 'running') { + await provider.powerOnServer!(node.providerResourceId); + } + await this.waitForServerStatus( + provider, + node.providerResourceId, + 'running', + ); + run.poweredOff = false; + await this.advanceStep(operationId, 3, 100); + } + + /** + * Step 4. For the master: a TCP probe of the public IP on port 80 + * (Traefik); when Traefik answers, k3s, ingress and master are all up. For a + * worker: its Node Ready condition through the master's k8s API. The space + * for copies grows once the node is back. + */ + private async waitHealthy(run: ScaleRun): Promise { + const { operationId, kubeconfig, node, cluster, meta } = run; + await this.advanceStep(operationId, 4, 0, { + message: run.isMaster + ? 'Waiting for master ingress (TCP :80) to respond' + : 'Waiting for worker node Ready via master k8s API', + }); + if (run.isMaster) { + const masterIp = cluster.masterIpAddress; + if (!masterIp) { + throw new Error( + 'Cluster has no masterIpAddress recorded — cannot wait for health', ); - await this.kubernetesService.waitForNodeReady( - kubeconfig, - node.serverName, - 480000, - 5000, + } + await this.waitForTcpOpen(masterIp, 80, 480000, 5000); + } else if (kubeconfig) { + await this.kubernetesService.waitForNodeReady( + kubeconfig, + node.serverName, + 480000, + 5000, + ); + try { + await this.kubernetesService.uncordonNode(kubeconfig, node.serverName); + } catch (err) { + this.logger.warn( + `Uncordon failed for ${node.serverName}: ${(err as Error).message}`, ); - try { - await this.kubernetesService.uncordonNode( + } + } + const grown = + meta.copiesGiB && kubeconfig && this.resizeHa + ? await this.resizeHa.growCopies( kubeconfig, node.serverName, - ); - } catch (err) { - this.logger.warn( - `Uncordon failed for ${node.serverName}: ${(err as Error).message}`, - ); - } - } - await this.advanceStep(saved.id, 4, 100); - - node.serverType = request.targetServerType; - node.hourlyPriceEur = await this.nodePriceService.resolveHourlyEur( - cluster.provider, - request.targetServerType, - cluster.region, - ); - await this.nodeRepository.save(node); + meta.copiesGiB, + ) + : null; + await this.advanceStep(operationId, 4, 100, { + ...(grown && { message: grown }), + }); + } - await this.billingIntervals.openNodeInterval({ - clusterId: cluster.id, - nodeId: node.id, - serverName: node.serverName, - providerResourceId: node.providerResourceId, - provider: cluster.provider, - region: cluster.region, - serverType: request.targetServerType, - nodeType: node.nodeType, - }); + private async recordNewType(run: ScaleRun): Promise { + const { cluster, node, meta } = run; + const region = node.region ?? cluster.region; + node.serverType = meta.targetServerType; + node.hourlyPriceEur = await this.nodePriceService.resolveHourlyEur( + cluster.provider, + meta.targetServerType, + region, + ); + await this.nodeRepository.save(node); + await this.billingIntervals.openNodeInterval({ + clusterId: cluster.id, + nodeId: node.id, + serverName: node.serverName, + providerResourceId: node.providerResourceId, + provider: cluster.provider, + region, + serverType: meta.targetServerType, + nodeType: node.nodeType, + }); + } - // Step 5: finalize - await this.advanceStep(saved.id, 5, 100, { - status: OperationStatus.COMPLETED, - message: `Node ${node.serverName} scaled to ${request.targetServerType}`, - completedAt: new Date().toISOString(), - }); - saved.status = OperationStatus.COMPLETED; - saved.completedAt = new Date(); - await this.operationRepository.save(saved); - return saved; - } catch (error) { - this.logger.error( - `scaleNode failed for node=${nodeId}: ${(error as Error).message}`, - ); - saved.status = OperationStatus.FAILED; - saved.metadata = { - ...saved.metadata, - error: (error as Error).message, - failedAt: new Date().toISOString(), - }; - await this.operationRepository.save(saved); - throw error; - } + private async failScaleNode(run: ScaleRun, error: Error): Promise { + const { operationId, node, provider } = run; + const message = error.message; + this.logger.error( + `scaleNode failed for node=${node.serverName}: ${message}`, + ); + const recovery = run.poweredOff + ? await provider.powerOnServer!(node.providerResourceId) + .then(() => 'powered back on') + .catch((err: Error) => `could not be powered back on: ${err.message}`) + : null; + const current = await this.operationRepository.findOneOrFail({ + where: { id: operationId }, + }); + current.status = OperationStatus.FAILED; + current.errorMessage = recovery + ? `${message} The node ${recovery}.` + : message; + current.completedAt = new Date(); + current.metadata = { + ...current.metadata, + error: message, + ...(recovery && { recovery }), + failedAt: new Date().toISOString(), + } as InfrastructureOperationEntity['metadata']; + await this.operationRepository.save(current); } // ─── Expand shared storage volume ────────────────────────────────────────── diff --git a/src/modules/infrastructure/clusters/services/cluster-orchestration.master-script.spec.ts b/src/modules/infrastructure/clusters/services/cluster-orchestration.master-script.spec.ts new file mode 100644 index 00000000..6533b7ed --- /dev/null +++ b/src/modules/infrastructure/clusters/services/cluster-orchestration.master-script.spec.ts @@ -0,0 +1,100 @@ +jest.mock('@kubernetes/client-node', () => ({})); +jest.mock('jose', () => ({})); +jest.mock('ip-cidr', () => ({})); + +import { ClusterOrchestrationService } from './cluster-orchestration.service'; +import { ClusterType } from '../entities/cluster.entity'; + +const STOP = new Error('stop after the boot script'); + +describe('the boot script a new master receives', () => { + function build(plan: Record) { + const generateMasterScript = jest.fn().mockResolvedValue('#!/bin/bash'); + const service = Object.create( + ClusterOrchestrationService.prototype, + ) as ClusterOrchestrationService; + Object.assign(service, { + logger: { log: jest.fn(), warn: jest.fn(), error: jest.fn() }, + updateOperationProgress: jest.fn(), + encryptionService: { decrypt: () => 'k3s-token' }, + loadCaKeyPair: jest + .fn() + .mockResolvedValue({ caPublicKey: 'ca-pub', caPrivateKey: 'ca-priv' }), + resolveControlClusterIp: jest.fn().mockResolvedValue('203.0.113.5'), + capabilitiesFactory: { isProviderSupported: () => false }, + ensureNodeRecord: jest.fn().mockResolvedValue({ id: 'node-1' }), + privateNetworksOf: jest.fn().mockResolvedValue([]), + overlayPlanFor: jest.fn().mockResolvedValue(plan), + hostFirewallAtBoot: jest.fn().mockResolvedValue(undefined), + k3sScriptService: { generateMasterScript }, + ensureBootstrapKey: jest + .fn() + .mockResolvedValue({ id: 'key-1', publicKey: 'ssh-ed25519 AAAA' }), + providerFactory: { getProvider: () => ({}) }, + labelService: { generateServerLabels: () => ({}) }, + operationRepository: { + create: () => { + throw STOP; + }, + }, + }); + const create = () => + service.createMasterNode( + { + id: 'c1', + name: 'lab', + provider: 'hetzner', + region: 'fsn1', + clusterType: ClusterType.WORKLOAD, + sharedStorageEnabled: false, + } as never, + 'op-1', + ); + return { create, generateMasterScript }; + } + + it('keeps the Flui network settings alongside the bootstrap key', async () => { + const control = { + publicKey: 'k-control', + address: '10.250.0.1', + endpoint: '198.51.100.7:51821', + }; + const peers = [ + { publicKey: 'k-peer', address: '10.250.0.2', endpoint: 'x:51821' }, + ]; + const { create, generateMasterScript } = build({ + address: '10.250.0.9', + mode: 'overlay', + peers, + control, + }); + + await expect(create()).rejects.toBe(STOP); + + expect(generateMasterScript).toHaveBeenCalledTimes(1); + expect(generateMasterScript.mock.calls[0][0]).toMatchObject({ + wgAddress: '10.250.0.9', + wgControl: control, + wgMode: 'overlay', + wgPeers: peers, + bootstrapPublicKey: 'ssh-ed25519 AAAA', + }); + }); + + /** In a network Flui builds, the private address is the tunnel one and the boot script refuses to guess it. */ + it('tells a mesh master its private address', async () => { + const { create, generateMasterScript } = build({ + address: '10.250.0.9', + mode: 'mesh', + peers: [], + }); + + await expect(create()).rejects.toBe(STOP); + + expect(generateMasterScript.mock.calls[0][0]).toMatchObject({ + wgMode: 'mesh', + privateIp: '10.250.0.9', + bootstrapPublicKey: 'ssh-ed25519 AAAA', + }); + }); +}); diff --git a/src/modules/infrastructure/clusters/services/cluster-orchestration.service.spec.ts b/src/modules/infrastructure/clusters/services/cluster-orchestration.service.spec.ts index d252fce9..f8954692 100644 --- a/src/modules/infrastructure/clusters/services/cluster-orchestration.service.spec.ts +++ b/src/modules/infrastructure/clusters/services/cluster-orchestration.service.spec.ts @@ -188,6 +188,123 @@ describe('ClusterOrchestrationService — retry safety', () => { ).rejects.toThrow('did not become Ready'); expect(onTick).toHaveBeenCalledTimes(1); }); + + it('hands back the labels of the node that joined', async () => { + const readNode = jest.fn().mockResolvedValue({ + metadata: { + labels: { 'node-role.kubernetes.io/control-plane': 'true' }, + }, + status: { conditions: [{ type: 'Ready', status: 'True' }] }, + }); + const service = build({ + encryptionService: { decrypt: () => 'kubeconfig' }, + kubernetesService: { getKubeClient: () => ({ coreApi: { readNode } }) }, + }); + + await expect( + call(service, { ...CLUSTER, kubeconfigEncrypted: 'x' }, 'w-1', 1000, 0), + ).resolves.toEqual({ 'node-role.kubernetes.io/control-plane': 'true' }); + }); + }); + + describe('labelsOnceServer', () => { + type Labels = ( + c: unknown, + name: string, + role: 'server' | 'agent', + labels: Record | undefined, + ) => Promise | undefined>; + const call = ( + service: ClusterOrchestrationService, + ...args: Parameters + ) => + (service as unknown as { labelsOnceServer: Labels }).labelsOnceServer( + ...args, + ); + const plane = { 'node-role.kubernetes.io/control-plane': 'true' }; + + it('waits for the label K3s adds to a server shortly after it turns Ready', async () => { + const waitForNodeReady = jest + .fn() + .mockResolvedValueOnce({}) + .mockResolvedValueOnce(plane); + const service = build({ waitForNodeReady }); + + await expect( + call( + service, + { ...CLUSTER, kubeconfigEncrypted: 'x' }, + 'w-1', + 'server', + {}, + ), + ).resolves.toEqual(plane); + expect(waitForNodeReady).toHaveBeenCalledTimes(2); + }); + + it('does not wait for an agent or a server already labelled', async () => { + const waitForNodeReady = jest.fn(); + const service = build({ waitForNodeReady }); + + await call( + service, + { ...CLUSTER, kubeconfigEncrypted: 'x' }, + 'a', + 'agent', + {}, + ); + await call( + service, + { ...CLUSTER, kubeconfigEncrypted: 'x' }, + 's', + 'server', + plane, + ); + expect(waitForNodeReady).not.toHaveBeenCalled(); + }); + }); + + describe('refuseUnlessServer', () => { + type Refuse = ( + node: unknown, + k3sRole: 'server' | 'agent', + labels: Record | undefined, + ) => Promise; + const call = ( + service: ClusterOrchestrationService, + ...args: Parameters + ) => + (service as unknown as { refuseUnlessServer: Refuse }).refuseUnlessServer( + ...args, + ); + + it('fails a server that joined as an agent and leaves it an ordinary worker', async () => { + const save = jest.fn(); + const service = build({ nodeRepository: { save } }); + const node = { + serverName: 'wl-worker-2', + status: 'provisioning', + metadata: { workerIndex: 2, haRole: 'server' }, + }; + + await expect(call(service, node, 'server', {})).rejects.toThrow( + 'wl-worker-2 joined as an ordinary worker', + ); + expect(node.status).toBe('error'); + expect(node.metadata).not.toHaveProperty('haRole'); + expect(node.metadata).toMatchObject({ workerIndex: 2 }); + expect(save).toHaveBeenCalledWith(node); + }); + + it('lets a real server and every agent through', async () => { + const save = jest.fn(); + const service = build({ nodeRepository: { save } }); + const plane = { 'node-role.kubernetes.io/control-plane': 'true' }; + + await call(service, { serverName: 's', metadata: {} }, 'server', plane); + await call(service, { serverName: 'a', metadata: {} }, 'agent', {}); + expect(save).not.toHaveBeenCalled(); + }); }); describe('fetchKubeconfigFromMaster', () => { diff --git a/src/modules/infrastructure/clusters/services/cluster-orchestration.service.ts b/src/modules/infrastructure/clusters/services/cluster-orchestration.service.ts index fe3d9781..bd7bfcda 100644 --- a/src/modules/infrastructure/clusters/services/cluster-orchestration.service.ts +++ b/src/modules/infrastructure/clusters/services/cluster-orchestration.service.ts @@ -15,7 +15,17 @@ import { NodeType, NodeStatus, } from '../entities/cluster-node.entity'; -import { HostFirewallBoot, K3sScriptService } from './k3s-script.service'; +import { + haRoleMetadata, + HaNodeRole, + k3sRoleOf, + serverJoinRefusal, +} from '../utils/ha-node-role'; +import { + HostFirewallBoot, + K3sMasterConfig, + K3sScriptService, +} from './k3s-script.service'; import { calculateOperationProgressFromSaved } from '../../operations/helpers/operation-steps.helper'; import { EncryptionService } from '../../../shared/encryption/services/encryption.service'; import { CloudProvider } from 'src/modules/providers/enums/cloud-provider.enum'; @@ -36,6 +46,7 @@ import { SubnetsService } from '../../vnets/services/subnets.service'; import { VNetsService } from '../../vnets/services/vnets.service'; import { NativeSSHConnectionService } from 'src/modules/terminal/services/native-ssh-connection.service'; import { CapabilitiesProviderFactory } from 'src/modules/providers/core/factories/capabilities-provider.factory'; +import { NodeTopology } from 'src/modules/providers/core/node-topology'; import { DEFAULT_INTERNAL_CIDRS } from 'src/modules/providers/core/firewall/nftables-ruleset'; import { getFirewallRulesForClusterType } from '../../firewalls/templates/firewall-rules.template'; import { InstallLogService } from '../../operations/services/install-log.service'; @@ -54,6 +65,53 @@ import { installLogTarget, } from '../utils/install-log-target.util'; import { managementNetworkOn } from '../../networking/management-network.state'; +import { PlannedServer } from '../utils/ha-creation-plan.core'; + +// A server first syncs etcd from the cluster, so it is given longer than a worker. +const SERVER_JOIN_TIMEOUT_MS = 600_000; +const SERVER_LABEL_WAIT_MS = 120_000; +const SERVER_LABEL_POLL_MS = 5_000; +function labelsOf(node: { + metadata?: { labels?: Record }; +}): Record { + return node?.metadata?.labels ?? {}; +} + +/** Shared storage for a new worker; one bought to join later learns the master's address with the join details. */ +function workerSharedStorageOf( + cluster: ClusterEntity, + joinLater: boolean, +): { enabled: boolean; masterPrivateIp?: string } | undefined { + if (cluster.sharedStorageEnabled === false) return undefined; + if (!cluster.masterPrivateIp && !joinLater) return undefined; + return { + enabled: true, + masterPrivateIp: cluster.masterPrivateIp ?? undefined, + }; +} + +function workerNodeMetadata( + index: number, + haRole: HaNodeRole | null, + joinLater: boolean, +): Record { + const metadata: Record = { + workerIndex: index, + ...haRoleMetadata(haRole), + }; + if (joinLater) metadata.joinLater = true; + return metadata; +} + +interface WorkerNodeOptions { + providerFirewallIds?: string[]; + serverType?: string | null; + region?: string | null; + haRole?: HaNodeRole | null; + captureInstallLog?: boolean; + /** Bought with the cluster: prepares itself and waits to be told where to join. */ + joinLater?: boolean; +} @Injectable() export class ClusterOrchestrationService { @@ -92,10 +150,12 @@ export class ClusterOrchestrationService { /** * Create master node for the cluster */ + /** `location` pins the server to one zone of the cluster's region; the region otherwise. */ async createMasterNode( cluster: ClusterEntity, operationId: string, providerFirewallIds?: string[], + location: string = cluster.region, ): Promise { this.logger.log(`Creating master node for cluster ${cluster.name}`); @@ -111,6 +171,7 @@ export class ClusterOrchestrationService { const { caPublicKey, caPrivateKey } = await this.loadCaKeyPair(); const controlClusterIp = await this.resolveControlClusterIp(cluster); + const topology = this.topologyOf(cluster.provider, location); // Create node record FIRST so we have the node.id for SERVER_ID in cloud-init const serverName = `${cluster.name}-master`; @@ -118,6 +179,9 @@ export class ClusterOrchestrationService { cluster, serverName, NodeType.MASTER, + {}, + null, + location, ); // Resolve Flui shared storage config (NFS+fscache, see scaling doc §14). @@ -142,7 +206,7 @@ export class ClusterOrchestrationService { controlClusterIp, ]); - const masterScript = await this.k3sScriptService.generateMasterScript({ + const masterConfig: K3sMasterConfig = { serverId: node.id, // IMPORTANT: Pass database node ID for observability metrics hostFirewall, wgAddress: overlay.address, @@ -161,6 +225,7 @@ export class ClusterOrchestrationService { instanceId: serverName, instanceName: serverName, provider: cluster.provider, + topology, caPublicKey, caPrivateKey, deployObservabilityStack: isControlClusterType(cluster.clusterType), @@ -175,7 +240,7 @@ export class ClusterOrchestrationService { privateNetworks: sharedStoragePrivateNetworks, } : undefined, - }); + }; // Bootstrap key for CA enrollment: used only for initial server access to // install the CA public key. @@ -210,34 +275,10 @@ export class ClusterOrchestrationService { ); } - // Re-generate master script with bootstrap key if needed (cloud-init injection) - const finalMasterScript = bootstrapPublicKeyForCloudInit - ? await this.k3sScriptService.generateMasterScript({ - serverId: node.id, - hostFirewall, - clusterId: cluster.id, - clusterName: cluster.name, - k3sToken, - k3sVersion: cluster.k3sVersion, - instanceId: serverName, - instanceName: serverName, - provider: cluster.provider, - caPublicKey, - caPrivateKey, - deployObservabilityStack: isControlClusterType(cluster.clusterType), - controlClusterIp, - deployMonitoringAgent: - cluster.clusterType === ClusterType.WORKLOAD && !!controlClusterIp, - bootstrapPublicKey: bootstrapPublicKeyForCloudInit, - sharedStorage: sharedStorageEnabled - ? { - enabled: true, - volumeSizeGb: sharedStorageVolumeSizeGb, - privateNetworks: sharedStoragePrivateNetworks, - } - : undefined, - }) - : masterScript; + const finalMasterScript = await this.k3sScriptService.generateMasterScript({ + ...masterConfig, + bootstrapPublicKey: bootstrapPublicKeyForCloudInit, + }); // Generate labels for cluster master node const labels = this.labelService.generateServerLabels({ @@ -266,7 +307,7 @@ export class ClusterOrchestrationService { name: serverName, provider: cluster.provider as CloudProvider, server_type: cluster.nodeSize, - location: cluster.region, + location, region: cluster.region, size: cluster.nodeSize, image: cluster.image || 'ubuntu-22.04', @@ -291,7 +332,7 @@ export class ClusterOrchestrationService { provider: cluster.provider as CloudProvider, server_type: cluster.nodeSize, image: cluster.image || 'ubuntu-24.04', - location: cluster.region, + location, ssh_keys: localSSHKeyIds.length > 0 ? localSSHKeyIds : undefined, user_data: finalMasterScript, cluster_name: cluster.name, @@ -567,9 +608,11 @@ export class ClusterOrchestrationService { providerFirewallIds?: string[], serverType?: string | null, region?: string | null, + haRole?: HaNodeRole | null, ): Promise { + const role = haRole ? ` (${haRole})` : ''; this.logger.log( - `Creating ${count} worker nodes for cluster ${cluster.name}`, + `Creating ${count} worker nodes for cluster ${cluster.name}${role}`, ); const k3sToken = this.encryptionService.decrypt(cluster.k3sTokenEncrypted); @@ -607,12 +650,15 @@ export class ClusterOrchestrationService { k3sToken, masterJoinIp, operationId, - providerFirewallIds, - serverType, - region, - // One log per operation: several workers booting at once would - // interleave into the same stream at conflicting offsets. - count === 1, + { + providerFirewallIds, + serverType, + region, + haRole: haRole ?? null, + // One log per operation: several workers booting at once would + // interleave into the same stream at conflicting offsets. + captureInstallLog: count === 1, + }, ), ); } @@ -630,7 +676,45 @@ export class ClusterOrchestrationService { return workers; } - private async nextWorkerIndex(cluster: ClusterEntity): Promise { + /** Mints the cluster's bootstrap key once, before several nodes are bought at the same time. */ + async prepareBootstrapKey(cluster: ClusterEntity): Promise { + await this.ensureBootstrapKey(cluster, 'master'); + } + + /** Buys one server of a cluster across three locations that waits to be told where to join. */ + async buyWorkerToJoinLater( + cluster: ClusterEntity, + index: number, + server: PlannedServer, + operationId: string, + providerFirewallIds: string[], + ): Promise { + return this.createWorkerNode(cluster, index, '', '', operationId, { + providerFirewallIds, + serverType: server.shape, + region: server.zone, + haRole: 'server', + joinLater: true, + }); + } + + /** Waits for a server told where to join to be in the control plane. */ + async awaitServerJoin( + cluster: ClusterEntity, + node: ClusterNodeEntity, + operationId: string, + ): Promise { + return this.awaitJoin(cluster, node, 'server', operationId, { + timeoutMs: SERVER_JOIN_TIMEOUT_MS, + }); + } + + /** The key every node of the cluster is first reached with over SSH. */ + async bootstrapPrivateKey(cluster: ClusterEntity): Promise { + return (await this.ensureBootstrapKey(cluster, 'worker')).privateKey; + } + + async nextWorkerIndex(cluster: ClusterEntity): Promise { const rows = await this.nodeRepository.find({ where: { clusterId: cluster.id }, select: ['serverName'], @@ -653,14 +737,19 @@ export class ClusterOrchestrationService { k3sToken: string, masterIp: string, operationId: string, - providerFirewallIds?: string[], - serverType?: string | null, - region?: string | null, - captureInstallLog = false, + { + providerFirewallIds, + serverType, + region, + haRole = null, + captureInstallLog = false, + joinLater = false, + }: WorkerNodeOptions = {}, ): Promise { const serverName = `${cluster.name}-worker-${index}`; // Anywhere the cluster's private network reaches; the caller checked that. const location = region || cluster.region; + const topology = this.topologyOf(cluster.provider, location); // A fleet stops being uniform the moment something chooses a shape per // node, so the shape travels with the request; the cluster's own size is // the default for every caller that never had one to give. @@ -679,23 +768,20 @@ export class ClusterOrchestrationService { cluster, serverName, NodeType.WORKER, - { workerIndex: index }, + workerNodeMetadata(index, haRole, joinLater), shape, location, ); + const k3sRole = k3sRoleOf(haRole); + // A retried creation finds the machine it already bought waiting. + if (joinLater && node.providerResourceId) return node; const controlClusterIp = await this.resolveWorkerObservabilityIp(cluster); // Resolve Flui shared storage config for the worker (NFS+fscache, §14). // Workers mount the master's NFS export and run cachefilesd. We need // master's private IP for the mount. - const workerSharedStorage = - cluster.sharedStorageEnabled !== false && cluster.masterPrivateIp - ? { - enabled: true, - masterPrivateIp: cluster.masterPrivateIp, - } - : undefined; + const workerSharedStorage = workerSharedStorageOf(cluster, joinLater); const overlay = await this.overlayPlanFor(cluster, node.id); const overlayConfig = { @@ -720,10 +806,13 @@ export class ClusterOrchestrationService { clusterName: cluster.name, k3sToken, masterIp, + k3sRole, + joinLater, k3sVersion: cluster.k3sVersion, instanceId: serverName, instanceName: serverName, provider: cluster.provider, + topology, caPublicKey, // Multi-cluster observability controlClusterIp, @@ -768,10 +857,13 @@ export class ClusterOrchestrationService { clusterName: cluster.name, k3sToken, masterIp, + k3sRole, + joinLater, k3sVersion: cluster.k3sVersion, instanceId: serverName, instanceName: serverName, provider: cluster.provider, + topology, caPublicKey, controlClusterIp, bootstrapPublicKey: workerBootstrapPublicKeyForCloudInit, @@ -922,37 +1014,56 @@ export class ClusterOrchestrationService { nodeType: node.nodeType, }); + if (joinLater) { + this.logger.log( + `Server ${serverName} bought (privateIp=${node.privateIp ?? 'none'}); it waits to be told where to join`, + ); + return node; + } this.logger.log( `Worker node ${serverName} provisioned (privateIp=${node.privateIp ?? 'none'}). Waiting for K3s join...`, ); - await this.updateOperationProgress( - operationId, - 60, - `Server ${serverName} is up; waiting for it to join the cluster...`, - ); - const logHost = installLogTarget({ + return this.awaitJoin(cluster, node, k3sRole, operationId, { + tailWith: captureInstallLog ? bootstrapKey.privateKey : undefined, + }); + } + + /** Where Flui reaches a node over SSH from where this API runs. */ + nodeSshHost(cluster: ClusterEntity, node: ClusterNodeEntity): string | null { + return installLogTarget({ publicIp: node.ipAddress, privateIp: node.privateIp, controlCluster: isControlClusterType(cluster.clusterType), apiInCluster: apiRunsInCluster(), }); + } + + private async awaitJoin( + cluster: ClusterEntity, + node: ClusterNodeEntity, + k3sRole: 'server' | 'agent', + operationId: string, + { tailWith, timeoutMs }: { tailWith?: string; timeoutMs?: number } = {}, + ): Promise { + const serverName = node.serverName; + await this.updateOperationProgress( + operationId, + 60, + `Server ${serverName} is up; waiting for it to join the cluster...`, + ); + const logHost = this.nodeSshHost(cluster, node); const tail = - captureInstallLog && logHost - ? () => - this.tailInstallLog( - operationId, - cluster.id, - logHost, - bootstrapKey.privateKey, - ) + tailWith && logHost + ? () => this.tailInstallLog(operationId, cluster.id, logHost, tailWith) : undefined; + let joinedLabels: Record | undefined; try { - await this.waitForNodeReady( + joinedLabels = await this.waitForNodeReady( cluster, serverName, - undefined, + timeoutMs, undefined, tail, ); @@ -975,12 +1086,56 @@ export class ClusterOrchestrationService { } await tail?.(); + await this.refuseUnlessServer( + node, + k3sRole, + await this.labelsOnceServer(cluster, serverName, k3sRole, joinedLabels), + ); node.status = NodeStatus.READY; await this.nodeRepository.save(node); this.logger.log(`Worker node ${serverName} joined cluster and is Ready`); return node; } + /** + * K3s labels a server as part of the control plane shortly after the node + * turns Ready, so a server read at that instant can look like an agent. + */ + private async labelsOnceServer( + cluster: ClusterEntity, + nodeName: string, + k3sRole: 'server' | 'agent', + labels: Record | undefined, + ): Promise | undefined> { + let current = labels; + const deadline = Date.now() + SERVER_LABEL_WAIT_MS; + while ( + serverJoinRefusal(nodeName, k3sRole, current) && + Date.now() < deadline + ) { + await this.sleep(SERVER_LABEL_POLL_MS); + current = await this.waitForNodeReady(cluster, nodeName); + } + return current; + } + + private async refuseUnlessServer( + node: ClusterNodeEntity, + k3sRole: 'server' | 'agent', + labels: Record | undefined, + ): Promise { + const refusal = serverJoinRefusal(node.serverName, k3sRole, labels); + if (!refusal) return; + const { haRole: _dropped, ...rest } = node.metadata ?? {}; + node.status = NodeStatus.ERROR; + node.metadata = { + ...rest, + joinError: { message: refusal, timestamp: new Date().toISOString() }, + }; + await this.nodeRepository.save(node); + throw new Error(refusal); + } + /** * Poll the cluster until the named node appears Ready in the Kubernetes API. * Throws an explicit error on timeout — caller should surface it as the @@ -992,7 +1147,7 @@ export class ClusterOrchestrationService { timeoutMs = 240000, intervalMs = 10000, onTick?: () => Promise, - ): Promise { + ): Promise> { if (!cluster.kubeconfigEncrypted) { throw new Error( `Cluster ${cluster.name} has no kubeconfig — cannot verify node ${nodeName} join`, @@ -1016,7 +1171,7 @@ export class ClusterOrchestrationService { this.logger.log( `Node ${nodeName} joined cluster and is Ready (took ~${Math.round((Date.now() - (deadline - timeoutMs)) / 1000)}s)`, ); - return; + return labelsOf(node); } lastErr = ready ? `Ready=${ready.status} (${ready.reason ?? 'no reason'}: ${ready.message ?? ''})` @@ -1443,6 +1598,17 @@ export class ClusterOrchestrationService { }; } + private topologyOf( + provider: string, + location?: string | null, + ): NodeTopology | null { + const cloud = provider as CloudProvider; + if (!this.capabilitiesFactory.isProviderSupported(cloud)) return null; + return this.capabilitiesFactory + .getCapabilitiesService(cloud) + .nodeTopology(location); + } + /** * Idempotent by (clusterId, serverName): the queue re-runs node creation from * the top on retry, while server creation is idempotent by name. A blind insert diff --git a/src/modules/infrastructure/clusters/services/cluster-scaling.service.spec.ts b/src/modules/infrastructure/clusters/services/cluster-scaling.service.spec.ts index 0ec2916c..5f21e102 100644 --- a/src/modules/infrastructure/clusters/services/cluster-scaling.service.spec.ts +++ b/src/modules/infrastructure/clusters/services/cluster-scaling.service.spec.ts @@ -1,3 +1,5 @@ +jest.mock('@kubernetes/client-node', () => ({})); + import { ClusterBoundsRegistry } from './cluster-bounds.registry'; import { BadRequestException, diff --git a/src/modules/infrastructure/clusters/services/cluster-scaling.service.ts b/src/modules/infrastructure/clusters/services/cluster-scaling.service.ts index 83e090fb..b9d72a52 100644 --- a/src/modules/infrastructure/clusters/services/cluster-scaling.service.ts +++ b/src/modules/infrastructure/clusters/services/cluster-scaling.service.ts @@ -29,6 +29,7 @@ import { import { ByosNodeRemovalService } from './byos-node-removal.service'; import { CloudProvider } from '../../../providers/enums/cloud-provider.enum'; import { NodeLifeEventsService } from './node-life-events.service'; +import { HaNodeRole } from '../utils/ha-node-role'; import { ProviderFactory } from '../../../providers/core/factories/provider.factory'; import { ARM_SERVER_WARNING_CODE, @@ -47,6 +48,8 @@ export interface AddWorkerJobData { * has to be one the cluster's private network reaches. */ region?: string | null; + /** Added by an operator for a role in a cluster across three locations. */ + haRole?: HaNodeRole | null; } export interface RemoveWorkerJobData { @@ -150,6 +153,7 @@ export class ClusterScalingService { serverType?: string | null, region?: string | null, by?: string | null, + haRole?: HaNodeRole | null, ): Promise { if (count < 1 || count > MAX_WORKERS_PER_CALL) { throw new BadRequestException( @@ -226,6 +230,7 @@ export class ClusterScalingService { providerFirewallIds, serverType: serverType ?? cluster.nodeSize, region: region ?? cluster.region, + ...(haRole && { haRole }), operationSteps: steps, estimatedDurationInSeconds: 240 * count, initiatedBy: by ?? null, @@ -245,6 +250,7 @@ export class ClusterScalingService { providerFirewallIds, serverType: serverType ?? null, region: region ?? null, + haRole: haRole ?? null, }; await this.infrastructureQueue.add('add-worker', jobData, { diff --git a/src/modules/infrastructure/clusters/services/cluster-storage.service.ts b/src/modules/infrastructure/clusters/services/cluster-storage.service.ts index 1ae9b96f..3eb235d3 100644 --- a/src/modules/infrastructure/clusters/services/cluster-storage.service.ts +++ b/src/modules/infrastructure/clusters/services/cluster-storage.service.ts @@ -17,6 +17,7 @@ import { FLUI_SHARED_VOLUME_FS_LABEL, FluiStorageClass, NFS_EXPORT_OPTIONS, + nfsDurability, NFS_MOUNT_OPTIONS, } from '../constants/storage-conventions'; import { CloudProvider } from '../../../providers/enums/cloud-provider.enum'; @@ -67,6 +68,7 @@ export class ClusterStorageService { exportPath: FLUI_SHARED_STORAGE_PATH, exportOptions: NFS_EXPORT_OPTIONS, mountOptions: NFS_MOUNT_OPTIONS, + ...nfsDurability(NFS_EXPORT_OPTIONS), }; result.volume = volume; diff --git a/src/modules/infrastructure/clusters/services/cluster-validation.service.spec.ts b/src/modules/infrastructure/clusters/services/cluster-validation.service.spec.ts index 3fcba6ea..32b868f2 100644 --- a/src/modules/infrastructure/clusters/services/cluster-validation.service.spec.ts +++ b/src/modules/infrastructure/clusters/services/cluster-validation.service.spec.ts @@ -126,6 +126,8 @@ describe('ClusterValidationService.checkNameAvailability', () => { const whereArg = clusterRepo.findOne.mock.calls[0][0].where; expect(whereArg.status._type).toBe('not'); expect(whereArg.status._value).toBe(ClusterStatus.DELETED); + // A destroyed cluster a late job left in `error`. + expect(whereArg.deletedAt._type).toBe('isNull'); }); it('reuses the provider server list across calls instead of re-scanning per candidate name', async () => { diff --git a/src/modules/infrastructure/clusters/services/cluster-validation.service.ts b/src/modules/infrastructure/clusters/services/cluster-validation.service.ts index f2968733..84c4be3d 100644 --- a/src/modules/infrastructure/clusters/services/cluster-validation.service.ts +++ b/src/modules/infrastructure/clusters/services/cluster-validation.service.ts @@ -5,7 +5,7 @@ import { Logger, } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { Repository, Not } from 'typeorm'; +import { Repository, Not, IsNull } from 'typeorm'; import { ClusterEntity, ClusterStatus } from '../entities/cluster.entity'; import { CreateClusterDto } from '../dto/create-cluster.dto'; import { NameAvailabilityResponseDto } from '../dto/name-availability.dto'; @@ -52,8 +52,9 @@ export class ClusterValidationService { name: string, provider: CloudProvider, ): Promise { + // A destroyed cluster frees its name even when a late job rewrote its status. const existingCluster = await this.clusterRepository.findOne({ - where: { name, status: Not(ClusterStatus.DELETED) }, + where: { name, status: Not(ClusterStatus.DELETED), deletedAt: IsNull() }, }); if (existingCluster) { return { diff --git a/src/modules/infrastructure/clusters/services/flui-local-direct-io.reconciler.spec.ts b/src/modules/infrastructure/clusters/services/flui-local-direct-io.reconciler.spec.ts new file mode 100644 index 00000000..923e3e49 --- /dev/null +++ b/src/modules/infrastructure/clusters/services/flui-local-direct-io.reconciler.spec.ts @@ -0,0 +1,160 @@ +jest.mock('@kubernetes/client-node', () => ({})); + +import * as yaml from 'js-yaml'; +import { Repository } from 'typeorm'; +import { + DIRECT_IO_ANNOTATION, + FluiLocalDirectIoReconciler, + directIoHostScript, + directIoJobManifest, + parseDirectIoOutput, +} from './flui-local-direct-io.reconciler'; +import { ClusterEntity } from '../entities/cluster.entity'; +import { KubernetesService } from '../../shared/services/kubernetes.service'; +import { EncryptionService } from '../../../shared/encryption/services/encryption.service'; + +describe('what a node says about direct I/O on its flui-local loop', () => { + it('is on once the loop reports it', () => { + expect(parseDirectIoOutput('noise\nDIO /dev/loop3 1\n')).toEqual({ + kind: 'on', + device: '/dev/loop3', + }); + }); + + it('is still off when the switch did not take', () => { + expect(parseDirectIoOutput('DIO /dev/loop3 0')).toEqual({ + kind: 'off', + device: '/dev/loop3', + }); + }); + + /** A node whose flui-local is a Volume or a plain folder has nothing to switch. */ + it('is not a loop when flui-local sits on something else', () => { + expect(parseDirectIoOutput('NOTLOOP /dev/sdb')).toEqual({ + kind: 'not-a-loop', + source: '/dev/sdb', + }); + }); + + it('is unknown when the job printed neither', () => { + expect(parseDirectIoOutput('losetup: failed').kind).toBe('unknown'); + }); +}); + +describe('the job that switches it on', () => { + const manifest = yaml.load(directIoJobManifest('job-1', 'lab-worker-2')) as { + metadata: { namespace: string }; + spec: { + template: { + spec: { + hostPID: boolean; + nodeSelector: Record; + containers: Array<{ + securityContext: { privileged: boolean }; + args: string[]; + }>; + }; + }; + }; + }; + const pod = manifest.spec.template.spec; + + it('runs privileged in the host pid namespace on the one node it is for', () => { + expect(manifest.metadata.namespace).toBe('flui-local-storage'); + expect(pod.hostPID).toBe(true); + expect(pod.nodeSelector).toEqual({ + 'kubernetes.io/hostname': 'lab-worker-2', + }); + expect(pod.containers[0].securityContext.privileged).toBe(true); + }); + + /** The heredocs only close if their terminators survive the YAML block at column zero. */ + it('carries the host script intact through the YAML block', () => { + const args = pod.containers[0].args[0]; + expect(args).toContain(directIoHostScript()); + for (const end of ['FLUIPROBE', 'FLUIHELPER', 'FLUIUNIT']) { + expect(args.split('\n')).toContain(end); + } + }); + + it('installs the same helper and unit the bootstrap scripts do', () => { + const script = directIoHostScript(); + expect(script).toContain('/usr/local/sbin/flui-local-direct-io'); + expect(script).toContain('RequiresMountsFor=/var/lib/flui/local'); + expect(script).toContain('systemctl enable flui-local-direct-io.service'); + }); +}); + +describe('switching direct I/O on for nodes that booted without it', () => { + function setup( + annotations: Record>, + outputs: Record, + ) { + const annotated: Array<{ node: string; value: string }> = []; + const kubernetes = { + listNodeAnnotations: jest.fn( + async () => new Map(Object.entries(annotations)), + ), + addNodeAnnotations: jest.fn( + async (_k: string, node: string, a: Record) => { + annotated.push({ node, value: a[DIRECT_IO_ANNOTATION] }); + }, + ), + } as unknown as KubernetesService; + const clusters = { + findOne: jest.fn(async () => ({ + id: 'c1', + name: 'lab', + kubeconfigEncrypted: 'enc', + })), + } as unknown as Repository; + const encryption = { + decrypt: () => 'kubeconfig', + } as unknown as EncryptionService; + const reconciler = new FluiLocalDirectIoReconciler( + clusters, + kubernetes, + encryption, + ); + Object.assign(reconciler, { + runOnNode: jest.fn(async (_k: string, node: string) => { + const out = outputs[node]; + if (out === undefined) throw new Error('job failed'); + return out; + }), + }); + return { reconciler, annotated }; + } + + it('marks a node once its loop is on, and one with no loop as settled', async () => { + const { reconciler, annotated } = setup( + { 'lab-master': {}, 'lab-worker-1': {} }, + { 'lab-master': 'DIO /dev/loop3 1', 'lab-worker-1': 'NOTLOOP none' }, + ); + await reconciler.reconcile('c1'); + expect(annotated).toEqual([ + { node: 'lab-master', value: 'on' }, + { node: 'lab-worker-1', value: 'not-a-loop' }, + ]); + }); + + it('runs nothing on a node already marked', async () => { + const { reconciler, annotated } = setup( + { 'lab-master': { [DIRECT_IO_ANNOTATION]: 'on' } }, + {}, + ); + expect(await reconciler.reconcile('c1')).toEqual([]); + expect(annotated).toEqual([]); + }); + + /** Left unmarked, the node is tried again on the next run. */ + it('does not mark a node where the switch failed or did not take', async () => { + const { reconciler, annotated } = setup( + { 'lab-master': {}, 'lab-worker-1': {} }, + { 'lab-worker-1': 'DIO /dev/loop3 0' }, + ); + const results = await reconciler.reconcile('c1'); + expect(annotated).toEqual([]); + expect(results.map((r) => r.state.kind)).toEqual(['unknown', 'off']); + }); +}); diff --git a/src/modules/infrastructure/clusters/services/flui-local-direct-io.reconciler.ts b/src/modules/infrastructure/clusters/services/flui-local-direct-io.reconciler.ts new file mode 100644 index 00000000..8612007a --- /dev/null +++ b/src/modules/infrastructure/clusters/services/flui-local-direct-io.reconciler.ts @@ -0,0 +1,237 @@ +import { randomUUID } from 'node:crypto'; +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import * as k8s from '@kubernetes/client-node'; +import { ClusterEntity } from '../entities/cluster.entity'; +import { KubernetesService } from '../../shared/services/kubernetes.service'; +import { EncryptionService } from '../../../shared/encryption/services/encryption.service'; + +const FLUI_LOCAL_PATH = '/var/lib/flui/local'; +const JOB_NAMESPACE = 'flui-local-storage'; +const JOB_TIMEOUT_MS = 180_000; +const JOB_POLL_MS = 2_000; +export const DIRECT_IO_ANNOTATION = 'flui.cloud/local-direct-io'; + +export type DirectIoState = + | { kind: 'on'; device: string } + | { kind: 'not-a-loop'; source: string } + | { kind: 'off'; device: string } + | { kind: 'unknown'; detail: string }; + +export interface NodeDirectIoResult { + node: string; + state: DirectIoState; +} + +/** + * The same helper and unit the bootstrap scripts install, so a node fixed here + * ends up exactly like one that booted with them. Runs in the host's mount + * namespace, where `findmnt` sees the loop device and not a bind mount. + */ +export function directIoHostScript(): string { + return [ + 'set -e', + `ROOT=${FLUI_LOCAL_PATH}`, + 'DEV=$(findmnt -n -o SOURCE "$ROOT" 2>/dev/null || true)', + 'case "$DEV" in /dev/loop*) ;; *) echo "NOTLOOP ${DEV:-none}"; exit 0 ;; esac', + "cat > /usr/local/sbin/flui-local-direct-io <<'FLUIHELPER'", + '#!/bin/sh', + `dev=$(findmnt -n -o SOURCE "${FLUI_LOCAL_PATH}" 2>/dev/null)`, + 'case "$dev" in /dev/loop*) exec losetup --direct-io=on "$dev" ;; esac', + 'FLUIHELPER', + 'chmod 755 /usr/local/sbin/flui-local-direct-io', + "cat > /etc/systemd/system/flui-local-direct-io.service <<'FLUIUNIT'", + '[Unit]', + 'Description=Direct I/O for the flui-local loop device', + `RequiresMountsFor=${FLUI_LOCAL_PATH}`, + '', + '[Service]', + 'Type=oneshot', + 'ExecStart=/usr/local/sbin/flui-local-direct-io', + 'RemainAfterExit=yes', + '', + '[Install]', + 'WantedBy=multi-user.target', + 'FLUIUNIT', + 'systemctl daemon-reload >/dev/null 2>&1 || true', + 'systemctl enable flui-local-direct-io.service >/dev/null 2>&1 || true', + '/usr/local/sbin/flui-local-direct-io', + 'echo "DIO $DEV $(cat /sys/block/${DEV#/dev/}/loop/dio)"', + ].join('\n'); +} + +export function parseDirectIoOutput(logs: string): DirectIoState { + for (const raw of logs.split('\n')) { + const line = raw.trim(); + const notLoop = /^NOTLOOP (.+)$/.exec(line); + if (notLoop) return { kind: 'not-a-loop', source: notLoop[1] }; + const dio = /^DIO (\/dev\/loop\d+) ([01])$/.exec(line); + if (dio) { + return dio[2] === '1' + ? { kind: 'on', device: dio[1] } + : { kind: 'off', device: dio[1] }; + } + } + return { kind: 'unknown', detail: logs.trim().slice(-200) }; +} + +/** + * Turns direct I/O on for the `flui-local` loop of nodes that booted before + * the bootstrap scripts did it. Without it every write crosses the host's page + * cache a second time and sequential writes stall for over a second; on a + * mounted loop the change takes effect at once, with nothing to move. + * + * A node is marked once it is settled, so each node runs the job once. + */ +@Injectable() +export class FluiLocalDirectIoReconciler { + private readonly logger = new Logger(FluiLocalDirectIoReconciler.name); + + constructor( + @InjectRepository(ClusterEntity) + private readonly clusters: Repository, + private readonly kubernetes: KubernetesService, + private readonly encryption: EncryptionService, + ) {} + + async reconcile(clusterId: string): Promise { + const cluster = await this.clusters.findOne({ where: { id: clusterId } }); + if (!cluster?.kubeconfigEncrypted) return []; + const kubeconfig = this.encryption.decrypt(cluster.kubeconfigEncrypted); + + const annotations = await this.kubernetes.listNodeAnnotations(kubeconfig); + const results: NodeDirectIoResult[] = []; + for (const [node, current] of annotations) { + if (current[DIRECT_IO_ANNOTATION]) continue; + let state: DirectIoState; + try { + state = parseDirectIoOutput(await this.runOnNode(kubeconfig, node)); + } catch (error) { + state = { kind: 'unknown', detail: (error as Error).message }; + } + results.push({ node, state }); + if (state.kind === 'on' || state.kind === 'not-a-loop') { + await this.kubernetes.addNodeAnnotations(kubeconfig, node, { + [DIRECT_IO_ANNOTATION]: state.kind, + }); + } else { + const detail = + state.kind === 'off' + ? `${state.device} still without direct I/O` + : state.detail; + this.logger.warn( + `[local-direct-io] ${cluster.name}/${node}: ${detail}`, + ); + } + } + const changed = results.filter((r) => r.state.kind === 'on'); + if (changed.length) { + this.logger.log( + `[local-direct-io] ${cluster.name}: direct I/O on ${changed.map((r) => r.node).join(', ')}`, + ); + } + return results; + } + + private async runOnNode(kubeconfig: string, node: string): Promise { + const jobName = `flui-local-direct-io-${Date.now()}-${randomUUID().slice(0, 5)}`; + await this.kubernetes.applyManifest( + kubeconfig, + directIoJobManifest(jobName, node), + ); + try { + await this.awaitJob(kubeconfig, jobName); + return await this.readJobLogs(kubeconfig, jobName); + } finally { + await this.kubernetes + .deleteResource(kubeconfig, 'Job', jobName, JOB_NAMESPACE) + .catch(() => undefined); + } + } + + private async awaitJob(kubeconfig: string, jobName: string): Promise { + const start = Date.now(); + while (Date.now() - start < JOB_TIMEOUT_MS) { + const job = await this.kubernetes.getResource( + kubeconfig, + 'Job', + jobName, + JOB_NAMESPACE, + ); + if ((job?.status?.succeeded ?? 0) > 0) return; + if ((job?.status?.failed ?? 0) > 0) { + throw new Error(`direct I/O job ${jobName} failed`); + } + await new Promise((r) => setTimeout(r, JOB_POLL_MS)); + } + throw new Error(`direct I/O job ${jobName} timed out`); + } + + private async readJobLogs( + kubeconfig: string, + jobName: string, + ): Promise { + const coreApi = this.kubernetes + .makeKubeConfig(kubeconfig) + .makeApiClient(k8s.CoreV1Api); + const pods = await coreApi.listNamespacedPod({ + namespace: JOB_NAMESPACE, + labelSelector: `flui-local-direct-io-job=${jobName}`, + }); + const podName = pods.items?.[0]?.metadata?.name; + if (!podName) throw new Error(`no pod found for job ${jobName}`); + return this.kubernetes.getPodLogs(kubeconfig, podName, JOB_NAMESPACE); + } +} + +export function directIoJobManifest(jobName: string, node: string): string { + const script = [ + 'apk add --no-cache util-linux >/dev/null 2>&1 || true', + "SCRIPT=$(cat <<'FLUIPROBE'", + directIoHostScript(), + 'FLUIPROBE', + ')', + 'nsenter -t 1 -m -- /bin/sh -c "$SCRIPT"', + ] + .join('\n') + .split('\n') + .map((line) => ` ${line}`) + .join('\n'); + return [ + 'apiVersion: batch/v1', + 'kind: Job', + 'metadata:', + ` name: ${jobName}`, + ` namespace: ${JOB_NAMESPACE}`, + ' labels:', + ' flui.cloud/managed-by: flui-cloud', + ' flui-resource-type: local-direct-io', + 'spec:', + ' ttlSecondsAfterFinished: 60', + ' backoffLimit: 0', + ' template:', + ' metadata:', + ' labels:', + ' flui.cloud/managed-by: flui-cloud', + ' flui-resource-type: local-direct-io', + ` flui-local-direct-io-job: ${jobName}`, + ' spec:', + ' restartPolicy: Never', + ' hostPID: true', + ' nodeSelector:', + ` kubernetes.io/hostname: ${node}`, + ' tolerations:', + ' - operator: Exists', + ' containers:', + ' - name: direct-io', + ' image: alpine:3.20', + ' securityContext:', + ' privileged: true', + ' command: ["sh","-c"]', + ' args:', + ' - |', + script, + '', + ].join('\n'); +} diff --git a/src/modules/infrastructure/clusters/services/ha-creation-plan.service.spec.ts b/src/modules/infrastructure/clusters/services/ha-creation-plan.service.spec.ts new file mode 100644 index 00000000..f5b76637 --- /dev/null +++ b/src/modules/infrastructure/clusters/services/ha-creation-plan.service.spec.ts @@ -0,0 +1,56 @@ +import { HaCreationPlanService } from './ha-creation-plan.service'; +import { CloudProvider } from '../../../providers/enums/cloud-provider.enum'; + +describe('HaCreationPlanService on a region with three zones', () => { + const zones = ['fr-par-1', 'fr-par-2', 'fr-par-3']; + const service = new HaCreationPlanService( + { + getCapabilitiesService: () => ({ + nodeTopology: (location: string) => + location === 'fr-par' + ? { zone: 'fr-par-1', region: 'fr-par' } + : { zone: location, region: 'fr-par' }, + haTopologies: () => [ + { + region: 'fr-par', + zones, + dataZones: zones, + validated: true, + measured: false, + }, + ], + }), + } as never, + { + getProvider: () => ({ + getNodeSizes: jest.fn().mockResolvedValue([ + { + name: 'PRO2-XXS', + architecture: 'x86', + availability: [{ location: 'fr-par', available: true, zones }], + }, + ]), + }), + } as never, + ); + + it('starts in the first zone of the region the person picked, with a copy of the data in every zone', async () => { + const plan = await service.preview({ + provider: CloudProvider.SCALEWAY, + region: 'fr-par', + shapes: ['PRO2-XXS'], + completeLater: false, + }); + + expect(plan).toMatchObject({ + possible: true, + region: 'fr-par', + measured: false, + servers: [ + { zone: 'fr-par-1', role: 'data', first: true }, + { zone: 'fr-par-2', role: 'data', first: false }, + { zone: 'fr-par-3', role: 'data', first: false }, + ], + }); + }); +}); diff --git a/src/modules/infrastructure/clusters/services/ha-creation-plan.service.ts b/src/modules/infrastructure/clusters/services/ha-creation-plan.service.ts new file mode 100644 index 00000000..64e4b05d --- /dev/null +++ b/src/modules/infrastructure/clusters/services/ha-creation-plan.service.ts @@ -0,0 +1,184 @@ +import { + BadRequestException, + Injectable, + Logger, + Optional, +} from '@nestjs/common'; +import { CapabilitiesProviderFactory } from '../../../providers/core/factories/capabilities-provider.factory'; +import { ProviderFactory } from '../../../providers/core/factories/provider.factory'; +import { CloudProvider } from '../../../providers/enums/cloud-provider.enum'; +import { shapeNameOf } from '../../../providers/core/catalogue/shape-name'; +import { + HaTopologyResolution, + resolveHaTopology, +} from '../../../providers/core/ha-topology'; +import { CreateClusterDto } from '../dto/create-cluster.dto'; +import { HaPlanResponseDto } from '../dto/ha-creation-plan.dto'; +import { + CatalogueShape, + HaCreationAnswer, + HaCreationPlan, + planHaCreation, +} from '../utils/ha-creation-plan.core'; + +/** + * The size as the provider's catalogue names it: a client may send the id + * (`109` on Hetzner), and every later reader — the scaling group, the node + * list, billing — expects the name. Unchanged when the catalogue is unread. + */ +export async function shapeNameVia( + providers: ProviderFactory | undefined, + provider: string, + nodeSize: string, +): Promise { + if (!nodeSize) return nodeSize; + try { + const service = providers?.getProvider(provider as CloudProvider); + return shapeNameOf(nodeSize, (await service?.getNodeSizes?.(false)) ?? []); + } catch { + return nodeSize; + } +} + +/** Only the forced add of a node accepts a region Flui has not measured, so creating one never offers it. */ +function unsupportedReason( + region: string | undefined, + resolution: HaTopologyResolution, +): string | undefined { + if (region) { + return `Flui has not validated a highly available layout for ${region} yet.`; + } + return 'reason' in resolution ? resolution.reason : undefined; +} + +/** + * What a cluster across three locations is bought as, from the provider's + * declared layout and the sizes it has available now in each location. + */ +@Injectable() +export class HaCreationPlanService { + private readonly logger = new Logger(HaCreationPlanService.name); + + constructor( + private readonly capabilitiesFactory: CapabilitiesProviderFactory, + @Optional() private readonly providers?: ProviderFactory, + ) {} + + /** + * The servers of a cluster across three locations, decided before anything + * is bought: a refusal here leaves nothing behind. + */ + async plan(dto: CreateClusterDto): Promise { + if (dto.highAvailability !== 'three-locations') return null; + const { answer } = await this.answer({ + provider: dto.provider, + region: dto.region, + shapes: dto.shapes?.length ? dto.shapes : [dto.nodeSize], + completeLater: dto.completeLater === true, + workerCount: dto.workerCount, + }); + if ('refusal' in answer) throw new BadRequestException(answer.refusal); + const servers = [answer.plan.first, ...answer.plan.servers] + .map((s) => s.zone + ' ' + s.shape) + .join(', '); + const later = answer.plan.later.length + ? '; later: ' + answer.plan.later.join(', ') + : ''; + this.logger.log( + `Cluster ${dto.name} across three locations: ${servers}${later}`, + ); + return answer.plan; + } + + /** What creating a cluster across three locations would buy now, without buying it. */ + async preview(query: { + provider: CloudProvider; + region: string; + shapes: string[]; + completeLater: boolean; + }): Promise { + const { answer, dataZones, measured } = await this.answer({ + ...query, + workerCount: 0, + }); + const layoutAvailable = dataZones.length > 0; + if ('refusal' in answer) { + return { + layoutAvailable, + possible: false, + refusal: answer.refusal, + completeLaterHelps: answer.completeLaterHelps === true, + region: null, + measured, + servers: [], + later: [], + }; + } + const role = (zone: string) => + dataZones.includes(zone) ? ('data' as const) : ('vote' as const); + return { + layoutAvailable, + possible: true, + refusal: null, + completeLaterHelps: false, + region: answer.plan.region, + measured, + servers: [ + { + ...answer.plan.first, + role: role(answer.plan.first.zone), + first: true, + }, + ...answer.plan.servers.map((s) => ({ + ...s, + role: role(s.zone), + first: false, + })), + ], + later: answer.plan.later, + }; + } + + private async answer(input: { + provider: CloudProvider; + region: string; + shapes: string[]; + completeLater: boolean; + workerCount: number; + }): Promise<{ + answer: HaCreationAnswer; + dataZones: string[]; + measured: boolean; + }> { + const capabilities = this.capabilitiesFactory.getCapabilitiesService( + input.provider, + ); + const where = capabilities.nodeTopology(input.region); + const region = where?.region; + const resolution = resolveHaTopology( + capabilities.haTopologies(), + region, + [], + ); + const catalogue = (await this.providers + ?.getProvider(input.provider) + .getNodeSizes?.(true)) as CatalogueShape[] | undefined; + const shapes = await Promise.all( + input.shapes.map((s) => shapeNameVia(this.providers, input.provider, s)), + ); + const answer = planHaCreation({ + topology: resolution.topology, + unsupported: unsupportedReason(region, resolution), + firstZone: where?.zone ?? input.region, + workerCount: input.workerCount, + shapes, + catalogue: catalogue ?? [], + completeLater: input.completeLater, + }); + return { + answer, + dataZones: resolution.topology?.dataZones ?? [], + measured: resolution.topology?.measured ?? true, + }; + } +} diff --git a/src/modules/infrastructure/clusters/services/ha-nodes.service.spec.ts b/src/modules/infrastructure/clusters/services/ha-nodes.service.spec.ts new file mode 100644 index 00000000..93fab94e --- /dev/null +++ b/src/modules/infrastructure/clusters/services/ha-nodes.service.spec.ts @@ -0,0 +1,59 @@ +jest.mock('@kubernetes/client-node', () => ({})); + +import { BadRequestException } from '@nestjs/common'; +import { ClusterHaService } from '../../../cluster-ha/services/cluster-ha.service'; +import { ClusterScalingService } from './cluster-scaling.service'; +import { HaNodesService } from './ha-nodes.service'; + +function setup() { + const addWorkers = jest.fn(async () => ({ id: 'op-1' })); + const service = new HaNodesService( + { + inspect: jest.fn(async () => ({ + cluster: { name: 'lab' }, + state: { + unsupported: null, + topology: { + zones: [ + { zone: 'fsn1', role: 'data', servers: ['m'], dataHosts: ['m'] }, + { zone: 'nbg1', role: 'data', servers: [], dataHosts: ['w1'] }, + { zone: 'hel1', role: 'vote', servers: [], dataHosts: [] }, + ], + }, + missing: [{ role: 'server', zone: 'hel1', why: '' }], + }, + })), + } as unknown as ClusterHaService, + { addWorkers } as unknown as ClusterScalingService, + ); + return { service, addWorkers }; +} + +describe('HaNodesService', () => { + it('buys one node in the location, joined with its role', async () => { + const { service, addWorkers } = setup(); + await expect( + service.add( + 'c1', + { role: 'server', zone: 'hel1', serverType: 'cax21' }, + 'dawit', + ), + ).resolves.toEqual({ operationId: 'op-1', role: 'server', zone: 'hel1' }); + expect(addWorkers).toHaveBeenCalledWith( + 'c1', + 1, + 'cax21', + 'hel1', + 'dawit', + 'server', + ); + }); + + it('buys nothing when the role is not missing there', async () => { + const { service, addWorkers } = setup(); + await expect( + service.add('c1', { role: 'server', zone: 'fsn1' }, 'dawit'), + ).rejects.toThrow(BadRequestException); + expect(addWorkers).not.toHaveBeenCalled(); + }); +}); diff --git a/src/modules/infrastructure/clusters/services/ha-nodes.service.ts b/src/modules/infrastructure/clusters/services/ha-nodes.service.ts new file mode 100644 index 00000000..8865a9cb --- /dev/null +++ b/src/modules/infrastructure/clusters/services/ha-nodes.service.ts @@ -0,0 +1,48 @@ +import { BadRequestException, Injectable, Logger } from '@nestjs/common'; +import { ClusterHaService } from '../../../cluster-ha/services/cluster-ha.service'; +import { ClusterScalingService } from './cluster-scaling.service'; +import { AddHaNodeDto } from '../dto/add-ha-node.dto'; +import { haNodeRefusal } from '../utils/ha-node-request.core'; + +/** + * An operator adds the node a cluster across three locations still lacks, + * with its role and its location. Nothing adds one on its own: a server + * changes the votes the control plane counts, and the person who pays for + * it decides when. + */ +@Injectable() +export class HaNodesService { + private readonly logger = new Logger(HaNodesService.name); + + constructor( + private readonly clusterHa: ClusterHaService, + private readonly scaling: ClusterScalingService, + ) {} + + async add( + clusterId: string, + dto: AddHaNodeDto, + by: string, + ): Promise<{ operationId: string; role: string; zone: string }> { + const ha = await this.clusterHa.inspect(clusterId, dto.force ?? false); + const refusal = haNodeRefusal( + ha.cluster.name, + ha.state, + dto.role, + dto.zone, + ); + if (refusal) throw new BadRequestException(refusal); + const operation = await this.scaling.addWorkers( + clusterId, + 1, + dto.serverType ?? null, + dto.zone, + by, + dto.role, + ); + this.logger.log( + `[ha] ${ha.cluster.name}: adding a ${dto.role} in ${dto.zone} (operation ${operation.id})`, + ); + return { operationId: operation.id, role: dto.role, zone: dto.zone }; + } +} diff --git a/src/modules/infrastructure/clusters/services/ha-server-join.service.spec.ts b/src/modules/infrastructure/clusters/services/ha-server-join.service.spec.ts new file mode 100644 index 00000000..c85ecec7 --- /dev/null +++ b/src/modules/infrastructure/clusters/services/ha-server-join.service.spec.ts @@ -0,0 +1,306 @@ +// Pulled in transitively and ship ESM that jest won't parse; unused on these paths. +jest.mock('@kubernetes/client-node', () => ({})); +jest.mock('jose', () => ({})); +jest.mock('ip-cidr', () => ({})); + +import { HaServerJoinService, haJoinOutcome } from './ha-server-join.service'; + +describe('HaServerJoinService', () => { + const CLUSTER = { + id: 'c-1', + name: 'ha-hz', + clusterType: 'workload', + masterPrivateIp: '10.0.1.2', + masterIpAddress: '203.0.113.2', + masterNodeId: 'n-master', + k3sTokenEncrypted: 'enc', + }; + + function build({ + orchestration = {}, + nodeRepository = {}, + execCommand = jest.fn(), + }: { + orchestration?: Record; + nodeRepository?: Record; + execCommand?: jest.Mock; + } = {}) { + const service = new HaServerJoinService( + { + prepareBootstrapKey: jest.fn(), + bootstrapPrivateKey: jest.fn().mockResolvedValue('priv'), + nodeSshHost: (_c: unknown, n: { serverName: string }) => n.serverName, + ...orchestration, + } as never, + nodeRepository as never, + { decrypt: () => 'K10token::server:x' } as never, + { execCommand } as never, + ); + Object.assign(service, { + logger: { log: jest.fn(), warn: jest.fn(), error: jest.fn() }, + sleep: jest.fn().mockResolvedValue(undefined), + }); + return service; + } + + describe('buyServersToJoinLater', () => { + it('buys every server at once, each in its location with its size', async () => { + const buyWorkerToJoinLater = jest.fn(async (...args: unknown[]) => ({ + serverName: `ha-hz-worker-${String(args[1])}`, + })); + const service = build({ + orchestration: { + nextWorkerIndex: jest.fn().mockResolvedValue(1), + buyWorkerToJoinLater, + }, + nodeRepository: { find: jest.fn().mockResolvedValue([]) }, + }); + + const result = await service.buyServersToJoinLater( + CLUSTER as never, + [ + { zone: 'nbg1', shape: 'cpx22' }, + { zone: 'hel1', shape: 'cx23' }, + ], + 'op-1', + ['fw-1'], + ); + + expect(buyWorkerToJoinLater.mock.calls.map((c) => [c[1], c[2]])).toEqual([ + [1, { zone: 'nbg1', shape: 'cpx22' }], + [2, { zone: 'hel1', shape: 'cx23' }], + ]); + expect(result.map((r) => r.zone)).toEqual(['nbg1', 'hel1']); + }); + + it('reuses the machine a retried creation already bought for a location', async () => { + const buyWorkerToJoinLater = jest.fn().mockResolvedValue({}); + const service = build({ + orchestration: { + nextWorkerIndex: jest.fn().mockResolvedValue(2), + buyWorkerToJoinLater, + }, + nodeRepository: { + find: jest + .fn() + .mockResolvedValue([ + { region: 'nbg1', metadata: { joinLater: true, workerIndex: 1 } }, + ]), + }, + }); + + await service.buyServersToJoinLater( + CLUSTER as never, + [ + { zone: 'nbg1', shape: 'cpx22' }, + { zone: 'hel1', shape: 'cpx22' }, + ], + 'op-1', + [], + ); + + expect(buyWorkerToJoinLater.mock.calls.map((c) => c[1])).toEqual([1, 2]); + }); + + it('reports a location the provider would not sell, without stopping the others', async () => { + const service = build({ + orchestration: { + nextWorkerIndex: jest.fn().mockResolvedValue(1), + buyWorkerToJoinLater: jest + .fn() + .mockResolvedValueOnce({ serverName: 'w-1' }) + .mockRejectedValueOnce(new Error('resource_unavailable')), + }, + nodeRepository: { find: jest.fn().mockResolvedValue([]) }, + }); + + const result = await service.buyServersToJoinLater( + CLUSTER as never, + [ + { zone: 'nbg1', shape: 'cpx22' }, + { zone: 'hel1', shape: 'cpx22' }, + ], + 'op-1', + [], + ); + + expect(result).toEqual([ + { zone: 'nbg1', node: { serverName: 'w-1' } }, + { zone: 'hel1', error: 'resource_unavailable' }, + ]); + }); + }); + + describe('joinWaitingServers', () => { + const waiting = [ + { + serverName: 'ha-hz-worker-2', + region: 'hel1', + status: 'joining', + providerResourceId: 'p2', + metadata: { joinLater: true, workerIndex: 2 }, + }, + { + serverName: 'ha-hz-worker-1', + region: 'nbg1', + status: 'joining', + providerResourceId: 'p1', + metadata: { joinLater: true, workerIndex: 1 }, + }, + { + serverName: 'ha-hz-worker-9', + region: 'fsn1', + status: 'joining', + providerResourceId: 'p9', + metadata: { workerIndex: 9 }, + }, + ]; + + it('hands each server where to join only after the one before it joined, in the planned order', async () => { + const order: string[] = []; + const execCommand = jest.fn(async (...args: string[]) => { + order.push(`hand ${args[0]}`); + return 'join-details-handed-over'; + }); + const awaitServerJoin = jest.fn(async (...args: unknown[]) => { + const node = args[1] as { serverName: string }; + order.push(`joined ${node.serverName}`); + return node; + }); + const service = build({ + orchestration: { awaitServerJoin }, + nodeRepository: { find: jest.fn().mockResolvedValue(waiting) }, + execCommand, + }); + + const { joined, failed } = await service.joinWaitingServers( + CLUSTER as never, + 'op-1', + ); + + expect(order).toEqual([ + 'hand ha-hz-worker-1', + 'joined ha-hz-worker-1', + 'hand ha-hz-worker-2', + 'joined ha-hz-worker-2', + ]); + expect(joined).toHaveLength(2); + expect(failed).toEqual([]); + const script = execCommand.mock.calls[0][3] as string; + expect(script).toContain("K3S_URL='https://10.0.1.2:6443'"); + expect(script).toContain("FLUI_SHARED_STORAGE_MASTER_IP='10.0.1.2'"); + }); + + it('goes on to the next server when one does not join', async () => { + const service = build({ + orchestration: { + awaitServerJoin: jest + .fn() + .mockRejectedValueOnce(new Error('did not become Ready')) + .mockImplementation(async (_c, node) => node), + }, + nodeRepository: { find: jest.fn().mockResolvedValue(waiting) }, + execCommand: jest.fn().mockResolvedValue('join-details-handed-over'), + }); + + const { joined, failed } = await service.joinWaitingServers( + CLUSTER as never, + 'op-1', + ); + + expect(failed).toEqual([ + { + name: 'ha-hz-worker-1', + zone: 'nbg1', + error: 'did not become Ready', + }, + ]); + expect(joined.map((n) => n.serverName)).toEqual(['ha-hz-worker-2']); + }); + + it('keeps trying to reach a server that is still booting', async () => { + const execCommand = jest + .fn() + .mockRejectedValueOnce(new Error('ECONNREFUSED')) + .mockResolvedValue('join-details-handed-over'); + const service = build({ + orchestration: { awaitServerJoin: jest.fn(async (_c, node) => node) }, + nodeRepository: { find: jest.fn().mockResolvedValue([waiting[1]]) }, + execCommand, + }); + + const { joined } = await service.joinWaitingServers( + CLUSTER as never, + 'op-1', + ); + + expect(execCommand).toHaveBeenCalledTimes(2); + expect(joined).toHaveLength(1); + }); + }); + + describe('waitForPlatformInstalled', () => { + const master = { + findOne: jest.fn().mockResolvedValue({ serverName: 'master' }), + }; + + it('waits until the first server marks the install finished', async () => { + const execCommand = jest + .fn() + .mockResolvedValueOnce('platform-installing') + .mockResolvedValueOnce('platform-installed'); + const service = build({ nodeRepository: master, execCommand }); + + await expect( + service.waitForPlatformInstalled(CLUSTER as never, 60_000), + ).resolves.toBe('installed'); + expect(execCommand).toHaveBeenCalledTimes(2); + }); + + it('stops waiting as soon as the boot script has ended without finishing', async () => { + const execCommand = jest + .fn() + .mockResolvedValueOnce('platform-installing') + .mockResolvedValueOnce('platform-stopped'); + const service = build({ nodeRepository: master, execCommand }); + + await expect( + service.waitForPlatformInstalled(CLUSTER as never, 60_000), + ).resolves.toBe('stopped'); + expect(execCommand.mock.calls[0][3]).toContain('cloud-init status'); + }); + + it('says so when it has not finished in time', async () => { + const service = build({ + nodeRepository: master, + execCommand: jest.fn().mockResolvedValue('platform-installing'), + }); + + await expect( + service.waitForPlatformInstalled(CLUSTER as never, 0), + ).resolves.toBe('timeout'); + }); + }); + + describe('haJoinOutcome', () => { + const plan = { + region: 'eu-central', + first: { zone: 'fsn1', shape: 'cpx22' }, + servers: [], + later: [], + }; + + it('names every location when every server joined', () => { + expect(haJoinOutcome(plan, ['nbg1', 'hel1'], [])).toEqual({ + joinedMessage: 'Joined the servers in nbg1 and hel1', + readyMessage: 'Cluster ready across fsn1, nbg1, hel1', + }); + }); + + it('names the location still missing a server', () => { + expect(haJoinOutcome(plan, ['nbg1'], ['hel1']).readyMessage).toBe( + 'Cluster ready in fsn1, nbg1; a server is still missing in hel1: add it from the High availability tab', + ); + }); + }); +}); diff --git a/src/modules/infrastructure/clusters/services/ha-server-join.service.ts b/src/modules/infrastructure/clusters/services/ha-server-join.service.ts new file mode 100644 index 00000000..46c6124b --- /dev/null +++ b/src/modules/infrastructure/clusters/services/ha-server-join.service.ts @@ -0,0 +1,382 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { ClusterEntity } from '../entities/cluster.entity'; +import { + ClusterNodeEntity, + NodeStatus, + NodeType, +} from '../entities/cluster-node.entity'; +import { EncryptionService } from '../../../shared/encryption/services/encryption.service'; +import { NativeSSHConnectionService } from '../../../terminal/services/native-ssh-connection.service'; +import { ClusterOrchestrationService } from './cluster-orchestration.service'; +import { HaCreationPlan, PlannedServer } from '../utils/ha-creation-plan.core'; +import { + JOIN_DETAILS_ALREADY_JOINED, + JOIN_DETAILS_HANDED_OVER, + JoinDetails, + joinDetailsScript, +} from '../utils/join-details.util'; + +const HAND_OVER_ATTEMPTS = 30; +const HAND_OVER_INTERVAL_MS = 10_000; +const PLATFORM_INSTALL_TIMEOUT_MS = 20 * 60_000; +const PLATFORM_INSTALL_POLL_MS = 15_000; + +// cloud-init done or in error with no marker: the boot script ended before the end. +const PLATFORM_INSTALL_STATE = [ + 'if test -f /var/log/flui-bootstrap-complete; then echo platform-installed', + "elif cloud-init status 2>/dev/null | grep -q -E 'status: (done|error)'; then echo platform-stopped", + 'else echo platform-installing; fi', +].join('; '); + +function platformInstallState( + out: string | undefined, +): 'installed' | 'stopped' | 'installing' { + if (out?.includes('platform-installed')) return 'installed'; + if (out?.includes('platform-stopped')) return 'stopped'; + return 'installing'; +} + +export interface HaPurchase { + zone: string; + node?: ClusterNodeEntity; + error?: string; +} + +export interface HaJoined { + joined: number; + joinedAt: string; + missing: string[]; + joinedMessage: string; + readyMessage: string; +} + +/** Throws when the creation must stop; checked between servers and while waiting. */ +export type StopCheck = () => Promise; + +const keepGoing: StopCheck = async () => undefined; + +/** Writes a step of the creation's operation; the job owns the operation. */ +export type OperationStepWriter = ( + step: number, + progress: number, + data: Record, +) => Promise; + +/** What the creation says once the servers joined, or did not. */ +export function haJoinOutcome( + plan: HaCreationPlan, + joinedZones: Array, + missing: string[], +): Pick { + const where = [plan.first.zone, ...joinedZones]; + return { + joinedMessage: joinedZones.length + ? `Joined the servers in ${joinedZones.join(' and ')}` + : 'No other server joined', + readyMessage: missing.length + ? `Cluster ready in ${where.join(', ')}; a server is still missing in ${missing.join(' and ')}: add it from the High availability tab` + : `Cluster ready across ${where.join(', ')}`, + }; +} + +/** + * The servers of a cluster across three locations: bought with its first + * server, then told where to join one at a time, because etcd takes a single + * new member at a time and K3s gives up instead of waiting when a second one + * asks. + */ +@Injectable() +export class HaServerJoinService { + private readonly logger = new Logger(HaServerJoinService.name); + + constructor( + private readonly orchestration: ClusterOrchestrationService, + @InjectRepository(ClusterNodeEntity) + private readonly nodeRepository: Repository, + private readonly encryptionService: EncryptionService, + private readonly nativeSsh: NativeSSHConnectionService, + ) {} + + /** + * Starts buying the other servers while the caller buys the first one. + * Returned unawaited, so the purchases run alongside the first server's own. + */ + async startBuying( + cluster: ClusterEntity, + plan: HaCreationPlan | undefined, + operationId: string, + providerFirewallIds: string[], + ): Promise<{ purchases: Promise } | null> { + if (!plan) return null; + await this.orchestration.prepareBootstrapKey(cluster); + const purchases = this.buyServersToJoinLater( + cluster, + plan.servers, + operationId, + providerFirewallIds, + ).catch((err) => + plan.servers.map((s) => ({ + zone: s.zone, + error: err instanceof Error ? err.message : String(err), + })), + ); + return { purchases }; + } + + /** Joins what was bought, in the planned order, and says what is still missing. */ + async joinBought( + cluster: ClusterEntity, + plan: HaCreationPlan, + bought: { purchases: Promise }, + operationId: string, + step: OperationStepWriter, + stop: StopCheck = keepGoing, + ): Promise { + const purchases = await bought.purchases; + const notBought = purchases.filter((p) => p.error); + for (const p of notBought) { + this.logger.warn( + `[ha] ${cluster.name}: the server in ${p.zone} was not bought: ${p.error}`, + ); + } + await step(3, 10, { + message: `Joining the servers in ${plan.servers.map((s) => s.zone).join(' and ')}, one at a time`, + }); + const { joined, failed } = await this.joinWaitingServers( + cluster, + operationId, + stop, + ); + for (const f of failed) { + this.logger.warn( + `[ha] ${cluster.name}: ${f.name} did not join: ${f.error}`, + ); + } + const missing = [ + ...plan.later, + ...notBought.map((p) => p.zone), + ...failed.map((f) => f.zone ?? f.name), + ]; + return { + joined: joined.length, + joinedAt: new Date().toISOString(), + missing, + ...haJoinOutcome( + plan, + joined.map((n) => n.region), + missing, + ), + }; + } + + /** + * When Flui finished installing on the first server, or why it did not: + * the cluster exists either way, so this is said, not undone. + */ + async waitForPlatform( + cluster: ClusterEntity, + step: OperationStepWriter, + stepIndex = 4, + stop: StopCheck = keepGoing, + ): Promise<{ at: string | null; problem: string | null }> { + await step(stepIndex, 20, { + message: 'Waiting for Flui to finish installing on the first server', + }); + const state = await this.waitForPlatformInstalled( + cluster, + PLATFORM_INSTALL_TIMEOUT_MS, + stop, + ); + if (state === 'installed') { + return { at: new Date().toISOString(), problem: null }; + } + const problem = + state === 'stopped' + ? 'The install on the first server stopped before the end (its log is /var/log/k3s-master-init.log)' + : 'Flui had not finished installing on the first server in time'; + this.logger.warn(`[ha] ${cluster.name}: ${problem}`); + return { at: null, problem }; + } + + /** + * Buys the other servers while the first one is still being made. Each + * prepares itself and waits; none joins until `joinWaitingServers` tells it + * where. A retried creation reuses the machine it already bought for a + * location. + */ + async buyServersToJoinLater( + cluster: ClusterEntity, + servers: PlannedServer[], + operationId: string, + providerFirewallIds: string[], + ): Promise { + const bought = await this.nodeRepository.find({ + where: { clusterId: cluster.id, nodeType: NodeType.WORKER }, + }); + let next = await this.orchestration.nextWorkerIndex(cluster); + const indexes = servers.map( + (s) => + bought.find((n) => n.metadata?.joinLater && n.region === s.zone) + ?.metadata?.workerIndex ?? next++, + ); + const settled = await Promise.allSettled( + servers.map((s, i) => + this.orchestration.buyWorkerToJoinLater( + cluster, + indexes[i], + s, + operationId, + providerFirewallIds, + ), + ), + ); + return settled.map((result, i) => + result.status === 'fulfilled' + ? { zone: servers[i].zone, node: result.value } + : { + zone: servers[i].zone, + error: + result.reason instanceof Error + ? result.reason.message + : String(result.reason), + }, + ); + } + + /** Joins the servers bought to join later, one at a time and in the order they were planned. */ + async joinWaitingServers( + cluster: ClusterEntity, + operationId: string, + stop: StopCheck = keepGoing, + ): Promise<{ + joined: ClusterNodeEntity[]; + failed: Array<{ name: string; zone: string | null; error: string }>; + }> { + const masterIp = cluster.masterPrivateIp || cluster.masterIpAddress; + if (!masterIp) throw new Error('Master IP address not set'); + const details: JoinDetails = { + masterIp, + k3sToken: this.encryptionService.decrypt(cluster.k3sTokenEncrypted), + sharedStorageMasterIp: + cluster.sharedStorageEnabled === false ? null : cluster.masterPrivateIp, + }; + const privateKey = await this.orchestration.bootstrapPrivateKey(cluster); + const waiting = ( + await this.nodeRepository.find({ + where: { clusterId: cluster.id, nodeType: NodeType.WORKER }, + }) + ) + .filter((n) => n.metadata?.joinLater && n.providerResourceId) + .filter((n) => n.status !== NodeStatus.READY) + .sort( + (a, b) => + (a.metadata?.workerIndex ?? 0) - (b.metadata?.workerIndex ?? 0), + ); + + const joined: ClusterNodeEntity[] = []; + const failed: Array<{ name: string; zone: string | null; error: string }> = + []; + for (const node of waiting) { + await stop(); + try { + await this.handOverJoinDetails(cluster, node, privateKey, details); + joined.push( + await this.orchestration.awaitServerJoin(cluster, node, operationId), + ); + } catch (err) { + failed.push({ + name: node.serverName, + zone: node.region ?? null, + error: err instanceof Error ? err.message : String(err), + }); + } + } + return { joined, failed }; + } + + /** + * Waits for the first server's boot script to finish installing what Flui + * runs on the cluster, which it marks after K3s itself has long been ready. + * `stopped` when the script ended without marking it: waiting longer would + * change nothing. + */ + async waitForPlatformInstalled( + cluster: ClusterEntity, + timeoutMs = PLATFORM_INSTALL_TIMEOUT_MS, + stop: StopCheck = keepGoing, + ): Promise<'installed' | 'stopped' | 'timeout'> { + const master = cluster.masterNodeId + ? await this.nodeRepository.findOne({ + where: { id: cluster.masterNodeId }, + }) + : null; + const host = master + ? this.orchestration.nodeSshHost(cluster, master) + : null; + if (!host) return 'timeout'; + const privateKey = await this.orchestration.bootstrapPrivateKey(cluster); + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + await stop(); + try { + const out = await this.nativeSsh.execCommand( + host, + 'root', + privateKey, + PLATFORM_INSTALL_STATE, + 15000, + ); + const state = platformInstallState(out); + if (state !== 'installing') return state; + } catch (err) { + this.logger.warn( + `Could not read ${cluster.name}'s install state yet: ${err instanceof Error ? err.message : String(err)}`, + ); + } + await this.sleep(PLATFORM_INSTALL_POLL_MS); + } + return 'timeout'; + } + + private async handOverJoinDetails( + cluster: ClusterEntity, + node: ClusterNodeEntity, + bootstrapPrivateKey: string, + details: JoinDetails, + ): Promise { + const host = this.orchestration.nodeSshHost(cluster, node); + if (!host) throw new Error(`${node.serverName} has no address to reach`); + const script = joinDetailsScript(details); + let lastError = ''; + for (let attempt = 1; attempt <= HAND_OVER_ATTEMPTS; attempt++) { + try { + const out = await this.nativeSsh.execCommand( + host, + 'root', + bootstrapPrivateKey, + script, + 20000, + ); + if ( + out?.includes(JOIN_DETAILS_HANDED_OVER) || + out?.includes(JOIN_DETAILS_ALREADY_JOINED) + ) { + return; + } + lastError = 'the server did not confirm'; + } catch (err) { + lastError = err instanceof Error ? err.message : String(err); + } + await this.sleep(HAND_OVER_INTERVAL_MS); + } + throw new Error( + `Could not tell ${node.serverName} where to join: ${lastError}`, + ); + } + + private sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } +} diff --git a/src/modules/infrastructure/clusters/services/k3s-script.service.ts b/src/modules/infrastructure/clusters/services/k3s-script.service.ts index 7e959c36..2930cabe 100644 --- a/src/modules/infrastructure/clusters/services/k3s-script.service.ts +++ b/src/modules/infrastructure/clusters/services/k3s-script.service.ts @@ -5,7 +5,7 @@ import { installedReleaseOf, } from 'src/config/bootstrap.config'; import { RELEASE } from 'src/config/release.config'; -import { K3S_DEFAULT_VERSION } from '../constants'; +import { K3S_DEFAULT_VERSION, POD_NETWORK } from '../constants'; import { FirewallRule } from 'src/modules/providers/interfaces/firewall-provider.interface'; import { renderHostFirewallPreamble } from 'src/modules/providers/core/firewall/host-firewall-preamble'; import { overlayRulesetOptions } from 'src/modules/providers/core/firewall/overlay-ruleset-policy'; @@ -13,6 +13,12 @@ import { BootstrapPeer, encodeBootstrapPeers, } from '../../networking/wireguard-config'; +import { + NodeTopology, + TOPOLOGY_REGION_LABEL, + TOPOLOGY_ZONE_LABEL, + topologyLabels, +} from 'src/modules/providers/core/node-topology'; export interface HostFirewallBoot { rules: FirewallRule[]; @@ -137,6 +143,27 @@ export interface K3sMasterConfig { device?: string; sizeGb?: number; }; + + /** Labelled on the node when it registers, so placement can tell sites apart. */ + topology?: NodeTopology | null; +} + +/** Where a worker joins; one bought to join later is told by Flui instead, so its boot script carries no token. */ +function joinEnv(config: K3sWorkerConfig): Record { + if (config.joinLater) { + return { + K3S_TOKEN: '', + K3S_URL: '', + MASTER_IP: '', + FLUI_K3S_JOIN: 'later', + }; + } + return { + K3S_TOKEN: config.k3sToken, + K3S_URL: `https://${config.masterIp}:6443`, + MASTER_IP: config.masterIp, + FLUI_K3S_JOIN: 'now', + }; } export interface K3sWorkerConfig { @@ -150,6 +177,13 @@ export interface K3sWorkerConfig { clusterName: string; k3sToken: string; masterIp: string; + /** `server` joins the control plane and its datastore; absent is an agent, as always. */ + k3sRole?: 'agent' | 'server'; + /** + * Bought before the first server exists: the node prepares itself and waits + * for Flui to write where to join, so the boot script carries no token. + */ + joinLater?: boolean; k3sVersion?: string; instanceId: string; instanceName: string; @@ -198,6 +232,9 @@ export interface K3sWorkerConfig { device?: string; sizeGb?: number; }; + + /** Labelled on the node when it registers, so placement can tell sites apart. */ + topology?: NodeTopology | null; } /** @@ -215,10 +252,21 @@ export interface K3sWorkerConfig { */ const DEFAULT_LOCAL_STORAGE_GB = 20; -const POD_NETWORK = '10.42.0.0/16'; const CIDR_RE = /^[0-9a-fA-F:.]+\/\d{1,3}$/; const ANYWHERE = new Set(['0.0.0.0/0', '::/0']); +/** Empty when the provider declares no topology, so the node joins unlabelled as before. */ +export function topologyEnv(topology?: NodeTopology | null): { + FLUI_NODE_ZONE: string; + FLUI_NODE_REGION: string; +} { + const labels = topologyLabels(topology); + return { + FLUI_NODE_ZONE: labels[TOPOLOGY_ZONE_LABEL] ?? '', + FLUI_NODE_REGION: labels[TOPOLOGY_REGION_LABEL] ?? '', + }; +} + /** What the installer records as the release a master was built from. */ export function installedReleaseEnv( scriptsBaseUrl: string, @@ -428,6 +476,7 @@ export class K3sScriptService { FLUI_LOCAL_STORAGE_SIZE_GB: String( this.resolveLocalStorage(config.localStorage).sizeGb, ), + ...topologyEnv(config.topology), }, config.bootstrapPublicKey, config.hostFirewall, @@ -476,10 +525,9 @@ export class K3sScriptService { CLOUD_PROVIDER: config.provider, CLUSTER_ID: config.clusterId, CLUSTER_NAME: config.clusterName, - K3S_TOKEN: config.k3sToken, - K3S_URL: `https://${config.masterIp}:6443`, + ...joinEnv(config), + FLUI_K3S_ROLE: config.k3sRole ?? 'agent', K3S_VERSION: config.k3sVersion || K3S_DEFAULT_VERSION, - MASTER_IP: config.masterIp, FLUI_CA_PUBLIC_KEY: config.caPublicKey || '', // Multi-cluster observability OBSERVABILITY_CLUSTER_IP: config.controlClusterIp || '', @@ -502,6 +550,7 @@ export class K3sScriptService { FLUI_LOCAL_STORAGE_SIZE_GB: String( this.resolveLocalStorage(config.localStorage).sizeGb, ), + ...topologyEnv(config.topology), }, config.bootstrapPublicKey, config.hostFirewall, diff --git a/src/modules/infrastructure/clusters/services/k3s-script.topology.spec.ts b/src/modules/infrastructure/clusters/services/k3s-script.topology.spec.ts new file mode 100644 index 00000000..e842d9e3 --- /dev/null +++ b/src/modules/infrastructure/clusters/services/k3s-script.topology.spec.ts @@ -0,0 +1,79 @@ +jest.mock('@kubernetes/client-node', () => ({})); + +import { + K3sScriptService, + K3sWorkerConfig, + topologyEnv, +} from './k3s-script.service'; + +const worker: K3sWorkerConfig = { + clusterId: 'c1', + clusterName: 'lab', + masterIp: '10.0.0.2', + instanceId: 'lab-worker-1', + instanceName: 'lab-worker-1', + provider: 'hetzner', + k3sToken: 'token', +}; + +describe('the topology a bootstrap script hands the node', () => { + it('is the zone and region the provider declares', () => { + expect(topologyEnv({ zone: 'nbg1', region: 'eu-central' })).toEqual({ + FLUI_NODE_ZONE: 'nbg1', + FLUI_NODE_REGION: 'eu-central', + }); + }); + + /** A node of a provider that declares nothing joins exactly as it did before labels existed. */ + it('is empty when the provider declares none', () => { + expect(topologyEnv(null)).toEqual({ + FLUI_NODE_ZONE: '', + FLUI_NODE_REGION: '', + }); + expect(topologyEnv()).toEqual({ + FLUI_NODE_ZONE: '', + FLUI_NODE_REGION: '', + }); + }); + + it('reaches the worker script as exported variables', async () => { + const script = await new K3sScriptService().generateWorkerScript({ + ...worker, + topology: { zone: 'nbg1', region: 'eu-central' }, + }); + expect(script).toContain("export FLUI_NODE_ZONE='nbg1'"); + expect(script).toContain("export FLUI_NODE_REGION='eu-central'"); + }); + + it('exports empty values for a worker with no declared topology', async () => { + const script = await new K3sScriptService().generateWorkerScript(worker); + expect(script).toContain("export FLUI_NODE_ZONE=''"); + expect(script).toContain("export FLUI_NODE_REGION=''"); + }); + + it('carries no token and no master for a server bought to join later', async () => { + const script = await new K3sScriptService().generateWorkerScript({ + ...worker, + k3sRole: 'server', + joinLater: true, + }); + expect(script).toContain("export FLUI_K3S_JOIN='later'"); + expect(script).toContain("export K3S_TOKEN=''"); + expect(script).toContain("export K3S_URL=''"); + expect(script).toContain("export MASTER_IP=''"); + expect(script).not.toContain('token'); + expect(await new K3sScriptService().generateWorkerScript(worker)).toContain( + "export FLUI_K3S_JOIN='now'", + ); + }); + + it('joins a server to the control plane only when asked, an agent otherwise', async () => { + const service = new K3sScriptService(); + expect( + await service.generateWorkerScript({ ...worker, k3sRole: 'server' }), + ).toContain("export FLUI_K3S_ROLE='server'"); + expect(await service.generateWorkerScript(worker)).toContain( + "export FLUI_K3S_ROLE='agent'", + ); + }); +}); diff --git a/src/modules/infrastructure/clusters/services/kubeconfig-endpoint.promoter.spec.ts b/src/modules/infrastructure/clusters/services/kubeconfig-endpoint.promoter.spec.ts index 6a5b1749..0f826fd7 100644 --- a/src/modules/infrastructure/clusters/services/kubeconfig-endpoint.promoter.spec.ts +++ b/src/modules/infrastructure/clusters/services/kubeconfig-endpoint.promoter.spec.ts @@ -1,3 +1,8 @@ +jest.mock('../../networking/api-server-probe', () => ({ + apiServerAnswers: jest.fn(), +})); + +import { apiServerAnswers } from '../../networking/api-server-probe'; import { KubeconfigEndpointPromoter } from './kubeconfig-endpoint.promoter'; const kubeconfig = (host: string) => @@ -37,8 +42,7 @@ describe('KubeconfigEndpointPromoter', () => { } as any, { enrolOverlayAddress: enrolSan } as any, ); - (promoter as unknown as { answers: () => Promise }).answers = - async () => answers; + (apiServerAnswers as jest.Mock).mockResolvedValue(answers); return { promoter, save, enrolSan }; }; diff --git a/src/modules/infrastructure/clusters/services/kubeconfig-endpoint.promoter.ts b/src/modules/infrastructure/clusters/services/kubeconfig-endpoint.promoter.ts index 9d77b76a..90372630 100644 --- a/src/modules/infrastructure/clusters/services/kubeconfig-endpoint.promoter.ts +++ b/src/modules/infrastructure/clusters/services/kubeconfig-endpoint.promoter.ts @@ -1,13 +1,13 @@ import { Injectable, Logger } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { IsNull, Repository } from 'typeorm'; -import * as https from 'node:https'; import { ClusterEntity, ClusterType } from '../entities/cluster.entity'; import { NodeType } from '../entities/cluster-node.entity'; import { EncryptionService } from '../../../shared/encryption/services/encryption.service'; import { WireGuardPeerService } from '../../networking/services/wireguard-peer.service'; import { ApiServerSanService } from '../../networking/services/api-server-san.service'; import { managementNetworkOn } from '../../networking/management-network.state'; +import { apiServerAnswers } from '../../networking/api-server-probe'; import { SERVER_LINE, overlayDemotionOf, @@ -122,7 +122,7 @@ export class KubeconfigEndpointPromoter { } const port = current[3] ?? ':6443'; - if (!(await this.answers(overlay.nodeAddress, port.slice(1), ca))) { + if (!(await apiServerAnswers(overlay.nodeAddress, port.slice(1), ca))) { this.logger.log( `[kubeconfig] ${cluster.name}: ${overlay.nodeAddress} not answering yet — leaving ${current[2]} in place`, ); @@ -138,51 +138,4 @@ export class KubeconfigEndpointPromoter { ); return true; } - - /** - * Verified against the cluster's own CA, taken from the kubeconfig being - * moved — which makes this more than a reachability check. The API server's - * certificate has to name the overlay address for a kubeconfig pointing there - * to work at all, so a probe that skipped verification could happily promote - * a cluster into a permanent TLS failure. - * - * Any HTTP answer proves the path: 401 is what an unauthenticated probe - * should get, and getting it means TCP, TLS and the certificate's names all - * worked over the tunnel. Only silence is a failure. - */ - private answers( - host: string, - port: string, - caBase64: string, - ): Promise { - return new Promise((resolve) => { - let ca: Buffer; - try { - ca = Buffer.from(caBase64, 'base64'); - } catch { - resolve(false); - return; - } - const req = https.request( - { - host, - port, - path: '/readyz', - method: 'GET', - timeout: 5000, - ca, - }, - (res) => { - res.resume(); - resolve(true); - }, - ); - req.on('timeout', () => { - req.destroy(); - resolve(false); - }); - req.on('error', () => resolve(false)); - req.end(); - }); - } } diff --git a/src/modules/infrastructure/clusters/services/node-disks.service.ts b/src/modules/infrastructure/clusters/services/node-disks.service.ts new file mode 100644 index 00000000..55a40e59 --- /dev/null +++ b/src/modules/infrastructure/clusters/services/node-disks.service.ts @@ -0,0 +1,87 @@ +import { Injectable, NotFoundException } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { ClusterEntity } from '../entities/cluster.entity'; +import { EncryptionService } from '../../../shared/encryption/services/encryption.service'; +import { KubernetesService } from '../../shared/services/kubernetes.service'; +import { TOPOLOGY_ZONE_LABEL } from '../../../providers/core/node-topology'; +import { + LINSTOR_CONTROLLER_DEPLOYMENT, + LINSTOR_CONTROLLER_SELECTOR, + PIRAEUS_NAMESPACE, + REPLICATED_POOL, +} from '../../../replicated-volumes/replicated-volumes.constants'; +import { + DiskSpace, + NodeDisk, + copiesFromStoragePools, + diskFromSummary, + diskWarnings, +} from '../utils/node-disks.core'; + +/** + * How much room is left on each node: its own disk, where databases and + * dedicated data live, and the space set aside for copies of replicated + * volumes. Both have a ceiling, the node's disk, and nothing warns when it + * comes close unless this is read. + */ +@Injectable() +export class NodeDisksService { + constructor( + @InjectRepository(ClusterEntity) + private readonly clusters: Repository, + private readonly encryption: EncryptionService, + private readonly kubernetes: KubernetesService, + ) {} + + async view( + clusterId: string, + ): Promise<{ measuredAt: string; nodes: NodeDisk[]; warnings: string[] }> { + const cluster = await this.clusters.findOne({ where: { id: clusterId } }); + if (!cluster?.kubeconfigEncrypted) { + throw new NotFoundException(`Cluster ${clusterId} not found`); + } + const kubeconfig = this.encryption.decrypt(cluster.kubeconfigEncrypted); + const [k8sNodes, copies] = await Promise.all([ + this.kubernetes.listNodes(kubeconfig), + this.copies(kubeconfig), + ]); + const nodes: NodeDisk[] = await Promise.all( + k8sNodes.map(async (n) => { + const name = n.metadata?.name ?? ''; + return { + node: name, + zone: n.metadata?.labels?.[TOPOLOGY_ZONE_LABEL] ?? null, + disk: diskFromSummary( + await this.kubernetes.nodeStatsSummary(kubeconfig, name), + ), + copies: copies.get(name) ?? null, + }; + }), + ); + nodes.sort((a, b) => a.node.localeCompare(b.node)); + return { + measuredAt: new Date().toISOString(), + nodes, + warnings: diskWarnings(nodes), + }; + } + + /** Empty on a cluster without replicated volumes. */ + private async copies(kubeconfig: string): Promise> { + try { + return copiesFromStoragePools( + await this.kubernetes.execInPod( + kubeconfig, + PIRAEUS_NAMESPACE, + LINSTOR_CONTROLLER_SELECTOR, + LINSTOR_CONTROLLER_DEPLOYMENT, + ['linstor', '-m', '--output-version', 'v1', 'storage-pool', 'list'], + ), + REPLICATED_POOL, + ); + } catch { + return new Map(); + } + } +} diff --git a/src/modules/infrastructure/clusters/services/node-price.service.spec.ts b/src/modules/infrastructure/clusters/services/node-price.service.spec.ts index 9a861a87..c31dc4d2 100644 --- a/src/modules/infrastructure/clusters/services/node-price.service.spec.ts +++ b/src/modules/infrastructure/clusters/services/node-price.service.spec.ts @@ -45,6 +45,28 @@ describe('NodePriceService', () => { ).resolves.toBe(0.0071); }); + it('prices a zone as the region it belongs to', async () => { + const { service } = build([ + size({ + prices: [ + { + location: 'GRA', + priceHourly: { net: '0.05', gross: '0.06' }, + priceMonthly: { net: '36', gross: '43' }, + }, + { + location: 'EU-WEST-PAR', + priceHourly: { net: '0.0457', gross: '0.0548' }, + priceMonthly: { net: '33', gross: '40' }, + }, + ], + }), + ]); + await expect( + service.resolveHourlyEur('ovh', 'cx32', 'eu-west-par-b'), + ).resolves.toBe(0.0457); + }); + it('falls back to the first location when none is given', async () => { const { service } = build([size()]); await expect(service.resolveHourlyEur('hetzner', 'cx32')).resolves.toBe( diff --git a/src/modules/infrastructure/clusters/services/node-price.service.ts b/src/modules/infrastructure/clusters/services/node-price.service.ts index 06e8a57c..4a9a4c92 100644 --- a/src/modules/infrastructure/clusters/services/node-price.service.ts +++ b/src/modules/infrastructure/clusters/services/node-price.service.ts @@ -10,6 +10,11 @@ const CACHE_TTL_MS = 60 * 60 * 1000; * a provider without a catalogue, an unknown size and a failed lookup all * answer null, so a node row that carries a price carries a real one. */ +/** `fr-par-2` in `fr-par`, `eu-west-par-a` in `EU-WEST-PAR`: a zone priced as its region. */ +function isZoneOf(zone: string, location: string): boolean { + return zone.toLowerCase().startsWith(`${location.toLowerCase()}-`); +} + @Injectable() export class NodePriceService { private readonly logger = new Logger(NodePriceService.name); @@ -35,7 +40,8 @@ export class NodePriceService { if (!size?.prices?.length) return null; const match = location - ? size.prices.find((p) => p.location === location) + ? (size.prices.find((p) => p.location === location) ?? + size.prices.find((p) => isZoneOf(location, p.location))) : undefined; const raw = (match ?? size.prices[0]).priceHourly?.net; if (!raw) return null; diff --git a/src/modules/infrastructure/clusters/services/node-resize-ha.service.ts b/src/modules/infrastructure/clusters/services/node-resize-ha.service.ts new file mode 100644 index 00000000..8d7824fb --- /dev/null +++ b/src/modules/infrastructure/clusters/services/node-resize-ha.service.ts @@ -0,0 +1,235 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { ClusterEntity } from '../entities/cluster.entity'; +import { ClusterNodeEntity } from '../entities/cluster-node.entity'; +import { + InfrastructureOperationEntity, + OperationStatus, + OperationType, +} from '../../servers/entities/infrastructure-operations.entity'; +import { KubernetesService } from '../../shared/services/kubernetes.service'; +import { ClusterHaService } from '../../../cluster-ha/services/cluster-ha.service'; +import { ReplicatedPoolPreparer } from '../../../replicated-volumes/services/replicated-pool.preparer'; +import { clusterHaNodesFrom } from '../../../cluster-ha/utils/cluster-ha.core'; +import { CNPG_API_VERSION } from '../../../postgres-ha/postgres-ha.constants'; +import { + LINSTOR_CONTROLLER_DEPLOYMENT, + LINSTOR_CONTROLLER_SELECTOR, + PIRAEUS_NAMESPACE, + REPLICATED_POOL_FILE, + REPLICATED_VOLUME_GROUP, +} from '../../../replicated-volumes/replicated-volumes.constants'; +import { + staleCopyWarnings, + volumeCopiesFrom, +} from '../../../replicated-volumes/utils/replicated-volumes.core'; +import { + growCopiesScript, + resizeHaRefusal, + switchoverTarget, +} from '../utils/node-resize-ha.core'; + +export interface PlannedSwitchover { + namespace: string; + cluster: string; + target: string; +} + +export interface HaResizeReadiness { + refusal: string | null; + switchovers: PlannedSwitchover[]; +} + +/** + * What resizing a node of a cluster across three locations takes beyond the + * resize itself: the checks before it, the primaries handed over first, and + * the space for copies grown once the node is back. + */ +@Injectable() +export class NodeResizeHaService { + constructor( + @InjectRepository(InfrastructureOperationEntity) + private readonly operationRepository: Repository, + private readonly kubernetesService: KubernetesService, + private readonly clusterHa: ClusterHaService, + private readonly hostScripts: ReplicatedPoolPreparer, + ) {} + + /** + * Null on a cluster that is not across three locations, or that could not + * be read: the resize then works as it always has. + */ + async readiness( + cluster: ClusterEntity, + node: ClusterNodeEntity, + ): Promise { + const ha = await this.clusterHa.inspect(cluster.id).catch(() => null); + if (ha?.state.level !== 'three-locations' || !ha.topology) return null; + const nodes = clusterHaNodesFrom(ha.nodes); + const running = await this.operationRepository + .createQueryBuilder('op') + .where('op.operationType = :type', { type: OperationType.SCALE_NODE }) + .andWhere('op.status IN (:...statuses)', { + statuses: [OperationStatus.PENDING, OperationStatus.IN_PROGRESS], + }) + .andWhere(`op.metadata ->> 'clusterId' = :clusterId`, { + clusterId: cluster.id, + }) + .andWhere('op.resourceId != :nodeId', { nodeId: node.id }) + .getOne(); + const zoneOf = new Map(nodes.map((n) => [n.name, n.zone ?? ''])); + return { + refusal: resizeHaRefusal({ + clusterName: cluster.name, + nodeName: node.serverName, + serversNotReady: nodes + .filter((n) => n.server && !n.ready) + .map((n) => n.name), + otherResizeRunning: running?.resourceName ?? null, + staleCopies: await this.staleCopies(ha.kubeconfig), + }), + switchovers: await this.plannedSwitchovers( + ha.kubeconfig, + node.serverName, + zoneOf, + ha.topology.dataZones, + ), + }; + } + + private async staleCopies(kubeconfig: string): Promise { + try { + return staleCopyWarnings( + volumeCopiesFrom( + await this.kubernetesService.execInPod( + kubeconfig, + PIRAEUS_NAMESPACE, + LINSTOR_CONTROLLER_SELECTOR, + LINSTOR_CONTROLLER_DEPLOYMENT, + ['linstor', '-m', '--output-version', 'v1', 'resource', 'list'], + ), + ), + new Map(), + ); + } catch { + return []; + } + } + + private async plannedSwitchovers( + kubeconfig: string, + nodeName: string, + zoneOf: Map, + preferredZones: string[], + ): Promise { + const clusters = await this.kubernetesService + .listObjectsEverywhere(kubeconfig, CNPG_API_VERSION, 'Cluster') + .catch(() => []); + const planned: PlannedSwitchover[] = []; + for (const c of clusters) { + const namespace = c.metadata?.namespace; + const name = c.metadata?.name; + if (!namespace || !name) continue; + const pods = await this.kubernetesService.listPodsByLabel( + kubeconfig, + namespace, + `cnpg.io/cluster=${name}`, + ); + const target = switchoverTarget({ + primary: c.status?.currentPrimary, + nodeName, + instances: pods.map((p) => ({ + name: p.metadata?.name ?? '', + node: p.spec?.nodeName, + ready: (p.status?.conditions ?? []).some( + (cond) => cond.type === 'Ready' && cond.status === 'True', + ), + })), + zoneOf, + preferredZones, + }); + if (target) planned.push({ namespace, cluster: name, target }); + } + return planned; + } + + /** The same planned switchover CloudNativePG's own plugin asks for; writes pause for seconds. */ + async switchover(kubeconfig: string, s: PlannedSwitchover): Promise { + await this.kubernetesService.jsonPatchObjectStatus( + kubeconfig, + { + apiVersion: CNPG_API_VERSION, + plural: 'clusters', + name: s.cluster, + namespace: s.namespace, + }, + [ + { op: 'replace', path: '/status/targetPrimary', value: s.target }, + { + op: 'replace', + path: '/status/targetPrimaryTimestamp', + value: new Date().toISOString(), + }, + { + op: 'replace', + path: '/status/phase', + value: 'Switchover in progress', + }, + { + op: 'replace', + path: '/status/phaseReason', + value: `Switching over to ${s.target} before a node is resized`, + }, + ], + ); + const deadline = Date.now() + 3 * 60 * 1000; + while (Date.now() < deadline) { + const c = await this.kubernetesService.readObject( + kubeconfig, + CNPG_API_VERSION, + 'Cluster', + s.cluster, + s.namespace, + ); + if ( + c?.status?.currentPrimary === s.target && + c?.status?.phase === 'Cluster in healthy state' + ) { + return; + } + await new Promise((r) => setTimeout(r, 3000)); + } + throw new Error( + `The primary of ${s.namespace}/${s.cluster} did not move to ${s.target} within 3 minutes; the node was left on.`, + ); + } + + /** A message for the operation, or null on a node that holds no copies. */ + async growCopies( + kubeconfig: string, + nodeName: string, + sizeGiB: number, + ): Promise { + const output = await this.hostScripts.runHostScript( + kubeconfig, + nodeName, + growCopiesScript(REPLICATED_POOL_FILE, REPLICATED_VOLUME_GROUP, sizeGiB), + ); + const line = + output + .trim() + .split('\n') + .find((l) => /^(GROWN|NOSPACE|UNCHANGED|NOPOOL)/.test(l)) ?? ''; + if (line.startsWith('GROWN')) { + return `Space for copies on ${nodeName} grown to ${line.split(' ')[1]} GiB`; + } + if (line.startsWith('NOSPACE')) { + return `Space for copies on ${nodeName} kept: the disk has not enough room for ${sizeGiB} GiB`; + } + if (line.startsWith('UNCHANGED')) { + return `Space for copies on ${nodeName} already ${sizeGiB} GiB or more`; + } + return null; + } +} diff --git a/src/modules/infrastructure/clusters/services/node-topology.reconciler.spec.ts b/src/modules/infrastructure/clusters/services/node-topology.reconciler.spec.ts new file mode 100644 index 00000000..b3db2627 --- /dev/null +++ b/src/modules/infrastructure/clusters/services/node-topology.reconciler.spec.ts @@ -0,0 +1,183 @@ +jest.mock('@kubernetes/client-node', () => ({})); + +import { ConfigService } from '@nestjs/config'; +import { Repository } from 'typeorm'; +import { NodeTopologyReconciler } from './node-topology.reconciler'; +import { ClusterEntity } from '../entities/cluster.entity'; +import { ClusterNodeEntity, NodeType } from '../entities/cluster-node.entity'; +import { KubernetesService } from '../../shared/services/kubernetes.service'; +import { EncryptionService } from '../../../shared/encryption/services/encryption.service'; +import { CapabilitiesProviderFactory } from '../../../providers/core/factories/capabilities-provider.factory'; +import { CloudProvider } from '../../../providers/enums/cloud-provider.enum'; +import { HetznerCapabilitiesService } from '../../../providers/implementations/hetzner/hetzner-capabilities.service'; +import { ByosCapabilitiesService } from '../../../providers/implementations/byos/byos-capabilities.service'; +import { ICredentialProvider } from '../../../providers/interfaces/credential-provider.interface'; + +const config = { + get: (_key: string, fallback?: unknown) => fallback, +} as unknown as ConfigService; + +const capabilities = new CapabilitiesProviderFactory([ + { + provider: CloudProvider.HETZNER, + service: new HetznerCapabilitiesService( + config, + {} as unknown as ICredentialProvider, + ), + }, + { provider: CloudProvider.BYOS, service: new ByosCapabilitiesService() }, +]); + +const node = ( + serverName: string, + nodeType: NodeType, + region: string | null, +): ClusterNodeEntity => + ({ serverName, nodeType, region }) as unknown as ClusterNodeEntity; + +function setup( + cluster: Partial, + labels: Record>, +) { + const added: Array<{ node: string; labels: Record }> = []; + const kubernetes = { + listNodeLabels: jest.fn(async () => new Map(Object.entries(labels))), + addNodeLabels: jest.fn( + async (_k: string, nodeName: string, l: Record) => { + added.push({ node: nodeName, labels: l }); + }, + ), + } as unknown as KubernetesService; + const clusters = { + findOne: jest.fn(async () => ({ + id: 'c1', + name: 'lab', + kubeconfigEncrypted: 'enc', + ...cluster, + })), + } as unknown as Repository; + const encryption = { + decrypt: () => 'kubeconfig', + } as unknown as EncryptionService; + const reconciler = new NodeTopologyReconciler( + clusters, + kubernetes, + encryption, + capabilities, + ); + return { reconciler, added }; +} + +describe('labelling the nodes that joined before topology labels existed', () => { + it('gives each Hetzner node its own location, and the master the cluster one', async () => { + const { reconciler, added } = setup( + { + provider: 'hetzner', + region: 'fsn1', + nodes: [ + node('lab-master', NodeType.MASTER, null), + node('lab-worker-1', NodeType.WORKER, 'nbg1'), + node('lab-worker-3', NodeType.WORKER, 'hel1'), + ], + }, + { 'lab-master': {}, 'lab-worker-1': {}, 'lab-worker-3': {} }, + ); + const outcome = await reconciler.reconcile('c1'); + expect(outcome).toEqual({ + state: 'done', + labelled: ['lab-master', 'lab-worker-1', 'lab-worker-3'], + conflicts: [], + }); + expect( + added.map((a) => [a.node, a.labels['topology.kubernetes.io/zone']]), + ).toEqual([ + ['lab-master', 'fsn1'], + ['lab-worker-1', 'nbg1'], + ['lab-worker-3', 'hel1'], + ]); + expect( + added.every( + (a) => a.labels['topology.kubernetes.io/region'] === 'eu-central', + ), + ).toBe(true); + }); + + /** A scaling group can buy in several locations, so a worker with no record is not placed by guess. */ + it('skips a worker whose location was never recorded', async () => { + const { reconciler, added } = setup( + { + provider: 'hetzner', + region: 'fsn1', + nodes: [node('lab-worker-9', NodeType.WORKER, null)], + }, + { 'lab-worker-9': {} }, + ); + await reconciler.reconcile('c1'); + expect(added).toEqual([]); + }); + + it('leaves a zone someone else set and reports it, still adding what is missing', async () => { + const { reconciler, added } = setup( + { + provider: 'hetzner', + region: 'fsn1', + nodes: [node('lab-master', NodeType.MASTER, 'fsn1')], + }, + { 'lab-master': { 'topology.kubernetes.io/zone': 'fsn1-dc14' } }, + ); + const outcome = await reconciler.reconcile('c1'); + expect(added).toEqual([ + { + node: 'lab-master', + labels: { 'topology.kubernetes.io/region': 'eu-central' }, + }, + ]); + expect(outcome).toEqual({ + state: 'done', + labelled: ['lab-master'], + conflicts: ['lab-master: topology.kubernetes.io/zone=fsn1-dc14'], + }); + }); + + it('touches nothing on a node that already carries its labels', async () => { + const { reconciler, added } = setup( + { + provider: 'hetzner', + region: 'fsn1', + nodes: [node('lab-master', NodeType.MASTER, 'fsn1')], + }, + { + 'lab-master': { + 'topology.kubernetes.io/zone': 'fsn1', + 'topology.kubernetes.io/region': 'eu-central', + }, + }, + ); + await reconciler.reconcile('c1'); + expect(added).toEqual([]); + }); + + it('labels nothing on a provider that declares no topology', async () => { + const { reconciler, added } = setup( + { + provider: 'byos', + region: null as unknown as string, + nodes: [node('box', NodeType.MASTER, null)], + }, + { box: {} }, + ); + await reconciler.reconcile('c1'); + expect(added).toEqual([]); + }); + + it('skips a cluster it cannot reach', async () => { + const { reconciler } = setup( + { kubeconfigEncrypted: null as unknown as string }, + {}, + ); + expect(await reconciler.reconcile('c1')).toEqual({ + state: 'skipped', + reason: 'no kubeconfig', + }); + }); +}); diff --git a/src/modules/infrastructure/clusters/services/node-topology.reconciler.ts b/src/modules/infrastructure/clusters/services/node-topology.reconciler.ts new file mode 100644 index 00000000..5fa4985c --- /dev/null +++ b/src/modules/infrastructure/clusters/services/node-topology.reconciler.ts @@ -0,0 +1,93 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { ClusterEntity } from '../entities/cluster.entity'; +import { ClusterNodeEntity, NodeType } from '../entities/cluster-node.entity'; +import { KubernetesService } from '../../shared/services/kubernetes.service'; +import { EncryptionService } from '../../../shared/encryption/services/encryption.service'; +import { CapabilitiesProviderFactory } from '../../../providers/core/factories/capabilities-provider.factory'; +import { CloudProvider } from '../../../providers/enums/cloud-provider.enum'; +import { + missingTopologyLabels, + topologyLabels, +} from '../../../providers/core/node-topology'; + +export type NodeTopologyOutcome = + | { state: 'done'; labelled: string[]; conflicts: string[] } + | { state: 'skipped'; reason: string }; + +/** + * Gives nodes that joined before topology labels existed the zone and region + * their provider declares. A node registers its labels only once, when it + * first joins, so only the API can label the nodes that are already there. + */ +@Injectable() +export class NodeTopologyReconciler { + private readonly logger = new Logger(NodeTopologyReconciler.name); + + constructor( + @InjectRepository(ClusterEntity) + private readonly clusters: Repository, + private readonly kubernetes: KubernetesService, + private readonly encryption: EncryptionService, + private readonly capabilities: CapabilitiesProviderFactory, + ) {} + + async reconcile(clusterId: string): Promise { + const cluster = await this.clusters.findOne({ + where: { id: clusterId }, + relations: ['nodes'], + }); + if (!cluster?.kubeconfigEncrypted) { + return { state: 'skipped', reason: 'no kubeconfig' }; + } + const provider = cluster.provider as CloudProvider; + if (!this.capabilities.isProviderSupported(provider)) { + return { state: 'skipped', reason: `unknown provider ${provider}` }; + } + const declares = this.capabilities.getCapabilitiesService(provider); + + const kubeconfig = this.encryption.decrypt(cluster.kubeconfigEncrypted); + const current = await this.kubernetes.listNodeLabels(kubeconfig); + const labelled: string[] = []; + const conflicts: string[] = []; + for (const node of cluster.nodes ?? []) { + const labels = current.get(node.serverName); + if (!labels) continue; + const desired = topologyLabels( + declares.nodeTopology(this.locationOf(cluster, node)), + ); + const diff = missingTopologyLabels(labels, desired); + if (diff.conflicts.length) { + conflicts.push(`${node.serverName}: ${diff.conflicts.join(', ')}`); + } + if (!Object.keys(diff.add).length) continue; + await this.kubernetes.addNodeLabels( + kubeconfig, + node.serverName, + diff.add, + ); + labelled.push(node.serverName); + } + if (labelled.length) { + this.logger.log( + `[node-topology] ${cluster.name}: labelled ${labelled.join(', ')}`, + ); + } + if (conflicts.length) { + this.logger.warn( + `[node-topology] ${cluster.name}: left as they are, set to another value: ${conflicts.join('; ')}`, + ); + } + return { state: 'done', labelled, conflicts }; + } + + /** A worker with no recorded region is skipped rather than assumed to sit with the master. */ + private locationOf( + cluster: ClusterEntity, + node: ClusterNodeEntity, + ): string | null { + if (node.region) return node.region; + return node.nodeType === NodeType.MASTER ? cluster.region : null; + } +} diff --git a/src/modules/infrastructure/clusters/services/shared-storage-export.reconciler.ts b/src/modules/infrastructure/clusters/services/shared-storage-export.reconciler.ts index 1196700e..6daa7a3d 100644 --- a/src/modules/infrastructure/clusters/services/shared-storage-export.reconciler.ts +++ b/src/modules/infrastructure/clusters/services/shared-storage-export.reconciler.ts @@ -19,10 +19,11 @@ import { } from '../../../providers/core/host/cluster-private-networks'; import { CloudProvider } from '../../../providers/enums/cloud-provider.enum'; import { nfsAllowedNetworks } from './k3s-script.service'; +import { NFS_EXPORT_OPTIONS } from '../constants/storage-conventions'; export const SHARED_STORAGE_PATH = '/var/lib/flui/storage'; const EXPORTS_FILE = '/etc/exports'; -const EXPORT_OPTIONS = 'rw,async,no_subtree_check,no_root_squash'; +const EXPORT_OPTIONS = NFS_EXPORT_OPTIONS; const SCRIPT_TIMEOUT_MS = 6 * 60_000; const CERT_TTL_SECONDS = 600; diff --git a/src/modules/infrastructure/clusters/utils/cluster-metadata-merge.util.ts b/src/modules/infrastructure/clusters/utils/cluster-metadata-merge.util.ts new file mode 100644 index 00000000..cb0fddf9 --- /dev/null +++ b/src/modules/infrastructure/clusters/utils/cluster-metadata-merge.util.ts @@ -0,0 +1,23 @@ +import { Repository } from 'typeorm'; +import { ClusterEntity } from '../entities/cluster.entity'; + +/** + * Sets a few metadata keys in one statement. A sweep that saved the whole + * entity would put back what it read, undoing keys another sweep wrote since. + */ +export async function mergeClusterMetadata( + clusters: Repository, + clusterId: string, + keys: Record, +): Promise { + await clusters + .createQueryBuilder() + .update(ClusterEntity) + .set({ + metadata: () => + `(COALESCE("metadata"::jsonb, '{}'::jsonb) || CAST(:keys AS jsonb))::json`, + }) + .setParameter('keys', JSON.stringify(keys)) + .where('id = :id', { id: clusterId }) + .execute(); +} diff --git a/src/modules/infrastructure/clusters/utils/ha-creation-plan.core.spec.ts b/src/modules/infrastructure/clusters/utils/ha-creation-plan.core.spec.ts new file mode 100644 index 00000000..138958e0 --- /dev/null +++ b/src/modules/infrastructure/clusters/utils/ha-creation-plan.core.spec.ts @@ -0,0 +1,217 @@ +import { CatalogueShape, planHaCreation } from './ha-creation-plan.core'; +import { HaTopology } from '../../../providers/core/ha-topology'; + +const EU_CENTRAL: HaTopology = { + region: 'eu-central', + zones: ['fsn1', 'nbg1', 'hel1'], + dataZones: ['fsn1', 'nbg1'], + validated: true, + measured: true, +}; + +function shape( + name: string, + architecture: string, + zones: Record, + monthly: string, +): CatalogueShape { + return { + name, + architecture, + availability: Object.entries(zones).map(([location, available]) => ({ + location, + available, + })), + prices: Object.keys(zones).map((location) => ({ + location, + priceMonthly: { net: monthly }, + })), + }; +} + +// Some sizes out in some locations, as Hetzner reports them. +const CATALOGUE = [ + shape('cax11', 'arm', { fsn1: false, nbg1: false, hel1: false }, '5.99'), + shape('cx23', 'x86', { fsn1: true, nbg1: false, hel1: false }, '3.49'), + shape('cpx22', 'x86', { fsn1: true, nbg1: true, hel1: true }, '19.49'), + shape('ccx13', 'x86', { fsn1: true, nbg1: true, hel1: true }, '42.99'), +]; + +const base = { + topology: EU_CENTRAL, + firstZone: 'fsn1', + workerCount: 0, + catalogue: CATALOGUE, + completeLater: false, +}; + +describe('planHaCreation', () => { + it('puts one server in each location, the other location for data before the one that votes', () => { + const answer = planHaCreation({ ...base, shapes: ['cpx22'] }); + + expect(answer.plan).toEqual({ + region: 'eu-central', + first: { zone: 'fsn1', shape: 'cpx22' }, + servers: [ + { zone: 'nbg1', shape: 'cpx22' }, + { zone: 'hel1', shape: 'cpx22' }, + ], + later: [], + }); + }); + + it('gives each location the first size it has available', () => { + const answer = planHaCreation({ ...base, shapes: ['cx23', 'cpx22'] }); + + expect(answer.plan?.first).toEqual({ zone: 'fsn1', shape: 'cx23' }); + expect(answer.plan?.servers).toEqual([ + { zone: 'nbg1', shape: 'cpx22' }, + { zone: 'hel1', shape: 'cpx22' }, + ]); + }); + + it('buys nothing when a location has none of the sizes, and names the sizes every location has', () => { + const answer = planHaCreation({ ...base, shapes: ['cx23'] }); + + expect(answer.plan).toBeNull(); + expect(answer).toMatchObject({ + completeLaterHelps: true, + refusal: + 'nbg1 and hel1 have none of cx23 available now. Sizes every location has: cpx22 (€19.49/month), ccx13 (€42.99/month).', + }); + }); + + it('only proposes sizes of the same architecture', () => { + const answer = planHaCreation({ ...base, shapes: ['cax11'] }); + + expect(answer).toMatchObject({ + refusal: + 'fsn1, nbg1 and hel1 have none of cax11 available now. No arm size is available in all three locations right now.', + }); + }); + + it('leaves a location for later only when asked, and never the first one', () => { + const later = planHaCreation({ + ...base, + shapes: ['cx23'], + completeLater: true, + }); + expect(later.plan).toMatchObject({ + first: { zone: 'fsn1', shape: 'cx23' }, + servers: [], + later: ['nbg1', 'hel1'], + }); + + const noFirst = planHaCreation({ + ...base, + firstZone: 'nbg1', + shapes: ['cx23'], + completeLater: true, + }); + expect(noFirst.plan).toBeNull(); + }); + + it('refuses a first server in the location that only votes', () => { + const answer = planHaCreation({ + ...base, + firstZone: 'hel1', + shapes: ['cpx22'], + }); + + expect(answer).toMatchObject({ + refusal: 'The first server goes in a location for data: fsn1 or nbg1.', + }); + }); + + it('refuses sizes of two architectures', () => { + const answer = planHaCreation({ ...base, shapes: ['cax11', 'cpx22'] }); + + expect(answer).toMatchObject({ + refusal: + 'These sizes mix arm and x86; a cluster runs on one architecture.', + }); + }); + + it('refuses workers at creation and a region with no layout', () => { + expect( + planHaCreation({ ...base, workerCount: 1, shapes: ['cpx22'] }).plan, + ).toBeNull(); + expect( + planHaCreation({ + ...base, + topology: null, + unsupported: + 'Flui has not validated a highly available layout for us-east yet.', + shapes: ['cpx22'], + }), + ).toMatchObject({ + refusal: + 'Flui has not validated a highly available layout for us-east yet.', + }); + }); +}); + +describe('planHaCreation where the catalogue is by region', () => { + const PAR: HaTopology = { + region: 'fr-par', + zones: ['fr-par-1', 'fr-par-2', 'fr-par-3'], + dataZones: ['fr-par-1', 'fr-par-2', 'fr-par-3'], + validated: true, + measured: true, + }; + const byRegion = ( + name: string, + location: string, + zones?: string[], + ): CatalogueShape => ({ + name, + architecture: 'x86', + availability: [{ location, available: true, zones }], + prices: [{ location, priceMonthly: { net: '28.47' } }], + }); + + it('counts a size sold in the region for every zone the region names', () => { + const answer = planHaCreation({ + topology: PAR, + firstZone: 'fr-par-1', + workerCount: 0, + completeLater: false, + shapes: ['BASIC3-X2C-4G', 'PRO2-XXS'], + catalogue: [ + byRegion('BASIC3-X2C-4G', 'fr-par', ['fr-par-1', 'fr-par-2']), + byRegion('PRO2-XXS', 'fr-par', ['fr-par-1', 'fr-par-2', 'fr-par-3']), + ], + }); + + expect(answer.plan).toEqual({ + region: 'fr-par', + first: { zone: 'fr-par-1', shape: 'BASIC3-X2C-4G' }, + servers: [ + { zone: 'fr-par-2', shape: 'BASIC3-X2C-4G' }, + { zone: 'fr-par-3', shape: 'PRO2-XXS' }, + ], + later: [], + }); + }); + + it('counts a region that does not say which zones sell a size for all of them, whatever its case', () => { + const answer = planHaCreation({ + topology: { + ...PAR, + region: 'eu-west-par', + zones: ['eu-west-par-a', 'eu-west-par-b', 'eu-west-par-c'], + dataZones: ['eu-west-par-a', 'eu-west-par-b', 'eu-west-par-c'], + }, + firstZone: 'eu-west-par-a', + workerCount: 0, + completeLater: false, + shapes: ['c3-4'], + catalogue: [byRegion('c3-4', 'EU-WEST-PAR')], + }); + + expect(answer.plan?.servers.map((s) => s.zone)).toEqual([ + 'eu-west-par-b', + 'eu-west-par-c', + ]); + }); +}); diff --git a/src/modules/infrastructure/clusters/utils/ha-creation-plan.core.ts b/src/modules/infrastructure/clusters/utils/ha-creation-plan.core.ts new file mode 100644 index 00000000..110c8c5e --- /dev/null +++ b/src/modules/infrastructure/clusters/utils/ha-creation-plan.core.ts @@ -0,0 +1,211 @@ +import { HaTopology, locationList } from '../../../providers/core/ha-topology'; + +/** A size as the provider's catalogue reports it, live for availability. */ +export interface CatalogueShape { + name: string; + architecture?: string; + availability?: Array<{ + location: string; + available: boolean; + zones?: string[]; + }>; + prices?: Array<{ location: string; priceMonthly?: { net?: string } }>; +} + +export interface PlannedServer { + zone: string; + shape: string; +} + +export interface HaCreationPlan { + region: string; + first: PlannedServer; + /** The other servers, locations for data first: the order they join in. */ + servers: PlannedServer[]; + /** Locations left without a server, to add once they have one available. */ + later: string[]; +} + +export type HaCreationAnswer = + | { plan: HaCreationPlan } + | { plan: null; refusal: string; completeLaterHelps?: boolean }; + +const ALTERNATIVES_SHOWN = 3; + +/** + * A catalogue entry for the zone itself (a Hetzner location), or for its + * region (Scaleway, OVH); a region's entry names the zones that sell the size + * when the provider says so, and otherwise counts for every zone. + */ +function coversZone( + location: string, + zone: string, + region: string, + zones?: string[], +): boolean { + if (location === zone) return true; + return ( + location.toLowerCase() === region.toLowerCase() && + (!zones || zones.includes(zone)) + ); +} + +function availableIn( + shape: CatalogueShape, + zone: string, + region: string, +): boolean { + return ( + shape.availability?.some( + (a) => a.available && coversZone(a.location, zone, region, a.zones), + ) ?? false + ); +} + +function monthlyIn( + shape: CatalogueShape, + zone: string, + region: string, +): number | null { + const net = shape.prices?.find((p) => coversZone(p.location, zone, region)) + ?.priceMonthly?.net; + const value = net === undefined ? Number.NaN : Number(net); + return Number.isFinite(value) ? value : null; +} + +function everywhere( + catalogue: CatalogueShape[], + topology: HaTopology, + architecture: string | undefined, +): string[] { + const { zones, region } = topology; + return catalogue + .filter((s) => !architecture || s.architecture === architecture) + .filter((s) => zones.every((z) => availableIn(s, z, region))) + .map((s) => ({ name: s.name, price: monthlyIn(s, zones[0], region) })) + .sort( + (a, b) => + (a.price ?? Number.POSITIVE_INFINITY) - + (b.price ?? Number.POSITIVE_INFINITY), + ) + .slice(0, ALTERNATIVES_SHOWN) + .map((s) => + s.price === null ? s.name : `${s.name} (€${s.price.toFixed(2)}/month)`, + ); +} + +interface PlanInput { + topology: HaTopology | null; + unsupported?: string; + firstZone: string; + workerCount: number; + shapes: string[]; + catalogue: CatalogueShape[]; + completeLater: boolean; +} + +/** Why the request cannot be planned before any size is looked at. */ +function requestRefusal(input: PlanInput): string | null { + const { topology } = input; + if (!topology) { + return ( + input.unsupported ?? + 'Flui has no highly available layout for this region.' + ); + } + if (input.workerCount > 0) { + return 'A cluster across three locations starts with its three servers; add workers once it is ready.'; + } + if (!topology.dataZones.includes(input.firstZone)) { + return `The first server goes in a location for data: ${topology.dataZones.join(' or ')}.`; + } + return input.shapes.length ? null : 'Name at least one size.'; +} + +function architecturesOf( + shapes: string[], + catalogue: CatalogueShape[], +): string[] { + const known = shapes + .map((name) => catalogue.find((s) => s.name === name)) + .filter((s): s is CatalogueShape => !!s); + return [ + ...new Set( + known.map((s) => s.architecture).filter((a): a is string => !!a), + ), + ]; +} + +function noneAvailable( + missing: string[], + shapes: string[], + alternatives: string[], + architecture: string | undefined, +): string { + const verb = missing.length > 1 ? 'have' : 'has'; + const none = `${locationList(missing)} ${verb} none of ${shapes.join(', ')} available now.`; + if (alternatives.length) { + return `${none} Sizes every location has: ${alternatives.join(', ')}.`; + } + const kind = architecture ? `${architecture} ` : ''; + return `${none} No ${kind}size is available in all three locations right now.`; +} + +/** + * What a cluster across three locations is bought as, decided before anything + * is bought: one server per location, each on the first size it has + * available. The first server goes in a location for data, because that is + * where the cluster's leader and primaries belong. + */ +export function planHaCreation(input: PlanInput): HaCreationAnswer { + const refusal = requestRefusal(input); + if (refusal) return { plan: null, refusal }; + const topology = input.topology!; + const { firstZone, shapes, catalogue } = input; + + const architectures = architecturesOf(shapes, catalogue); + if (architectures.length > 1) { + return { + plan: null, + refusal: `These sizes mix ${architectures.join(' and ')}; a cluster runs on one architecture.`, + }; + } + + const pick = (zone: string) => + shapes.find((name) => { + const shape = catalogue.find((s) => s.name === name); + return shape ? availableIn(shape, zone, topology.region) : false; + }); + const others = [ + ...topology.zones.filter( + (z) => z !== firstZone && topology.dataZones.includes(z), + ), + ...topology.zones.filter((z) => !topology.dataZones.includes(z)), + ]; + const firstShape = pick(firstZone); + const missing = [firstZone, ...others].filter((z) => !pick(z)); + + if (missing.length && (!firstShape || !input.completeLater)) { + return { + plan: null, + completeLaterHelps: !!firstShape, + refusal: noneAvailable( + missing, + shapes, + everywhere(catalogue, topology, architectures[0]), + architectures[0], + ), + }; + } + + return { + plan: { + region: topology.region, + first: { zone: firstZone, shape: firstShape! }, + servers: others + .filter((z) => !missing.includes(z)) + .map((zone) => ({ zone, shape: pick(zone)! })), + later: missing, + }, + }; +} diff --git a/src/modules/infrastructure/clusters/utils/ha-node-request.core.spec.ts b/src/modules/infrastructure/clusters/utils/ha-node-request.core.spec.ts new file mode 100644 index 00000000..d352822a --- /dev/null +++ b/src/modules/infrastructure/clusters/utils/ha-node-request.core.spec.ts @@ -0,0 +1,72 @@ +import { ClusterHaState } from '../../../cluster-ha/utils/cluster-ha.core'; +import { haNodeRefusal } from './ha-node-request.core'; + +const state = (over: Partial = {}): ClusterHaState => + ({ + level: 'standard', + region: 'eu-central', + unsupported: null, + topology: { + validated: true, + measured: true, + zones: [ + { + zone: 'fsn1', + role: 'data', + servers: ['lab-master'], + dataHosts: ['lab-master', 'lab-worker-2'], + }, + { + zone: 'nbg1', + role: 'data', + servers: [], + dataHosts: ['lab-worker-1'], + }, + { zone: 'hel1', role: 'vote', servers: [], dataHosts: [] }, + ], + }, + missing: [ + { role: 'server', zone: 'nbg1', why: '' }, + { role: 'server', zone: 'hel1', why: '' }, + ], + ...over, + }) as unknown as ClusterHaState; + +describe('which node an operator may add for high availability', () => { + it('adds what the view lists as missing', () => { + expect(haNodeRefusal('lab', state(), 'server', 'hel1')).toBeNull(); + expect(haNodeRefusal('lab', state(), 'server', 'nbg1')).toBeNull(); + }); + + it('refuses a second server in a location, and says how a lost one is replaced', () => { + expect(haNodeRefusal('lab', state(), 'server', 'fsn1')).toContain( + 'fsn1 already has a server (lab-master)', + ); + }); + + it('keeps nodes for data out of the voting location', () => { + expect(haNodeRefusal('lab', state(), 'data-host', 'hel1')).toContain( + 'hel1 only votes', + ); + expect(haNodeRefusal('lab', state(), 'data-host', 'nbg1')).toContain( + 'nbg1 already has a node for data', + ); + }); + + it('refuses a location outside the layout, and a region without one', () => { + expect(haNodeRefusal('lab', state(), 'server', 'ash')).toContain( + 'ash is not one of lab', + ); + expect( + haNodeRefusal( + 'lab', + state({ + unsupported: + 'Flui has not validated a highly available layout for us-east yet.', + }), + 'server', + 'ash', + ), + ).toContain('not validated'); + }); +}); diff --git a/src/modules/infrastructure/clusters/utils/ha-node-request.core.ts b/src/modules/infrastructure/clusters/utils/ha-node-request.core.ts new file mode 100644 index 00000000..a7fbbcdb --- /dev/null +++ b/src/modules/infrastructure/clusters/utils/ha-node-request.core.ts @@ -0,0 +1,35 @@ +import { ClusterHaState } from '../../../cluster-ha/utils/cluster-ha.core'; +import { HaNodeRole } from './ha-node-role'; + +/** + * Whether the cluster still needs this role in this location, as its high + * availability view says. Only what is missing is added: a fourth server, or + * a second one in a location, would change the votes the control plane counts + * without making the cluster survive anything more. + */ +export function haNodeRefusal( + clusterName: string, + state: ClusterHaState, + role: HaNodeRole, + zone: string, +): string | null { + if (state.unsupported) return state.unsupported; + const zones = state.topology?.zones ?? []; + const where = zones.find((z) => z.zone === zone); + if (!where) { + return `${zone} is not one of ${clusterName}'s locations for high availability (${zones.map((z) => z.zone).join(', ')}).`; + } + if (role === 'data-host' && where.role !== 'data') { + return `${zone} only votes: copies of data live in ${zones + .filter((z) => z.role === 'data') + .map((z) => z.zone) + .join(' and ')}.`; + } + if (state.missing.some((m) => m.role === role && m.zone === zone)) { + return null; + } + if (role === 'server') { + return `${zone} already has a server (${where.servers.join(', ')}). A lost server is replaced once it has been removed from the cluster.`; + } + return `${zone} already has a node for data (${where.dataHosts.join(', ')}).`; +} diff --git a/src/modules/infrastructure/clusters/utils/ha-node-role.spec.ts b/src/modules/infrastructure/clusters/utils/ha-node-role.spec.ts new file mode 100644 index 00000000..86174753 --- /dev/null +++ b/src/modules/infrastructure/clusters/utils/ha-node-role.spec.ts @@ -0,0 +1,36 @@ +import { + haRoleMetadata, + haRoleOf, + k3sRoleOf, + serverJoinRefusal, +} from './ha-node-role'; + +describe('ha-node-role', () => { + it('reads only the roles Flui records', () => { + expect(haRoleOf({ metadata: { haRole: 'server' } })).toBe('server'); + expect(haRoleOf({ metadata: { haRole: 'data-host' } })).toBe('data-host'); + expect(haRoleOf({ metadata: { haRole: 'master' } })).toBeNull(); + expect(haRoleOf({ metadata: null })).toBeNull(); + }); + + it('joins only a server to the control plane', () => { + expect(k3sRoleOf('server')).toBe('server'); + expect(k3sRoleOf('data-host')).toBe('agent'); + expect(k3sRoleOf(null)).toBe('agent'); + }); + + it('refuses a server that joined without the control-plane label', () => { + const plane = { 'node-role.kubernetes.io/control-plane': 'true' }; + expect(serverJoinRefusal('w-2', 'server', plane)).toBeNull(); + expect(serverJoinRefusal('w-2', 'server', {})).toMatch( + /w-2 joined as an ordinary worker/, + ); + expect(serverJoinRefusal('w-2', 'server', undefined)).not.toBeNull(); + expect(serverJoinRefusal('w-2', 'agent', {})).toBeNull(); + }); + + it('records the role, and nothing for an ordinary worker', () => { + expect(haRoleMetadata('data-host')).toEqual({ haRole: 'data-host' }); + expect(haRoleMetadata(null)).toEqual({}); + }); +}); diff --git a/src/modules/infrastructure/clusters/utils/ha-node-role.ts b/src/modules/infrastructure/clusters/utils/ha-node-role.ts new file mode 100644 index 00000000..6ed7fbeb --- /dev/null +++ b/src/modules/infrastructure/clusters/utils/ha-node-role.ts @@ -0,0 +1,47 @@ +/** + * The part a node plays in a cluster across three locations, recorded when an + * operator adds it for that part. A `server` also runs the control plane and + * holds a vote in its datastore; a `data-host` holds copies of databases and + * volumes. Neither is the master, which stays the first server. + */ +export type HaNodeRole = 'server' | 'data-host'; + +export const HA_NODE_ROLES: readonly HaNodeRole[] = ['server', 'data-host']; + +export function haRoleOf(node: { + metadata?: Record | null; +}): HaNodeRole | null { + const role = node.metadata?.haRole; + return HA_NODE_ROLES.includes(role as HaNodeRole) + ? (role as HaNodeRole) + : null; +} + +/** How K3s joins the node: only a `server` joins the control plane. */ +export function k3sRoleOf(role: HaNodeRole | null): 'server' | 'agent' { + return role === 'server' ? 'server' : 'agent'; +} + +const CONTROL_PLANE_LABEL = 'node-role.kubernetes.io/control-plane'; + +/** + * Why a node asked to join as a server did not become one. The join script + * of an older release ignores the role and joins an agent, which K3s accepts + * without complaint, so only the labels of the node that joined tell. + */ +export function serverJoinRefusal( + nodeName: string, + k3sRole: 'server' | 'agent', + labels: Record | null | undefined, +): string | null { + if (k3sRole !== 'server') return null; + if (labels?.[CONTROL_PLANE_LABEL] === 'true') return null; + return `${nodeName} joined as an ordinary worker, not as a server of the control plane: the bootstrap scripts this installation uses do not join servers. It stays a worker and can be removed.`; +} + +/** The node record's metadata for its role; nothing for an ordinary worker. */ +export function haRoleMetadata(role: HaNodeRole | null): { + haRole?: HaNodeRole; +} { + return role ? { haRole: role } : {}; +} diff --git a/src/modules/infrastructure/clusters/utils/join-details.util.spec.ts b/src/modules/infrastructure/clusters/utils/join-details.util.spec.ts new file mode 100644 index 00000000..4850f421 --- /dev/null +++ b/src/modules/infrastructure/clusters/utils/join-details.util.spec.ts @@ -0,0 +1,46 @@ +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { joinDetailsScript } from './join-details.util'; + +describe('joinDetailsScript', () => { + const details = { + masterIp: '10.0.1.2', + k3sToken: "K10ab'cd::server:ef", + sharedStorageMasterIp: '10.0.1.2', + }; + + it('writes a file the boot script can source back to the same values', () => { + const root = mkdtempSync(join(tmpdir(), 'join-')); + const script = joinDetailsScript(details) + .replaceAll('/var/lib/flui', `${root}/flui`) + .replace('/etc/systemd/system/k3s.service', `${root}/none`); + + const out = execFileSync('bash', ['-c', script]).toString(); + const read = execFileSync('bash', [ + '-c', + `. ${root}/flui/k3s-join.env; printf '%s|%s|%s|%s' "$K3S_URL" "$K3S_TOKEN" "$MASTER_IP" "$FLUI_SHARED_STORAGE_MASTER_IP"`, + ]).toString(); + + expect(out.trim()).toBe('join-details-handed-over'); + expect(read).toBe( + "https://10.0.1.2:6443|K10ab'cd::server:ef|10.0.1.2|10.0.1.2", + ); + expect(readFileSync(`${root}/flui/k3s-join.env`, 'utf8')).not.toContain( + 'tmp', + ); + }); + + it('hands nothing to a server that already joined', () => { + const root = mkdtempSync(join(tmpdir(), 'join-')); + const script = joinDetailsScript(details) + .replaceAll('/var/lib/flui', `${root}/flui`) + .replace('/etc/systemd/system/k3s.service', root); + + const out = execFileSync('bash', ['-c', script]).toString(); + + expect(out.trim()).toBe('join-details-already-joined'); + expect(() => readFileSync(`${root}/flui/k3s-join.env`)).toThrow(); + }); +}); diff --git a/src/modules/infrastructure/clusters/utils/join-details.util.ts b/src/modules/infrastructure/clusters/utils/join-details.util.ts new file mode 100644 index 00000000..d7d92bd7 --- /dev/null +++ b/src/modules/infrastructure/clusters/utils/join-details.util.ts @@ -0,0 +1,44 @@ +export interface JoinDetails { + masterIp: string; + k3sToken: string; + sharedStorageMasterIp?: string | null; +} + +export const JOIN_DETAILS_HANDED_OVER = 'join-details-handed-over'; +export const JOIN_DETAILS_ALREADY_JOINED = 'join-details-already-joined'; + +const QUOTED_QUOTE = String.raw`'\''`; + +function quoted(value: string): string { + return "'" + value.replaceAll("'", QUOTED_QUOTE) + "'"; +} + +/** + * The shell that hands a waiting server where to join. The file appears whole, + * so the boot script never reads half of it, and a server that already joined + * gets nothing, because the boot script deletes the file once it has read it + * and a second copy would leave the token on its disk. + */ +export function joinDetailsScript(details: JoinDetails): string { + const lines = [ + `K3S_URL=${quoted('https://' + details.masterIp + ':6443')}`, + `K3S_TOKEN=${quoted(details.k3sToken)}`, + `MASTER_IP=${quoted(details.masterIp)}`, + ...(details.sharedStorageMasterIp + ? [ + `FLUI_SHARED_STORAGE_MASTER_IP=${quoted(details.sharedStorageMasterIp)}`, + ] + : []), + ]; + return [ + 'set -e', + 'umask 077', + 'mkdir -p /var/lib/flui', + `if [ -e /etc/systemd/system/k3s.service ]; then echo ${JOIN_DETAILS_ALREADY_JOINED}; exit 0; fi`, + "cat > /var/lib/flui/k3s-join.env.tmp <<'FLUI_JOIN_DETAILS'", + ...lines, + 'FLUI_JOIN_DETAILS', + 'mv /var/lib/flui/k3s-join.env.tmp /var/lib/flui/k3s-join.env', + `echo ${JOIN_DETAILS_HANDED_OVER}`, + ].join('\n'); +} diff --git a/src/modules/infrastructure/clusters/utils/node-disks.core.spec.ts b/src/modules/infrastructure/clusters/utils/node-disks.core.spec.ts new file mode 100644 index 00000000..3b47c688 --- /dev/null +++ b/src/modules/infrastructure/clusters/utils/node-disks.core.spec.ts @@ -0,0 +1,63 @@ +import { + copiesFromStoragePools, + diskFromSummary, + diskWarnings, +} from './node-disks.core'; + +describe('the space left on the nodes', () => { + it('reads the disk from the kubelet summary', () => { + expect( + diskFromSummary({ + node: { + fs: { + capacityBytes: 40 * 1024 ** 3, + availableBytes: 21.5 * 1024 ** 3, + }, + }, + }), + ).toEqual({ totalGiB: 40, freeGiB: 21.5 }); + expect(diskFromSummary({ node: {} })).toBeNull(); + }); + + it('reads the space for copies from LINSTOR, the pool of replicated volumes only', () => { + const json = JSON.stringify([ + [ + { + storage_pool_name: 'flui-replicated', + node_name: 'w1', + free_capacity: 15 * 1024 ** 2, + total_capacity: 20 * 1024 ** 2, + }, + { + storage_pool_name: 'DfltDisklessStorPool', + node_name: 'w3', + free_capacity: 0, + total_capacity: 0, + }, + ], + ]); + expect([...copiesFromStoragePools(json, 'flui-replicated')]).toEqual([ + ['w1', { totalGiB: 20, freeGiB: 15 }], + ]); + }); + + it('warns on a disk or a space for copies running short, and stays quiet otherwise', () => { + const warnings = diskWarnings([ + { + node: 'w1', + zone: 'nbg1', + disk: { totalGiB: 40, freeGiB: 4 }, + copies: { totalGiB: 20, freeGiB: 1.5 }, + }, + { + node: 'w2', + zone: 'fsn1', + disk: { totalGiB: 40, freeGiB: 20 }, + copies: { totalGiB: 20, freeGiB: 12 }, + }, + ]); + expect(warnings).toHaveLength(2); + expect(warnings[0]).toContain('w1 has 4 of 40 GiB free on its disk'); + expect(warnings[1]).toContain('w1 has 1.5 of 20 GiB free for copies'); + }); +}); diff --git a/src/modules/infrastructure/clusters/utils/node-disks.core.ts b/src/modules/infrastructure/clusters/utils/node-disks.core.ts new file mode 100644 index 00000000..6a35d201 --- /dev/null +++ b/src/modules/infrastructure/clusters/utils/node-disks.core.ts @@ -0,0 +1,77 @@ +const GIB = 1024 ** 3; + +export interface DiskSpace { + totalGiB: number; + freeGiB: number; +} + +export interface NodeDisk { + node: string; + zone: string | null; + /** The node's own disk, where `flui-local` keeps databases and dedicated data; null when it did not answer. */ + disk: DiskSpace | null; + /** The space set aside for copies of replicated volumes; null on a node that holds none. */ + copies: DiskSpace | null; +} + +const round = (n: number) => Math.round(n * 10) / 10; + +/** The kubelet's `stats/summary`: the filesystem its root lives on. */ +export function diskFromSummary(summary: unknown): DiskSpace | null { + const fs = (summary as { node?: { fs?: Record } })?.node?.fs; + if (!fs?.capacityBytes || fs.availableBytes === undefined) return null; + return { + totalGiB: round(fs.capacityBytes / GIB), + freeGiB: round(fs.availableBytes / GIB), + }; +} + +/** `linstor -m --output-version v1 storage-pool list`, capacities in KiB, for one pool. */ +export function copiesFromStoragePools( + linstorJson: string, + pool: string, +): Map { + const parsed = JSON.parse(linstorJson) as unknown[]; + const rows = (Array.isArray(parsed[0]) ? parsed[0] : parsed) as Array<{ + storage_pool_name?: string; + node_name?: string; + free_capacity?: number; + total_capacity?: number; + }>; + const byNode = new Map(); + for (const r of rows) { + if (r.storage_pool_name !== pool || !r.node_name || !r.total_capacity) { + continue; + } + byNode.set(r.node_name, { + totalGiB: round(r.total_capacity / 1024 ** 2), + freeGiB: round((r.free_capacity ?? 0) / 1024 ** 2), + }); + } + return byNode; +} + +const LOW_SHARE = 0.15; +const LOW_DISK_GIB = 3; +const LOW_COPIES_GIB = 2; + +const low = (space: DiskSpace, floorGiB: number) => + space.freeGiB < floorGiB || space.freeGiB < space.totalGiB * LOW_SHARE; + +/** One sentence per node running short, on its disk or in its space for copies. */ +export function diskWarnings(nodes: NodeDisk[]): string[] { + const warnings: string[] = []; + for (const n of nodes) { + if (n.disk && low(n.disk, LOW_DISK_GIB)) { + warnings.push( + `${n.node} has ${n.disk.freeGiB} of ${n.disk.totalGiB} GiB free on its disk: databases and other data kept on the node stop growing when it is full.`, + ); + } + if (n.copies && low(n.copies, LOW_COPIES_GIB)) { + warnings.push( + `${n.node} has ${n.copies.freeGiB} of ${n.copies.totalGiB} GiB free for copies of replicated volumes: a new or larger replicated volume needs that room on both nodes holding its copies.`, + ); + } + } + return warnings; +} diff --git a/src/modules/infrastructure/clusters/utils/node-resize-ha.core.spec.ts b/src/modules/infrastructure/clusters/utils/node-resize-ha.core.spec.ts new file mode 100644 index 00000000..d7b220e5 --- /dev/null +++ b/src/modules/infrastructure/clusters/utils/node-resize-ha.core.spec.ts @@ -0,0 +1,80 @@ +import { + growCopiesScript, + resizeHaRefusal, + switchoverTarget, +} from './node-resize-ha.core'; + +describe('resizing a node of a cluster across three locations', () => { + const base = { + clusterName: 'lab', + nodeName: 'w1', + serversNotReady: [], + otherResizeRunning: null, + staleCopies: [], + }; + + it('lets one healthy node at a time go', () => { + expect(resizeHaRefusal(base)).toBeNull(); + expect(resizeHaRefusal({ ...base, serversNotReady: ['w1'] })).toBeNull(); + }); + + it('refuses a second node at once, a server already down, or a copy behind', () => { + expect(resizeHaRefusal({ ...base, otherResizeRunning: 'w3' })).toContain( + 'one node is switched off at a time', + ); + expect(resizeHaRefusal({ ...base, serversNotReady: ['w3'] })).toContain( + 'w3 is not ready', + ); + expect( + resizeHaRefusal({ + ...base, + staleCopies: [ + 'The copy of p/data on w2 is not up to date (Inconsistent): until it is, losing w1 stops the volume.', + ], + }), + ).toContain('once every copy is up to date'); + }); + + it('hands a primary on the node to a ready copy elsewhere, in a location for data first', () => { + const zoneOf = new Map([ + ['w1', 'nbg1'], + ['m', 'fsn1'], + ['w3', 'hel1'], + ]); + const instances = [ + { name: 'db-1', node: 'w1', ready: true }, + { name: 'db-2', node: 'w3', ready: true }, + { name: 'db-3', node: 'm', ready: true }, + ]; + expect( + switchoverTarget({ + primary: 'db-1', + nodeName: 'w1', + instances, + zoneOf, + preferredZones: ['fsn1', 'nbg1'], + }), + ).toBe('db-3'); + expect( + switchoverTarget({ + primary: 'db-2', + nodeName: 'w1', + instances, + zoneOf, + preferredZones: ['fsn1', 'nbg1'], + }), + ).toBeNull(); + }); + + it('grows the space for copies without ever shrinking it', () => { + const script = growCopiesScript( + '/var/lib/flui/replicated.img', + 'flui-replicated', + 40, + ); + expect(script).toContain('[ "$WANT" -gt "$NOW" ] || { echo "UNCHANGED'); + expect(script).toContain('losetup -c "$DEV"'); + expect(script).toContain('pvresize -q "$DEV"'); + expect(() => growCopiesScript('/f', 'vg', 0)).toThrow(); + }); +}); diff --git a/src/modules/infrastructure/clusters/utils/node-resize-ha.core.ts b/src/modules/infrastructure/clusters/utils/node-resize-ha.core.ts new file mode 100644 index 00000000..15b7c38e --- /dev/null +++ b/src/modules/infrastructure/clusters/utils/node-resize-ha.core.ts @@ -0,0 +1,87 @@ +/** + * What a cluster across three locations needs before one of its nodes is + * switched off to change its size. A resize is a planned loss of that node, + * which the cluster survives only while everything else stands. + */ +export function resizeHaRefusal(input: { + clusterName: string; + nodeName: string; + serversNotReady: string[]; + otherResizeRunning: string | null; + staleCopies: string[]; +}): string | null { + if (input.otherResizeRunning) { + return `${input.otherResizeRunning} of ${input.clusterName} is being resized: in a cluster across three locations one node is switched off at a time.`; + } + const others = input.serversNotReady.filter((s) => s !== input.nodeName); + if (others.length) { + return `${others.join(', ')} ${others.length === 1 ? 'is' : 'are'} not ready: switching off ${input.nodeName} now would leave the cluster without the majority it needs to keep running.`; + } + if (input.staleCopies.length) { + return `${input.staleCopies[0]} Resize ${input.nodeName} once every copy is up to date.`; + } + return null; +} + +export interface CnpgInstance { + name: string; + node?: string; + ready: boolean; +} + +/** + * A primary that would go down with the node is handed over first: a planned + * switchover pauses writes for seconds, the failover that would follow the + * node otherwise takes about a minute. The copy that takes over is a ready one + * on another node, in a location for data when there is one. + */ +export function switchoverTarget(input: { + primary: string | undefined; + nodeName: string; + instances: CnpgInstance[]; + zoneOf: Map; + preferredZones: string[]; +}): string | null { + const primary = input.instances.find((i) => i.name === input.primary); + if (primary?.node !== input.nodeName) return null; + const candidates = input.instances + .filter((i) => i.ready && i.node && i.node !== input.nodeName) + .sort( + (a, b) => + Number(input.preferredZones.includes(input.zoneOf.get(b.node!) ?? '')) - + Number( + input.preferredZones.includes(input.zoneOf.get(a.node!) ?? ''), + ) || a.name.localeCompare(b.name), + ); + return candidates[0]?.name ?? null; +} + +/** + * Grows the space for copies on a node whose disk grew: the file is allocated + * to the new size, the loop device rereads it and the volume group takes the + * new space. Never shrinks. + */ +export function growCopiesScript( + file: string, + volumeGroup: string, + sizeGiB: number, +): string { + if (!Number.isInteger(sizeGiB) || sizeGiB < 1) { + throw new Error(`invalid size ${sizeGiB}`); + } + return [ + 'set -e', + `FILE=${file}`, + '[ -f "$FILE" ] || { echo "NOPOOL"; exit 0; }', + 'NOW=$(stat -c %s "$FILE")', + `WANT=$(( ${sizeGiB} * 1024 * 1024 * 1024 ))`, + '[ "$WANT" -gt "$NOW" ] || { echo "UNCHANGED $NOW"; exit 0; }', + 'FREE=$(df -B1 --output=avail "$(dirname "$FILE")" | tail -1 | tr -d " ")', + '[ $(( WANT - NOW + 5 * 1024 * 1024 * 1024 )) -le "$FREE" ] || { echo "NOSPACE $FREE"; exit 0; }', + 'fallocate -l "$WANT" "$FILE"', + 'DEV=$(losetup -j "$FILE" -O NAME -n | head -1)', + 'losetup -c "$DEV"', + 'pvresize -q "$DEV"', + `echo "GROWN $(vgs --noheadings --units g --nosuffix -o vg_size ${volumeGroup} | tr -d ' ')"`, + ].join('\n'); +} diff --git a/src/modules/infrastructure/networking/api-server-probe.ts b/src/modules/infrastructure/networking/api-server-probe.ts new file mode 100644 index 00000000..20ccb3c0 --- /dev/null +++ b/src/modules/infrastructure/networking/api-server-probe.ts @@ -0,0 +1,36 @@ +import * as https from 'node:https'; + +/** + * Whether an API server answers at this address with a certificate the + * cluster's own CA vouches for, under this address. Any HTTP answer proves the + * path: 401 is what an unauthenticated probe gets, and getting it means TCP, + * TLS and the certificate's names all worked. Only silence is a failure. + */ +export function apiServerAnswers( + host: string, + port: string, + caBase64: string, +): Promise { + return new Promise((resolve) => { + let ca: Buffer; + try { + ca = Buffer.from(caBase64, 'base64'); + } catch { + resolve(false); + return; + } + const req = https.request( + { host, port, path: '/readyz', method: 'GET', timeout: 5000, ca }, + (res) => { + res.resume(); + resolve(true); + }, + ); + req.on('timeout', () => { + req.destroy(); + resolve(false); + }); + req.on('error', () => resolve(false)); + req.end(); + }); +} diff --git a/src/modules/infrastructure/networking/services/api-server-san.service.ts b/src/modules/infrastructure/networking/services/api-server-san.service.ts index 5d6ede8d..a12dfb82 100644 --- a/src/modules/infrastructure/networking/services/api-server-san.service.ts +++ b/src/modules/infrastructure/networking/services/api-server-san.service.ts @@ -133,8 +133,14 @@ export class ApiServerSanService { * certificate for an address nothing has ever handshaken on would restart a * live master to reach an endpoint that does not answer. */ - async enrolOverlayAddress(clusterId: string): Promise { - const { cluster, master, address } = await this.preflight(clusterId); + async enrolOverlayAddress( + clusterId: string, + nodeId?: string, + ): Promise { + const { cluster, master, address } = await this.preflight( + clusterId, + nodeId, + ); const target = this.targetFor(cluster, master); const out = await this.hostCommand.run( @@ -166,9 +172,13 @@ export class ApiServerSanService { * Everything that must be true before a live master is restarted. * * Shared by the queued path and the direct one so the two can never disagree - * about what makes this safe. + * about what makes this safe. `nodeId` names another server of the + * cluster; without it, the master. */ - private async preflight(clusterId: string): Promise<{ + private async preflight( + clusterId: string, + nodeId?: string, + ): Promise<{ cluster: ClusterEntity; master: ClusterNodeEntity; address: string; @@ -192,12 +202,14 @@ export class ApiServerSanService { ); } - const master = (cluster.nodes ?? []).find( - (n) => n.nodeType === NodeType.MASTER, + const master = (cluster.nodes ?? []).find((n) => + nodeId ? n.id === nodeId : n.nodeType === NodeType.MASTER, ); if (!master) { throw new BadRequestException( - `Cluster ${cluster.name} has no master node recorded`, + nodeId + ? `Cluster ${cluster.name} has no node ${nodeId} recorded` + : `Cluster ${cluster.name} has no master node recorded`, ); } diff --git a/src/modules/infrastructure/networking/services/wireguard-peer.service.spec.ts b/src/modules/infrastructure/networking/services/wireguard-peer.service.spec.ts index 1730a320..c945a32a 100644 --- a/src/modules/infrastructure/networking/services/wireguard-peer.service.spec.ts +++ b/src/modules/infrastructure/networking/services/wireguard-peer.service.spec.ts @@ -304,6 +304,28 @@ describe('WireGuardPeerService', () => { expect(peer.publicKey).toBeNull(); }); + it('gives nodes bought together an address each, though they ask at the same moment', async () => { + await enrolControl(); + const plainSave = peers.save; + peers.save = async (x: any) => { + const clash = peers.rows.some( + (r) => + r !== x && r.revokedAt == null && r.managementIp === x.managementIp, + ); + if (!x.id && clash) + throw Object.assign(new Error('duplicate key'), { code: '23505' }); + return plainSave(x); + }; + + const reserved = await Promise.all( + ['master', 'w1', 'w2'].map((nodeId) => + svc.reserveAddress({ clusterId: 'w', nodeId }), + ), + ); + + expect(new Set(reserved.map((p) => p.managementIp)).size).toBe(3); + }); + it('is idempotent — a second call returns the same address', async () => { await enrolControl(); const first = await svc.reserveAddress({ clusterId: 'w', nodeId: 'n1' }); diff --git a/src/modules/infrastructure/networking/services/wireguard-peer.service.ts b/src/modules/infrastructure/networking/services/wireguard-peer.service.ts index 7982ae0f..1dfbdb34 100644 --- a/src/modules/infrastructure/networking/services/wireguard-peer.service.ts +++ b/src/modules/infrastructure/networking/services/wireguard-peer.service.ts @@ -58,6 +58,8 @@ export interface HandshakeTransition { peer: WireGuardPeerEntity; } +const ADDRESS_ATTEMPTS = 5; + @Injectable() export class WireGuardPeerService { private readonly logger = new Logger(WireGuardPeerService.name); @@ -284,21 +286,46 @@ export class WireGuardPeerService { if (existing) return existing; const pool = await this.poolFor(params.subnetId); - const taken = await this.allocatedAddresses(); - const peer = this.peers.create({ - clusterId: params.clusterId, - nodeId: params.nodeId, - role: WireGuardPeerRole.MEMBER, - publicKey: null, - managementIp: pool.allocate(taken), - subnetId: params.subnetId ?? null, - listenPort: params.subnetId ? WG_DEFAULT_PORT : null, - status: WireGuardPeerStatus.PENDING, - }); + const peer = await this.saveOnFreeAddress(params.nodeId, (taken) => + this.peers.create({ + clusterId: params.clusterId, + nodeId: params.nodeId, + role: WireGuardPeerRole.MEMBER, + publicKey: null, + managementIp: pool.allocate(taken), + subnetId: params.subnetId ?? null, + listenPort: params.subnetId ? WG_DEFAULT_PORT : null, + status: WireGuardPeerStatus.PENDING, + }), + ); this.logger.log( `[wg] address ${peer.managementIp} reserved for node ${params.nodeId}`, ); - return this.peers.save(peer); + return peer; + } + + /** + * Nodes bought together reserve their addresses at the same moment, and two + * can pick the same free one; the unique index refuses the second. It reads + * the taken addresses again and takes the next, or the peer the same node + * got meanwhile. + */ + private async saveOnFreeAddress( + nodeId: string, + build: (taken: string[]) => WireGuardPeerEntity, + ): Promise { + for (let attempt = 1; ; attempt++) { + try { + return await this.peers.save(build(await this.allocatedAddresses())); + } catch (err) { + if ((err as { code?: string })?.code !== '23505') throw err; + const mine = await this.peers.findOne({ + where: { nodeId, revokedAt: IsNull() }, + }); + if (mine) return mine; + if (attempt >= ADDRESS_ATTEMPTS) throw err; + } + } } /** @@ -327,24 +354,25 @@ export class WireGuardPeerService { } const pool = await this.poolFor(input.subnetId); - const taken = await this.allocatedAddresses(); - const peer = this.peers.create({ - clusterId: input.clusterId, - nodeId: input.nodeId, - role: WireGuardPeerRole.MEMBER, - publicKey: input.publicKey, - managementIp: pool.allocate(taken), - endpointHost: input.endpointHost ?? null, - subnetId: input.subnetId ?? null, - // A mesh member listens; a management-overlay member does not. Recording - // it here is what lets the renderer tell the two apart later. - listenPort: input.subnetId ? WG_DEFAULT_PORT : null, - status: WireGuardPeerStatus.PENDING, - }); + const peer = await this.saveOnFreeAddress(input.nodeId, (taken) => + this.peers.create({ + clusterId: input.clusterId, + nodeId: input.nodeId, + role: WireGuardPeerRole.MEMBER, + publicKey: input.publicKey, + managementIp: pool.allocate(taken), + endpointHost: input.endpointHost ?? null, + subnetId: input.subnetId ?? null, + // A mesh member listens; a management-overlay member does not. Recording + // it here is what lets the renderer tell the two apart later. + listenPort: input.subnetId ? WG_DEFAULT_PORT : null, + status: WireGuardPeerStatus.PENDING, + }), + ); this.logger.log( `[wg] node ${input.nodeId} enrolled at ${peer.managementIp}`, ); - return this.peers.save(peer); + return peer; } /** Withdraws a peer. Its address stays claimed — see allocatedAddresses. */ diff --git a/src/modules/infrastructure/operations/helpers/operation-steps.helper.ts b/src/modules/infrastructure/operations/helpers/operation-steps.helper.ts index 26085f82..31d0ad2a 100644 --- a/src/modules/infrastructure/operations/helpers/operation-steps.helper.ts +++ b/src/modules/infrastructure/operations/helpers/operation-steps.helper.ts @@ -40,6 +40,105 @@ export function estimateCreateDurationSeconds(workerCount = 0): number { ); } +/** The steps of a cluster's creation: single node, with workers, or across three locations. */ +function createClusterSteps( + context?: Record, +): OperationStepConfig[] { + const workerCount = context?.workerCount || 0; + const haServers: string[] = context?.haServers ?? []; + + if (haServers.length) { + return [ + { + step: OperationStep.CLUSTER_CREATE_INIT, + description: 'Initializing cluster creation', + weight: 5, + }, + { + step: OperationStep.CLUSTER_CREATE_MASTER, + description: + 'Buying the servers for every location; the first one starts the cluster', + weight: 40, + }, + { + step: OperationStep.CLUSTER_CREATE_KUBECONFIG, + description: 'Fetching kubeconfig from the first server', + weight: 5, + }, + { + step: OperationStep.CLUSTER_CREATE_WORKERS, + description: `Joining the servers in ${haServers.join(' and ')}, one at a time`, + weight: 30, + }, + { + step: OperationStep.CLUSTER_CREATE_FINALIZING, + description: 'Waiting for Flui to finish installing, then finalizing', + weight: 20, + }, + ]; + } + + // The master step owns the whole bootstrap — server create, cloud-init, + // k3s, and the kubeconfig fetch all happen inside createMasterNode. The + // kubeconfig step that follows only marks what already happened (~25ms), + // so it is weighted as the formality it is; the 30 it used to carry made + // the bar jump a third of its length in a single frame. + if (workerCount === 0) { + // Single-node cluster (no workers) + return [ + { + step: OperationStep.CLUSTER_CREATE_INIT, + description: 'Initializing cluster creation', + weight: 5, + }, + { + step: OperationStep.CLUSTER_CREATE_MASTER, + description: 'Creating master node and waiting for K3s', + weight: 80, + }, + { + step: OperationStep.CLUSTER_CREATE_KUBECONFIG, + description: 'Fetching kubeconfig from master node', + weight: 5, + }, + { + step: OperationStep.CLUSTER_CREATE_FINALIZING, + description: 'Finalizing cluster setup', + weight: 10, + }, + ]; + } else { + // Multi-node cluster (with workers) + return [ + { + step: OperationStep.CLUSTER_CREATE_INIT, + description: 'Initializing cluster creation', + weight: 5, + }, + { + step: OperationStep.CLUSTER_CREATE_MASTER, + description: 'Creating master node and waiting for K3s', + weight: 35, + }, + { + step: OperationStep.CLUSTER_CREATE_KUBECONFIG, + description: 'Fetching kubeconfig from master node', + weight: 5, + }, + { + step: OperationStep.CLUSTER_CREATE_WORKERS, + description: `Creating ${workerCount} worker node${workerCount > 1 ? 's' : ''}`, + weight: 45, + }, + { + step: OperationStep.CLUSTER_CREATE_FINALIZING, + description: 'Finalizing cluster setup', + weight: 10, + }, + ]; + } +} + /** * Generate step configuration dynamically based on operation context * This allows steps to adapt to different scenarios (e.g., single-node vs multi-node clusters) @@ -49,69 +148,8 @@ export function getOperationSteps( context?: Record, ): OperationStepConfig[] { switch (operationType) { - case OperationType.CREATE_CLUSTER: { - const workerCount = context?.workerCount || 0; - - // The master step owns the whole bootstrap — server create, cloud-init, - // k3s, and the kubeconfig fetch all happen inside createMasterNode. The - // kubeconfig step that follows only marks what already happened (~25ms), - // so it is weighted as the formality it is; the 30 it used to carry made - // the bar jump a third of its length in a single frame. - if (workerCount === 0) { - // Single-node cluster (no workers) - return [ - { - step: OperationStep.CLUSTER_CREATE_INIT, - description: 'Initializing cluster creation', - weight: 5, - }, - { - step: OperationStep.CLUSTER_CREATE_MASTER, - description: 'Creating master node and waiting for K3s', - weight: 80, - }, - { - step: OperationStep.CLUSTER_CREATE_KUBECONFIG, - description: 'Fetching kubeconfig from master node', - weight: 5, - }, - { - step: OperationStep.CLUSTER_CREATE_FINALIZING, - description: 'Finalizing cluster setup', - weight: 10, - }, - ]; - } else { - // Multi-node cluster (with workers) - return [ - { - step: OperationStep.CLUSTER_CREATE_INIT, - description: 'Initializing cluster creation', - weight: 5, - }, - { - step: OperationStep.CLUSTER_CREATE_MASTER, - description: 'Creating master node and waiting for K3s', - weight: 35, - }, - { - step: OperationStep.CLUSTER_CREATE_KUBECONFIG, - description: 'Fetching kubeconfig from master node', - weight: 5, - }, - { - step: OperationStep.CLUSTER_CREATE_WORKERS, - description: `Creating ${workerCount} worker node${workerCount > 1 ? 's' : ''}`, - weight: 45, - }, - { - step: OperationStep.CLUSTER_CREATE_FINALIZING, - description: 'Finalizing cluster setup', - weight: 10, - }, - ]; - } - } + case OperationType.CREATE_CLUSTER: + return createClusterSteps(context); case OperationType.REINSTALL_CLUSTER: { return [ diff --git a/src/modules/infrastructure/scaling/dto/scaling-response.dto.ts b/src/modules/infrastructure/scaling/dto/scaling-response.dto.ts index a2c6087d..f3021d05 100644 --- a/src/modules/infrastructure/scaling/dto/scaling-response.dto.ts +++ b/src/modules/infrastructure/scaling/dto/scaling-response.dto.ts @@ -635,6 +635,8 @@ export class GiveBackReasonDto { 'no-interruption', 'placed-by-machine', 'master', + 'server', + 'copies', 'runs-flui-api', ], }) diff --git a/src/modules/infrastructure/scaling/engine/drain-feasibility.service.spec.ts b/src/modules/infrastructure/scaling/engine/drain-feasibility.service.spec.ts index 952a8e37..118bee60 100644 --- a/src/modules/infrastructure/scaling/engine/drain-feasibility.service.spec.ts +++ b/src/modules/infrastructure/scaling/engine/drain-feasibility.service.spec.ts @@ -213,7 +213,11 @@ describe('whether a node can be emptied', () => { status: { phase }, }); - function checking(pods: unknown[], apiPods: unknown[] = []) { + function checking( + pods: unknown[], + apiPods: unknown[] = [], + labels: Record = {}, + ) { const coreApi = { listPodForAllNamespaces: jest.fn(async () => ({ items: pods })), listPersistentVolume: jest.fn(async () => ({ items: [] })), @@ -221,6 +225,7 @@ describe('whether a node can be emptied', () => { items: [], })), listNamespacedPod: jest.fn(async () => ({ items: apiPods })), + readNode: jest.fn(async () => ({ metadata: { labels } })), }; const service = new DrainFeasibilityService( { find: async () => [] } as never, @@ -289,6 +294,7 @@ describe('whether a node can be emptied', () => { items: [], })), listNamespacedPod: jest.fn(async () => ({ items: apiPods })), + readNode: jest.fn(async () => ({ metadata: { labels: {} } })), }; const service = new DrainFeasibilityService( { find } as never, @@ -317,3 +323,42 @@ describe('whether a node can be emptied', () => { expect(check?.blockers.map((b) => b.kind)).toEqual(['no-controller']); }); }); + +describe('what the cluster itself says about a node', () => { + const coreFor = (labels: Record) => ({ + listPodForAllNamespaces: jest.fn(async () => ({ items: [] })), + listPersistentVolume: jest.fn(async () => ({ items: [] })), + listPodDisruptionBudgetForAllNamespaces: jest.fn(async () => ({ + items: [], + })), + listNamespacedPod: jest.fn(async () => ({ items: [] })), + readNode: jest.fn(async () => ({ metadata: { labels } })), + }); + const service = (labels: Record) => + new DrainFeasibilityService( + { find: async () => [] } as never, + { + makeKubeConfig: () => ({ makeApiClient: () => coreFor(labels) }), + } as never, + { decrypt: () => 'kubeconfig' } as never, + ); + const worker = { + serverName: 'worker-3', + nodeType: NodeType.WORKER, + } as ClusterNodeEntity; + const cluster = { id: 'c1', kubeconfigEncrypted: 'x' } as ClusterEntity; + + it('refuses a server Flui recorded as a worker, from its control-plane label', async () => { + const check = await service({ + 'node-role.kubernetes.io/control-plane': 'true', + }).check(cluster, worker); + expect(check?.blockers.map((b) => b.kind)).toEqual(['is-server']); + }); + + it('holds a node prepared for copies of replicated volumes', async () => { + const check = await service({ + 'flui.cloud/replicated-storage': 'true', + }).check(cluster, worker); + expect(check?.blockers.map((b) => b.kind)).toContain('holds-copies'); + }); +}); diff --git a/src/modules/infrastructure/scaling/engine/drain-feasibility.service.ts b/src/modules/infrastructure/scaling/engine/drain-feasibility.service.ts index a2af1013..a735d95a 100644 --- a/src/modules/infrastructure/scaling/engine/drain-feasibility.service.ts +++ b/src/modules/infrastructure/scaling/engine/drain-feasibility.service.ts @@ -1,6 +1,8 @@ import { Injectable, Logger } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { In, IsNull, Repository } from 'typeorm'; +import { haRoleOf } from '../../clusters/utils/ha-node-role'; +import { REPLICATED_STORAGE_NODE_LABEL } from '../../../replicated-volumes/replicated-volumes.constants'; import * as k8s from '@kubernetes/client-node'; import { ApplicationEntity } from '../../../applications/entities/application.entity'; import { @@ -34,6 +36,8 @@ import { nodeRoomInputs } from './room-inputs'; */ const USAGE_TIMEOUT_MS = 6_000; +const CONTROL_PLANE_LABEL = 'node-role.kubernetes.io/control-plane'; + /** * Whether a node can be emptied, answered from the cluster itself. * @@ -77,7 +81,7 @@ export class DrainFeasibilityService { const coreApi = kc.makeApiClient(k8s.CoreV1Api); const policyApi = kc.makeApiClient(k8s.PolicyV1Api); - const [pods, volumes, budgets, api] = await Promise.all([ + const [pods, volumes, budgets, api, k8sNode] = await Promise.all([ coreApi.listPodForAllNamespaces({ fieldSelector: `spec.nodeName=${node.serverName},status.phase!=Succeeded,status.phase!=Failed`, }), @@ -87,7 +91,9 @@ export class DrainFeasibilityService { namespace: 'flui-system', labelSelector: 'app=flui-api', }), + coreApi.readNode({ name: node.serverName }).catch(() => null), ]); + const labels = k8sNode?.metadata?.labels ?? {}; const pinned = pinnedClaims(volumes.items ?? [], node.serverName); const running = (pods.items ?? []).filter((pod) => !finished(pod)); @@ -102,6 +108,9 @@ export class DrainFeasibilityService { apiCopiesElsewhere: (api.items ?? []).filter( (pod) => pod.spec?.nodeName !== node.serverName && ready(pod), ).length, + isServer: + haRoleOf(node) === 'server' || labels[CONTROL_PLANE_LABEL] === 'true', + holdsCopies: labels[REPLICATED_STORAGE_NODE_LABEL] === 'true', }); } catch (err) { this.logger.warn( diff --git a/src/modules/infrastructure/scaling/engine/drain.core.spec.ts b/src/modules/infrastructure/scaling/engine/drain.core.spec.ts index 644126d0..916981d2 100644 --- a/src/modules/infrastructure/scaling/engine/drain.core.spec.ts +++ b/src/modules/infrastructure/scaling/engine/drain.core.spec.ts @@ -245,4 +245,19 @@ describe('what emptying the node would move', () => { { what: 'flui-web', system: true, application: null }, ]); }); + + it('refuses a server of the control plane by what it is, whatever runs on it', () => { + const check = checkDrain(subject({ isServer: true })); + expect(check.ok).toBe(false); + expect(check.blockers.map((b) => b.kind)).toEqual(['is-server']); + }); + + it('refuses a node holding copies of replicated volumes, with the reason', () => { + const check = checkDrain(subject({ holdsCopies: true })); + expect(check.ok).toBe(false); + expect(check.blockers[0]).toMatchObject({ + kind: 'holds-copies', + what: 'prod-eu-worker-2', + }); + }); }); diff --git a/src/modules/infrastructure/scaling/engine/drain.core.ts b/src/modules/infrastructure/scaling/engine/drain.core.ts index 7dc737f4..2c37b219 100644 --- a/src/modules/infrastructure/scaling/engine/drain.core.ts +++ b/src/modules/infrastructure/scaling/engine/drain.core.ts @@ -19,6 +19,8 @@ export type DrainBlockerKind = | 'disruption-budget' | 'not-evictable' | 'is-master' + | 'is-server' + | 'holds-copies' | 'runs-flui-api'; /** The application something on the node belongs to, when Flui deployed it. */ @@ -107,6 +109,10 @@ export interface DrainSubject { * copy here is just another replica to move; absent is read as none. */ apiCopiesElsewhere?: number; + /** A server of the control plane, by Flui's record or by the cluster's own label. */ + isServer?: boolean; + /** Prepared to hold copies of replicated volumes. */ + holdsCopies?: boolean; } const DAEMONSET = 'DaemonSet'; @@ -129,9 +135,34 @@ export function checkDrain(subject: DrainSubject): DrainCheck { }; } + // A server holds one of the votes the control plane needs, and nothing + // shows that in its pods either. + if (subject.isServer) { + return { + ok: false, + node: subject.nodeName, + blockers: [ + { + kind: 'is-server', + what: subject.nodeName, + fix: 'A server of the control plane is not removed by scaling. A lost one is replaced by adding a server in the same location.', + }, + ], + cleared: [], + }; + } + const blockers: DrainBlocker[] = []; const cleared: string[] = []; + if (subject.holdsCopies) { + blockers.push({ + kind: 'holds-copies', + what: subject.nodeName, + fix: 'This node holds copies of replicated volumes. Removing it leaves those volumes with one copy; add a node for data in the same location first.', + }); + } + for (const app of subject.dedicatedApps) { blockers.push({ kind: 'dedicated-app', diff --git a/src/modules/infrastructure/scaling/engine/give-back.core.ts b/src/modules/infrastructure/scaling/engine/give-back.core.ts index 1d98639d..04607eac 100644 --- a/src/modules/infrastructure/scaling/engine/give-back.core.ts +++ b/src/modules/infrastructure/scaling/engine/give-back.core.ts @@ -15,6 +15,8 @@ export type GiveBackReasonKind = | 'no-interruption' | 'placed-by-machine' | 'master' + | 'server' + | 'copies' | 'runs-flui-api'; export interface GiveBackItem { @@ -38,6 +40,8 @@ const REASON_OF: Record = { 'disruption-budget': 'no-interruption', 'not-evictable': 'placed-by-machine', 'is-master': 'master', + 'is-server': 'server', + 'holds-copies': 'copies', 'runs-flui-api': 'runs-flui-api', }; @@ -72,6 +76,16 @@ const WORDING: Record< fix: 'Nothing replaces the master of a cluster in place. Rebuild the cluster if it has to change.', sentence: () => "it is the cluster's master", }, + server: { + title: 'Is a server of the control plane', + fix: 'A server of the control plane is not removed by scaling. A lost one is replaced by adding a server in the same location.', + sentence: () => 'it is a server of the control plane', + }, + copies: { + title: 'Holds copies of replicated volumes', + fix: 'Removing it leaves those volumes with one copy. Add a node for data in the same location first.', + sentence: () => 'it holds copies of replicated volumes', + }, 'runs-flui-api': { title: 'Runs the Flui API, which carries out the removal', fix: 'Nothing is changed while the Flui API runs here. The removal waits until it runs on another machine.', diff --git a/src/modules/infrastructure/scaling/engine/removable.core.spec.ts b/src/modules/infrastructure/scaling/engine/removable.core.spec.ts index 54989369..0214822d 100644 --- a/src/modules/infrastructure/scaling/engine/removable.core.spec.ts +++ b/src/modules/infrastructure/scaling/engine/removable.core.spec.ts @@ -46,4 +46,17 @@ describe('pickRemovable', () => { it('never names the master', () => { expect(pickRemovable([fleet[0]], null, true)).toBeNull(); }); + + it('never names a node an operator added for high availability', () => { + const server = { + ...node('worker-5', 'hel1', 0.05), + metadata: { haRole: 'server' }, + } as unknown as ClusterNodeEntity; + const dataHost = { + ...node('worker-6', 'nbg1', 0.04), + metadata: { haRole: 'data-host' }, + } as unknown as ClusterNodeEntity; + expect(pickRemovable([fleet[0], server, dataHost], null)).toBeNull(); + expect(pickRemovable([...fleet, server], null)?.name).toBe('worker-4'); + }); }); diff --git a/src/modules/infrastructure/scaling/engine/removable.core.ts b/src/modules/infrastructure/scaling/engine/removable.core.ts index e2f68ce6..b0dc6fcf 100644 --- a/src/modules/infrastructure/scaling/engine/removable.core.ts +++ b/src/modules/infrastructure/scaling/engine/removable.core.ts @@ -3,6 +3,7 @@ import { NodeStatus, NodeType, } from '../../clusters/entities/cluster-node.entity'; +import { haRoleOf } from '../../clusters/utils/ha-node-role'; /** A node the fleet could give back, named so something can actually remove it. */ export interface RemovableNode { @@ -19,7 +20,8 @@ export interface RemovableNode { * cheapest machine of an oversized fleet is the version of this that costs * money to run. A group that spreads its nodes gives back from the region * holding the most of them first, so coming down keeps the fleet spread. Never - * the master, and never a node already leaving. The preview a person reads + * the master, never a node an operator added for high availability, and never + * a node already leaving. The preview a person reads * before saving asks the same function, so the two never name different * machines. */ @@ -30,7 +32,9 @@ export function pickRemovable( ): RemovableNode | null { const workers = rows.filter( (row) => - row.nodeType !== NodeType.MASTER && row.status !== NodeStatus.DELETING, + row.nodeType !== NodeType.MASTER && + row.status !== NodeStatus.DELETING && + !haRoleOf(row), ); if (!workers.length) return null; diff --git a/src/modules/infrastructure/servers/entities/infrastructure-operations.entity.ts b/src/modules/infrastructure/servers/entities/infrastructure-operations.entity.ts index 58ef1f49..11a5ce6c 100644 --- a/src/modules/infrastructure/servers/entities/infrastructure-operations.entity.ts +++ b/src/modules/infrastructure/servers/entities/infrastructure-operations.entity.ts @@ -1,6 +1,7 @@ // File: src/modules/infrastructure/servers/entities/infrastructure-operations.entity.ts import { CloudProvider } from 'src/modules/providers/enums/cloud-provider.enum'; +import type { HaCreationPlan } from '../../clusters/utils/ha-creation-plan.core'; import { BeforeInsert, Entity, @@ -81,6 +82,9 @@ export enum OperationType { APP_BACKUP_CREATE = 'app_backup_create', // Database lifecycle MIGRATE_DATABASE = 'migrate_database', + ACTIVATE_DATABASE_HA = 'activate_database_ha', + // Replicated volumes + ENABLE_REPLICATED_VOLUMES = 'enable_replicated_volumes', // Application lifecycle MIGRATE_APPLICATION = 'migrate_application', // Full-app lifecycle (app + DB) @@ -324,6 +328,16 @@ export enum OperationStep { PLATFORM_UPDATE_BACKUP = 'platform_update_backup', PLATFORM_UPDATE_MANIFESTS = 'platform_update_manifests', PLATFORM_UPDATE_K3S = 'platform_update_k3s', + // Highly available Postgres + DB_HA_PREPARE = 'db_ha_prepare', + DB_HA_COPY = 'db_ha_copy', + DB_HA_FENCE = 'db_ha_fence', + DB_HA_PROMOTE = 'db_ha_promote', + DB_HA_SWITCH = 'db_ha_switch', + // Replicated volumes + REPLICATED_PREPARE_NODES = 'replicated_prepare_nodes', + REPLICATED_INSTALL_ENGINE = 'replicated_install_engine', + REPLICATED_STORAGE_CLASS = 'replicated_storage_class', } // Base metadata interface - allows additional runtime properties @@ -371,6 +385,8 @@ export interface CreateClusterOperationMetadata extends BaseOperationMetadata { * egress IPs, not just what's on the request DTO). */ desiredFirewallRules?: FirewallRuleDto[]; + /** A cluster across three locations: the servers decided before any was bought. */ + haPlan?: HaCreationPlan; } export interface DeleteClusterOperationMetadata extends BaseOperationMetadata { diff --git a/src/modules/infrastructure/shared/services/kubernetes.service.ts b/src/modules/infrastructure/shared/services/kubernetes.service.ts index 3244d280..77e740db 100644 --- a/src/modules/infrastructure/shared/services/kubernetes.service.ts +++ b/src/modules/infrastructure/shared/services/kubernetes.service.ts @@ -1,4 +1,5 @@ import { randomBytes } from 'node:crypto'; +import { TOPOLOGY_ZONE_LABEL } from '../../../providers/core/node-topology'; import { ForbiddenException, Injectable, Logger } from '@nestjs/common'; import * as k8s from '@kubernetes/client-node'; import { Writable, Readable } from 'node:stream'; @@ -15,6 +16,7 @@ import { WaitingForRoomError, roomWaitSeconds, } from '../utils/waits-for-room.util'; +import { runningPodForDatabase } from '../utils/database-pod.util'; export interface ContainerResources { cpu: string | null; @@ -106,6 +108,11 @@ const timeOfEvent = (event: k8s.CoreV1Event): number => /** Long enough for the scheduler to have tried every node, short enough not to look like a hang. */ const ROOM_SETTLE_SECONDS = 45; +/** The busybox K3s already keeps on every node. */ +export const NODE_PROBE_IMAGE = 'rancher/mirrored-library-busybox:1.36.1'; + +const DNS_RESPREAD_ANNOTATION = 'flui.cloud/dns-respread-at'; + @Injectable() export class KubernetesService { private readonly logger = new Logger(KubernetesService.name); @@ -243,6 +250,32 @@ export class KubernetesService { return JSON.stringify(spec); } + /** + * Applies every document server-side, as `kubectl apply --server-side` + * does. For manifests whose objects are too large to carry a + * last-applied-configuration annotation, such as operator CRDs. + */ + async serverSideApply( + kubeconfigContent: string, + manifestYaml: string, + ): Promise { + const client = this.objectApi(kubeconfigContent); + let applied = 0; + for (const spec of k8s.loadAllYaml(manifestYaml)) { + if (!spec || Object.keys(spec).length === 0) continue; + await client.patch( + spec, + undefined, + undefined, + 'flui-api', + true, + k8s.PatchStrategy.ServerSideApply, + ); + applied++; + } + return applied; + } + /** * Apply manifest from YAML string * Supports multiple documents separated by --- @@ -1270,7 +1303,7 @@ export class KubernetesService { ].filter(Boolean), } : await coreApi.listNamespacedPod({ namespace, labelSelector }); - const pod = (pods.items ?? []).find((p) => p.status?.phase === 'Running'); + const pod = runningPodForDatabase(pods.items ?? []); if (!pod?.metadata?.name) { throw new Error( `No running pod found with selector "${labelSelector}" in namespace "${namespace}"`, @@ -1338,7 +1371,7 @@ export class KubernetesService { const kc = this.loadKubeconfig(kubeconfigContent); const coreApi = kc.makeApiClient(k8s.CoreV1Api); const pods = await coreApi.listNamespacedPod({ namespace, labelSelector }); - const pod = (pods.items ?? []).find((p) => p.status?.phase === 'Running'); + const pod = runningPodForDatabase(pods.items ?? []); if (!pod?.metadata?.name) { throw new Error( `No running pod found with selector "${labelSelector}" in namespace "${namespace}"`, @@ -2313,6 +2346,82 @@ export class KubernetesService { ); } + async listNodes(kubeconfigContent: string): Promise { + const { coreApi } = this.getKubeClient(kubeconfigContent); + return (await coreApi.listNode()).items ?? []; + } + + /** The labels of every node, by node name. */ + async listNodeLabels( + kubeconfigContent: string, + ): Promise>> { + const { coreApi } = this.getKubeClient(kubeconfigContent); + const nodes = (await coreApi.listNode()).items ?? []; + return new Map( + nodes + .filter((node) => node.metadata?.name) + .map((node) => [node.metadata!.name!, node.metadata?.labels ?? {}]), + ); + } + + /** The annotations of every node, by node name. */ + async listNodeAnnotations( + kubeconfigContent: string, + ): Promise>> { + const { coreApi } = this.getKubeClient(kubeconfigContent); + const nodes = (await coreApi.listNode()).items ?? []; + return new Map( + nodes + .filter((node) => node.metadata?.name) + .map((node) => [ + node.metadata!.name!, + node.metadata?.annotations ?? {}, + ]), + ); + } + + /** Adds labels to a node; labels not named are kept. */ + async addNodeLabels( + kubeconfigContent: string, + nodeName: string, + labels: Record, + ): Promise { + await this.mergeNodeMetadata(kubeconfigContent, nodeName, { labels }); + } + + /** Adds annotations to a node; annotations not named are kept. */ + async addNodeAnnotations( + kubeconfigContent: string, + nodeName: string, + annotations: Record, + ): Promise { + await this.mergeNodeMetadata(kubeconfigContent, nodeName, { annotations }); + } + + private async mergeNodeMetadata( + kubeconfigContent: string, + nodeName: string, + metadata: { + labels?: Record; + annotations?: Record; + }, + ): Promise { + const client = this.objectApi(kubeconfigContent); + const patch = { + apiVersion: 'v1', + kind: 'Node', + metadata: { name: nodeName, ...metadata }, + } as k8s.KubernetesObject; + await client.patch( + patch, + undefined, + undefined, + 'flui-api', + undefined, + k8s.PatchStrategy.MergePatch, + ); + } + /** * Mark a node unschedulable. Existing pods continue to run; new ones are * not placed on it. Used as a precaution before power-cycling a node during @@ -2612,20 +2721,69 @@ export class KubernetesService { } /** The cluster DNS (K3s' `coredns` Deployment): its replica count and whether it spreads across nodes. */ - async readClusterDns( - kubeconfigContent: string, - ): Promise<{ replicas: number; spreadAcrossNodes: boolean } | null> { - const { appsApi } = this.getKubeClient(kubeconfigContent); + async readClusterDns(kubeconfigContent: string): Promise<{ + replicas: number; + spreadAcrossNodes: boolean; + spreadAcrossZones: boolean; + /** Locations of the nodes running a ready copy. */ + podZones: string[]; + /** Locations with a ready node that takes workloads. */ + readyZones: string[]; + respreadAt?: string; + } | null> { + const { appsApi, coreApi } = this.getKubeClient(kubeconfigContent); try { const deployment = await appsApi.readNamespacedDeployment({ name: 'coredns', namespace: 'kube-system', }); + const constraints = + deployment.spec?.template?.spec?.topologySpreadConstraints ?? []; + const [nodes, pods] = await Promise.all([ + coreApi.listNode(), + coreApi.listNamespacedPod({ + namespace: 'kube-system', + labelSelector: 'k8s-app=kube-dns', + }), + ]); + const zoneOf = new Map( + (nodes.items ?? []).map((n) => [ + n.metadata?.name, + n.metadata?.labels?.[TOPOLOGY_ZONE_LABEL], + ]), + ); + const ready = (conditions?: Array<{ type: string; status: string }>) => + (conditions ?? []).some( + (c) => c.type === 'Ready' && c.status === 'True', + ); return { replicas: deployment.spec?.replicas ?? 1, - spreadAcrossNodes: ( - deployment.spec?.template?.spec?.topologySpreadConstraints ?? [] - ).some((c) => c.topologyKey === 'kubernetes.io/hostname'), + spreadAcrossNodes: constraints.some( + (c) => c.topologyKey === 'kubernetes.io/hostname', + ), + spreadAcrossZones: constraints.some( + (c) => c.topologyKey === TOPOLOGY_ZONE_LABEL, + ), + podZones: (pods.items ?? []) + .filter((p) => ready(p.status?.conditions)) + .map((p) => zoneOf.get(p.spec?.nodeName)) + .filter((z): z is string => !!z), + readyZones: [ + ...new Set( + (nodes.items ?? []) + .filter( + (n) => + ready(n.status?.conditions) && + !n.spec?.unschedulable && + !(n.spec?.taints ?? []).some( + (t) => t.effect === 'NoSchedule', + ), + ) + .map((n) => n.metadata?.labels?.[TOPOLOGY_ZONE_LABEL]) + .filter((z): z is string => !!z), + ), + ], + respreadAt: deployment.metadata?.annotations?.[DNS_RESPREAD_ANNOTATION], }; } catch (err) { if ((err as { code?: number })?.code === 404) return null; @@ -2634,8 +2792,11 @@ export class KubernetesService { } /** - * Runs the cluster DNS as `replicas` copies placed on different nodes when - * they can be, so losing one node does not take name resolution with it. + * Runs the cluster DNS as `replicas` copies placed on different nodes, and in + * different locations, when they can be: losing one node, or a location, + * does not take name resolution with it. The volume engine finds its own + * controller by name, so a location that held every copy also held up + * every volume that had to move. */ async spreadClusterDns( kubeconfigContent: string, @@ -2652,6 +2813,12 @@ export class KubernetesService { template: { spec: { topologySpreadConstraints: [ + { + maxSkew: 1, + topologyKey: TOPOLOGY_ZONE_LABEL, + whenUnsatisfiable: 'ScheduleAnyway', + labelSelector: { matchLabels: { 'k8s-app': 'kube-dns' } }, + }, { maxSkew: 1, topologyKey: 'kubernetes.io/hostname', @@ -2671,6 +2838,40 @@ export class KubernetesService { ); } + /** + * Restarts the cluster DNS one copy at a time, so the scheduler places the + * copies again: a spread rule only applies when a copy is placed, and copies + * placed before the nodes said where they are stay where they landed. + */ + async respreadClusterDns( + kubeconfigContent: string, + at: string, + ): Promise { + await this.objectApi(kubeconfigContent).patch( + { + apiVersion: 'apps/v1', + kind: 'Deployment', + metadata: { + name: 'coredns', + namespace: 'kube-system', + annotations: { [DNS_RESPREAD_ANNOTATION]: at }, + }, + spec: { + template: { + metadata: { + annotations: { 'kubectl.kubernetes.io/restartedAt': at }, + }, + }, + } as unknown as k8s.V1DeploymentSpec, + } as k8s.V1Deployment, + undefined, + undefined, + 'flui-api', + undefined, + k8s.PatchStrategy.StrategicMergePatch, + ); + } + /** * Runs one short shell command on a given node, with the node's * `hostDir` mounted at `/host`, and returns what it printed. Pinned with @@ -2682,6 +2883,30 @@ export class KubernetesService { nodeName: string, hostDir: string, script: string, + ): Promise { + return this.runImageOnNode(kubeconfigContent, nodeName, { + image: NODE_PROBE_IMAGE, + command: ['sh', '-c', script], + hostDir, + mountPath: '/host', + }); + } + + /** + * Runs one container to completion on a given node and returns what it + * printed, for images without a shell. With `hostNetwork` it reaches what + * the node only serves on its loopback, and measures the node's own links. + */ + async runImageOnNode( + kubeconfigContent: string, + nodeName: string, + run: { + image: string; + command: string[]; + hostDir?: string; + mountPath?: string; + hostNetwork?: boolean; + }, ): Promise { const { coreApi } = this.getKubeClient(kubeconfigContent); const namespace = 'kube-system'; @@ -2703,14 +2928,19 @@ export class KubernetesService { nodeName, restartPolicy: 'Never', enableServiceLinks: false, + hostNetwork: run.hostNetwork ?? false, tolerations: [{ operator: 'Exists' }], - volumes: [{ name: 'host', hostPath: { path: hostDir } }], + volumes: run.hostDir + ? [{ name: 'host', hostPath: { path: run.hostDir } }] + : [], containers: [ { name: 'probe', - image: 'rancher/mirrored-library-busybox:1.36.1', - command: ['sh', '-c', script], - volumeMounts: [{ name: 'host', mountPath: '/host' }], + image: run.image, + command: run.command, + volumeMounts: run.hostDir + ? [{ name: 'host', mountPath: run.mountPath ?? '/host' }] + : [], }, ], }, @@ -2744,6 +2974,40 @@ export class KubernetesService { } } + /** Every object of a kind in every namespace; empty when the kind is not installed. */ + async listObjectsEverywhere( + kubeconfigContent: string, + apiVersion: string, + kind: string, + ): Promise { + const client = this.objectApi(kubeconfigContent); + try { + const response = await client.list(apiVersion, kind); + const body = (response as any).body ?? response; + return body.items ?? []; + } catch (error) { + if (this.httpCode(error) === 404) return []; + throw error; + } + } + + /** The kubelet's own summary of a node (`stats/summary`), as it publishes it; null when the node does not answer. */ + async nodeStatsSummary( + kubeconfigContent: string, + nodeName: string, + ): Promise { + const { coreApi } = this.getKubeClient(kubeconfigContent); + try { + const raw: unknown = await coreApi.connectGetNodeProxyWithPath({ + name: nodeName, + path: 'stats/summary', + }); + return typeof raw === 'string' ? JSON.parse(raw) : raw; + } catch { + return null; + } + } + /** One key of a ConfigMap's data, or null when the ConfigMap or the key is missing. */ async readConfigMapKey( kubeconfigContent: string, diff --git a/src/modules/infrastructure/shared/utils/database-pod.util.spec.ts b/src/modules/infrastructure/shared/utils/database-pod.util.spec.ts new file mode 100644 index 00000000..705f004b --- /dev/null +++ b/src/modules/infrastructure/shared/utils/database-pod.util.spec.ts @@ -0,0 +1,42 @@ +import { runningPodForDatabase } from './database-pod.util'; + +const pod = (name: string, phase: string, role?: string) => ({ + name, + metadata: { labels: role ? { 'cnpg.io/instanceRole': role } : {} }, + status: { phase }, +}); + +describe('the pod a database command reaches', () => { + it('is the first running pod when nothing says which one leads', () => { + expect( + runningPodForDatabase([ + pod('a', 'Pending'), + pod('b', 'Running'), + pod('c', 'Running'), + ])?.name, + ).toBe('b'); + }); + + /** A copy refuses writes and reports another disk, so it is never the one a person means. */ + it('is the primary of a highly available Postgres, wherever it sits in the list', () => { + expect( + runningPodForDatabase([ + pod('db-1', 'Running', 'replica'), + pod('db-2', 'Running', 'primary'), + ])?.name, + ).toBe('db-2'); + }); + + it('falls back to a running copy while the primary is not running', () => { + expect( + runningPodForDatabase([ + pod('db-1', 'Running', 'replica'), + pod('db-2', 'Pending', 'primary'), + ])?.name, + ).toBe('db-1'); + }); + + it('is nothing when no pod runs', () => { + expect(runningPodForDatabase([pod('a', 'Failed')])).toBeUndefined(); + }); +}); diff --git a/src/modules/infrastructure/shared/utils/database-pod.util.ts b/src/modules/infrastructure/shared/utils/database-pod.util.ts new file mode 100644 index 00000000..ea7b600a --- /dev/null +++ b/src/modules/infrastructure/shared/utils/database-pod.util.ts @@ -0,0 +1,22 @@ +export const CNPG_INSTANCE_ROLE_LABEL = 'cnpg.io/instanceRole'; + +interface PodLike { + metadata?: { labels?: Record }; + status?: { phase?: string }; +} + +/** + * The running pod a command, a tunnel or a probe should reach. For a highly + * available Postgres that is the primary, since a copy refuses writes and + * reports a different disk; everything else keeps the first running pod. + */ +export function runningPodForDatabase( + pods: T[], +): T | undefined { + const running = pods.filter((p) => p.status?.phase === 'Running'); + return ( + running.find( + (p) => p.metadata?.labels?.[CNPG_INSTANCE_ROLE_LABEL] === 'primary', + ) ?? running[0] + ); +} diff --git a/src/modules/mask/sensitivity-sentinel.spec.ts b/src/modules/mask/sensitivity-sentinel.spec.ts index 05a630b5..e629feb1 100644 --- a/src/modules/mask/sensitivity-sentinel.spec.ts +++ b/src/modules/mask/sensitivity-sentinel.spec.ts @@ -1491,9 +1491,11 @@ const MASK_EXEMPT: string[] = [ 'CreateClusterDatasourcesDto.clusterName', // modules/grafana/dto/grafana-datasource.dto.ts 'CreateClusterDatasourcesDto.masterIpAddress', // modules/grafana/dto/grafana-datasource.dto.ts 'CreateClusterDto.autoscalingEnabled', // modules/infrastructure/clusters/dto/create-cluster.dto.ts + 'CreateClusterDto.completeLater', // modules/infrastructure/clusters/dto/create-cluster.dto.ts 'CreateClusterDto.diskSizeGb', // modules/infrastructure/clusters/dto/create-cluster.dto.ts 'CreateClusterDto.endpointHostnameMode', // modules/infrastructure/clusters/dto/create-cluster.dto.ts 'CreateClusterDto.firewallRules', // modules/infrastructure/clusters/dto/create-cluster.dto.ts + 'CreateClusterDto.highAvailability', // modules/infrastructure/clusters/dto/create-cluster.dto.ts 'CreateClusterDto.image', // modules/infrastructure/clusters/dto/create-cluster.dto.ts 'CreateClusterDto.k3sVersion', // modules/infrastructure/clusters/dto/create-cluster.dto.ts 'CreateClusterDto.maxNodes', // modules/infrastructure/clusters/dto/create-cluster.dto.ts @@ -1504,6 +1506,7 @@ const MASK_EXEMPT: string[] = [ 'CreateClusterDto.nodeSize', // modules/infrastructure/clusters/dto/create-cluster.dto.ts 'CreateClusterDto.provider', // modules/infrastructure/clusters/dto/create-cluster.dto.ts 'CreateClusterDto.region', // modules/infrastructure/clusters/dto/create-cluster.dto.ts + 'CreateClusterDto.shapes', // modules/infrastructure/clusters/dto/create-cluster.dto.ts 'CreateClusterDto.sharedStorageEnabled', // modules/infrastructure/clusters/dto/create-cluster.dto.ts 'CreateClusterDto.sharedStorageVolumeSizeGb', // modules/infrastructure/clusters/dto/create-cluster.dto.ts 'CreateClusterDto.sshKeys', // modules/infrastructure/clusters/dto/create-cluster.dto.ts @@ -1746,6 +1749,14 @@ const MASK_EXEMPT: string[] = [ 'DistributionAppDto.id', // modules/visualizations/dto/cluster-app-distribution.dto.ts 'DistributionAppDto.imageRef', // modules/visualizations/dto/cluster-app-distribution.dto.ts 'DistributionAppDto.k8sNamespace', // modules/visualizations/dto/cluster-app-distribution.dto.ts + 'ActivateDatabaseHaDto.durability', // modules/postgres-ha/dto/activate-database-ha.dto.ts + 'ActivateDatabaseHaDto.force', // modules/postgres-ha/dto/activate-database-ha.dto.ts + 'AddHaNodeDto.force', // modules/infrastructure/clusters/dto/add-ha-node.dto.ts + 'AddHaNodeDto.role', // modules/infrastructure/clusters/dto/add-ha-node.dto.ts + 'AddHaNodeDto.serverType', // modules/infrastructure/clusters/dto/add-ha-node.dto.ts + 'AddHaNodeDto.zone', // modules/infrastructure/clusters/dto/add-ha-node.dto.ts + 'ClusterStorageNfsDto.confirmsBeforeDisk', // modules/infrastructure/clusters/dto/cluster-storage.dto.ts + 'ClusterStorageNfsDto.durability', // modules/infrastructure/clusters/dto/cluster-storage.dto.ts 'DistributionAppDto.name', // modules/visualizations/dto/cluster-app-distribution.dto.ts 'DistributionAppDto.replicas', // modules/visualizations/dto/cluster-app-distribution.dto.ts 'DistributionAppDto.slug', // modules/visualizations/dto/cluster-app-distribution.dto.ts @@ -1817,6 +1828,9 @@ const MASK_EXEMPT: string[] = [ 'DrainCheckDto.cleared', // modules/infrastructure/scaling/dto/scaling-response.dto.ts 'DrainCheckDto.ok', // modules/infrastructure/scaling/dto/scaling-response.dto.ts 'EnableProviderDto.enabled', // modules/management/dto/enable-provider.dto.ts + 'EnableReplicatedVolumesDto.force', // modules/replicated-volumes/dto/enable-replicated-volumes.dto.ts + 'EnableReplicatedVolumesDto.nodes', // modules/replicated-volumes/dto/enable-replicated-volumes.dto.ts + 'EnableReplicatedVolumesDto.sizeGiB', // modules/replicated-volumes/dto/enable-replicated-volumes.dto.ts 'EnsureByosVNetDto.ipRange', // modules/infrastructure/clusters/dto/byos-vnet.dto.ts 'EnvVarCandidateDto.detectedPattern', // modules/frameworks/env-var-detection/dto/env-var-detection.dto.ts 'EnvVarCandidateDto.sourceFile', // modules/frameworks/env-var-detection/dto/env-var-detection.dto.ts @@ -2007,6 +2021,7 @@ const MASK_EXEMPT: string[] = [ 'GrafanaDatasourceResponseDto.type', // modules/grafana/dto/grafana-datasource.dto.ts 'GrafanaDatasourceResponseDto.uid', // modules/grafana/dto/grafana-datasource.dto.ts 'GrafanaDatasourceResponseDto.url', // modules/grafana/dto/grafana-datasource.dto.ts + 'HaPlanResponseDto.servers', // modules/infrastructure/clusters/dto/ha-creation-plan.dto.ts 'HealthCheckConfigDto.enabled', // modules/repositories/dto/analyze-repository.dto.ts 'HealthCheckConfigDto.failureThreshold', // modules/repositories/dto/analyze-repository.dto.ts 'HealthCheckConfigDto.initialDelaySeconds', // modules/repositories/dto/analyze-repository.dto.ts @@ -3104,6 +3119,7 @@ const MASK_EXEMPT: string[] = [ 'SandboxVisitorAdmissionRequestDto.turnstileToken', // modules/sandbox/dto/sandbox-visitor-admission.dto.ts 'SaveGhcrPatDto.expiresAt', // modules/repositories/dto/ghcr-pat.dto.ts 'SaveGhcrPatDto.token', // modules/repositories/dto/ghcr-pat.dto.ts + 'ScaleNodeDto.copiesGiB', // modules/infrastructure/clusters/dto/scale-node.dto.ts 'ScaleNodeDto.targetServerType', // modules/infrastructure/clusters/dto/scale-node.dto.ts 'ScaleNodeDto.upgradeDisk', // modules/infrastructure/clusters/dto/scale-node.dto.ts 'ScalingActuationDto.acts', // modules/infrastructure/scaling/dto/scaling-response.dto.ts diff --git a/src/modules/mcp/services/__fixtures__/mcp-tools-list.json b/src/modules/mcp/services/__fixtures__/mcp-tools-list.json index 64968c51..51fd290f 100644 --- a/src/modules/mcp/services/__fixtures__/mcp-tools-list.json +++ b/src/modules/mcp/services/__fixtures__/mcp-tools-list.json @@ -87,6 +87,8 @@ "db_session_terminate": "c5b96a4451a928ca", "db_pooler_client_close": "5a641978b56c1d45", "db_pooler_get": "439e190c60cc01b6", + "db_ha_get": "0ae26a9a7c213108", + "db_ha_enable": "828737d7ff6192b5", "migrate_app": "3b899e7cb73d45cc", "migrate_db": "78a5944802c57cc5", "migrate_full": "87e8c5c1cff174dc", @@ -183,6 +185,8 @@ "db_session_terminate": "c5b96a4451a928ca", "db_pooler_client_close": "5a641978b56c1d45", "db_pooler_get": "439e190c60cc01b6", + "db_ha_get": "0ae26a9a7c213108", + "db_ha_enable": "828737d7ff6192b5", "schedule_list": "72d4bd63af07ed57", "schedule_create": "1408fb67251fba8d", "schedule_update": "777fe1f8585edf8e", diff --git a/src/modules/mcp/tools/db-sessions.tools.ts b/src/modules/mcp/tools/db-sessions.tools.ts index 14c192f7..b6b71161 100644 --- a/src/modules/mcp/tools/db-sessions.tools.ts +++ b/src/modules/mcp/tools/db-sessions.tools.ts @@ -144,4 +144,31 @@ export const DB_SESSIONS_TOOLS: ToolDef[] = [ inputSchema: { id: z.string() }, run: (args, ctx) => ctx.api.get(`/applications/${enc(args.id)}/db/pooler`), }), + defineTool({ + name: 'db_ha_get', + routes: ['GET /applications/:id/db/high-availability'], + description: + 'Whether a managed PostgreSQL is highly available (pass the database application id). Null `state` is a single instance: losing its server takes the database down until it returns or is restored. Otherwise the highly available cluster (one instance in each of three locations, surviving the loss of a server or of a location), the durability (`required`: no confirmed write is lost, writes wait without a copy; `preferred`: writes go on and the copy catches up) and where a move stands (`copying`, `active`, `failed` with its detail). While active, `placement` is read live: the primary and each copy with its location, whether it can confirm commits, and `flushLagMs` (how long the latest writes took to be safe on it; null when nothing was written lately). Flui moves the primary back to a location for data after a failover.', + scope: MCP_SCOPE.APP_READ, + inputSchema: { id: z.string() }, + run: (args, ctx) => + ctx.api.get(`/applications/${enc(args.id)}/db/high-availability`), + }), + defineTool({ + name: 'db_ha_enable', + routes: ['POST /applications/:id/db/high-availability'], + description: + "Make a managed PostgreSQL highly available (pass the database application id). Flui builds three instances, one in each of the cluster's three locations, as live copies of the running one, and moves the application to them without stopping it; writes pause for a few seconds and the single instance is kept, stopped, as the way back. A commit waits for whichever copy answers first, which is the close location in practice. Refused with the reason when the database is not Flui's own PostgreSQL, or the cluster is not yet highly available across three locations of a validated region; `force` accepts a region Flui has not validated. A person approves it before it happens. Returns the operation; follow it with db_ha_get.", + scope: MCP_SCOPE.APP_WRITE, + inputSchema: { + id: z.string(), + durability: z.enum(['required', 'preferred']).optional(), + force: z.boolean().optional(), + }, + run: (args, ctx) => + ctx.api.post(`/applications/${enc(args.id)}/db/high-availability`, { + ...(args.durability && { durability: args.durability }), + ...(args.force && { force: args.force }), + }), + }), ]; diff --git a/src/modules/mcp/tools/infrastructure-operations.tools.ts b/src/modules/mcp/tools/infrastructure-operations.tools.ts index fde34ff0..622a716f 100644 --- a/src/modules/mcp/tools/infrastructure-operations.tools.ts +++ b/src/modules/mcp/tools/infrastructure-operations.tools.ts @@ -278,7 +278,7 @@ export const INFRASTRUCTURE_OPERATION_TOOLS: ToolDef[] = [ name: 'cluster_node_scale_preview', routes: ['GET /infrastructure/clusters/:id/nodes/:nodeId/scale/preview'], description: - 'What resizing one node would take down: its current server type, the dedicated workloads on it and the expected downtime. This is the estimate attached to cluster_node_resize — read it first and say the downtime out loud, because everything scheduled on that node, dedicated databases included, is unavailable for the whole window.', + 'What resizing one node would take down: its current server type, the dedicated workloads on it, the expected downtime and the machine types it could move to (`candidates`, against its own type). On a cluster across three locations `highAvailability.refusal` says why the resize would be refused now (another node being resized, a server down, a copy of a replicated volume behind) and `switchovers` the database primaries handed over first. This is the estimate attached to cluster_node_resize — read it first and say the downtime out loud, because everything scheduled on that node, dedicated databases included, is unavailable for the whole window.', scope: MCP_SCOPE.INFRA_READ, inputSchema: { ...clusterArg, nodeId: NODE_ID }, run: async (args, ctx) => { @@ -293,7 +293,7 @@ export const INFRASTRUCTURE_OPERATION_TOOLS: ToolDef[] = [ name: 'cluster_storage_status', routes: ['GET /infrastructure/clusters/:id/storage'], description: - "The cluster's shared-storage layer: the backing volume and its size, the NFS export and a summary of the claims on it. Read it before cluster_storage_expand — a volume can only ever grow, so the new size has to be chosen once and knowingly.", + "The cluster's shared-storage layer: the backing volume and its size, the NFS export, what it promises about confirmed writes (`nfs.durability`: when `confirmsBeforeDisk` is true the newest writes can be lost if the master stops abruptly, so databases belong on dedicated storage) and a summary of the claims on it. Read it before cluster_storage_expand — a volume can only ever grow, so the new size has to be chosen once and knowingly.", scope: MCP_SCOPE.INFRA_READ, inputSchema: clusterArg, run: async (args, ctx) => { @@ -386,6 +386,24 @@ export const INFRASTRUCTURE_OPERATION_TOOLS: ToolDef[] = [ .describe( 'Deprecated and ignored: use scaling_group_set to make a cluster grow on its own.', ), + highAvailability: z + .enum(['standard', 'three-locations']) + .optional() + .describe( + "`three-locations`: one server in each of the three locations the provider validated for this region (cluster_ha_get shows the layout once it exists), bought together and joined one at a time; `region` is the first server's location and must be one for data, and `workerCount` must be 0. It costs three machines: say so to the person. Refused with the reason where the provider declares no such layout.", + ), + shapes: z + .array(z.string()) + .optional() + .describe( + 'With `three-locations`: sizes each location may take, in order of preference, one architecture. Each location gets the first it has available. Omitted: `nodeSize`.', + ), + completeLater: z + .boolean() + .optional() + .describe( + 'With `three-locations`: when a location has none of the sizes, create the cluster in the others and leave that server to add later. Off (default), nothing is bought and the refusal names sizes every location has: relay it.', + ), }, run: async (args, ctx) => { const operation = await ctx.api.post( @@ -481,7 +499,7 @@ export const INFRASTRUCTURE_OPERATION_TOOLS: ToolDef[] = [ name: 'cluster_node_resize', routes: ['POST /infrastructure/clusters/:id/nodes/:nodeId/scale'], description: - 'Vertically resize one node: cordon, power off, change the server type at the provider (an arm type is accepted with a warning: "Arm server: apps deployed from a repository are built for x86 and will not run on it. Catalog apps run normally."), power on, wait for the node to be Ready again. This is a maintenance window of roughly 3-5 minutes during which every pod on that node — dedicated databases included — is down. Call cluster_node_scale_preview first and say the downtime out loud. Growing the local OS disk is deliberately not offered here: it is a one-way change that permanently blocks resizing back down.', + 'Vertically resize one node: cordon, power off, change the server type at the provider (an arm type is accepted with a warning: "Arm server: apps deployed from a repository are built for x86 and will not run on it. Catalog apps run normally."), power on, wait for the node to be Ready again. This is a maintenance window of roughly 3-5 minutes during which every pod on that node — dedicated databases included — is down. Call cluster_node_scale_preview first and say the downtime out loud. It is queued: the answer is the operation to follow with operation_status, and a job that an API restart interrupts carries on. On a cluster across three locations one node is resized at a time, only while the rest is healthy, and a database primary on the node is handed over first. `copiesGiB` grows the space the node sets aside for copies of replicated volumes once it is back. Growing the local OS disk is deliberately not offered here: it is a one-way change that permanently blocks resizing back down.', scope: MCP_SCOPE.INFRA_WRITE, inputSchema: { ...clusterArg, @@ -489,12 +507,29 @@ export const INFRASTRUCTURE_OPERATION_TOOLS: ToolDef[] = [ targetServerType: z .string() .describe('Provider server type to move to, e.g. `cx32`.'), + copiesGiB: z + .number() + .int() + .min(1) + .optional() + .describe( + 'New size of the space for copies of replicated volumes on this node, grown once it is back.', + ), }, run: async (args, ctx) => { const id = await resolveClusterId(ctx, args.clusterId); - return ctx.api.post( + const operation = await ctx.api.post<{ id: string; status?: string }>( `/infrastructure/clusters/${encoded(id)}/nodes/${encoded(args.nodeId)}/scale`, - { targetServerType: args.targetServerType }, + { + targetServerType: args.targetServerType, + ...(args.copiesGiB && { copiesGiB: args.copiesGiB }), + }, + ); + return startedOutcome( + ctx, + operation.id, + (operation.status ?? 'PENDING').toUpperCase(), + `Resize node ${args.nodeId} to ${args.targetServerType}`, ); }, }), diff --git a/src/modules/mcp/tools/infrastructure.tools.ts b/src/modules/mcp/tools/infrastructure.tools.ts index 41f067aa..8d159f77 100644 --- a/src/modules/mcp/tools/infrastructure.tools.ts +++ b/src/modules/mcp/tools/infrastructure.tools.ts @@ -2,6 +2,8 @@ import { z } from 'zod'; import { MCP_SCOPE } from '../constants/mcp-scopes'; import { defineTool, resolveClusterId, ToolDef } from './mcp-tool.util'; +const enc = encodeURIComponent; + /** Cluster discovery (read tier) — the entry point for any cluster-scoped tool. */ export const INFRASTRUCTURE_TOOLS: ToolDef[] = [ defineTool({ @@ -182,4 +184,82 @@ export const INFRASTRUCTURE_TOOLS: ToolDef[] = [ ...(args.includeStopped ? { includeStopped: true } : {}), }), }), + defineTool({ + name: 'cluster_replicated_volumes_get', + routes: ['GET /infrastructure/clusters/:id/replicated-volumes'], + description: + 'Whether a cluster offers replicated volumes (pass the cluster id). A replicated volume keeps a synchronous copy in each location for data, so an application with one writer comes back in another of them, with its data, when its server or its location is lost; where the layout has two locations for data, the third only votes. `plan.zones` are the locations holding copies and `plan.tieBreakers` the nodes that only vote (empty when every location holds data). `enabled` says whether the `flui-replicated` storage class exists; `storageNodes` are the nodes holding copies; `warnings` names copies an application could not follow; `plan` is what enabling would do now or why it cannot (`reason`), including a cluster not yet highly available across three locations. An application asks for it with `replicated: true` on a volume in flui.yaml, and moves only if every volume it mounts is replicated. Read-only; to turn it on call cluster_replicated_volumes_enable.', + scope: MCP_SCOPE.INFRA_READ, + inputSchema: { id: z.string() }, + run: (args, ctx) => + ctx.api.get( + `/infrastructure/clusters/${enc(args.id)}/replicated-volumes`, + ), + }), + defineTool({ + name: 'cluster_replicated_volumes_enable', + routes: ['POST /infrastructure/clusters/:id/replicated-volumes'], + description: + 'Enable replicated volumes on a cluster (pass the cluster id). A node in each location for data sets aside `sizeGiB` of its own disk (default 20) for copies, the volume engine is installed and the `flui-replicated` storage class appears; with two locations for data the third casts the tie-breaking vote, with three there is a copy in each and no vote; existing volumes do not change. `nodes` picks those nodes, one per location for data, otherwise Flui does. Refused with the reason when the cluster is not yet highly available across three locations of a validated region (`force` accepts a region Flui has not validated), or lacks a node that takes applications in a location for data. A person approves it before it happens. Returns the operation; follow it with operation_status.', + scope: MCP_SCOPE.INFRA_WRITE, + inputSchema: { + id: z.string(), + nodes: z.array(z.string()).min(2).max(3).optional(), + sizeGiB: z.number().int().min(5).max(2000).optional(), + force: z.boolean().optional(), + }, + run: (args, ctx) => + ctx.api.post( + `/infrastructure/clusters/${enc(args.id)}/replicated-volumes`, + { + ...(args.nodes && { nodes: args.nodes }), + ...(args.sizeGiB && { sizeGiB: args.sizeGiB }), + ...(args.force && { force: args.force }), + }, + ), + }), + defineTool({ + name: 'cluster_disks_get', + routes: ['GET /infrastructure/clusters/:id/storage/disks'], + description: + 'How much room is left on each node of a cluster (pass the cluster id): its own disk (`disk`), where databases and data kept on the node live, and the space set aside for copies of replicated volumes (`copies`, null where the node holds none), as total and free GiB. `warnings` names a node running short: a full disk stops what is kept on it from growing, a full space for copies refuses new or larger replicated volumes. Read-only.', + scope: MCP_SCOPE.INFRA_READ, + inputSchema: { id: z.string() }, + run: (args, ctx) => + ctx.api.get(`/infrastructure/clusters/${enc(args.id)}/storage/disks`), + }), + defineTool({ + name: 'cluster_ha_add_node', + routes: ['POST /infrastructure/clusters/:id/high-availability/nodes'], + description: + 'Add the node a cluster still needs to survive the loss of a location (pass the cluster id): `role` `server` (also runs the control plane and holds a vote) or `data-host` (takes applications where copies of data live), and `zone`, a location cluster_ha_get lists under `missing` for that role. Flui buys one machine there, of `serverType` or the cluster size, and joins it with its role; scaling never removes it. Anything not missing is refused with the reason. With two servers the cluster is more fragile than with one until the third joins, so add them one after the other. A person approves it before it happens. Returns the operation; follow it with operation_status.', + scope: MCP_SCOPE.INFRA_WRITE, + inputSchema: { + id: z.string(), + role: z.enum(['server', 'data-host']), + zone: z.string(), + serverType: z.string().optional(), + force: z.boolean().optional(), + }, + run: (args, ctx) => + ctx.api.post( + `/infrastructure/clusters/${enc(args.id)}/high-availability/nodes`, + { + role: args.role, + zone: args.zone, + ...(args.serverType && { serverType: args.serverType }), + ...(args.force && { force: args.force }), + }, + ), + }), + defineTool({ + name: 'cluster_ha_get', + routes: ['GET /infrastructure/clusters/:id/high-availability'], + description: + 'What a cluster survives with the nodes it has now (pass the cluster id), as plain sentences in `survives` and `doesNotSurvive`, and its `level`: `standard` or `three-locations` (a server in each of three locations of a validated region, copies of data in the two closest, the third voting only). `topology` names each location with its role, servers and hosts for data, or is null with the reason in `unsupported`; `missing` lists the roles still needed (`server`, `data-host`), where and why; `warnings` flags two servers, which are more fragile than one; cluster_ha_add_node adds what is missing. `observed` is what Flui last measured (as of `observed.at`): where the etcd leader is, the round trip in ms between each pair of locations, and the server Flui reaches the cluster through. Read-only.', + scope: MCP_SCOPE.INFRA_READ, + inputSchema: { id: z.string() }, + run: (args, ctx) => + ctx.api.get(`/infrastructure/clusters/${enc(args.id)}/high-availability`), + }), ]; diff --git a/src/modules/mcp/tools/wire-catalog.spec.ts b/src/modules/mcp/tools/wire-catalog.spec.ts index 4ff4482c..38c7b720 100644 --- a/src/modules/mcp/tools/wire-catalog.spec.ts +++ b/src/modules/mcp/tools/wire-catalog.spec.ts @@ -39,6 +39,11 @@ const ARGS: Record> = { cluster_orphaned_volumes: {}, control_restore_plan: {}, control_restore_apps: { includeStopped: true }, + cluster_replicated_volumes_get: { id: 'c1' }, + cluster_replicated_volumes_enable: { id: 'c1', sizeGiB: 20 }, + cluster_ha_get: { id: 'c1' }, + cluster_disks_get: { id: 'c1' }, + cluster_ha_add_node: { id: 'c1', role: 'server', zone: 'hel1' }, dns_wildcard_status: {}, dns_wildcard_publish: {}, dns_acme_resolvers: { clusterId: 'c1' }, @@ -148,6 +153,8 @@ const ARGS: Record> = { backendStart: '2026-10-12T07:00:00.123Z', }, db_pooler_get: { id: 'a1' }, + db_ha_get: { id: 'a1' }, + db_ha_enable: { id: 'a1', durability: 'required' }, db_pooler_client_close: { id: 'a1', clientId: 5, @@ -523,6 +530,8 @@ describe('strada B — the whole tool catalogue goes over the wire', () => { ['db_session_cancel', 'POST /applications/a1/db/sessions/42/cancel'], ['db_session_terminate', 'POST /applications/a1/db/sessions/42/terminate'], ['db_pooler_get', 'GET /applications/a1/db/pooler'], + ['db_ha_get', 'GET /applications/a1/db/high-availability'], + ['db_ha_enable', 'POST /applications/a1/db/high-availability'], [ 'db_pooler_client_close', 'POST /applications/a1/db/pooler/clients/5/close', @@ -579,6 +588,20 @@ describe('strada B — the whole tool catalogue goes over the wire', () => { 'GET /infrastructure/clusters/control-restore/plan', ], ['control_restore_apps', 'POST /infrastructure/clusters/control-restore'], + [ + 'cluster_replicated_volumes_get', + 'GET /infrastructure/clusters/c1/replicated-volumes', + ], + [ + 'cluster_replicated_volumes_enable', + 'POST /infrastructure/clusters/c1/replicated-volumes', + ], + ['cluster_ha_get', 'GET /infrastructure/clusters/c1/high-availability'], + ['cluster_disks_get', 'GET /infrastructure/clusters/c1/storage/disks'], + [ + 'cluster_ha_add_node', + 'POST /infrastructure/clusters/c1/high-availability/nodes', + ], ['access_revocation_preview', 'GET /iam/grants/g1/revocation-preview'], ['access_grant_list', 'GET /iam/grants'], ['access_grant_add', 'POST /iam/grants'], diff --git a/src/modules/postgres-ha/controllers/postgres-ha.controller.ts b/src/modules/postgres-ha/controllers/postgres-ha.controller.ts new file mode 100644 index 00000000..376705d2 --- /dev/null +++ b/src/modules/postgres-ha/controllers/postgres-ha.controller.ts @@ -0,0 +1,83 @@ +import { + Body, + Controller, + ForbiddenException, + Get, + HttpCode, + HttpStatus, + Param, + Post, + Req, + UseGuards, +} from '@nestjs/common'; +import { + ApiBearerAuth, + ApiOperation, + ApiParam, + ApiTags, +} from '@nestjs/swagger'; +import { Request } from 'express'; +import { + AppAccessGuard, + AppAction, +} from '../../applications/guards/app-access.guard'; +import { IAM_PERMISSION } from '../../iam/constants/iam-permissions'; +import { ActionCycle } from '../../action-cycle/action-cycle.decorator'; +import { DataDoor } from '../../iam/decorators/data-door.decorator'; +import { isSandboxGuestRequest } from '../../sandbox/guards/sandbox-guest-request'; +import { ActivateDatabaseHaDto } from '../dto/activate-database-ha.dto'; +import { PostgresHaService } from '../services/postgres-ha.service'; + +/** A second copy on another server, and an operator for the whole cluster: not a demo tenancy's to start. */ +const GUEST_HA_REFUSAL = + 'Highly available databases are not available to demo guests: they take a second server of the shared cluster.'; + +@ApiTags('Applications') +@ApiBearerAuth() +@Controller('applications/:id/db/high-availability') +@UseGuards(AppAccessGuard) +export class PostgresHaController { + constructor(private readonly service: PostgresHaService) {} + + @Get() + @AppAction(IAM_PERMISSION.APP_READ) + @ApiOperation({ + summary: 'Whether a managed PostgreSQL is highly available', + description: + 'Null `state` is the single instance. Otherwise the highly available cluster, one instance in each of three locations, what happens to writes when a copy is away (`durability`) and where the move stands.', + }) + @ApiParam({ name: 'id', description: 'Application id of the database' }) + view(@Param('id') id: string) { + return this.service.view(id); + } + + @Post() + @HttpCode(HttpStatus.ACCEPTED) + @AppAction(IAM_PERMISSION.APP_DEPLOY) + @DataDoor() + @ActionCycle({ + action: 'POST /applications/:id/db/high-availability', + bind: ['id'], + sentence: 'make database {id} highly available', + consequence: + "Three instances of the database are built, one in each of the cluster's three locations, and take storage there. Once they have caught up, writes stop for a few seconds while the application moves to the new primary; the single instance is then kept stopped, with its data, as the way back.", + }) + @ApiOperation({ + summary: 'Make a managed PostgreSQL highly available', + description: + "Builds a highly available cluster, one instance in each of the cluster's three locations, as a live copy of the database and moves the application to it, without stopping the application: writes pause for a few seconds. A commit waits for whichever copy answers first. " + + 'Refused, with the reason, when the database is not Flui-managed PostgreSQL or the cluster is not yet highly available across three locations of a validated region (`force` accepts a region Flui has not validated). Returns the operation to follow.', + }) + @ApiParam({ name: 'id', description: 'Application id of the database' }) + activate( + @Req() req: Request, + @Param('id') id: string, + @Body() dto: ActivateDatabaseHaDto, + ) { + if (isSandboxGuestRequest(req)) { + throw new ForbiddenException(GUEST_HA_REFUSAL); + } + const user = req.user as { id?: string; userId?: string } | undefined; + return this.service.activate(id, dto, user?.id ?? user?.userId); + } +} diff --git a/src/modules/postgres-ha/dto/activate-database-ha.dto.ts b/src/modules/postgres-ha/dto/activate-database-ha.dto.ts new file mode 100644 index 00000000..4fca7b7d --- /dev/null +++ b/src/modules/postgres-ha/dto/activate-database-ha.dto.ts @@ -0,0 +1,24 @@ +import { ApiPropertyOptional } from '@nestjs/swagger'; +import { IsBoolean, IsIn, IsOptional } from 'class-validator'; +import { PostgresHaDurability } from '../utils/cnpg-cluster.core'; + +export class ActivateDatabaseHaDto { + @ApiPropertyOptional({ + enum: ['required', 'preferred'], + default: 'required', + description: + '`required`: with no copy able to confirm, writes wait, and no confirmed write is ever lost. `preferred`: writes go on alone while no copy is reachable and the copies catch up later.', + }) + @IsOptional() + @IsIn(['required', 'preferred']) + durability?: PostgresHaDurability; + + @ApiPropertyOptional({ + default: false, + description: + 'Accept a region Flui has not validated for high availability. Its three locations are then treated as equal, with no promise about how far apart they are.', + }) + @IsOptional() + @IsBoolean() + force?: boolean; +} diff --git a/src/modules/postgres-ha/postgres-ha.constants.ts b/src/modules/postgres-ha/postgres-ha.constants.ts new file mode 100644 index 00000000..c9470e15 --- /dev/null +++ b/src/modules/postgres-ha/postgres-ha.constants.ts @@ -0,0 +1,29 @@ +export const CNPG_VERSION = '1.30.1'; +export const CNPG_MANIFEST_URL = `https://github.com/cloudnative-pg/cloudnative-pg/releases/download/v${CNPG_VERSION}/cnpg-${CNPG_VERSION}.yaml`; +export const CNPG_NAMESPACE = 'cnpg-system'; +export const CNPG_DEPLOYMENT = 'cnpg-controller-manager'; +export const CNPG_API_VERSION = 'postgresql.cnpg.io/v1'; +/** + * The same Postgres major and Debian base as Flui's own `flui-postgres`, so a + * single instance can be copied into a highly available one. The operator's + * own default is a newer major, which cannot start on that data. Fixed by + * digest: CloudNativePG rebuilds its images under the same tag. + */ +export const CNPG_POSTGRES_IMAGE = + 'ghcr.io/cloudnative-pg/postgresql:17.11-standard-trixie@sha256:4d0aac315c4d8c3202252bd06fea83b16951cc57234a7b610eeaa382ff4482d0'; +export const POSTGRES_HA_QUEUE = 'postgres-ha'; +export const POSTGRES_HA_JOB_ACTIVATE = 'activate-database-ha'; +/** The only range the copy may stream from. */ +export { POD_NETWORK as CLUSTER_POD_NETWORK } from '../infrastructure/clusters/constants'; + +/** + * A rolling restart hands the primary's role to a copy that confirmed every + * write before restarting it: writes pause for seconds, not for the restart. + */ +export const CNPG_PRIMARY_UPDATE_METHOD = 'switchover'; + +/** The phase CloudNativePG reports once every instance serves and nothing is under way. */ +export const CNPG_HEALTHY_PHASE = 'Cluster in healthy state'; + +/** The role the operator streams and rewinds with. */ +export const CNPG_REPLICATION_USER = 'streaming_replica'; diff --git a/src/modules/postgres-ha/postgres-ha.module.ts b/src/modules/postgres-ha/postgres-ha.module.ts new file mode 100644 index 00000000..7d0b8f0c --- /dev/null +++ b/src/modules/postgres-ha/postgres-ha.module.ts @@ -0,0 +1,46 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { BullModule } from '@nestjs/bull'; +import { ApplicationEntity } from '../applications/entities/application.entity'; +import { ClusterEntity } from '../infrastructure/clusters/entities/cluster.entity'; +import { InfrastructureOperationEntity } from '../infrastructure/servers/entities/infrastructure-operations.entity'; +import { SharedInfrastructureModule } from '../infrastructure/shared/shared-infrastructure.module'; +import { EncryptionModule } from '../shared/encryption/encryption.module'; +import { DbLifecycleModule } from '../db-lifecycle/db-lifecycle.module'; +import { ApplicationsModule } from '../applications/applications.module'; +import { ClusterHaModule } from '../cluster-ha/cluster-ha.module'; +import { CnpgOperatorInstallerService } from './services/cnpg-operator-installer.service'; +import { PostgresHaService } from './services/postgres-ha.service'; +import { PostgresHaPortsAdapter } from './services/postgres-ha-ports.adapter'; +import { PostgresHaProcessor } from './processors/postgres-ha.processor'; +import { PrimaryPlacementService } from './services/primary-placement.service'; +import { PrimaryPlacementScheduler } from './schedulers/primary-placement.scheduler'; +import { PostgresHaController } from './controllers/postgres-ha.controller'; +import { POSTGRES_HA_QUEUE } from './postgres-ha.constants'; + +@Module({ + imports: [ + TypeOrmModule.forFeature([ + ApplicationEntity, + ClusterEntity, + InfrastructureOperationEntity, + ]), + BullModule.registerQueue({ name: POSTGRES_HA_QUEUE }), + SharedInfrastructureModule, + EncryptionModule, + DbLifecycleModule, + ApplicationsModule, + ClusterHaModule, + ], + controllers: [PostgresHaController], + providers: [ + CnpgOperatorInstallerService, + PostgresHaService, + PostgresHaPortsAdapter, + PostgresHaProcessor, + PrimaryPlacementService, + PrimaryPlacementScheduler, + ], + exports: [CnpgOperatorInstallerService, PostgresHaService], +}) +export class PostgresHaModule {} diff --git a/src/modules/postgres-ha/postgres-ha.types.ts b/src/modules/postgres-ha/postgres-ha.types.ts new file mode 100644 index 00000000..a3750308 --- /dev/null +++ b/src/modules/postgres-ha/postgres-ha.types.ts @@ -0,0 +1,41 @@ +import { PostgresHaDurability } from './utils/cnpg-cluster.core'; +import { CNPG_INSTANCE_ROLE_LABEL } from '../infrastructure/shared/utils/database-pod.util'; + +/** + * A managed Postgres that has moved, or is moving, from its single instance to + * a highly available cluster. Null on the application row is the single + * instance every install has had so far. + */ +export interface DatabaseHaSpec { + engine: 'cnpg'; + /** The highly available cluster's name, in the application's namespace. */ + cluster: string; + durability: PostgresHaDurability; + instances: number; + /** + * `copying`: the cluster streams from the single instance, which still serves. + * `active`: the cluster serves; the single instance is kept at zero copies. + * `failed`: the move stopped; the single instance serves again. + */ + state: 'copying' | 'active' | 'failed'; + since: string; + detail?: string; + /** When Flui last moved the primary back to a location for data. */ + primaryMovedAt?: string; +} + +/** Whether the application's traffic goes to the highly available cluster. */ +export function databaseHaServes( + spec: DatabaseHaSpec | null | undefined, +): spec is DatabaseHaSpec { + return spec?.state === 'active'; +} + +export function databaseHaPrimarySelector( + spec: DatabaseHaSpec, +): Record { + return { + 'cnpg.io/cluster': spec.cluster, + [CNPG_INSTANCE_ROLE_LABEL]: 'primary', + }; +} diff --git a/src/modules/postgres-ha/processors/postgres-ha.processor.ts b/src/modules/postgres-ha/processors/postgres-ha.processor.ts new file mode 100644 index 00000000..2bf9d6d5 --- /dev/null +++ b/src/modules/postgres-ha/processors/postgres-ha.processor.ts @@ -0,0 +1,135 @@ +import { Logger } from '@nestjs/common'; +import { singleInstanceSelector } from '../utils/postgres-ha-migration.core'; +import { Process, Processor } from '@nestjs/bull'; +import { Job } from 'bull'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { randomBytes } from 'node:crypto'; +import { ApplicationEntity } from '../../applications/entities/application.entity'; +import { ClusterEntity } from '../../infrastructure/clusters/entities/cluster.entity'; +import { + InfrastructureOperationEntity, + OperationStatus, +} from '../../infrastructure/servers/entities/infrastructure-operations.entity'; +import { EncryptionService } from '../../shared/encryption/services/encryption.service'; +import { DbPodExecService } from '../../db-lifecycle/services/db-pod-exec.service'; +import { FluiStorageClass } from '../../infrastructure/clusters/constants/storage-conventions'; +import { + CLUSTER_POD_NETWORK, + POSTGRES_HA_JOB_ACTIVATE, + POSTGRES_HA_QUEUE, +} from '../postgres-ha.constants'; +import { + ActivateDatabaseHaJob, + HA_INSTANCES, +} from '../services/postgres-ha.service'; +import { PostgresHaPortsAdapter } from '../services/postgres-ha-ports.adapter'; +import { + HaActivationError, + PostgresHaActivationRunner, +} from '../services/postgres-ha-activation.runner'; +import { ModuleRef } from '@nestjs/core'; +import { DatabaseHaBackupHandoverService } from '../../backups/services/database-ha-backup-handover.service'; + +const DEFAULT_SIZE_GI = 10; + +function sizeGiOf(app: ApplicationEntity): number { + const size = app.volumes?.[0]?.size ?? ''; + const match = /^(\d+)(Gi|G)$/.exec(size); + return match ? Number(match[1]) : DEFAULT_SIZE_GI; +} + +@Processor(POSTGRES_HA_QUEUE) +export class PostgresHaProcessor { + private readonly logger = new Logger(PostgresHaProcessor.name); + + constructor( + @InjectRepository(ApplicationEntity) + private readonly apps: Repository, + @InjectRepository(ClusterEntity) + private readonly clusters: Repository, + @InjectRepository(InfrastructureOperationEntity) + private readonly operations: Repository, + private readonly encryption: EncryptionService, + private readonly exec: DbPodExecService, + private readonly adapter: PostgresHaPortsAdapter, + private readonly moduleRef: ModuleRef, + ) {} + + /** + * The database's continuous backup follows it to the cluster that now + * serves. Looked up at run time: backups depend on databases, not the + * other way round. A failure leaves the database highly available and says + * that its backup did not follow, which is the one thing to act on. + */ + private async handBackupOver(appId: string): Promise { + try { + const handover = this.moduleRef.get(DatabaseHaBackupHandoverService, { + strict: false, + }); + return await handover.handOver(appId); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + this.logger.error( + `[postgres-ha] ${appId}: backup not handed over: ${message}`, + ); + return `The database is highly available, but its continuous backup did not move to the new cluster: ${message}. Until it does, no new backup is taken.`; + } + } + + @Process(POSTGRES_HA_JOB_ACTIVATE) + async activate(job: Job): Promise { + const { operationId, appId, durability, preferredZones } = job.data; + await this.operations.update(operationId, { + status: OperationStatus.IN_PROGRESS, + startedAt: new Date(), + }); + try { + const app = await this.apps.findOneOrFail({ where: { id: appId } }); + const cluster = await this.clusters.findOneOrFail({ + where: { id: app.clusterId }, + }); + const kubeconfig = this.encryption.decrypt(cluster.kubeconfigEncrypted); + const resolved = await this.exec.resolve(appId); + const source = { + ...resolved, + labelSelector: singleInstanceSelector(resolved.labelSelector), + }; + const runner = new PostgresHaActivationRunner( + this.adapter.portsFor({ app, kubeconfig, source, operationId }), + ); + const spec = await runner.run({ + appId, + slug: app.slug, + namespace: app.k8sNamespace, + durability, + instances: HA_INSTANCES, + preferredZones, + storageClass: FluiStorageClass.LOCAL, + sizeGi: sizeGiOf(app), + podNetwork: CLUSTER_POD_NETWORK, + replicationPassword: randomBytes(24).toString('hex'), + }); + const backup = await this.handBackupOver(appId); + await this.operations.update(operationId, { + status: OperationStatus.COMPLETED, + progress: 100, + completedAt: new Date(), + ...(backup && { errorMessage: backup }), + }); + this.logger.log(`[postgres-ha] ${app.slug}: served by ${spec.cluster}`); + } catch (error) { + const message = (error as Error).message; + const sourceServes = + error instanceof HaActivationError ? error.sourceServes : true; + await this.operations.update(operationId, { + status: OperationStatus.FAILED, + errorMessage: sourceServes + ? message + : `${message} The database is held read-only; nothing reverts on its own.`, + completedAt: new Date(), + }); + this.logger.error(`[postgres-ha] ${appId}: ${message}`); + } + } +} diff --git a/src/modules/postgres-ha/schedulers/primary-placement.scheduler.ts b/src/modules/postgres-ha/schedulers/primary-placement.scheduler.ts new file mode 100644 index 00000000..c1dbfd93 --- /dev/null +++ b/src/modules/postgres-ha/schedulers/primary-placement.scheduler.ts @@ -0,0 +1,28 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { Cron, CronExpression } from '@nestjs/schedule'; +import { PrimaryPlacementService } from '../services/primary-placement.service'; + +@Injectable() +export class PrimaryPlacementScheduler { + private readonly logger = new Logger(PrimaryPlacementScheduler.name); + private running = false; + + constructor(private readonly placement: PrimaryPlacementService) {} + + @Cron( + process.env.FLUI_PRIMARY_PLACEMENT_CRON || CronExpression.EVERY_5_MINUTES, + ) + async tick(): Promise { + if (this.running) return; + this.running = true; + try { + await this.placement.sweep(); + } catch (err: any) { + this.logger.error( + `[postgres-ha] placement sweep failed: ${err?.message ?? err}`, + ); + } finally { + this.running = false; + } + } +} diff --git a/src/modules/postgres-ha/services/cnpg-operator-installer.service.spec.ts b/src/modules/postgres-ha/services/cnpg-operator-installer.service.spec.ts new file mode 100644 index 00000000..586d752f --- /dev/null +++ b/src/modules/postgres-ha/services/cnpg-operator-installer.service.spec.ts @@ -0,0 +1,98 @@ +jest.mock('@kubernetes/client-node', () => ({})); +jest.mock('../utils/cnpg-images', () => ({ + pinnedCnpgManifest: (manifest: string) => `pinned ${manifest}`, +})); + +import { CnpgOperatorInstallerService } from './cnpg-operator-installer.service'; +import { KubernetesService } from '../../infrastructure/shared/services/kubernetes.service'; +import { CNPG_MANIFEST_URL } from '../postgres-ha.constants'; + +const ready = { spec: { replicas: 1 }, status: { availableReplicas: 1 } }; + +function setup(deployments: Array) { + const kubernetes = { + getResource: jest.fn(async () => + deployments.length ? deployments.shift() : ready, + ), + serverSideApply: jest.fn(async () => 42), + applyManifest: jest.fn(), + mergePatchObject: jest.fn(), + } as unknown as KubernetesService; + const fetchMock = jest + .spyOn(global, 'fetch') + .mockResolvedValue(new Response('kind: Namespace', { status: 200 })); + return { + installer: new CnpgOperatorInstallerService(kubernetes), + kubernetes, + fetchMock, + }; +} + +describe('installing the Postgres HA operator only when it is needed', () => { + afterEach(() => jest.restoreAllMocks()); + + it('applies nothing on a cluster that already has it', async () => { + const { installer, kubernetes, fetchMock } = setup([ready, ready]); + await installer.ensureInstalled('kc'); + expect(kubernetes.serverSideApply).not.toHaveBeenCalled(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + /** Its resource definitions are too large for a client-side apply's annotation. */ + it('applies the pinned release server-side the first time', async () => { + const { installer, kubernetes, fetchMock } = setup([null, ready]); + await installer.ensureInstalled('kc'); + expect(fetchMock).toHaveBeenCalledWith(CNPG_MANIFEST_URL); + expect(kubernetes.serverSideApply).toHaveBeenCalledWith( + 'kc', + 'pinned kind: Namespace', + ); + expect(kubernetes.applyManifest).not.toHaveBeenCalled(); + }); + + it('runs the operator as two copies in different locations, on every pass', async () => { + const { installer, kubernetes } = setup([ready, ready]); + await installer.ensureInstalled('kc'); + expect(kubernetes.mergePatchObject).toHaveBeenCalledWith( + 'kc', + expect.objectContaining({ + kind: 'Deployment', + metadata: { name: 'cnpg-controller-manager', namespace: 'cnpg-system' }, + spec: expect.objectContaining({ + replicas: 2, + template: { + spec: expect.objectContaining({ + affinity: { + podAntiAffinity: { + requiredDuringSchedulingIgnoredDuringExecution: [ + expect.objectContaining({ + topologyKey: 'topology.kubernetes.io/zone', + }), + ], + }, + }, + }), + }, + }), + }), + ); + }); + + it('is ready as soon as one copy answers, so a location away does not stop it', async () => { + const { installer } = setup([ + ready, + { spec: { replicas: 2 }, status: { availableReplicas: 1 } }, + ]); + await expect(installer.ensureInstalled('kc')).resolves.toBeUndefined(); + }); + + it('says where the manifest could not be fetched from', async () => { + const { installer } = setup([null]); + jest + .spyOn(global, 'fetch') + .mockResolvedValue(new Response('', { status: 404 })); + await expect(installer.ensureInstalled('kc')).rejects.toThrow( + CNPG_MANIFEST_URL, + ); + }); +}); diff --git a/src/modules/postgres-ha/services/cnpg-operator-installer.service.ts b/src/modules/postgres-ha/services/cnpg-operator-installer.service.ts new file mode 100644 index 00000000..d8b280af --- /dev/null +++ b/src/modules/postgres-ha/services/cnpg-operator-installer.service.ts @@ -0,0 +1,86 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { KubernetesService } from '../../infrastructure/shared/services/kubernetes.service'; +import { + CNPG_DEPLOYMENT, + CNPG_MANIFEST_URL, + CNPG_NAMESPACE, + CNPG_VERSION, +} from '../postgres-ha.constants'; +import { cnpgOperatorPlacement } from '../utils/cnpg-operator-placement.core'; +import { pinnedCnpgManifest } from '../utils/cnpg-images'; + +const READY_TIMEOUT_MS = 5 * 60 * 1000; +const POLL_INTERVAL_MS = 3000; + +/** + * Installs the CloudNativePG operator on a cluster the first time an + * application asks for a highly available Postgres, and never before: a + * cluster where nobody asks for one carries nothing of it. + * + * Applied server-side, as the project requires: its resource definitions are + * too large for the annotation a client-side apply writes. + */ +@Injectable() +export class CnpgOperatorInstallerService { + private readonly logger = new Logger(CnpgOperatorInstallerService.name); + private manifestCache: string | null = null; + + constructor(private readonly kubernetesService: KubernetesService) {} + + async isInstalled(kubeconfig: string): Promise { + const existing = await this.kubernetesService.getResource( + kubeconfig, + 'Deployment', + CNPG_DEPLOYMENT, + CNPG_NAMESPACE, + ); + return Boolean(existing); + } + + async ensureInstalled(kubeconfig: string): Promise { + if (!(await this.isInstalled(kubeconfig))) { + this.logger.log(`Installing CloudNativePG ${CNPG_VERSION}`); + const applied = await this.kubernetesService.serverSideApply( + kubeconfig, + await this.fetchManifest(), + ); + this.logger.log(`CloudNativePG: ${applied} objects applied`); + } + await this.kubernetesService.mergePatchObject( + kubeconfig, + cnpgOperatorPlacement(), + ); + await this.waitReady(kubeconfig); + this.logger.log(`CloudNativePG ${CNPG_VERSION} ready`); + } + + private async fetchManifest(): Promise { + if (this.manifestCache) return this.manifestCache; + const response = await fetch(CNPG_MANIFEST_URL); + if (!response.ok) { + throw new Error( + `Failed to fetch the CloudNativePG manifest from ${CNPG_MANIFEST_URL}: ${response.status} ${response.statusText}`, + ); + } + this.manifestCache = pinnedCnpgManifest(await response.text()); + return this.manifestCache; + } + + private async waitReady(kubeconfig: string): Promise { + const start = Date.now(); + while (Date.now() - start < READY_TIMEOUT_MS) { + const resource = await this.kubernetesService.getResource( + kubeconfig, + 'Deployment', + CNPG_DEPLOYMENT, + CNPG_NAMESPACE, + ); + const body = resource?.body ?? resource; + if ((body?.status?.availableReplicas ?? 0) >= 1) return; + await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS)); + } + throw new Error( + `CloudNativePG did not become ready within ${READY_TIMEOUT_MS / 1000}s`, + ); + } +} diff --git a/src/modules/postgres-ha/services/pgbackrest-plugin-installer.service.ts b/src/modules/postgres-ha/services/pgbackrest-plugin-installer.service.ts new file mode 100644 index 00000000..c08668bc --- /dev/null +++ b/src/modules/postgres-ha/services/pgbackrest-plugin-installer.service.ts @@ -0,0 +1,91 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { KubernetesService } from '../../infrastructure/shared/services/kubernetes.service'; +import { CNPG_NAMESPACE } from '../postgres-ha.constants'; +import { + PGBACKREST_PLUGIN_DEPLOYMENT, + PGBACKREST_PLUGIN_MANIFEST_URL, + pgbackrestPluginPlacement, + pinnedPgbackrestPluginManifest, +} from '../utils/pgbackrest-plugin.core'; + +const READY_TIMEOUT_MS = 5 * 60 * 1000; +const POLL_INTERVAL_MS = 3000; + +/** + * Installs the pgBackRest plugin beside CloudNativePG the first time a highly + * available database is backed up, and never before. Its controller and the + * operator talk over TLS that cert-manager issues, which every Flui cluster + * has from its first boot. + */ +@Injectable() +export class PgbackrestPluginInstallerService { + private readonly logger = new Logger(PgbackrestPluginInstallerService.name); + private manifestCache: string | null = null; + + constructor(private readonly kubernetesService: KubernetesService) {} + + async ensureInstalled(kubeconfig: string): Promise { + const existing = await this.kubernetesService.getResource( + kubeconfig, + 'Deployment', + PGBACKREST_PLUGIN_DEPLOYMENT, + CNPG_NAMESPACE, + ); + if (!existing) { + const issuerKind = await this.kubernetesService + .readObject( + kubeconfig, + 'apiextensions.k8s.io/v1', + 'CustomResourceDefinition', + 'issuers.cert-manager.io', + ) + .catch(() => null); + if (!issuerKind) { + throw new Error( + 'This cluster has no cert-manager, which the backup of a highly available database needs to secure its components.', + ); + } + this.logger.log('Installing the pgBackRest plugin for CloudNativePG'); + const applied = await this.kubernetesService.serverSideApply( + kubeconfig, + await this.fetchManifest(), + ); + this.logger.log(`pgBackRest plugin: ${applied} objects applied`); + } + await this.kubernetesService.mergePatchObject( + kubeconfig, + pgbackrestPluginPlacement(), + ); + await this.waitReady(kubeconfig); + } + + private async fetchManifest(): Promise { + if (this.manifestCache) return this.manifestCache; + const response = await fetch(PGBACKREST_PLUGIN_MANIFEST_URL); + if (!response.ok) { + throw new Error( + `Failed to fetch the pgBackRest plugin manifest from ${PGBACKREST_PLUGIN_MANIFEST_URL}: ${response.status} ${response.statusText}`, + ); + } + this.manifestCache = pinnedPgbackrestPluginManifest(await response.text()); + return this.manifestCache; + } + + private async waitReady(kubeconfig: string): Promise { + const start = Date.now(); + while (Date.now() - start < READY_TIMEOUT_MS) { + const resource = await this.kubernetesService.getResource( + kubeconfig, + 'Deployment', + PGBACKREST_PLUGIN_DEPLOYMENT, + CNPG_NAMESPACE, + ); + const body = resource?.body ?? resource; + if ((body?.status?.availableReplicas ?? 0) >= 1) return; + await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS)); + } + throw new Error( + `The pgBackRest plugin did not become ready within ${READY_TIMEOUT_MS / 1000}s`, + ); + } +} diff --git a/src/modules/postgres-ha/services/postgres-ha-activation.runner.spec.ts b/src/modules/postgres-ha/services/postgres-ha-activation.runner.spec.ts new file mode 100644 index 00000000..1549bb15 --- /dev/null +++ b/src/modules/postgres-ha/services/postgres-ha-activation.runner.spec.ts @@ -0,0 +1,249 @@ +import { + HaActivationError, + HaActivationPlan, + HaActivationPorts, + PostgresHaActivationRunner, +} from './postgres-ha-activation.runner'; +import { DatabaseHaSpec } from '../postgres-ha.types'; +import { ResourcePair } from '../../shared/utils/resource-quantity.util'; + +const plan: HaActivationPlan = { + appId: 'app-1', + slug: 'orders-db', + namespace: 'p-orders', + durability: 'required', + instances: 3, + preferredZones: ['fsn1', 'nbg1'], + storageClass: 'flui-local', + sizeGi: 10, + podNetwork: '10.42.0.0/16', + replicationPassword: 'abcdefghij0123456789', +}; + +interface Script { + fenceFails?: boolean; + superuserPassword?: string | null; + readyAfterPolls?: number; + settlingPolls?: number; + caughtUp?: boolean; + promoted?: boolean; + sourceResources?: ResourcePair; +} + +function setup(script: Script = {}) { + const calls: string[] = []; + const states: DatabaseHaSpec[] = []; + let clock = 0; + let readyPolls = 0; + let settlePolls = 0; + const record = (call: string): void => { + calls.push(call); + }; + const ports: HaActivationPorts = { + ensureOperator: async () => record('operator'), + sourceSql: async (sql) => { + calls.push(`source-sql:${sql.slice(0, 30)}`); + return sql.includes('pg_current_wal_lsn') ? '0/14000000\n' : ''; + }, + sourceShell: async () => (calls.push('source-hba'), ''), + fenceSource: async () => { + record('fence'); + if (script.fenceFails) throw new Error('Unexpected server response: 400'); + }, + unfenceSource: async () => record('unfence'), + apply: async (objects) => + record(`apply:${objects.map((o) => o.kind).join('+')}`), + readyInstances: async () => { + readyPolls++; + if (calls.includes('scale:0')) { + settlePolls++; + calls.push('ready-after-switch'); + return settlePolls > (script.settlingPolls ?? 0) ? 3 : 2; + } + return readyPolls > (script.readyAfterPolls ?? 0) ? 3 : 1; + }, + primaryPod: async () => 'orders-db-ha-1', + clusterSql: async (_pod, sql) => { + if (sql.includes('pg_last_wal_replay_lsn')) { + return script.caughtUp === false ? 'f' : 't'; + } + if (sql.includes('pg_is_in_recovery')) { + return script.promoted === false ? 't' : 'f'; + } + if (sql.startsWith('GRANT')) calls.push('rewind-grants'); + if (sql.includes('has_function_privilege')) { + return calls.includes('rewind-grants') ? 't' : 'f'; + } + return ''; + }, + patchCluster: async (_c, patch) => + record(`promote:${JSON.stringify(patch)}`), + patchAppService: async () => record('switch-service'), + scaleSingleInstance: async (n) => record(`scale:${n}`), + superuserPassword: async () => script.superuserPassword ?? null, + sourceResources: async () => script.sourceResources ?? null, + saveState: async (spec) => { + states.push(spec); + calls.push(`state:${spec.state}`); + }, + progress: async (step) => record(`step:${step}`), + sleep: async (ms) => { + clock += ms; + }, + now: () => new Date(1_800_000_000_000 + clock), + }; + const runner = new PostgresHaActivationRunner(ports, { + copyMs: 10_000, + catchUpMs: 5_000, + promoteMs: 5_000, + pollMs: 1_000, + }); + return { runner, calls, states }; +} + +describe('moving a database to its highly available cluster', () => { + it('prepares, copies, holds writes, promotes and only then moves the application', async () => { + const { runner, calls, states } = setup({ readyAfterPolls: 2 }); + const result = await runner.run(plan); + expect(result).toMatchObject({ + engine: 'cnpg', + cluster: 'orders-db-ha', + state: 'active', + instances: 3, + }); + const order = (name: string) => calls.findIndex((c) => c.startsWith(name)); + expect(order('operator')).toBeLessThan(order('source-hba')); + expect(order('apply:Secret+Cluster')).toBeLessThan(order('state:copying')); + expect(order('state:copying')).toBeLessThan(order('fence')); + expect(order('fence')).toBeLessThan(order('promote')); + expect(order('promote')).toBeLessThan(order('state:active')); + expect(order('state:active')).toBeLessThan(order('switch-service')); + expect(calls.filter((c) => c !== 'ready-after-switch').at(-1)).toBe( + 'scale:0', + ); + expect(calls).not.toContain('unfence'); + expect(states.map((s) => s.state)).toEqual(['copying', 'active']); + }); + + it('ends once every instance serves again after the switch, not while the copies restart', async () => { + const { runner, calls } = setup({ settlingPolls: 2 }); + + await runner.run(plan); + + const settling = calls.filter((c) => c === 'ready-after-switch'); + expect(settling).toHaveLength(3); + expect(calls.at(-1)).toBe('ready-after-switch'); + }); + + it('never holds writes when the copy does not become ready', async () => { + const { runner, calls, states } = setup({ readyAfterPolls: 1000 }); + await expect(runner.run(plan)).rejects.toMatchObject({ + sourceServes: true, + }); + expect(calls).not.toContain('fence'); + expect(states[states.length - 1].state).toBe('failed'); + }); + + it('puts the single instance back in service when the copy does not catch up', async () => { + const { runner, calls, states } = setup({ caughtUp: false }); + const error = await runner.run(plan).catch((e: unknown) => e); + expect(error).toBeInstanceOf(HaActivationError); + expect((error as HaActivationError).sourceServes).toBe(true); + expect(calls.indexOf('unfence')).toBeGreaterThan(calls.indexOf('fence')); + expect(calls.some((c) => c.startsWith('promote'))).toBe(false); + expect(states[states.length - 1]).toMatchObject({ + state: 'failed', + detail: expect.stringContaining('single instance again'), + }); + }); + + it('keeps the postgres password through the promotion when the application signs in as postgres', async () => { + const applied: Array> = []; + const { runner } = setup({ superuserPassword: 's3cret' }); + const apply = (runner as unknown as { ports: HaActivationPorts }).ports + .apply; + (runner as unknown as { ports: HaActivationPorts }).ports.apply = async ( + objects, + ) => { + applied.push(...objects); + return apply(objects); + }; + await runner.run(plan); + const secret = applied.find( + (o) => (o.metadata as { name: string }).name === 'orders-db-ha-superuser', + ); + expect(secret).toMatchObject({ + type: 'kubernetes.io/basic-auth', + stringData: { username: 'postgres', password: 's3cret' }, + }); + const cluster = applied.find((o) => o.kind === 'Cluster') as { + spec: Record; + }; + expect(cluster.spec).toMatchObject({ + enableSuperuserAccess: true, + superuserSecret: { name: 'orders-db-ha-superuser' }, + }); + }); + + it('lets a former primary rewind and rejoin, granting what the operator grants only to a primary that starts as one', async () => { + const { runner, calls } = setup(); + await runner.run(plan); + expect(calls.indexOf('rewind-grants')).toBeGreaterThan( + calls.findIndex((c) => c.startsWith('promote:')), + ); + expect(calls.indexOf('rewind-grants')).toBeLessThan( + calls.indexOf('switch-service'), + ); + }); + + it('runs every instance with the requests and limits of the single instance', async () => { + const applied: Array> = []; + const resources = { + requests: { cpu: '250m', memory: '256Mi' }, + limits: { cpu: '2', memory: '2Gi' }, + }; + const { runner } = setup({ sourceResources: resources }); + const ports = (runner as unknown as { ports: HaActivationPorts }).ports; + const apply = ports.apply; + ports.apply = async (objects) => { + applied.push(...objects); + return apply(objects); + }; + await runner.run(plan); + const cluster = applied.find((o) => o.kind === 'Cluster') as { + spec: Record; + }; + expect(cluster.spec.resources).toEqual(resources); + }); + + it('a fence that fails leaves the single instance serving and the activation retryable', async () => { + const { runner, calls, states } = setup({ fenceFails: true }); + const error = await runner.run(plan).catch((e: unknown) => e); + expect(error).toBeInstanceOf(HaActivationError); + expect((error as HaActivationError).sourceServes).toBe(true); + expect(calls).toContain('unfence'); + expect(calls.some((c) => c.startsWith('promote'))).toBe(false); + expect(states[states.length - 1]).toMatchObject({ state: 'failed' }); + }); + + /** Two primaries must never take writes, so nothing reverts on its own after promotion. */ + it('keeps writes held and moves nothing when the promotion is not confirmed', async () => { + const { runner, calls } = setup({ promoted: false }); + await expect(runner.run(plan)).rejects.toMatchObject({ + sourceServes: false, + }); + expect(calls).not.toContain('unfence'); + expect(calls).not.toContain('switch-service'); + }); + + it('refuses a replication password that could break out of its SQL literal', async () => { + const { runner, calls } = setup(); + await expect( + runner.run({ + ...plan, + replicationPassword: "x'; drop role postgres; --", + }), + ).rejects.toThrow(/letters and digits/); + expect(calls).toEqual([]); + }); +}); diff --git a/src/modules/postgres-ha/services/postgres-ha-activation.runner.ts b/src/modules/postgres-ha/services/postgres-ha-activation.runner.ts new file mode 100644 index 00000000..3be9ea1d --- /dev/null +++ b/src/modules/postgres-ha/services/postgres-ha-activation.runner.ts @@ -0,0 +1,281 @@ +import { OperationStep } from '../../infrastructure/servers/entities/infrastructure-operations.entity'; +import { DatabaseHaSpec } from '../postgres-ha.types'; +import { PostgresHaDurability } from '../utils/cnpg-cluster.core'; +import { ResourcePair } from '../../shared/utils/resource-quantity.util'; +import { + caughtUpSql, + copyHbaScript, + pgRewindGrantedSql, + pgRewindGrantsSql, + primarySelectorPatch, + promotionPatch, + renderCnpgCopyCluster, + sourceRolesSql, +} from '../utils/postgres-ha-migration.core'; + +/** What the move needs from the cluster, kept apart so the order can be tested without one. */ +export interface HaActivationPorts { + ensureOperator(): Promise; + sourceSql(sql: string): Promise; + sourceShell(script: string): Promise; + fenceSource(): Promise; + unfenceSource(): Promise; + apply(objects: Record[]): Promise; + readyInstances(cluster: string): Promise; + primaryPod(cluster: string): Promise; + clusterSql(pod: string, sql: string): Promise; + patchCluster(cluster: string, patch: Record): Promise; + patchAppService(patch: Record): Promise; + scaleSingleInstance(replicas: number): Promise; + /** The application's password when it signs in as `postgres`, otherwise null. */ + superuserPassword(): Promise; + /** The requests and limits the single instance runs with. */ + sourceResources(): Promise; + saveState(spec: DatabaseHaSpec): Promise; + progress(step: OperationStep, message: string): Promise; + sleep(ms: number): Promise; + now(): Date; +} + +export interface HaActivationPlan { + appId: string; + slug: string; + namespace: string; + durability: PostgresHaDurability; + instances: number; + preferredZones: string[]; + storageClass: string; + sizeGi: number; + podNetwork: string; + /** Letters and digits only: it is written into SQL as a literal. */ + replicationPassword: string; +} + +export interface HaActivationTimeouts { + copyMs: number; + catchUpMs: number; + promoteMs: number; + pollMs: number; +} + +export const DEFAULT_HA_TIMEOUTS: HaActivationTimeouts = { + copyMs: 30 * 60_000, + catchUpMs: 60_000, + promoteMs: 120_000, + pollMs: 1_000, +}; + +export class HaActivationError extends Error { + constructor( + message: string, + readonly sourceServes: boolean, + ) { + super(message); + } +} + +/** + * The move from a single instance to a highly available cluster. Until the + * copy is promoted, any failure puts the single instance back in service; + * after it, nothing goes back on its own, because the single instance stays + * fenced and two primaries must never take writes. + */ +export class PostgresHaActivationRunner { + constructor( + private readonly ports: HaActivationPorts, + private readonly timeouts: HaActivationTimeouts = DEFAULT_HA_TIMEOUTS, + ) {} + + clusterName(plan: HaActivationPlan): string { + return `${plan.slug}-ha`; + } + + async run(plan: HaActivationPlan): Promise { + if (!/^[A-Za-z0-9]{16,}$/.test(plan.replicationPassword)) { + throw new Error( + 'The replication password must be 16 or more letters and digits', + ); + } + const cluster = this.clusterName(plan); + const state = ( + s: DatabaseHaSpec['state'], + detail?: string, + ): DatabaseHaSpec => ({ + engine: 'cnpg', + cluster, + durability: plan.durability, + instances: plan.instances, + state: s, + since: this.ports.now().toISOString(), + ...(detail && { detail }), + }); + + await this.ports.progress( + OperationStep.DB_HA_PREPARE, + 'Preparing the database for a copy', + ); + await this.ports.ensureOperator(); + for (const sql of sourceRolesSql(`'${plan.replicationPassword}'`)) { + await this.ports.sourceSql(sql); + } + await this.ports.sourceShell(copyHbaScript(plan.podNetwork)); + await this.ports.sourceSql('SELECT pg_reload_conf()'); + + await this.ports.progress( + OperationStep.DB_HA_COPY, + 'Copying the database into the new cluster', + ); + const superuserPassword = await this.ports.superuserPassword(); + const resources = await this.ports.sourceResources(); + const labels = { + 'flui-app-id': plan.appId, + 'flui.cloud/managed-by': 'flui-cloud', + }; + await this.ports.apply([ + ...(superuserPassword + ? [ + { + apiVersion: 'v1', + kind: 'Secret', + type: 'kubernetes.io/basic-auth', + metadata: { + name: `${cluster}-superuser`, + namespace: plan.namespace, + labels, + }, + stringData: { username: 'postgres', password: superuserPassword }, + }, + ] + : []), + { + apiVersion: 'v1', + kind: 'Secret', + metadata: { + name: `${cluster}-repl`, + namespace: plan.namespace, + labels: { + 'flui-app-id': plan.appId, + 'flui.cloud/managed-by': 'flui-cloud', + }, + }, + stringData: { password: plan.replicationPassword }, + }, + renderCnpgCopyCluster({ + name: cluster, + namespace: plan.namespace, + appId: plan.appId, + instances: plan.instances, + preferredZones: plan.preferredZones, + durability: plan.durability, + storageClass: plan.storageClass, + sizeGi: plan.sizeGi, + sourceHost: `${plan.slug}-svc`, + replicationSecret: `${cluster}-repl`, + ...(superuserPassword && { superuserSecret: `${cluster}-superuser` }), + ...(resources && { resources }), + }), + ]); + await this.ports.saveState(state('copying')); + const copied = await this.waitFor( + this.timeouts.copyMs, + async () => (await this.ports.readyInstances(cluster)) >= plan.instances, + ); + if (!copied) { + const failed = state( + 'failed', + 'The copy did not become ready in time; the database keeps serving from its single instance.', + ); + await this.ports.saveState(failed); + throw new HaActivationError(failed.detail!, true); + } + + await this.ports.progress( + OperationStep.DB_HA_FENCE, + 'Holding writes while the copy catches up', + ); + let primary: string | null = null; + try { + await this.ports.fenceSource(); + await this.ports.sourceSql('SELECT pg_switch_wal()'); + const lsn = ( + await this.ports.sourceSql('SELECT pg_current_wal_lsn()') + ).trim(); + primary = await this.ports.primaryPod(cluster); + const caught = + primary !== null && + (await this.waitFor( + this.timeouts.catchUpMs, + async () => + (await this.ports.clusterSql(primary!, caughtUpSql(lsn))).trim() === + 't', + )); + if (!caught) throw new Error('The copy did not catch up in time'); + } catch (error) { + await this.ports.unfenceSource(); + const failed = state( + 'failed', + `${(error as Error).message}; the database serves from its single instance again.`, + ); + await this.ports.saveState(failed); + throw new HaActivationError(failed.detail!, true); + } + + await this.ports.progress( + OperationStep.DB_HA_PROMOTE, + 'Promoting the copy', + ); + await this.ports.patchCluster(cluster, promotionPatch()); + const promoted = await this.waitFor( + this.timeouts.promoteMs, + async () => + ( + await this.ports.clusterSql(primary!, 'SELECT pg_is_in_recovery()') + ).trim() === 'f', + ); + if (!promoted) { + const failed = state( + 'failed', + 'The copy was asked to take over and has not confirmed it; writes stay held until a person decides.', + ); + await this.ports.saveState(failed); + throw new HaActivationError(failed.detail!, false); + } + await this.waitFor(this.timeouts.promoteMs, async () => { + await this.ports.clusterSql(primary!, pgRewindGrantsSql()); + return ( + (await this.ports.clusterSql(primary!, pgRewindGrantedSql())).trim() === + 't' + ); + }); + + await this.ports.progress( + OperationStep.DB_HA_SWITCH, + 'Moving the application to the new primary', + ); + // Recorded before the move: reconciliation reads a stopped single instance + // as intended only once the database is active. + const active = state('active'); + await this.ports.saveState(active); + await this.ports.patchAppService(primarySelectorPatch(plan.slug, cluster)); + await this.ports.scaleSingleInstance(0); + // The copies restart once the promotion reaches them; the operation ends + // when every instance serves again, not while they are still settling. + await this.waitFor( + this.timeouts.copyMs, + async () => (await this.ports.readyInstances(cluster)) >= plan.instances, + ); + return active; + } + + private async waitFor( + timeoutMs: number, + check: () => Promise, + ): Promise { + const deadline = this.ports.now().getTime() + timeoutMs; + for (;;) { + if (await check().catch(() => false)) return true; + if (this.ports.now().getTime() >= deadline) return false; + await this.ports.sleep(this.timeouts.pollMs); + } + } +} diff --git a/src/modules/postgres-ha/services/postgres-ha-ports.adapter.ts b/src/modules/postgres-ha/services/postgres-ha-ports.adapter.ts new file mode 100644 index 00000000..93e3f9d0 --- /dev/null +++ b/src/modules/postgres-ha/services/postgres-ha-ports.adapter.ts @@ -0,0 +1,181 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { dump } from 'js-yaml'; +import { randomBytes } from 'node:crypto'; +import { ApplicationEntity } from '../../applications/entities/application.entity'; +import { + InfrastructureOperationEntity, + OperationStep, +} from '../../infrastructure/servers/entities/infrastructure-operations.entity'; +import { KubernetesService } from '../../infrastructure/shared/services/kubernetes.service'; +import { + DbPodExecService, + ResolvedApp, +} from '../../db-lifecycle/services/db-pod-exec.service'; +import { DbWriteFenceService } from '../../db-lifecycle/services/db-write-fence.service'; +import { DbFenceRecord } from '../../db-lifecycle/entities/db-migration.entity'; +import { CnpgOperatorInstallerService } from './cnpg-operator-installer.service'; +import { HaActivationPorts } from './postgres-ha-activation.runner'; +import { CNPG_API_VERSION } from '../postgres-ha.constants'; + +const STEP_PROGRESS: Partial> = { + [OperationStep.DB_HA_PREPARE]: 5, + [OperationStep.DB_HA_COPY]: 20, + [OperationStep.DB_HA_FENCE]: 80, + [OperationStep.DB_HA_PROMOTE]: 90, + [OperationStep.DB_HA_SWITCH]: 95, +}; + +export interface PortsTarget { + app: ApplicationEntity; + kubeconfig: string; + source: ResolvedApp; + operationId: string; +} + +/** The cluster side of the move, for one database. */ +@Injectable() +export class PostgresHaPortsAdapter { + constructor( + @InjectRepository(ApplicationEntity) + private readonly apps: Repository, + @InjectRepository(InfrastructureOperationEntity) + private readonly operations: Repository, + private readonly kubernetes: KubernetesService, + private readonly exec: DbPodExecService, + private readonly fences: DbWriteFenceService, + private readonly installer: CnpgOperatorInstallerService, + ) {} + + portsFor(target: PortsTarget): HaActivationPorts { + const { app, kubeconfig, source, operationId } = target; + const namespace = app.k8sNamespace; + let fence: DbFenceRecord | null = null; + const persistFence = async (record: DbFenceRecord) => { + fence = record; + const op = await this.operations.findOne({ where: { id: operationId } }); + await this.operations.update(operationId, { + metadata: { + ...op?.metadata, + fence: record, + } as InfrastructureOperationEntity['metadata'], + }); + }; + const singleContainer = async () => + ( + await this.kubernetes.readObject( + kubeconfig, + 'apps/v1', + 'StatefulSet', + app.slug, + namespace, + ) + )?.spec?.template?.spec?.containers?.[0]; + const clusterObject = (name: string) => + this.kubernetes.readObject( + kubeconfig, + CNPG_API_VERSION, + 'Cluster', + name, + namespace, + ); + + return { + ensureOperator: () => this.installer.ensureInstalled(kubeconfig), + sourceSql: (sql) => this.exec.execSql(source, sql), + sourceShell: (script) => this.exec.execRaw(source, script), + fenceSource: async () => { + await this.fences.fence( + source, + `flui:fence:ha${randomBytes(4).toString('hex')}`, + persistFence, + ); + }, + unfenceSource: async () => { + if (fence) await this.fences.unfence(source, fence); + }, + apply: async (objects) => { + await this.kubernetes.serverSideApply( + kubeconfig, + objects.map((o) => dump(o)).join('---\n'), + ); + }, + readyInstances: async (name) => + Number((await clusterObject(name))?.status?.readyInstances ?? 0), + primaryPod: async (name) => + (await clusterObject(name))?.status?.currentPrimary ?? null, + clusterSql: (pod, sql) => + this.kubernetes.execInPod( + kubeconfig, + namespace, + '', + 'postgres', + ['psql', '-U', 'postgres', '-Atc', sql], + pod, + ), + patchCluster: (name, patch) => + this.kubernetes.mergePatchObject(kubeconfig, { + apiVersion: CNPG_API_VERSION, + kind: 'Cluster', + metadata: { name, namespace }, + ...patch, + }), + patchAppService: async (patch) => { + const { metadata, ...rest } = patch as { + metadata?: Record; + }; + await this.kubernetes.mergePatchObject(kubeconfig, { + apiVersion: 'v1', + kind: 'Service', + metadata: { name: `${app.slug}-svc`, namespace, ...metadata }, + ...rest, + }); + if (app.pooler?.enabled && app.pooler.upstream?.mode !== 'forward') { + await this.kubernetes.mergePatchObject(kubeconfig, { + apiVersion: 'v1', + kind: 'Service', + metadata: { name: `${app.slug}-pooler-upstream`, namespace }, + ...rest, + }); + } + }, + scaleSingleInstance: (replicas) => + this.kubernetes.scaleWorkload( + kubeconfig, + 'StatefulSet', + namespace, + app.slug, + replicas, + ), + superuserPassword: async () => { + if (source.pgUser !== 'postgres') return null; + const ref = ((await singleContainer())?.env ?? []).find( + (e: { name: string }) => e.name === 'POSTGRES_PASSWORD', + )?.valueFrom?.secretKeyRef as { name: string; key: string } | undefined; + if (!ref) return null; + const secret = await this.kubernetes.readObject( + kubeconfig, + 'v1', + 'Secret', + ref.name, + namespace, + ); + const value = secret?.data?.[ref.key]; + return value ? Buffer.from(value, 'base64').toString('utf8') : null; + }, + sourceResources: async () => (await singleContainer())?.resources ?? null, + saveState: async (spec) => { + await this.apps.update(app.id, { databaseHa: spec }); + }, + progress: async (step) => { + await this.operations.update(operationId, { + currentStep: step, + progress: STEP_PROGRESS[step] ?? 0, + }); + }, + sleep: (ms) => new Promise((r) => setTimeout(r, ms)), + now: () => new Date(), + }; + } +} diff --git a/src/modules/postgres-ha/services/postgres-ha.service.spec.ts b/src/modules/postgres-ha/services/postgres-ha.service.spec.ts new file mode 100644 index 00000000..a8cd7089 --- /dev/null +++ b/src/modules/postgres-ha/services/postgres-ha.service.spec.ts @@ -0,0 +1,257 @@ +jest.mock('@kubernetes/client-node', () => ({})); + +import { BadRequestException } from '@nestjs/common'; +import { Repository } from 'typeorm'; +import { Queue } from 'bull'; +import { ClusterHaService } from '../../cluster-ha/services/cluster-ha.service'; +import { PostgresHaService } from './postgres-ha.service'; +import { ApplicationEntity } from '../../applications/entities/application.entity'; +import { ClusterEntity } from '../../infrastructure/clusters/entities/cluster.entity'; +import { + InfrastructureOperationEntity, + OperationType, +} from '../../infrastructure/servers/entities/infrastructure-operations.entity'; +import { KubernetesService } from '../../infrastructure/shared/services/kubernetes.service'; +import { EncryptionService } from '../../shared/encryption/services/encryption.service'; +import { postgresHaEligibility } from '../utils/postgres-ha-eligibility.core'; + +const flui = (over: Record = {}) => + ({ + id: 'db-1', + slug: 'orders-db', + clusterId: 'c1', + k8sNamespace: 'p-orders', + workloadKind: 'StatefulSet', + dedicatedNodeName: 'lab-worker-2', + labels: { 'flui.cloud/db-engine': 'postgres' }, + sourceConfig: { imageRef: 'ghcr.io/flui-cloud/flui-postgres:17' }, + ...over, + }) as unknown as ApplicationEntity; + +const node = (name: string, zone: string) => ({ + metadata: { name, labels: { 'topology.kubernetes.io/zone': zone } }, + status: { conditions: [{ type: 'Ready', status: 'True' }] }, +}); + +describe('which databases can be made highly available', () => { + it("is Flui's own Postgres, and nothing else yet", () => { + expect(postgresHaEligibility(flui())).toEqual({ eligible: true }); + expect( + postgresHaEligibility( + flui({ labels: { 'flui.cloud/db-engine': 'mariadb' } }), + ), + ).toMatchObject({ eligible: false }); + expect( + postgresHaEligibility( + flui({ sourceConfig: { imageRef: 'pgvector/pgvector:pg17' } }), + ), + ).toMatchObject({ + eligible: false, + reason: expect.stringContaining('another image'), + }); + }); + + it('refuses a database already moving or moved, or on its way to another cluster', () => { + for (const state of ['active', 'copying'] as const) { + expect( + postgresHaEligibility(flui({ databaseHa: { state } })), + ).toMatchObject({ eligible: false }); + } + expect( + postgresHaEligibility( + flui({ pooler: { enabled: true, upstream: { mode: 'forward' } } }), + ), + ).toMatchObject({ eligible: false }); + expect( + postgresHaEligibility(flui({ databaseHa: { state: 'failed' } })), + ).toEqual({ eligible: true }); + }); + + it('lets a stopped attempt start again when nothing is running for it', () => { + expect( + postgresHaEligibility(flui({ databaseHa: { state: 'copying' } }), false), + ).toEqual({ eligible: true }); + }); +}); + +describe('asking for a highly available database', () => { + function setup(app: ApplicationEntity, nodes: unknown[]) { + const saved: Array> = []; + const queued: unknown[] = []; + const service = new PostgresHaService( + { + findOne: jest.fn(async () => app), + } as unknown as Repository, + { + findOne: jest.fn(async () => ({ + id: 'c1', + kubeconfigEncrypted: 'enc', + })), + } as unknown as Repository, + { + create: (o: Partial) => o, + count: jest.fn(async () => 0), + save: jest.fn(async (o: Partial) => { + saved.push(o); + return { ...o, id: 'op-1' }; + }), + } as unknown as Repository, + { + add: jest.fn(async (_name: string, data: unknown) => queued.push(data)), + } as unknown as Queue, + { listNodes: jest.fn(async () => nodes) } as unknown as KubernetesService, + { decrypt: () => 'kubeconfig' } as unknown as EncryptionService, + { + requireThreeLocations: jest.fn(async () => { + const zones = new Set( + ( + nodes as Array<{ metadata: { labels: Record } }> + ).map((n) => n.metadata.labels['topology.kubernetes.io/zone']), + ); + if (zones.size < 3) { + throw new BadRequestException( + 'lab is not highly available across three locations yet', + ); + } + return { + nodes, + topology: { + region: 'eu-central', + zones: ['fsn1', 'nbg1', 'hel1'], + dataZones: ['fsn1', 'nbg1'], + validated: true, + measured: true, + }, + }; + }), + } as unknown as ClusterHaService, + ); + return { service, saved, queued }; + } + + const threeLocations = [ + node('lab-master', 'fsn1'), + node('lab-worker-1', 'nbg1'), + node('lab-worker-2', 'fsn1'), + node('lab-worker-3', 'hel1'), + ]; + + it('queues the move with one instance per location and the primary in a close one', async () => { + const { service, saved, queued } = setup(flui(), threeLocations); + const result = await service.activate('db-1', {}, 'user-1'); + expect(result).toEqual({ operationId: 'op-1', durability: 'required' }); + expect(saved[0]).toMatchObject({ + operationType: OperationType.ACTIVATE_DATABASE_HA, + resourceId: 'db-1', + userId: 'user-1', + metadata: { durability: 'required', zones: ['fsn1', 'nbg1', 'hel1'] }, + }); + expect(queued).toEqual([ + { + operationId: 'op-1', + appId: 'db-1', + durability: 'required', + preferredZones: ['fsn1', 'nbg1'], + }, + ]); + }); + + it('refuses a cluster that is not highly available across three locations, and queues nothing', async () => { + const { service, queued } = setup(flui(), threeLocations.slice(0, 3)); + await expect(service.activate('db-1', {})).rejects.toThrow( + /three locations/, + ); + expect(queued).toEqual([]); + }); + + it('refuses a database it cannot move, before touching the cluster', async () => { + const { service, queued } = setup( + flui({ labels: { 'flui.cloud/db-engine': 'redis' } }), + [], + ); + await expect(service.activate('db-1', {})).rejects.toThrow( + /Only a PostgreSQL/, + ); + expect(queued).toEqual([]); + }); + + /** Its continuous backup follows it to the cluster once the move ends. */ + it('moves a database its continuous backups cover', async () => { + const { service, queued } = setup(flui(), threeLocations); + await expect(service.activate('db-1', {})).resolves.toMatchObject({ + operationId: 'op-1', + }); + expect(queued).toHaveLength(1); + }); +}); + +describe('the view of a highly available database', () => { + it('reads where the primary is and how far behind each copy is', async () => { + const execInPod = jest.fn(async () => 'orders-db-ha-2|quorum|3.1|3.4\n'); + const service = new PostgresHaService( + { + findOne: jest.fn(async () => + flui({ + databaseHa: { + engine: 'cnpg', + cluster: 'orders-db-ha', + durability: 'required', + instances: 3, + state: 'active', + since: 'x', + }, + }), + ), + } as unknown as Repository, + { + findOne: jest.fn(async () => ({ id: 'c1', kubeconfigEncrypted: 'e' })), + } as unknown as Repository, + {} as Repository, + {} as Queue, + { + readObject: jest.fn(async () => ({ + status: { currentPrimary: 'orders-db-ha-1' }, + })), + listPodsByLabel: jest.fn(async () => [ + { + metadata: { name: 'orders-db-ha-1' }, + spec: { nodeName: 'lab-master' }, + status: { conditions: [{ type: 'Ready', status: 'True' }] }, + }, + { + metadata: { name: 'orders-db-ha-2' }, + spec: { nodeName: 'lab-worker-1' }, + status: { conditions: [{ type: 'Ready', status: 'True' }] }, + }, + ]), + listNodes: jest.fn(async () => [ + node('lab-master', 'fsn1'), + node('lab-worker-1', 'nbg1'), + ]), + execInPod, + } as unknown as KubernetesService, + { decrypt: () => 'kubeconfig' } as unknown as EncryptionService, + {} as ClusterHaService, + ); + const view = await service.view('db-1'); + expect(view.placement).toMatchObject({ + primary: { instance: 'orders-db-ha-1', zone: 'fsn1' }, + copies: [ + { + instance: 'orders-db-ha-2', + zone: 'nbg1', + confirms: true, + flushLagMs: 3.1, + }, + ], + }); + expect(execInPod).toHaveBeenCalledWith( + 'kubeconfig', + 'p-orders', + '', + 'postgres', + expect.arrayContaining(['psql']), + 'orders-db-ha-1', + ); + }); +}); diff --git a/src/modules/postgres-ha/services/postgres-ha.service.ts b/src/modules/postgres-ha/services/postgres-ha.service.ts new file mode 100644 index 00000000..9d804ab1 --- /dev/null +++ b/src/modules/postgres-ha/services/postgres-ha.service.ts @@ -0,0 +1,227 @@ +import { + BadRequestException, + Injectable, + Logger, + NotFoundException, +} from '@nestjs/common'; +import { InjectQueue } from '@nestjs/bull'; +import { Queue } from 'bull'; +import { InjectRepository } from '@nestjs/typeorm'; +import { In, Repository } from 'typeorm'; +import { ApplicationEntity } from '../../applications/entities/application.entity'; +import { ClusterEntity } from '../../infrastructure/clusters/entities/cluster.entity'; +import { + InfrastructureOperationEntity, + OperationStatus, + OperationType, +} from '../../infrastructure/servers/entities/infrastructure-operations.entity'; +import { KubernetesService } from '../../infrastructure/shared/services/kubernetes.service'; +import { EncryptionService } from '../../shared/encryption/services/encryption.service'; +import { TOPOLOGY_ZONE_LABEL } from '../../providers/core/node-topology'; +import { ActivateDatabaseHaDto } from '../dto/activate-database-ha.dto'; +import { + CNPG_API_VERSION, + POSTGRES_HA_JOB_ACTIVATE, + POSTGRES_HA_QUEUE, +} from '../postgres-ha.constants'; +import { DatabaseHaSpec } from '../postgres-ha.types'; +import { ClusterHaService } from '../../cluster-ha/services/cluster-ha.service'; +import { postgresHaEligibility } from '../utils/postgres-ha-eligibility.core'; +import { + DatabaseHaPlacement, + REPLICATION_SQL, + databaseHaPlacement, +} from '../utils/database-ha-placement.core'; +import { + haCandidatesFrom, + postgresHaFeasibility, +} from '../utils/postgres-ha-feasibility.core'; + +/** One per location of the layout. */ +export const HA_INSTANCES = 3; + +export interface ActivateDatabaseHaJob { + operationId: string; + appId: string; + durability: 'required' | 'preferred'; + preferredZones: string[]; +} + +export interface DatabaseHaView { + appId: string; + highlyAvailable: boolean; + state: DatabaseHaSpec | null; + /** Read live while highly available; null when the cluster did not answer. */ + placement: DatabaseHaPlacement | null; +} + +@Injectable() +export class PostgresHaService { + private readonly logger = new Logger(PostgresHaService.name); + + constructor( + @InjectRepository(ApplicationEntity) + private readonly apps: Repository, + @InjectRepository(ClusterEntity) + private readonly clusters: Repository, + @InjectRepository(InfrastructureOperationEntity) + private readonly operations: Repository, + @InjectQueue(POSTGRES_HA_QUEUE) private readonly queue: Queue, + private readonly kubernetes: KubernetesService, + private readonly encryption: EncryptionService, + private readonly clusterHa: ClusterHaService, + ) {} + + async view(appId: string): Promise { + const app = await this.apps.findOne({ where: { id: appId } }); + if (!app) throw new NotFoundException(`Application ${appId} not found`); + const stopped = + app.databaseHa?.state === 'copying' && + !(await this.activationRunning(appId)); + return { + appId, + highlyAvailable: app.databaseHa?.state === 'active', + state: stopped + ? { + ...app.databaseHa!, + state: 'failed', + detail: 'it did not finish, and can be started again.', + } + : (app.databaseHa ?? null), + placement: + app.databaseHa?.state === 'active' + ? await this.placement(app, app.databaseHa).catch((err: any) => { + this.logger.warn( + `[postgres-ha] ${app.slug}: placement not read (${err?.message ?? err})`, + ); + return null; + }) + : null, + }; + } + + private async placement( + app: ApplicationEntity, + spec: DatabaseHaSpec, + ): Promise { + const cluster = await this.clusters.findOne({ + where: { id: app.clusterId }, + }); + if (!cluster?.kubeconfigEncrypted) return null; + const kubeconfig = this.encryption.decrypt(cluster.kubeconfigEncrypted); + const [object, pods, nodes] = await Promise.all([ + this.kubernetes.readObject( + kubeconfig, + CNPG_API_VERSION, + 'Cluster', + spec.cluster, + app.k8sNamespace, + ), + this.kubernetes.listPodsByLabel( + kubeconfig, + app.k8sNamespace, + `cnpg.io/cluster=${spec.cluster}`, + ), + this.kubernetes.listNodes(kubeconfig), + ]); + const primary: string | undefined = object?.status?.currentPrimary; + const replication = primary + ? await this.kubernetes.execInPod( + kubeconfig, + app.k8sNamespace, + '', + 'postgres', + ['psql', '-U', 'postgres', '-Atc', REPLICATION_SQL], + primary, + ) + : ''; + return databaseHaPlacement({ + primary, + pods: pods.map((p) => ({ + name: p.metadata?.name ?? '', + node: p.spec?.nodeName, + ready: (p.status?.conditions ?? []).some( + (c) => c.type === 'Ready' && c.status === 'True', + ), + })), + zoneOf: new Map( + nodes.map((n) => [ + n.metadata?.name ?? '', + n.metadata?.labels?.[TOPOLOGY_ZONE_LABEL] ?? '', + ]), + ), + replication, + now: new Date(), + }); + } + + private async activationRunning(appId: string): Promise { + const running = await this.operations.count({ + where: { + operationType: OperationType.ACTIVATE_DATABASE_HA, + resourceId: appId, + status: In([OperationStatus.PENDING, OperationStatus.IN_PROGRESS]), + }, + }); + return running > 0; + } + + /** + * Checks what can be checked before anything moves, so a request the cluster + * cannot honour is refused with its reason, then hands the move to the queue. + */ + async activate( + appId: string, + dto: ActivateDatabaseHaDto, + userId?: string, + ): Promise<{ operationId: string; durability: string }> { + const app = await this.apps.findOne({ where: { id: appId } }); + if (!app) throw new NotFoundException(`Application ${appId} not found`); + const eligibility = postgresHaEligibility( + app, + await this.activationRunning(appId), + ); + if (!eligibility.eligible) + throw new BadRequestException((eligibility as { reason: string }).reason); + + const ha = await this.clusterHa.requireThreeLocations( + app.clusterId, + dto.force, + ); + const durability = dto.durability ?? 'required'; + const feasibility = postgresHaFeasibility( + haCandidatesFrom(ha.nodes), + ha.topology, + ); + if (!feasibility.feasible) + throw new BadRequestException((feasibility as { reason: string }).reason); + + const operation = await this.operations.save( + this.operations.create({ + operationType: OperationType.ACTIVATE_DATABASE_HA, + status: OperationStatus.PENDING, + resourceType: 'application', + resourceId: appId, + resourceName: app.slug, + userId, + metadata: { + durability, + zones: ha.topology.zones, + ...(!ha.topology.validated && { forced: true }), + }, + totalSteps: 5, + }), + ); + const job: ActivateDatabaseHaJob = { + operationId: operation.id, + appId, + durability, + preferredZones: ha.topology.dataZones, + }; + await this.queue.add(POSTGRES_HA_JOB_ACTIVATE, job, { attempts: 1 }); + this.logger.log( + `[postgres-ha] ${app.slug}: activation queued (${durability}, ${ha.topology.zones.join(', ')})`, + ); + return { operationId: operation.id, durability }; + } +} diff --git a/src/modules/postgres-ha/services/primary-placement.service.spec.ts b/src/modules/postgres-ha/services/primary-placement.service.spec.ts new file mode 100644 index 00000000..27a6bcce --- /dev/null +++ b/src/modules/postgres-ha/services/primary-placement.service.spec.ts @@ -0,0 +1,128 @@ +jest.mock('@kubernetes/client-node', () => ({})); + +import { Repository } from 'typeorm'; +import { ApplicationEntity } from '../../applications/entities/application.entity'; +import { ClusterHaService } from '../../cluster-ha/services/cluster-ha.service'; +import { KubernetesService } from '../../infrastructure/shared/services/kubernetes.service'; +import { PrimaryPlacementService } from './primary-placement.service'; + +const node = (name: string, zone: string) => ({ + metadata: { name, labels: { 'topology.kubernetes.io/zone': zone } }, +}); + +const pod = (name: string, nodeName: string) => ({ + metadata: { name }, + spec: { nodeName }, + status: { conditions: [{ type: 'Ready', status: 'True' }] }, +}); + +const app = (over: Record = {}) => + ({ + id: 'db-1', + slug: 'orders-db', + clusterId: 'c1', + k8sNamespace: 'p-orders', + databaseHa: { + engine: 'cnpg', + cluster: 'orders-db-ha', + durability: 'required', + instances: 3, + state: 'active', + since: '2026-10-29T08:00:00.000Z', + }, + ...over, + }) as unknown as ApplicationEntity; + +function setup(primary: string, opts: { validated?: boolean } = {}) { + const validated = opts.validated ?? true; + const update = jest.fn(); + const patch = jest.fn(); + const apps = { update } as unknown as Repository; + const kubernetes = { + readObject: jest.fn().mockResolvedValue({ + spec: { instances: 3 }, + status: { + currentPrimary: primary, + phase: 'Cluster in healthy state', + readyInstances: 3, + }, + }), + listPodsByLabel: jest + .fn() + .mockResolvedValue([ + pod('orders-db-ha-1', 'master'), + pod('orders-db-ha-2', 'w1'), + pod('orders-db-ha-3', 'w3'), + ]), + jsonPatchObjectStatus: patch, + } as unknown as KubernetesService; + const clusterHa = { + inspect: jest.fn().mockResolvedValue({ + kubeconfig: 'kc', + nodes: [node('master', 'fsn1'), node('w1', 'nbg1'), node('w3', 'hel1')], + state: { level: 'three-locations' }, + topology: { + region: 'eu-central', + zones: ['fsn1', 'nbg1', 'hel1'], + dataZones: validated ? ['fsn1', 'nbg1'] : ['fsn1', 'nbg1', 'hel1'], + validated, + }, + }), + } as unknown as ClusterHaService; + return { + service: new PrimaryPlacementService(apps, kubernetes, clusterHa), + update, + patch, + }; +} + +describe('PrimaryPlacementService', () => { + it('switches a primary that sits in the voting location over to a copy in a location for data', async () => { + const { service, patch, update } = setup('orders-db-ha-3'); + await service.check(app()); + expect(patch).toHaveBeenCalledWith( + 'kc', + expect.objectContaining({ plural: 'clusters', name: 'orders-db-ha' }), + expect.arrayContaining([ + { + op: 'replace', + path: '/status/targetPrimary', + value: 'orders-db-ha-1', + }, + { + op: 'replace', + path: '/status/phase', + value: 'Switchover in progress', + }, + ]), + ); + expect(update).toHaveBeenCalledWith('db-1', { + databaseHa: expect.objectContaining({ + primaryMovedAt: expect.any(String), + }), + }); + }); + + it('leaves a primary already in a location for data alone', async () => { + const { service, patch } = setup('orders-db-ha-2'); + await service.check(app()); + expect(patch).not.toHaveBeenCalled(); + }); + + it('does not move anything on a forced region, where every location counts as one for data', async () => { + const { service, patch } = setup('orders-db-ha-3', { validated: false }); + await service.check(app()); + expect(patch).not.toHaveBeenCalled(); + }); + + it('waits after a move instead of chasing the primary again', async () => { + const { service, patch } = setup('orders-db-ha-3'); + const spec = app().databaseHa!; + await service.check( + app({ + databaseHa: { ...spec, primaryMovedAt: new Date().toISOString() }, + }), + ); + expect(patch).not.toHaveBeenCalled(); + }); +}); diff --git a/src/modules/postgres-ha/services/primary-placement.service.ts b/src/modules/postgres-ha/services/primary-placement.service.ts new file mode 100644 index 00000000..467f1b5c --- /dev/null +++ b/src/modules/postgres-ha/services/primary-placement.service.ts @@ -0,0 +1,117 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { ApplicationEntity } from '../../applications/entities/application.entity'; +import { KubernetesService } from '../../infrastructure/shared/services/kubernetes.service'; +import { ClusterHaService } from '../../cluster-ha/services/cluster-ha.service'; +import { TOPOLOGY_ZONE_LABEL } from '../../providers/core/node-topology'; +import { CNPG_API_VERSION } from '../postgres-ha.constants'; +import { primaryPlacement } from '../utils/primary-placement.core'; + +/** + * Brings the primary of each highly available Postgres back to a location for + * data after a failover put it in the voting one, where every write of the + * applications would cross the far link. A planned switchover: writes pause + * for a few seconds, and no confirmed write is lost. + */ +@Injectable() +export class PrimaryPlacementService { + private readonly logger = new Logger(PrimaryPlacementService.name); + + constructor( + @InjectRepository(ApplicationEntity) + private readonly apps: Repository, + private readonly kubernetes: KubernetesService, + private readonly clusterHa: ClusterHaService, + ) {} + + async sweep(): Promise { + const active = await this.apps + .createQueryBuilder('app') + .where(`app."databaseHa"->>'state' = 'active'`) + .getMany(); + for (const app of active) { + await this.check(app).catch((err: any) => + this.logger.warn(`[postgres-ha] ${app.slug}: ${err?.message ?? err}`), + ); + } + } + + async check(app: ApplicationEntity): Promise { + const spec = app.databaseHa; + if (spec?.state !== 'active') return; + const ha = await this.clusterHa.inspect(app.clusterId); + if (ha.state.level !== 'three-locations' || !ha.topology) return; + const namespace = app.k8sNamespace; + const cluster = await this.kubernetes.readObject( + ha.kubeconfig, + CNPG_API_VERSION, + 'Cluster', + spec.cluster, + namespace, + ); + const pods = await this.kubernetes.listPodsByLabel( + ha.kubeconfig, + namespace, + `cnpg.io/cluster=${spec.cluster}`, + ); + const zoneOf = new Map( + ha.nodes.map((n) => [ + n.metadata?.name ?? '', + n.metadata?.labels?.[TOPOLOGY_ZONE_LABEL] ?? '', + ]), + ); + const decision = primaryPlacement({ + primary: cluster?.status?.currentPrimary, + phase: cluster?.status?.phase, + instances: Number(cluster?.spec?.instances ?? spec.instances), + readyInstances: Number(cluster?.status?.readyInstances ?? 0), + pods: pods.map((p) => ({ + name: p.metadata?.name ?? '', + node: p.spec?.nodeName, + ready: (p.status?.conditions ?? []).some( + (c) => c.type === 'Ready' && c.status === 'True', + ), + })), + zoneOf, + preferredZones: ha.topology.dataZones, + lastMovedAt: spec.primaryMovedAt, + now: Date.now(), + }); + if (!decision.move) return; + const at = new Date().toISOString(); + await this.kubernetes.jsonPatchObjectStatus( + ha.kubeconfig, + { + apiVersion: CNPG_API_VERSION, + plural: 'clusters', + name: spec.cluster, + namespace, + }, + [ + { + op: 'replace', + path: '/status/targetPrimary', + value: decision.target, + }, + { op: 'replace', path: '/status/targetPrimaryTimestamp', value: at }, + { + op: 'replace', + path: '/status/phase', + value: 'Switchover in progress', + }, + { + op: 'replace', + path: '/status/phaseReason', + value: `Switching over to ${decision.target}`, + }, + ], + ); + await this.apps.update(app.id, { + databaseHa: { ...spec, primaryMovedAt: at }, + }); + this.logger.log( + `[postgres-ha] ${app.slug}: primary ${cluster?.status?.currentPrimary} sat in the voting location; switching over to ${decision.target}`, + ); + } +} diff --git a/src/modules/postgres-ha/utils/cnpg-cluster.core.ts b/src/modules/postgres-ha/utils/cnpg-cluster.core.ts new file mode 100644 index 00000000..6adf5026 --- /dev/null +++ b/src/modules/postgres-ha/utils/cnpg-cluster.core.ts @@ -0,0 +1,131 @@ +import { TOPOLOGY_ZONE_LABEL } from '../../providers/core/node-topology'; +import { ResourcePair } from '../../shared/utils/resource-quantity.util'; +import { + CNPG_API_VERSION, + CNPG_POSTGRES_IMAGE, + CNPG_PRIMARY_UPDATE_METHOD, +} from '../postgres-ha.constants'; + +/** + * `required`: with no copy able to confirm, writes wait (integrity first). + * `preferred`: writes go on alone and the copy catches up (availability first). + */ +export type PostgresHaDurability = 'required' | 'preferred'; + +export interface CnpgClusterInput { + name: string; + namespace: string; + appId: string; + instances: number; + /** Where the primary belongs: the locations close enough to confirm a write together. */ + preferredZones: string[]; + durability: PostgresHaDurability; + storageClass: string; + sizeGi: number; + database: string; + owner: string; + credentialsSecret: string; + imageName?: string; + /** The single instance's requests and limits, carried over as they are. */ + resources?: ResourcePair; +} + +/** The operator's own exporter; the cluster's metrics agent collects what carries these. */ +const METRICS_ANNOTATIONS = { + 'prometheus.io/scrape': 'true', + 'prometheus.io/port': '9187', + 'prometheus.io/path': '/metrics', +}; + +export function renderCnpgCluster( + input: CnpgClusterInput, +): Record { + if (input.instances < 2) { + throw new Error('A highly available Postgres needs at least two instances'); + } + const labels = { + 'flui-app-id': input.appId, + 'flui.cloud/db-engine': 'postgres', + 'flui.cloud/managed-by': 'flui-cloud', + }; + // One instance per location. The first one is placed first and becomes the + // primary, so the preference puts it, and its closest copy, where writes + // are confirmed fastest. + const affinity = { + enablePodAntiAffinity: true, + podAntiAffinityType: 'required', + topologyKey: TOPOLOGY_ZONE_LABEL, + nodeAffinity: { + preferredDuringSchedulingIgnoredDuringExecution: [ + { + weight: 100, + preference: { + matchExpressions: [ + { + key: TOPOLOGY_ZONE_LABEL, + operator: 'In', + values: input.preferredZones, + }, + ], + }, + }, + ], + }, + }; + return { + apiVersion: CNPG_API_VERSION, + kind: 'Cluster', + metadata: { name: input.name, namespace: input.namespace, labels }, + spec: { + instances: input.instances, + imageName: input.imageName ?? CNPG_POSTGRES_IMAGE, + primaryUpdateStrategy: 'unsupervised', + primaryUpdateMethod: CNPG_PRIMARY_UPDATE_METHOD, + inheritedMetadata: { labels, annotations: METRICS_ANNOTATIONS }, + storage: { storageClass: input.storageClass, size: `${input.sizeGi}Gi` }, + bootstrap: { + initdb: { + database: input.database, + owner: input.owner, + secret: { name: input.credentialsSecret }, + }, + }, + postgresql: { + // A commit waits for whichever copy answers first, the close one in + // practice; the far one only when the close one is away. + synchronous: { + method: 'any', + number: 1, + dataDurability: input.durability, + failoverQuorum: true, + }, + }, + affinity, + ...(input.resources && { resources: input.resources }), + }, + }; +} + +/** The owner credentials in the shape the operator reads at first start. */ +export function renderCnpgCredentials(input: { + name: string; + namespace: string; + appId: string; + username: string; + password: string; +}): Record { + return { + apiVersion: 'v1', + kind: 'Secret', + type: 'kubernetes.io/basic-auth', + metadata: { + name: input.name, + namespace: input.namespace, + labels: { + 'flui-app-id': input.appId, + 'flui.cloud/managed-by': 'flui-cloud', + }, + }, + stringData: { username: input.username, password: input.password }, + }; +} diff --git a/src/modules/postgres-ha/utils/cnpg-images.spec.ts b/src/modules/postgres-ha/utils/cnpg-images.spec.ts new file mode 100644 index 00000000..0541418d --- /dev/null +++ b/src/modules/postgres-ha/utils/cnpg-images.spec.ts @@ -0,0 +1,26 @@ +import { createHash } from 'node:crypto'; +import { CNPG_OPERATOR_IMAGE_DIGEST, pinnedCnpgManifest } from './cnpg-images'; + +describe('the Postgres HA operator pinned by digest', () => { + const manifest = [ + ' - name: OPERATOR_IMAGE_NAME', + ' value: ghcr.io/cloudnative-pg/cloudnative-pg:1.30.1', + ' image: ghcr.io/cloudnative-pg/cloudnative-pg:1.30.1', + ].join('\n'); + const sha = createHash('sha256').update(manifest).digest('hex'); + + it('refuses a manifest that is not the one checked', () => { + expect(() => pinnedCnpgManifest(manifest)).toThrow( + 'is not the one Flui checked', + ); + }); + + it('fixes the operator, and the image it hands to the instances it starts', () => { + const pinned = pinnedCnpgManifest(manifest, sha); + expect( + pinned.split( + `ghcr.io/cloudnative-pg/cloudnative-pg:1.30.1@${CNPG_OPERATOR_IMAGE_DIGEST}`, + ), + ).toHaveLength(3); + }); +}); diff --git a/src/modules/postgres-ha/utils/cnpg-images.ts b/src/modules/postgres-ha/utils/cnpg-images.ts new file mode 100644 index 00000000..06cf34fd --- /dev/null +++ b/src/modules/postgres-ha/utils/cnpg-images.ts @@ -0,0 +1,32 @@ +import { createHash } from 'node:crypto'; +import { CNPG_VERSION } from '../postgres-ha.constants'; + +/** The SHA-256 of the 1.30.1 release manifest: a manifest that differs is refused. */ +export const CNPG_MANIFEST_SHA256 = + '37237f145d8138256ea25ae830f87759255665ff08f8d552fdd8224a5ec032fb'; + +const OPERATOR_IMAGE = `ghcr.io/cloudnative-pg/cloudnative-pg:${CNPG_VERSION}`; + +/** The operator's image by the digest GHCR served on 8 November 2026; a tag can be moved, a digest cannot. */ +export const CNPG_OPERATOR_IMAGE_DIGEST = + 'sha256:923c267ec29636db3bee20f993d0ec4973fa22998e1adad37da79e4d32b5bc07'; + +/** + * The release manifest with the operator's image fixed by digest, in its + * container and in the variable it hands to every instance it starts, after + * checking it is the manifest that digest was read for. + */ +export function pinnedCnpgManifest( + manifest: string, + expectedSha256 = CNPG_MANIFEST_SHA256, +): string { + const sha = createHash('sha256').update(manifest).digest('hex'); + if (sha !== expectedSha256) { + throw new Error( + `The CloudNativePG manifest is not the one Flui checked (SHA-256 ${sha}): refusing to install it`, + ); + } + return manifest + .split(OPERATOR_IMAGE) + .join(`${OPERATOR_IMAGE}@${CNPG_OPERATOR_IMAGE_DIGEST}`); +} diff --git a/src/modules/postgres-ha/utils/cnpg-operator-placement.core.ts b/src/modules/postgres-ha/utils/cnpg-operator-placement.core.ts new file mode 100644 index 00000000..356a9b9f --- /dev/null +++ b/src/modules/postgres-ha/utils/cnpg-operator-placement.core.ts @@ -0,0 +1,36 @@ +import { TOPOLOGY_ZONE_LABEL } from '../../providers/core/node-topology'; +import { CNPG_DEPLOYMENT, CNPG_NAMESPACE } from '../postgres-ha.constants'; + +export const CNPG_OPERATOR_REPLICAS = 2; + +/** + * The operator decides every failover. As shipped it runs one copy, and a lost + * location that held it leaves the database without a primary for as long as + * the location is away. Two copies in different locations, with the leader + * election the operator already runs, leave one to act. + */ +export function cnpgOperatorPlacement(): Record { + const selector = { + matchLabels: { 'app.kubernetes.io/name': 'cloudnative-pg' }, + }; + return { + apiVersion: 'apps/v1', + kind: 'Deployment', + metadata: { name: CNPG_DEPLOYMENT, namespace: CNPG_NAMESPACE }, + spec: { + replicas: CNPG_OPERATOR_REPLICAS, + template: { + spec: { + affinity: { + podAntiAffinity: { + requiredDuringSchedulingIgnoredDuringExecution: [ + { labelSelector: selector, topologyKey: TOPOLOGY_ZONE_LABEL }, + ], + }, + }, + tolerations: [{ operator: 'Exists', effect: 'NoSchedule' }], + }, + }, + }, + }; +} diff --git a/src/modules/postgres-ha/utils/database-ha-placement.core.spec.ts b/src/modules/postgres-ha/utils/database-ha-placement.core.spec.ts new file mode 100644 index 00000000..e5825cc0 --- /dev/null +++ b/src/modules/postgres-ha/utils/database-ha-placement.core.spec.ts @@ -0,0 +1,65 @@ +import { databaseHaPlacement } from './database-ha-placement.core'; + +describe('where a highly available database is and how far behind its copies are', () => { + it('places the primary and each copy, with the lag Postgres reports', () => { + const placement = databaseHaPlacement({ + primary: 'db-ha-1', + pods: [ + { name: 'db-ha-3', node: 'w3', ready: true }, + { name: 'db-ha-1', node: 'master', ready: true }, + { name: 'db-ha-2', node: 'w1', ready: false }, + ], + zoneOf: new Map([ + ['master', 'fsn1'], + ['w1', 'nbg1'], + ['w3', 'hel1'], + ]), + replication: 'db-ha-2|quorum|3.2|4.0\ndb-ha-3|quorum||\n', + now: new Date('2026-10-29T10:00:00Z'), + }); + expect(placement.primary).toEqual({ + instance: 'db-ha-1', + node: 'master', + zone: 'fsn1', + ready: true, + }); + expect(placement.copies).toEqual([ + { + instance: 'db-ha-2', + node: 'w1', + zone: 'nbg1', + ready: false, + confirms: true, + flushLagMs: 3.2, + replayLagMs: 4, + }, + { + instance: 'db-ha-3', + node: 'w3', + zone: 'hel1', + ready: true, + confirms: true, + flushLagMs: null, + replayLagMs: null, + }, + ]); + }); + + it('marks a copy the primary does not stream to as one that cannot confirm', () => { + const placement = databaseHaPlacement({ + primary: 'db-ha-1', + pods: [ + { name: 'db-ha-1', node: 'master', ready: true }, + { name: 'db-ha-2', node: 'w1', ready: true }, + ], + zoneOf: new Map(), + replication: '', + now: new Date(), + }); + expect(placement.copies[0]).toMatchObject({ + zone: null, + confirms: false, + flushLagMs: null, + }); + }); +}); diff --git a/src/modules/postgres-ha/utils/database-ha-placement.core.ts b/src/modules/postgres-ha/utils/database-ha-placement.core.ts new file mode 100644 index 00000000..2ad6ecf3 --- /dev/null +++ b/src/modules/postgres-ha/utils/database-ha-placement.core.ts @@ -0,0 +1,70 @@ +import { InstancePod } from './primary-placement.core'; + +export interface DatabaseHaInstance { + instance: string; + node: string | null; + zone: string | null; + ready: boolean; +} + +export interface DatabaseHaCopy extends DatabaseHaInstance { + /** `quorum` copies can confirm a commit; `async` ones only follow. */ + confirms: boolean; + /** How long the last writes took to be safe on this copy; null when nothing was written lately. */ + flushLagMs: number | null; + replayLagMs: number | null; +} + +export interface DatabaseHaPlacement { + readAt: string; + primary: DatabaseHaInstance | null; + copies: DatabaseHaCopy[]; +} + +export const REPLICATION_SQL = + 'SELECT application_name, sync_state, ' + + 'round((extract(epoch FROM flush_lag) * 1000)::numeric, 1), ' + + 'round((extract(epoch FROM replay_lag) * 1000)::numeric, 1) ' + + 'FROM pg_stat_replication ORDER BY application_name'; + +const ms = (v: string | undefined) => (v ? Number(v) : null); + +/** One row per copy from `psql -At`: name|sync_state|flush ms|replay ms. */ +export function databaseHaPlacement(input: { + primary: string | undefined; + pods: InstancePod[]; + zoneOf: Map; + replication: string; + now: Date; +}): DatabaseHaPlacement { + const instance = (p: InstancePod): DatabaseHaInstance => ({ + instance: p.name, + node: p.node ?? null, + zone: (p.node && input.zoneOf.get(p.node)) || null, + ready: p.ready, + }); + const rows = new Map( + input.replication + .split('\n') + .map((l) => l.trim().split('|')) + .filter((r) => r[0]) + .map((r) => [r[0], r] as const), + ); + const primary = input.pods.find((p) => p.name === input.primary); + return { + readAt: input.now.toISOString(), + primary: primary ? instance(primary) : null, + copies: input.pods + .filter((p) => p.name !== input.primary) + .sort((a, b) => a.name.localeCompare(b.name)) + .map((p) => { + const row = rows.get(p.name); + return { + ...instance(p), + confirms: row?.[1] === 'quorum' || row?.[1] === 'sync', + flushLagMs: ms(row?.[2]), + replayLagMs: ms(row?.[3]), + }; + }), + }; +} diff --git a/src/modules/postgres-ha/utils/pgbackrest-plugin.core.spec.ts b/src/modules/postgres-ha/utils/pgbackrest-plugin.core.spec.ts new file mode 100644 index 00000000..9ad85a0f --- /dev/null +++ b/src/modules/postgres-ha/utils/pgbackrest-plugin.core.spec.ts @@ -0,0 +1,173 @@ +import { createHash } from 'node:crypto'; +import * as yaml from 'js-yaml'; +import { + PGBACKREST_PLUGIN_IMAGE_DIGESTS, + archiveThroughPluginPatch, + pgbackrestPluginPlacement, + pinnedPgbackrestPluginManifest, + pluginBackupState, + promotedReplicaPatch, + renderPgbackrestStanza, + renderPluginBackup, + stopArchivingPatch, +} from './pgbackrest-plugin.core'; + +const sidecar = 'registry.hub.docker.com/dalibo/cnpg-pgbackrest-sidecar:0.0.3'; +const controller = + 'registry.hub.docker.com/dalibo/cnpg-pgbackrest-controller:0.0.3'; + +describe('the pgBackRest plugin, pinned', () => { + const manifest = [ + yaml.dump({ + apiVersion: 'v1', + kind: 'Secret', + data: { SIDECAR_IMAGE: Buffer.from(`${sidecar}\n`).toString('base64') }, + }), + yaml.dump({ + apiVersion: 'apps/v1', + kind: 'Deployment', + spec: { template: { spec: { containers: [{ image: controller }] } } }, + }), + ].join('---\n'); + const sha = createHash('sha256').update(manifest).digest('hex'); + + it('refuses a manifest that is not the one checked', () => { + expect(() => pinnedPgbackrestPluginManifest(manifest)).toThrow( + 'is not the one Flui checked', + ); + }); + + it('fixes the controller and the sidecar it hands to every instance', () => { + const [secret, deployment] = yaml.loadAll( + pinnedPgbackrestPluginManifest(manifest, sha), + ) as any[]; + expect(Buffer.from(secret.data.SIDECAR_IMAGE, 'base64').toString()).toBe( + `${sidecar}@${PGBACKREST_PLUGIN_IMAGE_DIGESTS[sidecar]}`, + ); + expect(deployment.spec.template.spec.containers[0].image).toBe( + `${controller}@${PGBACKREST_PLUGIN_IMAGE_DIGESTS[controller]}`, + ); + }); +}); + +describe('the repository of a highly available database', () => { + const stanza = renderPgbackrestStanza({ + cluster: 'orders-db-ha', + namespace: 'ns', + stanza: 'main', + repository: { + bucket: 'flui-backups', + endpoint: 'fsn1.your-objectstorage.com', + region: 'fsn1', + repoPath: '/eu/pgbackrest/a1/ha-20261108/', + pathStyle: false, + retentionFull: 2, + }, + }) as any; + const repo = stanza.spec.stanzaConfiguration.s3Repositories[0]; + + it('is encrypted with the destination passphrase, from a Secret beside it', () => { + expect(repo.cipherConfig).toEqual({ + type: 'aes-256-cbc', + encryptionPass: { + name: 'orders-db-ha-pgbackrest', + key: 'ENCRYPTION_PASS', + }, + }); + expect(repo.secretRef.accessKeyId.name).toBe('orders-db-ha-pgbackrest'); + }); + + it('keeps stanza, compression and bundling of the single instance, and verifies TLS', () => { + expect(stanza.spec.stanzaConfiguration).toMatchObject({ + name: 'main', + compressConfig: { type: 'zst', level: 3 }, + customEnvVar: { + PGBACKREST_REPO1_BUNDLE: 'y', + PGBACKREST_REPO1_BLOCK: 'y', + }, + }); + expect(repo).toMatchObject({ uriStyle: 'host', verifyTLS: true }); + expect(repo.retentionPolicy).toEqual({ + full: 2, + fullType: 'count', + archiveType: 'full', + }); + }); + + it('archives through the plugin, and stops by removing it', () => { + const on = archiveThroughPluginPatch({ + cluster: 'orders-db-ha', + namespace: 'ns', + }) as any; + expect(on.spec.plugins).toEqual([ + { + name: 'pgbackrest.dalibo.com', + enabled: true, + isWALArchiver: true, + parameters: { stanzaRef: 'orders-db-ha-pgbackrest' }, + }, + ]); + expect( + (stopArchivingPatch({ cluster: 'orders-db-ha', namespace: 'ns' }) as any) + .spec, + ).toEqual({ + plugins: null, + }); + }); + + it('drops the replica section a promoted copy keeps, which the plugin cannot read', () => { + expect( + ( + promotedReplicaPatch({ + cluster: 'orders-db-ha', + namespace: 'ns', + }) as any + ).spec, + ).toEqual({ replica: null }); + }); + + it('takes a base on the primary and reads its label back', () => { + const backup = renderPluginBackup({ + name: 'b1', + cluster: 'orders-db-ha', + namespace: 'ns', + type: 'incr', + }) as any; + expect(backup.spec).toMatchObject({ + method: 'plugin', + target: 'primary', + pluginConfiguration: { parameters: { backupType: 'incr' } }, + }); + expect( + pluginBackupState({ + status: { phase: 'completed', backupId: '20261108-101010F' }, + }), + ).toEqual({ + phase: 'completed', + label: '20261108-101010F', + }); + expect( + pluginBackupState({ status: { phase: 'failed', error: 'no space' } }), + ).toEqual({ + phase: 'failed', + error: 'no space', + }); + expect(pluginBackupState({})).toEqual({ phase: 'running' }); + }); +}); + +describe('where the plugin controller runs', () => { + it('keeps a copy in two locations, so losing one does not stop the failover', () => { + const placement = pgbackrestPluginPlacement() as { + spec: { replicas: number; template: { spec: Record } }; + }; + expect(placement.spec.replicas).toBe(2); + expect( + placement.spec.template.spec.affinity.podAntiAffinity + .requiredDuringSchedulingIgnoredDuringExecution[0], + ).toEqual({ + labelSelector: { matchLabels: { app: 'pgbackrest-controller' } }, + topologyKey: 'topology.kubernetes.io/zone', + }); + }); +}); diff --git a/src/modules/postgres-ha/utils/pgbackrest-plugin.core.ts b/src/modules/postgres-ha/utils/pgbackrest-plugin.core.ts new file mode 100644 index 00000000..62fe216d --- /dev/null +++ b/src/modules/postgres-ha/utils/pgbackrest-plugin.core.ts @@ -0,0 +1,325 @@ +import { createHash } from 'node:crypto'; +import * as yaml from 'js-yaml'; +import { TOPOLOGY_ZONE_LABEL } from '../../providers/core/node-topology'; +import { CNPG_API_VERSION, CNPG_NAMESPACE } from '../postgres-ha.constants'; + +/** + * Dalibo's pgBackRest plugin for CloudNativePG: the same tool, repository + * format and client-side cipher Flui's single Postgres backs up with, so a + * highly available database keeps the promise that the bucket alone does not + * read its data. The manifest is the one of commit 1b88275: v0.0.3 with the + * controller image of the same release. + */ +export const PGBACKREST_PLUGIN_NAME = 'pgbackrest.dalibo.com'; +export const PGBACKREST_PLUGIN_API_VERSION = 'pgbackrest.dalibo.com/v1'; +export const PGBACKREST_PLUGIN_MANIFEST_URL = + 'https://raw.githubusercontent.com/dalibo/cnpg-plugin-pgbackrest/1b88275d3684f1b5f3e59d2bbc560044fe44efa6/manifest.yaml'; +export const PGBACKREST_PLUGIN_MANIFEST_SHA256 = + 'c36ab57a8e80814301c57d8696bdebfaaa99b2d3526e49d97934d34cd86e5dfe'; +export const PGBACKREST_PLUGIN_DEPLOYMENT = 'pgbackrest-controller'; +/** The container the plugin adds to every instance; pgBackRest reads its configuration from its env. */ +export const PGBACKREST_SIDECAR = 'plugin-pgbackrest'; + +/** By the digest Docker Hub served on 8 November 2026; a tag can be moved, a digest cannot. */ +export const PGBACKREST_PLUGIN_IMAGE_DIGESTS: Record = { + 'registry.hub.docker.com/dalibo/cnpg-pgbackrest-controller:0.0.3': + 'sha256:9563249502db7c1ae22c5e1979ab7ffd040559ea8c7544bfad25c5254c7ceb2e', + 'registry.hub.docker.com/dalibo/cnpg-pgbackrest-sidecar:0.0.3': + 'sha256:3cb4ab8de262147f800be40e0bb082d33728060665282f1524d95c9772ee77c3', + 'registry.hub.docker.com/dalibo/cnpg-pgbackrest-sidecar-exporter:0.0.3': + 'sha256:45cbe4898b1b16a3cab0263a1af26c4af2cfbf523de5312836ad4b3adc98fcf0', +}; + +function pinned(image: string): string { + if (image.includes('@')) return image; + const digest = PGBACKREST_PLUGIN_IMAGE_DIGESTS[image]; + if (!digest) throw new Error(`No pinned digest for ${image}`); + return `${image}@${digest}`; +} + +function pinSidecarImages(secret: Record): void { + for (const key of ['SIDECAR_IMAGE', 'SIDECAR_EXPORTER_IMAGE']) { + const value = secret.data?.[key]; + if (!value) continue; + const image = Buffer.from(value, 'base64').toString().trim(); + secret.data[key] = Buffer.from(pinned(image)).toString('base64'); + } +} + +function pinContainers(deployment: Record): void { + for (const c of deployment.spec?.template?.spec?.containers ?? []) { + if (c.image) c.image = pinned(c.image); + } +} + +/** + * The plugin's manifest with its controller and the images it hands to every + * instance fixed by digest, after checking it is the manifest they were read + * for. The sidecar images reach the controller through a Secret. + */ +export function pinnedPgbackrestPluginManifest( + manifest: string, + expectedSha256 = PGBACKREST_PLUGIN_MANIFEST_SHA256, +): string { + const sha = createHash('sha256').update(manifest).digest('hex'); + if (sha !== expectedSha256) { + throw new Error( + `The pgBackRest plugin manifest is not the one Flui checked (SHA-256 ${sha}): refusing to install it`, + ); + } + const docs = yaml.loadAll(manifest).filter(Boolean) as Array< + Record + >; + for (const doc of docs) { + if (doc.kind === 'Secret') pinSidecarImages(doc); + if (doc.kind === 'Deployment') pinContainers(doc); + } + return docs.map((d) => yaml.dump(d, { lineWidth: -1 })).join('---\n'); +} + +export const PGBACKREST_PLUGIN_REPLICAS = 2; + +/** + * Once a cluster names the plugin, the operator reconciles nothing, a failover + * included, while it cannot reach the plugin's controller. As shipped that is + * one copy, so losing its location stops the database. Two copies in + * different locations: the controller serves only while it holds its lease, + * and its readiness probe is that port, so traffic follows the holder. + */ +export function pgbackrestPluginPlacement(): Record { + return { + apiVersion: 'apps/v1', + kind: 'Deployment', + metadata: { name: PGBACKREST_PLUGIN_DEPLOYMENT, namespace: CNPG_NAMESPACE }, + spec: { + replicas: PGBACKREST_PLUGIN_REPLICAS, + template: { + spec: { + affinity: { + podAntiAffinity: { + requiredDuringSchedulingIgnoredDuringExecution: [ + { + labelSelector: { + matchLabels: { app: PGBACKREST_PLUGIN_DEPLOYMENT }, + }, + topologyKey: TOPOLOGY_ZONE_LABEL, + }, + ], + }, + }, + tolerations: [{ operator: 'Exists', effect: 'NoSchedule' }], + }, + }, + }, + }; +} + +export interface PluginRepository { + bucket: string; + /** Host, and port when there is one, without the scheme. */ + endpoint: string; + region: string; + /** `repo1-path`: the destination's prefix and the repository's own. */ + repoPath: string; + pathStyle: boolean; + retentionFull: number; +} + +const KEY_ID = 'ACCESS_KEY_ID'; +const KEY_SECRET = 'ACCESS_SECRET_KEY'; +const CIPHER_PASS = 'ENCRYPTION_PASS'; + +/** One name for the Stanza and its Secret, beside the cluster they serve. */ +export function pgbackrestStanzaName(cluster: string): string { + return `${cluster}-pgbackrest`; +} + +export function renderPgbackrestSecret(input: { + cluster: string; + namespace: string; + accessKey: string; + secretKey: string; + cipherPass: string; +}): Record { + return { + apiVersion: 'v1', + kind: 'Secret', + metadata: { + name: pgbackrestStanzaName(input.cluster), + namespace: input.namespace, + labels: { 'flui.cloud/managed-by': 'flui-cloud' }, + }, + type: 'Opaque', + stringData: { + [KEY_ID]: input.accessKey, + [KEY_SECRET]: input.secretKey, + [CIPHER_PASS]: input.cipherPass, + }, + }; +} + +/** + * The repository the highly available cluster ships to: stanza `main`, + * encrypted with the destination's passphrase, compressed and bundled the way + * Flui's single Postgres does, so a restore reads it with the same code. + */ +export function renderPgbackrestStanza(input: { + cluster: string; + namespace: string; + stanza: string; + repository: PluginRepository; +}): Record { + const secret = pgbackrestStanzaName(input.cluster); + const r = input.repository; + return { + apiVersion: PGBACKREST_PLUGIN_API_VERSION, + kind: 'Stanza', + metadata: { + name: pgbackrestStanzaName(input.cluster), + namespace: input.namespace, + labels: { 'flui.cloud/managed-by': 'flui-cloud' }, + }, + spec: { + stanzaConfiguration: { + name: input.stanza, + processMax: 4, + startFast: true, + compressConfig: { type: 'zst', level: 3 }, + customEnvVar: { + PGBACKREST_REPO1_BUNDLE: 'y', + PGBACKREST_REPO1_BLOCK: 'y', + }, + s3Repositories: [ + { + bucket: r.bucket, + endpoint: r.endpoint, + region: r.region, + repoPath: r.repoPath, + uriStyle: r.pathStyle ? 'path' : 'host', + verifyTLS: true, + retentionPolicy: { + full: r.retentionFull, + fullType: 'count', + archiveType: 'full', + }, + secretRef: { + accessKeyId: { name: secret, key: KEY_ID }, + secretAccessKey: { name: secret, key: KEY_SECRET }, + }, + cipherConfig: { + type: 'aes-256-cbc', + encryptionPass: { name: secret, key: CIPHER_PASS }, + }, + }, + ], + }, + }, + }; +} + +/** Patch that has the cluster's instances archive their log through the plugin. */ +export function archiveThroughPluginPatch(input: { + cluster: string; + namespace: string; +}): Record { + return { + apiVersion: CNPG_API_VERSION, + kind: 'Cluster', + metadata: { name: input.cluster, namespace: input.namespace }, + spec: { + plugins: [ + { + name: PGBACKREST_PLUGIN_NAME, + enabled: true, + isWALArchiver: true, + parameters: { stanzaRef: pgbackrestStanzaName(input.cluster) }, + }, + ], + }, + }; +} + +/** + * A cluster that started as a copy keeps its `replica` section, disabled, once + * promoted. The plugin reads the source it names and fails on one that has no + * plugin of its own, which stops CloudNativePG from reconciling the cluster; + * the section serves nothing once the copy serves. + */ +export function promotedReplicaPatch(input: { + cluster: string; + namespace: string; +}): Record { + return { + apiVersion: CNPG_API_VERSION, + kind: 'Cluster', + metadata: { name: input.cluster, namespace: input.namespace }, + spec: { replica: null }, + }; +} + +/** Patch that stops the archiving: a JSON merge patch removes the list. */ +export function stopArchivingPatch(input: { + cluster: string; + namespace: string; +}): Record { + return { + apiVersion: CNPG_API_VERSION, + kind: 'Cluster', + metadata: { name: input.cluster, namespace: input.namespace }, + spec: { plugins: null }, + }; +} + +/** A base backup taken on the primary, the only instance a pgBackRest backup can start from without a standby setup. */ +export function renderPluginBackup(input: { + name: string; + cluster: string; + namespace: string; + type: 'full' | 'incr' | 'diff'; +}): Record { + return { + apiVersion: CNPG_API_VERSION, + kind: 'Backup', + metadata: { + name: input.name, + namespace: input.namespace, + labels: { 'flui.cloud/managed-by': 'flui-cloud' }, + }, + spec: { + cluster: { name: input.cluster }, + method: 'plugin', + target: 'primary', + pluginConfiguration: { + name: PGBACKREST_PLUGIN_NAME, + parameters: { backupType: input.type }, + }, + }, + }; +} + +export type PluginBackupState = + | { phase: 'running' } + | { phase: 'completed'; label: string } + | { phase: 'failed'; error: string }; + +/** A `Backup`'s status: CloudNativePG records the label the plugin returned as its backup id. */ +export function pluginBackupState(backup: { + status?: { + phase?: string; + backupId?: string; + backupName?: string; + error?: string; + }; +}): PluginBackupState { + const s = backup.status ?? {}; + if (s.phase === 'completed') { + const label = s.backupId || s.backupName; + return label + ? { phase: 'completed', label } + : { phase: 'failed', error: 'The backup completed without a label' }; + } + if (s.phase === 'failed') { + return { phase: 'failed', error: s.error || 'The backup failed' }; + } + return { phase: 'running' }; +} diff --git a/src/modules/postgres-ha/utils/postgres-ha-eligibility.core.ts b/src/modules/postgres-ha/utils/postgres-ha-eligibility.core.ts new file mode 100644 index 00000000..05e1a50a --- /dev/null +++ b/src/modules/postgres-ha/utils/postgres-ha-eligibility.core.ts @@ -0,0 +1,63 @@ +import { DatabaseHaSpec } from '../postgres-ha.types'; + +interface AppLike { + labels?: Record | null; + workloadKind?: string | null; + imageRef?: string | null; + sourceConfig?: unknown; + databaseHa?: DatabaseHaSpec | null; + pooler?: { enabled?: boolean; upstream?: { mode?: string } } | null; +} + +export type HaEligibility = + | { eligible: true } + | { eligible: false; reason: string }; + +/** + * Which databases can move to a highly available cluster today: Flui's own + * Postgres, the one whose image, roles and access rules the move was proven + * on. A database that is moving somewhere else, or already moved, is refused. + */ +export function postgresHaEligibility( + app: AppLike, + /** False when no activation is queued or running: a `copying` state is then what a stopped attempt left. */ + activationRunning = true, +): HaEligibility { + if (app.labels?.['flui.cloud/db-engine'] !== 'postgres') { + return { + eligible: false, + reason: 'Only a PostgreSQL database can be made highly available.', + }; + } + const image = + app.imageRef ?? + (app.sourceConfig as { imageRef?: string } | undefined)?.imageRef ?? + ''; + if (app.workloadKind !== 'StatefulSet' || !/\/flui-postgres:/.test(image)) { + return { + eligible: false, + reason: + "Only Flui's own PostgreSQL can be made highly available for now; this one runs another image.", + }; + } + if (app.databaseHa?.state === 'active') { + return { + eligible: false, + reason: 'This database is already highly available.', + }; + } + if (app.databaseHa?.state === 'copying' && activationRunning) { + return { + eligible: false, + reason: 'This database is already being made highly available.', + }; + } + if (app.pooler?.enabled && app.pooler.upstream?.mode === 'forward') { + return { + eligible: false, + reason: + 'This database is being moved to another cluster; finish that first.', + }; + } + return { eligible: true }; +} diff --git a/src/modules/postgres-ha/utils/postgres-ha-feasibility.core.ts b/src/modules/postgres-ha/utils/postgres-ha-feasibility.core.ts new file mode 100644 index 00000000..bcd04802 --- /dev/null +++ b/src/modules/postgres-ha/utils/postgres-ha-feasibility.core.ts @@ -0,0 +1,62 @@ +import { HaTopology } from '../../providers/core/ha-topology'; +import { TOPOLOGY_ZONE_LABEL } from '../../providers/core/node-topology'; + +export interface HaCandidateNode { + name: string; + zone?: string; + schedulable: boolean; +} + +export type PostgresHaFeasibility = + | { feasible: true } + | { feasible: false; reason: string }; + +/** + * Whether a cluster can hold a highly available Postgres: one instance in each + * of the layout's three locations, on a node that takes workloads. A request + * it cannot honour is refused with the reason, never placed on fewer + * locations than it needs. + */ +export function postgresHaFeasibility( + nodes: HaCandidateNode[], + topology: HaTopology, +): PostgresHaFeasibility { + const missing = topology.zones.filter( + (zone) => !nodes.some((n) => n.schedulable && n.zone === zone), + ); + if (missing.length) { + return { + feasible: false, + reason: `A highly available Postgres keeps one instance in each of ${topology.zones.join(', ')}; ${missing.join(' and ')} ${missing.length === 1 ? 'has' : 'have'} no node that takes workloads.`, + }; + } + return { feasible: true }; +} + +interface NodeLike { + metadata?: { name?: string; labels?: Record }; + spec?: { + unschedulable?: boolean; + taints?: Array<{ key: string; effect: string }>; + }; + status?: { conditions?: Array<{ type: string; status: string }> }; +} + +/** A node counts when it is ready, not cordoned and takes ordinary workloads. */ +export function haCandidatesFrom(nodes: NodeLike[]): HaCandidateNode[] { + return nodes + .filter((node) => node.metadata?.name) + .map((node) => { + const ready = (node.status?.conditions ?? []).some( + (c) => c.type === 'Ready' && c.status === 'True', + ); + const reserved = (node.spec?.taints ?? []).some( + (t) => t.effect === 'NoSchedule' || t.effect === 'NoExecute', + ); + return { + name: node.metadata!.name!, + zone: node.metadata?.labels?.[TOPOLOGY_ZONE_LABEL], + schedulable: ready && !node.spec?.unschedulable && !reserved, + }; + }); +} diff --git a/src/modules/postgres-ha/utils/postgres-ha-migration.core.spec.ts b/src/modules/postgres-ha/utils/postgres-ha-migration.core.spec.ts new file mode 100644 index 00000000..cc2d32e4 --- /dev/null +++ b/src/modules/postgres-ha/utils/postgres-ha-migration.core.spec.ts @@ -0,0 +1,127 @@ +import { + singleInstanceSelector, + HA_REPLICATION_ROLE, + caughtUpSql, + copyHbaScript, + fenceSql, + pgRewindGrantedSql, + pgRewindGrantsSql, + primarySelectorPatch, + promotionPatch, + renderCnpgCopyCluster, + sourceRolesSql, +} from './postgres-ha-migration.core'; + +describe('moving a single Postgres into a highly available one', () => { + /** Flui's image has no `postgres` role, and the operator's agent connects as it. */ + it('gives the source the roles the operator expects in what it copies', () => { + const sql = sourceRolesSql(`'pw'`).join('\n'); + expect(sql).toContain('CREATE ROLE postgres WITH SUPERUSER LOGIN'); + expect(sql).toContain( + 'CREATE ROLE streaming_replica WITH REPLICATION LOGIN', + ); + expect(sql).toContain( + `CREATE ROLE ${HA_REPLICATION_ROLE} WITH REPLICATION LOGIN PASSWORD 'pw'`, + ); + expect(sql).toContain( + `ALTER ROLE ${HA_REPLICATION_ROLE} WITH PASSWORD 'pw'`, + ); + }); + + it('lets only the pod network stream from the source, once', () => { + const script = copyHbaScript('10.42.0.0/16'); + expect(script).toContain( + `host replication ${HA_REPLICATION_ROLE} 10.42.0.0/16 scram-sha-256`, + ); + expect(script).toContain(`grep -qx '# flui-ha-copy:begin'`); + expect(() => copyHbaScript('0.0.0.0/0; rm -rf /')).toThrow(/pod network/); + }); + + it('stops writes on the source and reports where its log ends', () => { + const sql = fenceSql(); + expect(sql[0]).toBe('ALTER SYSTEM SET default_transaction_read_only = on'); + expect(sql).toContain('SELECT pg_reload_conf()'); + expect(sql[sql.length - 1]).toBe('SELECT pg_current_wal_lsn()'); + }); + + it('asks the copy whether it has replayed that far, and nothing else', () => { + expect(caughtUpSql('0/14000000')).toBe( + "SELECT pg_last_wal_replay_lsn() >= '0/14000000'::pg_lsn", + ); + expect(() => caughtUpSql("0/1'; drop table x; --")).toThrow(/WAL position/); + }); + + it('starts the copy from the running instance and keeps it a replica until promoted', () => { + const cluster = renderCnpgCopyCluster({ + name: 'orders-db-ha', + namespace: 'p-orders', + appId: 'app-1', + instances: 3, + preferredZones: ['fsn1', 'nbg1'], + durability: 'required', + storageClass: 'flui-local', + sizeGi: 10, + sourceHost: 'orders-db-svc', + replicationSecret: 'orders-db-ha-repl', + }); + const spec = cluster.spec as Record; + expect(spec.bootstrap).toEqual({ + pg_basebackup: { source: 'flui-single' }, + }); + expect(spec.replica).toEqual({ enabled: true, source: 'flui-single' }); + expect(spec.externalClusters[0]).toEqual({ + name: 'flui-single', + connectionParameters: { + host: 'orders-db-svc', + user: HA_REPLICATION_ROLE, + dbname: 'postgres', + sslmode: 'disable', + }, + password: { name: 'orders-db-ha-repl', key: 'password' }, + }); + expect(spec.imageName).toBe( + 'ghcr.io/cloudnative-pg/postgresql:17.11-standard-trixie@sha256:4d0aac315c4d8c3202252bd06fea83b16951cc57234a7b610eeaa382ff4482d0', + ); + expect(spec.affinity.topologyKey).toBe('topology.kubernetes.io/zone'); + expect( + spec.affinity.nodeAffinity + .preferredDuringSchedulingIgnoredDuringExecution[0].preference + .matchExpressions[0].values, + ).toEqual(['fsn1', 'nbg1']); + }); + + it('promotes the copy and points the application at its primary', () => { + expect(promotionPatch()).toEqual({ spec: { replica: { enabled: false } } }); + expect(primarySelectorPatch('orders-db', 'orders-db-ha')).toEqual({ + spec: { + selector: { + app: null, + 'cnpg.io/cluster': 'orders-db-ha', + 'cnpg.io/instanceRole': 'primary', + }, + }, + metadata: { + annotations: { 'flui.cloud/ha-previous-selector': 'orders-db' }, + }, + }); + }); +}); + +describe('the single instance while its copy exists', () => { + it('leaves out the pods of the copy, which carry the same application id', () => { + expect(singleInstanceSelector('flui-app-id=a1')).toBe( + 'flui-app-id=a1,!cnpg.io/cluster', + ); + }); + + it('grants the replication user the four functions pg_rewind reads the primary with', () => { + const grants = pgRewindGrantsSql(); + expect(grants.match(/GRANT EXECUTE ON FUNCTION/g)).toHaveLength(4); + expect(grants).toContain( + 'pg_catalog.pg_read_binary_file(text, bigint, bigint, boolean) TO streaming_replica', + ); + expect(pgRewindGrantedSql().match(/has_function_privilege/g)).toHaveLength( + 4, + ); + }); +}); diff --git a/src/modules/postgres-ha/utils/postgres-ha-migration.core.ts b/src/modules/postgres-ha/utils/postgres-ha-migration.core.ts new file mode 100644 index 00000000..fd89e2a9 --- /dev/null +++ b/src/modules/postgres-ha/utils/postgres-ha-migration.core.ts @@ -0,0 +1,199 @@ +import { + CNPG_API_VERSION, + CNPG_POSTGRES_IMAGE, + CNPG_REPLICATION_USER, +} from '../postgres-ha.constants'; +import { CnpgClusterInput, renderCnpgCluster } from './cnpg-cluster.core'; + +/** + * Moving a single Postgres into a highly available one without stopping its + * application, in this order: the copy streams from + * the running instance, the instance turns read-only, the copy catches up and + * is promoted, and the application's address moves to the new primary. Writes + * pause for about ten seconds and no confirmed write is lost. + */ + +export const HA_REPLICATION_ROLE = 'flui_ha_repl'; +export const HA_HBA_MARKER = 'flui-ha-copy'; +const EXTERNAL_SOURCE = 'flui-single'; +const OCTET_RE = /^\d{1,3}$/; + +function isIpv4Cidr(value: string): boolean { + const [address, prefix, ...rest] = value.split('/'); + if (rest.length || !prefix || !/^\d{1,2}$/.test(prefix)) return false; + const octets = address.split('.'); + return ( + octets.length === 4 && + octets.every((o) => OCTET_RE.test(o) && Number(o) <= 255) && + Number(prefix) <= 32 + ); +} + +/** + * The roles the operator expects in the data it copies. Flui's own image makes + * the application's user the superuser and has no `postgres`, which the + * operator's agent connects as; without it the copy never leaves its first + * instance. + */ +export function sourceRolesSql(replicationPasswordLiteral: string): string[] { + return [ + `DO $$ BEGIN CREATE ROLE postgres WITH SUPERUSER LOGIN; EXCEPTION WHEN duplicate_object THEN NULL; END $$`, + `DO $$ BEGIN CREATE ROLE streaming_replica WITH REPLICATION LOGIN; EXCEPTION WHEN duplicate_object THEN NULL; END $$`, + `DO $$ BEGIN CREATE ROLE ${HA_REPLICATION_ROLE} WITH REPLICATION LOGIN PASSWORD ${replicationPasswordLiteral}; EXCEPTION WHEN duplicate_object THEN ALTER ROLE ${HA_REPLICATION_ROLE} WITH PASSWORD ${replicationPasswordLiteral}; END $$`, + ]; +} + +/** Lets the copy stream from inside the cluster's pod network, and nothing else. */ +export function copyHbaScript(podNetwork: string): string { + if (!isIpv4Cidr(podNetwork)) { + throw new Error(`Not a pod network range: ${podNetwork}`); + } + const begin = `# ${HA_HBA_MARKER}:begin`; + const end = `# ${HA_HBA_MARKER}:end`; + return [ + 'set -e', + 'HBA="$PGDATA/pg_hba.conf"', + `if ! grep -qx '${begin}' "$HBA"; then`, + String.raw` { printf '${begin}\nhost replication ${HA_REPLICATION_ROLE} ${podNetwork} scram-sha-256\n${end}\n'`, + ' cat "$HBA"; } > "$HBA.flui" && cat "$HBA.flui" > "$HBA" && rm -f "$HBA.flui"', + 'fi', + ].join('\n'); +} + +/** The source stops taking writes; sessions already open are ended so none slips through. */ +export function fenceSql(): string[] { + return [ + 'ALTER SYSTEM SET default_transaction_read_only = on', + 'SELECT pg_reload_conf()', + `SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity WHERE backend_type = 'client backend' AND pid <> pg_backend_pid()`, + 'SELECT pg_switch_wal()', + 'SELECT pg_current_wal_lsn()', + ]; +} + +/** Undoes the fence, for a move that is abandoned before the copy is promoted. */ +export function unfenceSql(): string[] { + return [ + 'ALTER SYSTEM RESET default_transaction_read_only', + 'SELECT pg_reload_conf()', + ]; +} + +const PG_REWIND_FUNCTIONS = [ + 'pg_catalog.pg_ls_dir(text, boolean, boolean)', + 'pg_catalog.pg_stat_file(text, boolean)', + 'pg_catalog.pg_read_binary_file(text)', + 'pg_catalog.pg_read_binary_file(text, bigint, bigint, boolean)', +]; + +/** + * What a former primary needs to rejoin as a copy: the operator grants it only + * when Postgres starts as a primary, and a promoted copy never does, so a + * switchover would leave the old primary unable to rewind. + */ +export function pgRewindGrantsSql(): string { + return PG_REWIND_FUNCTIONS.map( + (f) => `GRANT EXECUTE ON FUNCTION ${f} TO ${CNPG_REPLICATION_USER};`, + ).join(' '); +} + +export function pgRewindGrantedSql(): string { + return `SELECT ${PG_REWIND_FUNCTIONS.map( + (f) => + `has_function_privilege('${CNPG_REPLICATION_USER}', '${f}', 'execute')`, + ).join(' AND ')}`; +} + +export function caughtUpSql(sourceLsn: string): string { + if (!/^[0-9A-F]+\/[0-9A-F]+$/i.test(sourceLsn)) { + throw new Error(`Not a WAL position: ${sourceLsn}`); + } + return `SELECT pg_last_wal_replay_lsn() >= '${sourceLsn}'::pg_lsn`; +} + +export interface CopyClusterInput + extends Omit { + /** + * A basic-auth secret for `postgres`. Without one the operator removes that + * role's password when the copy is promoted, and an application that signs + * in as `postgres` is locked out. + */ + superuserSecret?: string; + /** The source's in-cluster address, the application's own Service. */ + sourceHost: string; + replicationSecret: string; +} + +/** + * The highly available cluster as a streaming copy of the running instance. + * It keeps the source's data, users and passwords, so the application's + * credentials keep working once it is promoted. + */ +export function renderCnpgCopyCluster( + input: CopyClusterInput, +): Record { + const base = renderCnpgCluster({ + ...input, + database: 'unused', + owner: 'unused', + credentialsSecret: 'unused', + }); + const spec = { ...(base.spec as Record) }; + delete spec.bootstrap; + return { + ...base, + apiVersion: CNPG_API_VERSION, + spec: { + ...spec, + imageName: input.imageName ?? CNPG_POSTGRES_IMAGE, + ...(input.superuserSecret && { + enableSuperuserAccess: true, + superuserSecret: { name: input.superuserSecret }, + }), + bootstrap: { pg_basebackup: { source: EXTERNAL_SOURCE } }, + replica: { enabled: true, source: EXTERNAL_SOURCE }, + externalClusters: [ + { + name: EXTERNAL_SOURCE, + connectionParameters: { + host: input.sourceHost, + user: HA_REPLICATION_ROLE, + dbname: 'postgres', + sslmode: 'disable', + }, + password: { name: input.replicationSecret, key: 'password' }, + }, + ], + }, + }; +} + +export function promotionPatch(): Record { + return { spec: { replica: { enabled: false } } }; +} + +/** Where the application's Service points once the copy is its primary. */ +export function primarySelectorPatch( + appSlug: string, + clusterName: string, +): Record { + return { + spec: { + selector: { + app: null, + 'cnpg.io/cluster': clusterName, + 'cnpg.io/instanceRole': 'primary', + }, + }, + metadata: { annotations: { 'flui.cloud/ha-previous-selector': appSlug } }, + }; +} + +/** + * The copies carry the application's id as well, and the one acting as + * primary is picked before any other pod: while the copy is being built, the + * single instance has to be named apart from them. + */ +export function singleInstanceSelector(appSelector: string): string { + return `${appSelector},!cnpg.io/cluster`; +} diff --git a/src/modules/postgres-ha/utils/postgres-ha.core.spec.ts b/src/modules/postgres-ha/utils/postgres-ha.core.spec.ts new file mode 100644 index 00000000..d15633b4 --- /dev/null +++ b/src/modules/postgres-ha/utils/postgres-ha.core.spec.ts @@ -0,0 +1,241 @@ +import { + renderCnpgCluster, + renderCnpgCredentials, + CnpgClusterInput, +} from './cnpg-cluster.core'; +import { + haCandidatesFrom, + postgresHaFeasibility, +} from './postgres-ha-feasibility.core'; +import { HaTopology } from '../../providers/core/ha-topology'; + +const hetzner: HaTopology = { + region: 'eu-central', + zones: ['fsn1', 'nbg1', 'hel1'], + dataZones: ['fsn1', 'nbg1'], + validated: true, + measured: true, +}; + +const base: CnpgClusterInput = { + name: 'orders-db', + namespace: 'flui-app-orders', + appId: 'app-1', + instances: 3, + preferredZones: ['fsn1', 'nbg1'], + durability: 'required', + storageClass: 'flui-local', + sizeGi: 10, + database: 'orders', + owner: 'orders', + credentialsSecret: 'orders-db-credentials', +}; + +describe('the highly available Postgres Flui asks the operator for', () => { + it('keeps one instance in each location, the primary in a close one', () => { + const spec = renderCnpgCluster(base).spec as Record; + expect(spec.instances).toBe(3); + expect(spec.affinity).toEqual({ + enablePodAntiAffinity: true, + podAntiAffinityType: 'required', + topologyKey: 'topology.kubernetes.io/zone', + nodeAffinity: { + preferredDuringSchedulingIgnoredDuringExecution: [ + { + weight: 100, + preference: { + matchExpressions: [ + { + key: 'topology.kubernetes.io/zone', + operator: 'In', + values: ['fsn1', 'nbg1'], + }, + ], + }, + }, + ], + }, + }); + expect(spec.storage).toEqual({ storageClass: 'flui-local', size: '10Gi' }); + }); + + /** The operator's default is a newer major that cannot start on a copy of Flui's Postgres. */ + it("pins the image to the major of Flui's own Postgres unless told otherwise", () => { + expect( + (renderCnpgCluster(base).spec as Record).imageName, + ).toBe( + 'ghcr.io/cloudnative-pg/postgresql:17.11-standard-trixie@sha256:4d0aac315c4d8c3202252bd06fea83b16951cc57234a7b610eeaa382ff4482d0', + ); + expect( + ( + renderCnpgCluster({ ...base, imageName: 'registry/pg:17.12' }) + .spec as Record + ).imageName, + ).toBe('registry/pg:17.12'); + }); + + it('confirms a write once one copy has it, and says what happens without one', () => { + expect( + (renderCnpgCluster(base).spec as Record).postgresql, + ).toEqual({ + synchronous: { + method: 'any', + number: 1, + dataDurability: 'required', + failoverQuorum: true, + }, + }); + expect( + ( + renderCnpgCluster({ ...base, durability: 'preferred' }).spec as Record< + string, + any + > + ).postgresql.synchronous.dataDurability, + ).toBe('preferred'); + }); + + /** The labels every Flui feature already uses to find an application's database. */ + it('carries the application and engine labels onto every instance', () => { + const cluster = renderCnpgCluster(base); + const labels = { + 'flui-app-id': 'app-1', + 'flui.cloud/db-engine': 'postgres', + 'flui.cloud/managed-by': 'flui-cloud', + }; + expect((cluster.metadata as Record).labels).toEqual(labels); + expect( + (cluster.spec as Record).inheritedMetadata.labels, + ).toEqual(labels); + }); + + /** What the copy-behind and copy-missing alerts read. */ + it("has every instance's replication figures collected", () => { + expect( + (renderCnpgCluster(base).spec as Record).inheritedMetadata + .annotations, + ).toEqual({ + 'prometheus.io/scrape': 'true', + 'prometheus.io/port': '9187', + 'prometheus.io/path': '/metrics', + }); + }); + + it('creates the database and its owner from the credentials Flui generated', () => { + expect( + (renderCnpgCluster(base).spec as Record).bootstrap, + ).toEqual({ + initdb: { + database: 'orders', + owner: 'orders', + secret: { name: 'orders-db-credentials' }, + }, + }); + }); + + it('refuses one instance', () => { + expect(() => renderCnpgCluster({ ...base, instances: 1 })).toThrow( + /at least two/, + ); + }); +}); + +describe('whether a cluster can hold a highly available Postgres', () => { + const lab = [ + { name: 'master', zone: 'fsn1', schedulable: true }, + { name: 'worker-1', zone: 'nbg1', schedulable: true }, + { name: 'worker-2', zone: 'fsn1', schedulable: true }, + { name: 'worker-3', zone: 'hel1', schedulable: true }, + ]; + + it('needs a node that takes workloads in each of the three locations', () => { + expect(postgresHaFeasibility(lab, hetzner)).toEqual({ feasible: true }); + }); + + it('names the locations that have none', () => { + expect(postgresHaFeasibility(lab.slice(0, 3), hetzner)).toMatchObject({ + feasible: false, + reason: expect.stringContaining('hel1 has no node'), + }); + expect( + postgresHaFeasibility( + [ + ...lab.slice(0, 3), + { name: 'worker-3', zone: 'hel1', schedulable: false }, + ], + hetzner, + ), + ).toMatchObject({ feasible: false }); + }); +}); + +describe('the owner credentials the operator reads', () => { + it('is a basic-auth secret labelled with its application', () => { + expect( + renderCnpgCredentials({ + name: 'orders-db-credentials', + namespace: 'flui-app-orders', + appId: 'app-1', + username: 'orders', + password: 'secret-value', + }), + ).toEqual({ + apiVersion: 'v1', + kind: 'Secret', + type: 'kubernetes.io/basic-auth', + metadata: { + name: 'orders-db-credentials', + namespace: 'flui-app-orders', + labels: { + 'flui-app-id': 'app-1', + 'flui.cloud/managed-by': 'flui-cloud', + }, + }, + stringData: { username: 'orders', password: 'secret-value' }, + }); + }); +}); + +describe('which nodes can hold an instance', () => { + const node = ( + name: string, + zone: string | undefined, + extra: Record = {}, + ) => ({ + metadata: { + name, + labels: zone ? { 'topology.kubernetes.io/zone': zone } : {}, + }, + status: { conditions: [{ type: 'Ready', status: 'True' }] }, + ...extra, + }); + + it('reads the location label and leaves out cordoned, tainted and unready nodes', () => { + expect( + haCandidatesFrom([ + node('a', 'fsn1'), + node('b', 'fsn1', { spec: { unschedulable: true } }), + node('c', 'nbg1', { + spec: { + taints: [ + { + key: 'node-role.kubernetes.io/control-plane', + effect: 'NoSchedule', + }, + ], + }, + }), + node('d', 'hel1', { + status: { conditions: [{ type: 'Ready', status: 'False' }] }, + }), + node('e', undefined), + ]), + ).toEqual([ + { name: 'a', zone: 'fsn1', schedulable: true }, + { name: 'b', zone: 'fsn1', schedulable: false }, + { name: 'c', zone: 'nbg1', schedulable: false }, + { name: 'd', zone: 'hel1', schedulable: false }, + { name: 'e', zone: undefined, schedulable: true }, + ]); + }); +}); diff --git a/src/modules/postgres-ha/utils/primary-placement.core.spec.ts b/src/modules/postgres-ha/utils/primary-placement.core.spec.ts new file mode 100644 index 00000000..b27ccc0a --- /dev/null +++ b/src/modules/postgres-ha/utils/primary-placement.core.spec.ts @@ -0,0 +1,66 @@ +import { primaryPlacement } from './primary-placement.core'; + +const zoneOf = new Map([ + ['s-fsn', 'fsn1'], + ['s-nbg', 'nbg1'], + ['s-hel', 'hel1'], +]); +const pods = [ + { name: 'db-ha-1', node: 's-hel', ready: true }, + { name: 'db-ha-2', node: 's-nbg', ready: true }, + { name: 'db-ha-3', node: 's-fsn', ready: true }, +]; +const base = { + primary: 'db-ha-1', + phase: 'Cluster in healthy state', + instances: 3, + readyInstances: 3, + pods, + zoneOf, + preferredZones: ['fsn1', 'nbg1'], + now: Date.parse('2026-10-29T10:00:00Z'), +}; + +describe('where the primary of a highly available Postgres belongs', () => { + it('switches a primary that landed in the voting location over to a copy in a location for data', () => { + expect(primaryPlacement(base)).toEqual({ move: true, target: 'db-ha-2' }); + }); + + it('leaves a primary already in a location for data', () => { + expect(primaryPlacement({ ...base, primary: 'db-ha-3' })).toMatchObject({ + move: false, + }); + }); + + it('never switches over while the cluster is failing over or an instance is missing', () => { + expect(primaryPlacement({ ...base, phase: 'Failing over' }).move).toBe( + false, + ); + expect(primaryPlacement({ ...base, readyInstances: 2 }).move).toBe(false); + }); + + it('waits half an hour after the last move', () => { + expect( + primaryPlacement({ ...base, lastMovedAt: '2026-10-29T09:45:00Z' }).move, + ).toBe(false); + expect( + primaryPlacement({ ...base, lastMovedAt: '2026-10-29T09:00:00Z' }).move, + ).toBe(true); + }); + + it('needs a ready copy in a location for data', () => { + expect( + primaryPlacement({ + ...base, + pods: [ + pods[0], + { ...pods[1], ready: false }, + { ...pods[2], ready: false }, + ], + }), + ).toMatchObject({ + move: false, + reason: 'no copy ready in a location for data', + }); + }); +}); diff --git a/src/modules/postgres-ha/utils/primary-placement.core.ts b/src/modules/postgres-ha/utils/primary-placement.core.ts new file mode 100644 index 00000000..62f263ae --- /dev/null +++ b/src/modules/postgres-ha/utils/primary-placement.core.ts @@ -0,0 +1,59 @@ +import { CNPG_HEALTHY_PHASE } from '../postgres-ha.constants'; +export interface InstancePod { + name: string; + node?: string; + ready: boolean; +} + +export type PrimaryPlacement = + | { move: false; reason: string } + | { move: true; target: string }; + +/** Long enough that a switchover never chases a primary that is still settling. */ +export const PRIMARY_MOVE_COOLDOWN_MS = 30 * 60 * 1000; + +/** + * Whether the primary sits outside the locations writes are confirmed fastest + * from, and which copy to switch over to. Only on a healthy cluster with every + * instance ready, so a switchover never runs in the middle of a failover. + */ +export function primaryPlacement(input: { + primary: string | undefined; + phase: string | undefined; + instances: number; + readyInstances: number; + pods: InstancePod[]; + zoneOf: Map; + preferredZones: string[]; + lastMovedAt?: string; + now: number; +}): PrimaryPlacement { + if (input.phase !== CNPG_HEALTHY_PHASE) { + return { move: false, reason: 'not healthy' }; + } + if (input.readyInstances < input.instances) { + return { move: false, reason: 'an instance is not ready' }; + } + const primary = input.pods.find((p) => p.name === input.primary); + const zone = primary?.node ? input.zoneOf.get(primary.node) : undefined; + if (!primary || !zone) return { move: false, reason: 'primary not placed' }; + if (input.preferredZones.includes(zone)) { + return { move: false, reason: 'already in place' }; + } + const last = input.lastMovedAt ? Date.parse(input.lastMovedAt) : Number.NaN; + if (!Number.isNaN(last) && input.now - last < PRIMARY_MOVE_COOLDOWN_MS) { + return { move: false, reason: 'moved recently' }; + } + const target = input.pods + .filter( + (p) => + p.name !== primary.name && + p.ready && + !!p.node && + input.preferredZones.includes(input.zoneOf.get(p.node) ?? ''), + ) + .sort((a, b) => a.name.localeCompare(b.name))[0]; + if (!target) + return { move: false, reason: 'no copy ready in a location for data' }; + return { move: true, target: target.name }; +} diff --git a/src/modules/providers/core/ha-topology.spec.ts b/src/modules/providers/core/ha-topology.spec.ts new file mode 100644 index 00000000..382c6d1e --- /dev/null +++ b/src/modules/providers/core/ha-topology.spec.ts @@ -0,0 +1,51 @@ +import { HaTopology, resolveHaTopology } from './ha-topology'; + +const hetzner: HaTopology = { + region: 'eu-central', + zones: ['fsn1', 'nbg1', 'hel1'], + dataZones: ['fsn1', 'nbg1'], + validated: true, + measured: true, +}; + +describe('which highly available layout applies to a cluster', () => { + it('takes the layout Flui validated for the region', () => { + expect(resolveHaTopology([hetzner], 'eu-central', ['fsn1'])).toEqual({ + topology: hetzner, + }); + }); + + it('refuses a region nobody validated, and says it can be forced', () => { + const r = resolveHaTopology([hetzner], 'fr-par', ['fr-par-1']); + expect(r.topology).toBeNull(); + expect((r as { reason: string }).reason).toContain('can be forced'); + }); + + it('a forced region treats its three locations as equal', () => { + expect( + resolveHaTopology( + [], + 'fr-par', + ['fr-par-2', 'fr-par-1', 'fr-par-3'], + true, + ), + ).toEqual({ + topology: { + region: 'fr-par', + zones: ['fr-par-1', 'fr-par-2', 'fr-par-3'], + dataZones: ['fr-par-1', 'fr-par-2', 'fr-par-3'], + validated: false, + measured: false, + }, + }); + }); + + it('a forced region still needs servers in three locations', () => { + const r = resolveHaTopology([], 'fr-par', ['fr-par-1', 'fr-par-2'], true); + expect((r as { reason: string }).reason).toContain('exactly three'); + }); + + it('a cluster whose nodes report no region gets no layout', () => { + expect(resolveHaTopology([hetzner], undefined, []).topology).toBeNull(); + }); +}); diff --git a/src/modules/providers/core/ha-topology.ts b/src/modules/providers/core/ha-topology.ts new file mode 100644 index 00000000..d218c13f --- /dev/null +++ b/src/modules/providers/core/ha-topology.ts @@ -0,0 +1,75 @@ +/** + * A layout a cluster can take to survive the loss of a whole location: one + * control-plane server in each of three locations, and copies of data in the + * locations close enough to confirm a write together. + */ +export interface HaTopology { + region: string; + zones: [string, string, string]; + /** Where copies of data live, and where leaders and primaries belong. */ + dataZones: string[]; + /** False when a person forced a region Flui has not validated. */ + validated: boolean; + /** + * True when Flui measured these locations itself; false when it took them on + * the provider's word that they are zones of one region, close together. + */ + measured: boolean; +} + +/** `fsn1`, `fsn1 and nbg1`, `a, b and c`. */ +export function locationList(zones: string[]): string { + return zones.length < 2 + ? zones.join('') + : `${zones.slice(0, -1).join(', ')} and ${zones.at(-1)}`; +} + +export type HaTopologyResolution = + | { topology: HaTopology } + | { topology: null; reason: string }; + +/** + * The validated layout for a cluster's region. A region Flui has not measured + * is refused unless forced, and a forced one treats its three locations as + * equal, because nothing says which of them are close. + */ +export function resolveHaTopology( + declared: HaTopology[], + region: string | undefined, + zonesWithServers: string[], + force = false, +): HaTopologyResolution { + if (!region) { + return { + topology: null, + reason: + 'No node of this cluster reports its region yet, so Flui cannot tell which layout applies.', + }; + } + const validated = declared.find((t) => t.region === region); + if (validated) return { topology: validated }; + const zones = [...new Set(zonesWithServers)].sort((a, b) => + a.localeCompare(b), + ); + if (!force) { + return { + topology: null, + reason: `Flui has not validated a highly available layout for ${region} yet. It can be forced, with no promise about how far apart its locations are.`, + }; + } + if (zones.length !== 3) { + return { + topology: null, + reason: `A forced layout needs servers in exactly three locations of ${region}; this cluster has them in ${zones.length}.`, + }; + } + return { + topology: { + region, + zones: zones as [string, string, string], + dataZones: zones, + validated: false, + measured: false, + }, + }; +} diff --git a/src/modules/providers/core/node-topology.spec.ts b/src/modules/providers/core/node-topology.spec.ts new file mode 100644 index 00000000..d4ad6900 --- /dev/null +++ b/src/modules/providers/core/node-topology.spec.ts @@ -0,0 +1,178 @@ +import { ConfigService } from '@nestjs/config'; +import { + TOPOLOGY_REGION_LABEL, + TOPOLOGY_ZONE_LABEL, + missingTopologyLabels, + topologyLabels, +} from './node-topology'; +import { ByosCapabilitiesService } from '../implementations/byos/byos-capabilities.service'; +import { ContaboCapabilitiesService } from '../implementations/contabo/contabo-capabilities.service'; +import { HetznerCapabilitiesService } from '../implementations/hetzner/hetzner-capabilities.service'; +import { OvhCapabilitiesService } from '../implementations/ovh/ovh-capabilities.service'; +import { ovhAvailabilityZone } from '../implementations/ovh/ovh-region-metadata'; +import { ScalewayCapabilitiesService } from '../implementations/scaleway/scaleway-capabilities.service'; +import { ICredentialProvider } from '../interfaces/credential-provider.interface'; + +const config = { + get: (_key: string, fallback?: unknown) => fallback, +} as unknown as ConfigService; +const credentials = {} as unknown as ICredentialProvider; + +describe('where a node sits, as its provider declares it', () => { + const hetzner = new HetznerCapabilitiesService(config, credentials); + const scaleway = new ScalewayCapabilitiesService(credentials); + + it('puts a Hetzner node in its location, inside the network zone that covers it', () => { + expect(hetzner.nodeTopology('fsn1')).toEqual({ + zone: 'fsn1', + region: 'eu-central', + }); + expect(hetzner.nodeTopology('hel1')).toEqual({ + zone: 'hel1', + region: 'eu-central', + }); + expect(hetzner.nodeTopology('ash')).toEqual({ + zone: 'ash', + region: 'us-east', + }); + }); + + it('gives a Hetzner location no network zone covers no topology at all', () => { + expect(hetzner.nodeTopology('xyz1')).toBeNull(); + expect(hetzner.nodeTopology('')).toBeNull(); + expect(hetzner.nodeTopology(null)).toBeNull(); + }); + + it('puts a Scaleway node in its zone, and a bare region in its first zone', () => { + expect(scaleway.nodeTopology('fr-par-2')).toEqual({ + zone: 'fr-par-2', + region: 'fr-par', + }); + expect(scaleway.nodeTopology('nl-ams')).toEqual({ + zone: 'nl-ams-1', + region: 'nl-ams', + }); + expect(scaleway.nodeTopology(undefined)).toBeNull(); + }); + + it('puts an OVH node in its zone, a region with three in its first, and a single datacenter nowhere', () => { + const ovh = new OvhCapabilitiesService(config, credentials); + expect(ovh.nodeTopology('eu-west-par-b')).toEqual({ + zone: 'eu-west-par-b', + region: 'eu-west-par', + }); + expect(ovh.nodeTopology('EU-SOUTH-MIL')).toEqual({ + zone: 'eu-south-mil-a', + region: 'eu-south-mil', + }); + expect(ovh.nodeTopology('GRA11')).toBeNull(); + expect(ovh.nodeTopology('RBX-A')).toBeNull(); + }); + + it('pins an OVH server, and its disks, only to a zone the location names', () => { + expect(ovhAvailabilityZone('EU-WEST-PAR-C')).toBe('eu-west-par-c'); + expect(ovhAvailabilityZone('EU-WEST-PAR')).toBeNull(); + expect(ovhAvailabilityZone('GRA11')).toBeNull(); + }); + + it('declares nothing for providers whose failure domains are not mapped', () => { + expect(new ContaboCapabilitiesService(config).nodeTopology()).toBeNull(); + expect(new ByosCapabilitiesService().nodeTopology()).toBeNull(); + }); +}); + +describe('the regions Flui offers across three locations', () => { + it('keeps Hetzner with two locations for data and one that votes', () => { + expect( + new HetznerCapabilitiesService(config, credentials).haTopologies(), + ).toEqual([ + { + region: 'eu-central', + zones: ['fsn1', 'nbg1', 'hel1'], + dataZones: ['fsn1', 'nbg1'], + validated: true, + measured: true, + }, + ]); + }); + + it('puts data in every zone of OVH Paris and Milan, both measured', () => { + const regions = new OvhCapabilitiesService( + config, + credentials, + ).haTopologies(); + expect(regions.map((t) => [t.region, t.measured])).toEqual([ + ['eu-west-par', true], + ['eu-south-mil', true], + ]); + expect(regions[0].zones).toEqual(regions[0].dataZones); + expect(regions[0].zones).toEqual([ + 'eu-west-par-a', + 'eu-west-par-b', + 'eu-west-par-c', + ]); + }); + + it("takes Amsterdam and Warsaw on Scaleway's word until Flui measures them", () => { + const regions = new ScalewayCapabilitiesService(credentials).haTopologies(); + expect(regions.map((t) => [t.region, t.measured])).toEqual([ + ['fr-par', true], + ['it-mil', true], + ['nl-ams', false], + ['pl-waw', false], + ]); + expect(regions[1].dataZones).toEqual(['it-mil-1', 'it-mil-2', 'it-mil-3']); + }); +}); + +describe('the labels a topology becomes', () => { + it('is the standard zone and region pair', () => { + expect(topologyLabels({ zone: 'nbg1', region: 'eu-central' })).toEqual({ + [TOPOLOGY_ZONE_LABEL]: 'nbg1', + [TOPOLOGY_REGION_LABEL]: 'eu-central', + }); + }); + + it('is nothing when there is no topology or a value would not be a valid label', () => { + expect(topologyLabels(null)).toEqual({}); + expect(topologyLabels({ zone: 'fsn1 x', region: 'eu-central' })).toEqual( + {}, + ); + expect(topologyLabels({ zone: 'fsn1', region: 'EU' })).toEqual({}); + }); +}); + +describe('which topology labels a node still needs', () => { + const desired = { + [TOPOLOGY_ZONE_LABEL]: 'fsn1', + [TOPOLOGY_REGION_LABEL]: 'eu-central', + }; + + it('adds the ones that are missing and keeps the others', () => { + expect( + missingTopologyLabels( + { + 'kubernetes.io/hostname': 'n1', + [TOPOLOGY_REGION_LABEL]: 'eu-central', + }, + desired, + ), + ).toEqual({ add: { [TOPOLOGY_ZONE_LABEL]: 'fsn1' }, conflicts: [] }); + }); + + it('never overwrites a label someone set to another value, and says so', () => { + expect( + missingTopologyLabels({ [TOPOLOGY_ZONE_LABEL]: 'fsn1-dc14' }, desired), + ).toEqual({ + add: { [TOPOLOGY_REGION_LABEL]: 'eu-central' }, + conflicts: [`${TOPOLOGY_ZONE_LABEL}=fsn1-dc14`], + }); + }); + + it('asks for nothing when the node already carries them', () => { + expect(missingTopologyLabels({ ...desired }, desired)).toEqual({ + add: {}, + conflicts: [], + }); + }); +}); diff --git a/src/modules/providers/core/node-topology.ts b/src/modules/providers/core/node-topology.ts new file mode 100644 index 00000000..ad1407ce --- /dev/null +++ b/src/modules/providers/core/node-topology.ts @@ -0,0 +1,61 @@ +export const TOPOLOGY_ZONE_LABEL = 'topology.kubernetes.io/zone'; +export const TOPOLOGY_REGION_LABEL = 'topology.kubernetes.io/region'; + +/** + * Where a node sits, in the terms the standard Kubernetes topology labels use. + * `zone` is the place one outage takes down as a whole (a Hetzner location, a + * Scaleway zone); `region` is the group of zones a private network can span. + */ +export interface NodeTopology { + zone: string; + region: string; +} + +const LABEL_CHARS_RE = /^[a-z0-9._-]+$/; +const ALPHANUMERIC_RE = /^[a-z0-9]$/; + +/** Lowercase only, although Kubernetes allows capitals: every provider location Flui knows is lowercase. */ +export function isTopologyLabelValue(value: string): boolean { + return ( + value.length > 0 && + value.length <= 63 && + LABEL_CHARS_RE.test(value) && + ALPHANUMERIC_RE.test(value[0]) && + ALPHANUMERIC_RE.test(value.at(-1) ?? '') + ); +} + +/** The labels a node carries for its topology, or none when either part is not a valid label value. */ +export function topologyLabels( + topology: NodeTopology | null | undefined, +): Record { + if (!topology) return {}; + if ( + !isTopologyLabelValue(topology.zone) || + !isTopologyLabelValue(topology.region) + ) { + return {}; + } + return { + [TOPOLOGY_ZONE_LABEL]: topology.zone, + [TOPOLOGY_REGION_LABEL]: topology.region, + }; +} + +/** + * The labels to add so a node carries its topology. A label already set to + * another value is left alone and reported: it was put there by someone, and + * overwriting it would move workloads that rely on it. + */ +export function missingTopologyLabels( + current: Record, + desired: Record, +): { add: Record; conflicts: string[] } { + const add: Record = {}; + const conflicts: string[] = []; + for (const [key, value] of Object.entries(desired)) { + if (current[key] === undefined) add[key] = value; + else if (current[key] !== value) conflicts.push(`${key}=${current[key]}`); + } + return { add, conflicts }; +} diff --git a/src/modules/providers/dto/node-size.dto.ts b/src/modules/providers/dto/node-size.dto.ts index 760ad86a..3a1d69a5 100644 --- a/src/modules/providers/dto/node-size.dto.ts +++ b/src/modules/providers/dto/node-size.dto.ts @@ -36,6 +36,11 @@ export class NodeSizeLocationAvailabilityDto { * it behaves exactly as before. */ availabilityKnown?: boolean; + /** + * The availability zones of the location that sell this size, where the + * provider says so per zone. Absent when it does not: every zone may. + */ + zones?: string[]; } export class NodeSizeDto { diff --git a/src/modules/providers/implementations/byos/byos-capabilities.service.ts b/src/modules/providers/implementations/byos/byos-capabilities.service.ts index abd5bff9..4737abea 100644 --- a/src/modules/providers/implementations/byos/byos-capabilities.service.ts +++ b/src/modules/providers/implementations/byos/byos-capabilities.service.ts @@ -6,6 +6,8 @@ import { } from '../../interfaces/provider-capabilities.interface'; import { ProviderRegion } from '../../../management/entities/provider-region.entity'; import { CloudProvider } from '../../enums/cloud-provider.enum'; +import { NodeTopology } from '../../core/node-topology'; +import { HaTopology } from '../../core/ha-topology'; import { ProviderCapabilities } from '../../../management/entities/provider-capabilities.entity'; import { ProviderCredentials, @@ -59,6 +61,14 @@ export class ByosCapabilitiesService implements IProviderCapabilitiesService { return 'image/svg+xml'; } + haTopologies(): HaTopology[] { + return []; + } + + nodeTopology(): NodeTopology | null { + return null; + } + getStaticCapabilities(): ProviderCapabilities { return { supportedInstanceTypes: [], diff --git a/src/modules/providers/implementations/contabo/contabo-capabilities.service.ts b/src/modules/providers/implementations/contabo/contabo-capabilities.service.ts index 75ba82e6..8973855a 100644 --- a/src/modules/providers/implementations/contabo/contabo-capabilities.service.ts +++ b/src/modules/providers/implementations/contabo/contabo-capabilities.service.ts @@ -9,6 +9,8 @@ import { } from '../../interfaces/provider-capabilities.interface'; import { ProviderRegion } from '../../../management/entities/provider-region.entity'; import { CloudProvider } from '../../enums/cloud-provider.enum'; +import { NodeTopology } from '../../core/node-topology'; +import { HaTopology } from '../../core/ha-topology'; import { ProviderCapabilities } from '../../../management/entities/provider-capabilities.entity'; import { ProviderCredentials, @@ -230,6 +232,14 @@ export class ContaboCapabilitiesService return 'image/svg+xml'; } + haTopologies(): HaTopology[] { + return []; + } + + nodeTopology(): NodeTopology | null { + return null; + } + getStaticCapabilities(): ProviderCapabilities { return { supportedInstanceTypes: [], diff --git a/src/modules/providers/implementations/hetzner/hetzner-capabilities.service.ts b/src/modules/providers/implementations/hetzner/hetzner-capabilities.service.ts index 467bf072..cc6407bb 100644 --- a/src/modules/providers/implementations/hetzner/hetzner-capabilities.service.ts +++ b/src/modules/providers/implementations/hetzner/hetzner-capabilities.service.ts @@ -14,6 +14,9 @@ import { import { ValidationResultDto } from '../../../management/dto/validation-result.dto'; import { ProviderRegion } from '../../../management/entities/provider-region.entity'; import { CloudProvider } from '../../enums/cloud-provider.enum'; +import { NodeTopology } from '../../core/node-topology'; +import { HaTopology } from '../../core/ha-topology'; +import { networkZoneForRegion } from '../../core/network-zones'; import { ProviderCapabilities } from '../../../management/entities/provider-capabilities.entity'; import { ICredentialProvider } from '../../interfaces/credential-provider.interface'; import { @@ -192,6 +195,34 @@ export class HetznerCapabilitiesService }; } + /** + * Measured: Falkenstein and Nuremberg are 3 ms + * apart, Helsinki 35 ms from both, so Helsinki votes and never holds the + * copy a write waits for. + */ + haTopologies(): HaTopology[] { + return [ + { + region: 'eu-central', + zones: ['fsn1', 'nbg1', 'hel1'], + dataZones: ['fsn1', 'nbg1'], + validated: true, + measured: true, + }, + ]; + } + + /** Hetzner has no zones inside a location, so the location is the failure domain. */ + nodeTopology(location: string | null | undefined): NodeTopology | null { + const zone = location?.trim().toLowerCase(); + if (!zone) return null; + const region = networkZoneForRegion( + this.getStaticCapabilities().vnetTopology, + zone, + ); + return region ? { zone, region } : null; + } + getStaticCapabilities(): ProviderCapabilities { return { supportedInstanceTypes: [], diff --git a/src/modules/providers/implementations/ovh/openstack-volumes-client.ts b/src/modules/providers/implementations/ovh/openstack-volumes-client.ts index 4bf8ffb9..fa8987db 100644 --- a/src/modules/providers/implementations/ovh/openstack-volumes-client.ts +++ b/src/modules/providers/implementations/ovh/openstack-volumes-client.ts @@ -102,7 +102,12 @@ export class FluiOpenStackClient extends OpenStackClient { async createVolume( region: string, - spec: { name: string; sizeGb: number; metadata?: Record }, + spec: { + name: string; + sizeGb: number; + metadata?: Record; + availabilityZone?: string; + }, ): Promise { const cinder = await this.endpoint(CINDER_SERVICE_TYPE, region); const body = await this.post<{ volume: CinderVolume }>( @@ -112,6 +117,9 @@ export class FluiOpenStackClient extends OpenStackClient { name: spec.name, size: spec.sizeGb, ...(spec.metadata ? { metadata: spec.metadata } : {}), + ...(spec.availabilityZone + ? { availability_zone: spec.availabilityZone } + : {}), }, }, ); diff --git a/src/modules/providers/implementations/ovh/ovh-capabilities.service.ts b/src/modules/providers/implementations/ovh/ovh-capabilities.service.ts index 858fbde0..c741937b 100644 --- a/src/modules/providers/implementations/ovh/ovh-capabilities.service.ts +++ b/src/modules/providers/implementations/ovh/ovh-capabilities.service.ts @@ -17,9 +17,15 @@ import { } from '../../../management/entities/credentials.entity'; import { ValidationResultDto } from '../../../management/dto/validation-result.dto'; import { buildOvhOpenStackClient } from './ovh-openstack-client.factory'; -import { ovhRegionCode, ovhRegionMeta } from './ovh-region-metadata'; +import { + ovhNodeTopology, + ovhRegionCode, + ovhRegionMeta, +} from './ovh-region-metadata'; import { ICredentialProvider } from '../../interfaces/credential-provider.interface'; import { CloudProvider } from '../../enums/cloud-provider.enum'; +import { NodeTopology } from '../../core/node-topology'; +import { HaTopology } from '../../core/ha-topology'; /** * Flui-native OVH capabilities — delegates catalog/regions/pricing to @@ -196,6 +202,25 @@ export class OvhCapabilitiesService implements IProviderCapabilitiesService { return this.inner.getLogoContentType(); } + /** + * OVHcloud's regions with three availability zones, measured by Flui's own + * bench: a round trip under 1.2 ms between any two zones, so every zone + * holds a copy of the data. + */ + haTopologies(): HaTopology[] { + return ['eu-west-par', 'eu-south-mil'].map((region) => ({ + region, + zones: [`${region}-a`, `${region}-b`, `${region}-c`], + dataZones: [`${region}-a`, `${region}-b`, `${region}-c`], + validated: true, + measured: true, + })); + } + + nodeTopology(location: string | null | undefined): NodeTopology | null { + return ovhNodeTopology(location); + } + getStaticCapabilities(): ProviderCapabilities { return { supportedInstanceTypes: [], diff --git a/src/modules/providers/implementations/ovh/ovh-provider.service.spec.ts b/src/modules/providers/implementations/ovh/ovh-provider.service.spec.ts index 65721c80..9cf05638 100644 --- a/src/modules/providers/implementations/ovh/ovh-provider.service.spec.ts +++ b/src/modules/providers/implementations/ovh/ovh-provider.service.spec.ts @@ -566,6 +566,7 @@ describe('OvhProviderService.createServer — private NIC netplan injection', () expect(userData).toContain('dhcp4-overrides'); // Without this the private DHCP installs a competing default route. expect(userData).toContain('use-routes: false'); + expect(userData).toContain('use-dns: false'); // Without this a later boot blocks on systemd-networkd-wait-online. expect(userData).toContain('optional: true'); expect(userData).toContain("export CLUSTER_ID='abc'"); diff --git a/src/modules/providers/implementations/ovh/ovh-provider.service.ts b/src/modules/providers/implementations/ovh/ovh-provider.service.ts index 3ace2ed3..030e8ccd 100644 --- a/src/modules/providers/implementations/ovh/ovh-provider.service.ts +++ b/src/modules/providers/implementations/ovh/ovh-provider.service.ts @@ -35,6 +35,10 @@ import { ICredentialProvider } from '../../interfaces/credential-provider.interf import { buildOvhOpenStackClient } from './ovh-openstack-client.factory'; import { getOvhNodeSizesFromNova, FlavorPricing } from './ovh-nova-flavors'; import { FluiOpenStackClient } from './openstack-volumes-client'; +import { + ovhAvailabilityZone, + ovhComputeRegionFor, +} from './ovh-region-metadata'; /** * GPU flavors are real, orderable OVH products (@flui-cloud/infra's catalog @@ -79,7 +83,8 @@ function isOvhGpuFlavor(id: string): boolean { * * `use-routes: false` is the reason this is a separate file rather than a * plain `dhcp4: true`: the VNet's DHCP would otherwise install a second - * default route and blackhole egress. + * default route and blackhole egress. `use-dns: false` because the DNS + * servers that DHCP hands out may not answer, and pods then fail lookups. * * `netplan apply` reconfigures every link, not just the one whose file * changed, so the public interface loses its lease and its default route for @@ -157,6 +162,7 @@ network: dhcp4: true dhcp4-overrides: use-routes: false + use-dns: false accept-ra: false optional: true FLUI_NETPLAN_EOF @@ -440,7 +446,10 @@ export class OvhProviderService implements ICloudProvider { ): Promise { const client = await this.client(); const svc = new InfraOvhProviderService(this.configService, client); - const region = await client.resolveComputeRegion(config.location); + const zone = ovhAvailabilityZone(config.location) ?? undefined; + const region = await client.resolveComputeRegion( + ovhComputeRegionFor(config.location), + ); // Attaching a private VNet alongside Ext-Net at create time left the // instance stuck in BUILD with no IP ever assigned (OVH's own docs frame @@ -548,6 +557,7 @@ export class OvhProviderService implements ICloudProvider { name: vol.name, sizeGb: vol.sizeGb, metadata, + availabilityZone: zone, }); await client.waitForVolumeAvailable(region, created.id); const attachment = await client.attachVolumeToServer( diff --git a/src/modules/providers/implementations/ovh/ovh-region-metadata.ts b/src/modules/providers/implementations/ovh/ovh-region-metadata.ts index 463bc42f..b6b30397 100644 --- a/src/modules/providers/implementations/ovh/ovh-region-metadata.ts +++ b/src/modules/providers/implementations/ovh/ovh-region-metadata.ts @@ -1,4 +1,5 @@ import { toMacroRegion } from '@flui-cloud/infra'; +import { NodeTopology } from '../../core/node-topology'; /** * City-level decoration for OVH region codes. @@ -113,3 +114,38 @@ export function ovhRegionMeta(code: string): OvhRegionMeta | undefined { } export const OVH_REGION_METADATA = META; + +const THREE_ZONE_REGION = /^[a-z]+-[a-z]+-[a-z]+$/; + +/** + * Where an OVH server runs: a zone of a region that has three + * (`eu-west-par-a` in `eu-west-par`). A region code alone stands for its first + * zone; a region with a single datacenter (`GRA11`) has no zones, so none. + */ +export function ovhNodeTopology( + location: string | null | undefined, +): NodeTopology | null { + const raw = location?.trim().toLowerCase(); + if (!raw) return null; + const region = raw.slice(0, raw.lastIndexOf('-')); + if (/-[a-z]$/.test(raw) && THREE_ZONE_REGION.test(region)) { + return { zone: raw, region }; + } + return THREE_ZONE_REGION.test(raw) ? { zone: `${raw}-a`, region: raw } : null; +} + +/** The zone a location names explicitly, or null when it names only a region and Nova picks. */ +export function ovhAvailabilityZone( + location: string | null | undefined, +): string | null { + const zone = ovhNodeTopology(location)?.zone; + return zone && zone === location?.trim().toLowerCase() ? zone : null; +} + +/** The compute region to ask Nova for: a zone's region, or the location as given. */ +export function ovhComputeRegionFor( + location: string | null | undefined, +): string | undefined { + const zone = ovhAvailabilityZone(location); + return zone ? ovhNodeTopology(zone)?.region : (location ?? undefined); +} diff --git a/src/modules/providers/implementations/scaleway/scaleway-capabilities.service.ts b/src/modules/providers/implementations/scaleway/scaleway-capabilities.service.ts index c290a553..1de96e3e 100644 --- a/src/modules/providers/implementations/scaleway/scaleway-capabilities.service.ts +++ b/src/modules/providers/implementations/scaleway/scaleway-capabilities.service.ts @@ -9,6 +9,9 @@ import { import { ProviderRegion } from '../../../management/entities/provider-region.entity'; import { ProviderCapabilities } from '../../../management/entities/provider-capabilities.entity'; import { CloudProvider } from '../../enums/cloud-provider.enum'; +import { NodeTopology } from '../../core/node-topology'; +import { HaTopology } from '../../core/ha-topology'; +import { toScalewayZone } from './scaleway-zone.util'; import { ICredentialProvider } from '../../interfaces/credential-provider.interface'; import { ProviderCredentials, @@ -188,6 +191,31 @@ export class ScalewayCapabilitiesService } } + /** + * Scaleway's regions with three availability zones. Paris and Milan were + * measured by Flui's own bench (a round trip under 1.8 ms between zones); + * Amsterdam and Warsaw are taken on Scaleway's word. Every zone holds a copy + * of the data. + */ + haTopologies(): HaTopology[] { + const measured = new Set(['fr-par', 'it-mil']); + return ['fr-par', 'it-mil', 'nl-ams', 'pl-waw'].map((region) => ({ + region, + zones: [`${region}-1`, `${region}-2`, `${region}-3`], + dataZones: [`${region}-1`, `${region}-2`, `${region}-3`], + validated: true, + measured: measured.has(region), + })); + } + + nodeTopology(location: string | null | undefined): NodeTopology | null { + const raw = location?.trim().toLowerCase(); + if (!raw) return null; + const zone = toScalewayZone(raw); + const region = /^([a-z]{2}-[a-z]{3})-\d+$/.exec(zone)?.[1]; + return region ? { zone, region } : null; + } + getStaticCapabilities(): ProviderCapabilities { return { supportedInstanceTypes: [], diff --git a/src/modules/providers/implementations/scaleway/scaleway-firewall.service.ts b/src/modules/providers/implementations/scaleway/scaleway-firewall.service.ts index 671e6e6b..5977f540 100644 --- a/src/modules/providers/implementations/scaleway/scaleway-firewall.service.ts +++ b/src/modules/providers/implementations/scaleway/scaleway-firewall.service.ts @@ -34,6 +34,10 @@ type ScalewayFirewallId = { zone: string; sgId: string }; type ScalewayResourceId = { zone: string; serverId: string }; import { toScalewayZone } from './scaleway-zone.util'; +import { + ScalewayZoneSecurityGroups, + isZoneCopy, +} from './scaleway-zone-security-groups'; @Injectable() export class ScalewayFirewallService implements IFirewallProvider { private readonly logger = new Logger(ScalewayFirewallService.name); @@ -44,7 +48,11 @@ export class ScalewayFirewallService implements IFirewallProvider { private readonly labelService: LabelService, private readonly instancesAdapter: ScalewayInstancesAdapter, private readonly iamAdapter: ScalewayIamAdapter, - ) {} + ) { + this.zoneCopies = new ScalewayZoneSecurityGroups(instancesAdapter); + } + + private readonly zoneCopies: ScalewayZoneSecurityGroups; // ─── ID helpers ──────────────────────────────────────────────────────────── @@ -475,7 +483,7 @@ export class ScalewayFirewallService implements IFirewallProvider { } } - let filtered = allSgs; + let filtered = allSgs.filter((sg) => !isZoneCopy(sg.tags)); if (filters?.name) { filtered = filtered.filter((sg) => sg.name === filters.name); @@ -530,12 +538,17 @@ export class ScalewayFirewallService implements IFirewallProvider { CloudProvider.SCALEWAY, ); const scRules = this.fluiRulesToScaleway(rules); - await this.instancesAdapter.setSecurityGroupRules( - token, - parsed.zone, - parsed.sgId, - scRules, - ); + for (const group of [ + parsed, + ...(await this.zoneCopies.copies(token, parsed)), + ]) { + await this.instancesAdapter.setSecurityGroupRules( + token, + group.zone, + group.sgId, + scRules, + ); + } } async deleteFirewall(firewallId: string): Promise { @@ -550,6 +563,13 @@ export class ScalewayFirewallService implements IFirewallProvider { const token = await this.credentialProvider.getActiveApiToken( CloudProvider.SCALEWAY, ); + for (const copy of await this.zoneCopies.copies(token, parsed)) { + await this.instancesAdapter.deleteSecurityGroup( + token, + copy.zone, + copy.sgId, + ); + } await this.instancesAdapter.deleteSecurityGroup( token, parsed.zone, @@ -593,7 +613,7 @@ export class ScalewayFirewallService implements IFirewallProvider { token, parsed.zone, parsed.serverId, - parsedFw.sgId, + await this.zoneCopies.inZone(token, parsedFw, parsed.zone), ); } catch (err) { this.logger.error( diff --git a/src/modules/providers/implementations/scaleway/scaleway-instances.adapter.ts b/src/modules/providers/implementations/scaleway/scaleway-instances.adapter.ts index 13488496..eeb04f60 100644 --- a/src/modules/providers/implementations/scaleway/scaleway-instances.adapter.ts +++ b/src/modules/providers/implementations/scaleway/scaleway-instances.adapter.ts @@ -45,6 +45,9 @@ export const INSTANCE_ZONES: CreateServerZoneEnum[] = [ CreateServerZoneEnum.PlWaw2, CreateServerZoneEnum.PlWaw3, CreateServerZoneEnum.ItMil1, + // The generated client predates Milan's second and third zones; Scaleway's own SDK lists them. + 'it-mil-2' as CreateServerZoneEnum, + 'it-mil-3' as CreateServerZoneEnum, ]; export interface ScalewayInstanceServerType diff --git a/src/modules/providers/implementations/scaleway/scaleway-provider.service.ts b/src/modules/providers/implementations/scaleway/scaleway-provider.service.ts index 30f53e19..c526f7da 100644 --- a/src/modules/providers/implementations/scaleway/scaleway-provider.service.ts +++ b/src/modules/providers/implementations/scaleway/scaleway-provider.service.ts @@ -73,6 +73,10 @@ type ScalewayResourceId = { }; import { toScalewayZone } from './scaleway-zone.util'; +import { + ScalewayZoneSecurityGroups, + parseScalewayFirewallId, +} from './scaleway-zone-security-groups'; import { normaliseArchitecture } from '../../core/catalogue/server-architecture'; @Injectable() @@ -863,15 +867,17 @@ export class ScalewayProviderService implements ICloudProvider { ); } - // Extract the security group UUID from the first firewall ID (format: ":") + // The first firewall's security group, or its copy in this server's zone (id format: ":"). // Passing security_group at creation time is the recommended approach — it ensures the // firewall rules are applied before the instance boots for the first time. let securityGroupId: string | undefined; if (config.firewalls && config.firewalls.length > 0) { const firstFw = config.firewalls[0]; - const colonIdx = firstFw.indexOf(':'); - if (colonIdx > 0) { - securityGroupId = firstFw.substring(colonIdx + 1); + const original = parseScalewayFirewallId(firstFw); + if (original) { + securityGroupId = await new ScalewayZoneSecurityGroups( + this.instancesAdapter, + ).inZone(token, original, zone); this.logger.log( `[Scaleway] Attaching security group ${securityGroupId} at instance creation time (from firewall ${firstFw})`, ); @@ -990,7 +996,7 @@ export class ScalewayProviderService implements ICloudProvider { const parts = firewallId.split(':'); if (parts.length === 2) { const [fwZone, sgId] = parts; - if (i === 0 && securityGroupId === sgId) { + if (i === 0 && securityGroupId) { // Already set at creation time — just log confirmation this.logger.log( `[Scaleway] ✅ Security group ${sgId} was set at instance creation time (no extra call needed)`, @@ -1403,7 +1409,7 @@ export class ScalewayProviderService implements ICloudProvider { CloudProvider.SCALEWAY, ); const headers = { 'X-Auth-Token': token } as Record; - const zones = ['fr-par-1', 'fr-par-2', 'nl-ams-1', 'pl-waw-1', 'it-mil-1']; + const zones: string[] = INSTANCE_ZONES; const results: ProviderVolumeSummary[] = []; for (const zone of zones) { try { @@ -1524,6 +1530,7 @@ export class ScalewayProviderService implements ICloudProvider { { type: ReturnType; availableRegions: Set; + zonesByRegion: Map; pricesByRegion: Map< string, { hourly: number; monthlyCap?: number } @@ -1552,11 +1559,16 @@ export class ScalewayProviderService implements ICloudProvider { typeMap.set(t.name, { type: t, availableRegions: new Set(), + zonesByRegion: new Map(), pricesByRegion: new Map(), }); } const entry = typeMap.get(t.name); entry.availableRegions.add(region); + entry.zonesByRegion.set(region, [ + ...(entry.zonesByRegion.get(region) ?? []), + zone, + ]); if (t.hourly_price != null && !entry.pricesByRegion.has(region)) { entry.pricesByRegion.set(region, { hourly: t.hourly_price, @@ -1630,6 +1642,7 @@ export class ScalewayProviderService implements ICloudProvider { location: region, available: true, deprecated: t.end_of_service ?? false, + zones: entry.zonesByRegion.get(region), }), ); diff --git a/src/modules/providers/implementations/scaleway/scaleway-zone-security-groups.spec.ts b/src/modules/providers/implementations/scaleway/scaleway-zone-security-groups.spec.ts new file mode 100644 index 00000000..bea2de98 --- /dev/null +++ b/src/modules/providers/implementations/scaleway/scaleway-zone-security-groups.spec.ts @@ -0,0 +1,117 @@ +import { + ScalewayZoneSecurityGroups, + isZoneCopy, + parseScalewayFirewallId, +} from './scaleway-zone-security-groups'; + +describe('Scaleway security groups across the zones of a region', () => { + const original = { zone: 'fr-par-1', sgId: 'sg-1' }; + + function build( + groupsByZone: Record> = {}, + ) { + const adapter = { + listSecurityGroups: jest.fn( + async (_t: string, zone: string) => groupsByZone[zone] ?? [], + ), + getSecurityGroup: jest.fn().mockResolvedValue({ + id: 'sg-1', + name: 'flui-ha-sc', + tags: ['flui-cluster-id=c1'], + stateful: true, + inbound_default_policy: 'drop', + outbound_default_policy: 'accept', + project: 'p1', + }), + listSecurityGroupRules: jest.fn().mockResolvedValue([ + { + action: 'accept', + protocol: 'TCP', + direction: 'inbound', + ip_range: '0.0.0.0/0', + dest_port_from: 443, + position: 1, + editable: true, + }, + { + action: 'accept', + protocol: 'TCP', + direction: 'inbound', + ip_range: '0.0.0.0/0', + dest_port_from: 25, + position: 2, + editable: false, + }, + ]), + createSecurityGroup: jest.fn().mockResolvedValue({ id: 'sg-2' }), + setSecurityGroupRules: jest.fn().mockResolvedValue(undefined), + }; + return { + groups: new ScalewayZoneSecurityGroups(adapter as never), + adapter, + }; + } + + it("gives a server in the group's own zone the group itself", async () => { + const { groups, adapter } = build(); + await expect(groups.inZone('t', original, 'fr-par-1')).resolves.toBe( + 'sg-1', + ); + expect(adapter.createSecurityGroup).not.toHaveBeenCalled(); + }); + + it('copies the group, with its editable rules, the first time a server needs it in another zone', async () => { + const { groups, adapter } = build(); + + await expect(groups.inZone('t', original, 'fr-par-2')).resolves.toBe( + 'sg-2', + ); + + expect(adapter.createSecurityGroup).toHaveBeenCalledWith( + 't', + 'fr-par-2', + expect.objectContaining({ + name: 'flui-ha-sc', + tags: ['flui-cluster-id=c1', 'flui-copy-of=sg-1'], + project: 'p1', + }), + ); + const rules = adapter.setSecurityGroupRules.mock.calls[0][3]; + expect(rules).toHaveLength(1); + expect(rules[0]).toMatchObject({ dest_port_from: 443, editable: true }); + }); + + it('reuses the copy a zone already has', async () => { + const { groups, adapter } = build({ + 'fr-par-3': [{ id: 'sg-3', tags: ['flui-copy-of=sg-1'] }], + }); + await expect(groups.inZone('t', original, 'fr-par-3')).resolves.toBe( + 'sg-3', + ); + expect(adapter.createSecurityGroup).not.toHaveBeenCalled(); + }); + + it('finds the copies only in the other zones of the same region', async () => { + const { groups, adapter } = build({ + 'fr-par-3': [{ id: 'sg-3', tags: ['flui-copy-of=sg-1'] }], + 'nl-ams-1': [{ id: 'sg-9', tags: ['flui-copy-of=sg-1'] }], + }); + await expect(groups.copies('t', original)).resolves.toEqual([ + { zone: 'fr-par-3', sgId: 'sg-3' }, + ]); + expect(adapter.listSecurityGroups.mock.calls.map((c) => c[1])).toEqual([ + 'fr-par-2', + 'fr-par-3', + ]); + }); + + it('reads a firewall id and tells a copy apart', () => { + expect(parseScalewayFirewallId('it-mil-2:sg-7')).toEqual({ + zone: 'it-mil-2', + sgId: 'sg-7', + }); + expect(parseScalewayFirewallId('sg-7')).toBeNull(); + expect(isZoneCopy(['flui-copy-of=sg-1'])).toBe(true); + expect(isZoneCopy(['flui-cluster-id=c1'])).toBe(false); + }); +}); diff --git a/src/modules/providers/implementations/scaleway/scaleway-zone-security-groups.ts b/src/modules/providers/implementations/scaleway/scaleway-zone-security-groups.ts new file mode 100644 index 00000000..8d4e2311 --- /dev/null +++ b/src/modules/providers/implementations/scaleway/scaleway-zone-security-groups.ts @@ -0,0 +1,139 @@ +import { + CreateSecurityGroupRequest, + CreateSecurityGroupRequestInboundDefaultPolicyEnum, + CreateSecurityGroupRequestOutboundDefaultPolicyEnum, + ScalewayInstanceV1SecurityGroupRule, + ScalewayInstanceV1SetSecurityGroupRulesRequestRule, +} from './generated/instances'; +import { + INSTANCE_ZONES, + ScalewayInstancesAdapter, +} from './scaleway-instances.adapter'; + +/** Tag on a security group copied into another zone, naming the original group. */ +export const COPY_OF_TAG = 'flui-copy-of'; + +export interface ScalewaySecurityGroupRef { + zone: string; + sgId: string; +} + +/** `:`, the firewall id Flui keeps for a Scaleway security group. */ +export function parseScalewayFirewallId( + firewallId: string, +): ScalewaySecurityGroupRef | null { + const colon = firewallId.indexOf(':'); + if (colon <= 0) return null; + return { + zone: firewallId.substring(0, colon), + sgId: firewallId.substring(colon + 1), + }; +} + +export function isZoneCopy(tags: string[] | undefined): boolean { + return (tags ?? []).some((t) => t.startsWith(`${COPY_OF_TAG}=`)); +} + +function regionOf(zone: string): string { + return zone.replace(/-\d+$/, ''); +} + +/** Only the rules a person can edit: Scaleway adds its own and refuses them back. */ +function editableRules( + raw: ScalewayInstanceV1SecurityGroupRule[], +): ScalewayInstanceV1SetSecurityGroupRulesRequestRule[] { + return raw + .filter((r) => r.editable !== false) + .map((r) => ({ + action: r.action as never, + protocol: r.protocol as never, + direction: r.direction as never, + ip_range: r.ip_range, + dest_port_from: r.dest_port_from ?? undefined, + dest_port_to: r.dest_port_to ?? undefined, + position: r.position, + editable: true, + })); +} + +/** + * A Scaleway security group belongs to one zone, and a server takes only a + * group of its own zone. A cluster across three zones keeps one group as Flui + * sees it, plus a copy with the same rules in each other zone a server is made + * in, found again by the tag that names the original. + */ +export class ScalewayZoneSecurityGroups { + constructor(private readonly adapter: ScalewayInstancesAdapter) {} + + /** The group a server in `zone` takes: the original in its own zone, a copy elsewhere, made on first need. */ + async inZone( + token: string, + original: ScalewaySecurityGroupRef, + zone: string, + ): Promise { + if (original.zone === zone) return original.sgId; + const existing = await this.copyIn(token, original, zone); + if (existing) return existing; + + const sg = await this.adapter.getSecurityGroup( + token, + original.zone, + original.sgId, + ); + if (!sg) { + throw new Error( + `Security group ${original.sgId} is gone from ${original.zone}, so it cannot be copied to ${zone}`, + ); + } + const payload: CreateSecurityGroupRequest = { + name: sg.name ?? original.sgId, + description: sg.description ?? sg.name ?? original.sgId, + tags: [...(sg.tags ?? []), `${COPY_OF_TAG}=${original.sgId}`], + stateful: sg.stateful ?? true, + inbound_default_policy: (sg.inbound_default_policy ?? + CreateSecurityGroupRequestInboundDefaultPolicyEnum.Drop) as never, + outbound_default_policy: (sg.outbound_default_policy ?? + CreateSecurityGroupRequestOutboundDefaultPolicyEnum.Accept) as never, + project: sg.project, + }; + const copy = await this.adapter.createSecurityGroup(token, zone, payload); + const rules = editableRules( + await this.adapter.listSecurityGroupRules( + token, + original.zone, + original.sgId, + ), + ); + if (rules.length) { + await this.adapter.setSecurityGroupRules(token, zone, copy.id!, rules); + } + return copy.id!; + } + + /** The copies of a group in the other zones of its region. */ + async copies( + token: string, + original: ScalewaySecurityGroupRef, + ): Promise { + const zones = (INSTANCE_ZONES as string[]).filter( + (z) => z !== original.zone && regionOf(z) === regionOf(original.zone), + ); + const found = await Promise.all( + zones.map(async (zone) => { + const sgId = await this.copyIn(token, original, zone); + return sgId ? { zone, sgId } : null; + }), + ); + return found.filter((c): c is ScalewaySecurityGroupRef => !!c); + } + + private async copyIn( + token: string, + original: ScalewaySecurityGroupRef, + zone: string, + ): Promise { + const tag = `${COPY_OF_TAG}=${original.sgId}`; + const groups = await this.adapter.listSecurityGroups(token, zone); + return groups.find((g) => (g.tags ?? []).includes(tag))?.id ?? null; + } +} diff --git a/src/modules/providers/interfaces/provider-capabilities.interface.ts b/src/modules/providers/interfaces/provider-capabilities.interface.ts index 049df964..388f6456 100644 --- a/src/modules/providers/interfaces/provider-capabilities.interface.ts +++ b/src/modules/providers/interfaces/provider-capabilities.interface.ts @@ -2,6 +2,8 @@ import { ProviderCapabilities } from '../../management/entities/provider-capabil import { ProviderRegion } from '../../management/entities/provider-region.entity'; import { ProviderCredentials } from '../../management/entities/credentials.entity'; import { ValidationResultDto } from '../../management/dto/validation-result.dto'; +import { NodeTopology } from '../core/node-topology'; +import { HaTopology } from '../core/ha-topology'; export interface IProviderCapabilitiesService { getAvailableRegions(): Promise; @@ -17,6 +19,16 @@ export interface IProviderCapabilitiesService { * Safe to call without credentials. Use for listing all available providers. */ getStaticCapabilities(): ProviderCapabilities; + /** + * The zone and region a server placed at this location sits in. Null when + * the provider has no declared answer, so nothing guesses a failure domain. + */ + nodeTopology(location: string | null | undefined): NodeTopology | null; + /** + * The highly available layouts Flui has measured and validated on this + * provider, by region. Empty means none yet. + */ + haTopologies(): HaTopology[]; getLogo(): Buffer; getLogoContentType(): string; /** diff --git a/src/modules/providers/services/hetzner-provider.service.ts b/src/modules/providers/services/hetzner-provider.service.ts index 26c6b7b5..3f99b249 100644 --- a/src/modules/providers/services/hetzner-provider.service.ts +++ b/src/modules/providers/services/hetzner-provider.service.ts @@ -1275,6 +1275,18 @@ export class HetznerProviderService implements ICloudProvider { } } + /** + * Asked of Hetzner by name, not through `listSSHKeys`: that one keeps only + * keys carrying Flui's labels, and a key registered here carries none, so a + * second server bought at the same moment never found the first one's key. + */ + private async findSshKeyIdByName(name: string): Promise { + const sshKeysApi = await this.createSSHKeysApi(); + const response = await sshKeysApi.listSshKeys(undefined, name); + const found = response.data.ssh_keys.find((k) => k.name === name); + return found ? found.id.toString() : null; + } + async resolveSSHKeys(keys: SshKeyInfo[]): Promise { const providerIds: string[] = []; @@ -1290,13 +1302,9 @@ export class HetznerProviderService implements ICloudProvider { providerIds.push(result.id); } catch (error) { if (error.message?.includes('already exists')) { - // Key exists — find it by fingerprint or name - const existing = await this.listSSHKeys(); - const match = existing.find( - (k) => k.fingerprint === key.fingerprint || k.name === key.name, - ); + const match = await this.findSshKeyIdByName(key.name); if (match) { - providerIds.push(match.id); + providerIds.push(match); continue; } } diff --git a/src/modules/providers/services/hetzner-provider.ssh-keys.spec.ts b/src/modules/providers/services/hetzner-provider.ssh-keys.spec.ts new file mode 100644 index 00000000..97cdbbe1 --- /dev/null +++ b/src/modules/providers/services/hetzner-provider.ssh-keys.spec.ts @@ -0,0 +1,57 @@ +jest.mock('@kubernetes/client-node', () => ({})); + +import { HetznerProviderService } from './hetzner-provider.service'; + +describe('HetznerProviderService.resolveSSHKeys', () => { + function build(listed: Array<{ id: number; name: string; labels: object }>) { + const service = Object.create( + HetznerProviderService.prototype, + ) as HetznerProviderService; + const listSshKeys = jest.fn().mockResolvedValue({ + data: { ssh_keys: listed, meta: { pagination: { last_page: 1 } } }, + }); + Object.assign(service, { + logger: { log: jest.fn(), warn: jest.fn(), error: jest.fn() }, + createSSHKey: jest + .fn() + .mockRejectedValue( + new Error( + 'SSH key with name flui-bootstrap-x already exists on Hetzner', + ), + ), + createSSHKeysApi: jest.fn().mockResolvedValue({ listSshKeys }), + }); + return { service, listSshKeys }; + } + + it('finds the key another server registered a moment earlier, labels or not', async () => { + const { service, listSshKeys } = build([ + { id: 42, name: 'flui-bootstrap-x', labels: {} }, + ]); + + await expect( + service.resolveSSHKeys([ + { + name: 'flui-bootstrap-x', + publicKey: 'ssh-ed25519 AAA', + fingerprint: 'f', + }, + ] as never), + ).resolves.toEqual(['42']); + expect(listSshKeys).toHaveBeenCalledWith(undefined, 'flui-bootstrap-x'); + }); + + it('still fails when no key of that name is there', async () => { + const { service } = build([]); + + await expect( + service.resolveSSHKeys([ + { + name: 'flui-bootstrap-x', + publicKey: 'ssh-ed25519 AAA', + fingerprint: 'f', + }, + ] as never), + ).rejects.toThrow('already exists'); + }); +}); diff --git a/src/modules/replicated-volumes/controllers/replicated-volumes.controller.ts b/src/modules/replicated-volumes/controllers/replicated-volumes.controller.ts new file mode 100644 index 00000000..dcf2bf00 --- /dev/null +++ b/src/modules/replicated-volumes/controllers/replicated-volumes.controller.ts @@ -0,0 +1,67 @@ +import { + Body, + Controller, + Get, + HttpCode, + HttpStatus, + Param, + Post, + Req, +} from '@nestjs/common'; +import { + ApiBearerAuth, + ApiOperation, + ApiParam, + ApiTags, +} from '@nestjs/swagger'; +import { Request } from 'express'; +import { ActionCycle } from '../../action-cycle/action-cycle.decorator'; +import { RequireSection } from '../../iam/decorators/require-section.decorator'; +import { EnableReplicatedVolumesDto } from '../dto/enable-replicated-volumes.dto'; +import { ReplicatedVolumesService } from '../services/replicated-volumes.service'; + +@ApiTags('Infrastructure - Clusters') +@ApiBearerAuth() +@Controller('infrastructure/clusters/:id/replicated-volumes') +export class ReplicatedVolumesController { + constructor(private readonly service: ReplicatedVolumesService) {} + + @Get() + @RequireSection('clusters') + @ApiOperation({ + summary: 'Whether a cluster offers replicated volumes', + description: + 'A replicated volume keeps a synchronous copy in a second location, so an application with one writer comes back there when its server or its location is lost; the third location only votes. ' + + '`enabled` says whether the `flui-replicated` storage class exists; `storageNodes` are the nodes that hold copies; `warnings` names copies an application could not follow; `plan` is what enabling would do with the nodes the cluster has now, or why it cannot.', + }) + @ApiParam({ name: 'id', description: 'Cluster ID' }) + view(@Param('id') id: string) { + return this.service.view(id); + } + + @Post() + @HttpCode(HttpStatus.ACCEPTED) + @RequireSection('infrastructure') + @ActionCycle({ + action: 'POST /infrastructure/clusters/:id/replicated-volumes', + bind: ['id'], + sentence: 'enable replicated volumes on cluster {id}', + consequence: + 'A node in each of two locations sets aside disk space for copies of replicated volumes and the volume engine is installed on the cluster. Existing volumes do not change; only volumes created later with `replicated: true` are copied.', + }) + @ApiOperation({ + summary: 'Enable replicated volumes on a cluster', + description: + "Prepares a node in each of the layout's two locations for data to hold copies, installs the volume engine and creates the `flui-replicated` storage class. Nothing changes for existing volumes. " + + 'Refused, with the reason, when the cluster is not yet highly available across three locations of a validated region (`force` accepts a region Flui has not validated), or lacks a node that takes applications in a location for data. Returns the operation to follow.', + }) + @ApiParam({ name: 'id', description: 'Cluster ID' }) + enable( + @Req() req: Request, + @Param('id') id: string, + @Body() dto: EnableReplicatedVolumesDto, + ) { + const user = req.user as { id?: string; userId?: string } | undefined; + return this.service.enable(id, dto, user?.id ?? user?.userId); + } +} diff --git a/src/modules/replicated-volumes/dto/enable-replicated-volumes.dto.ts b/src/modules/replicated-volumes/dto/enable-replicated-volumes.dto.ts new file mode 100644 index 00000000..7cb28d54 --- /dev/null +++ b/src/modules/replicated-volumes/dto/enable-replicated-volumes.dto.ts @@ -0,0 +1,48 @@ +import { ApiPropertyOptional } from '@nestjs/swagger'; +import { + ArrayMaxSize, + ArrayMinSize, + IsArray, + IsInt, + IsOptional, + IsString, + Max, + Min, + IsBoolean, +} from 'class-validator'; + +export class EnableReplicatedVolumesDto { + @ApiPropertyOptional({ + type: [String], + description: + "The nodes that hold the copies, one in each of the layout's locations for data: two where the third location only votes, three where every location holds data. Omitted: the nodes already prepared, otherwise one that takes applications in each location for data.", + }) + @IsOptional() + @IsArray() + @ArrayMinSize(2) + @ArrayMaxSize(3) + @IsString({ each: true }) + nodes?: string[]; + + @ApiPropertyOptional({ + default: 20, + minimum: 5, + maximum: 2000, + description: + 'Space each of those nodes sets aside for replicated volumes, in GiB, taken from its own disk. Every replicated volume takes its size on each of them.', + }) + @IsOptional() + @IsInt() + @Min(5) + @Max(2000) + sizeGiB?: number; + + @ApiPropertyOptional({ + default: false, + description: + 'Accept a region Flui has not validated for high availability. Its three locations are then treated as equal, with no promise about how far apart they are.', + }) + @IsOptional() + @IsBoolean() + force?: boolean; +} diff --git a/src/modules/replicated-volumes/processors/replicated-volumes.processor.spec.ts b/src/modules/replicated-volumes/processors/replicated-volumes.processor.spec.ts new file mode 100644 index 00000000..f357ca9d --- /dev/null +++ b/src/modules/replicated-volumes/processors/replicated-volumes.processor.spec.ts @@ -0,0 +1,94 @@ +jest.mock('@kubernetes/client-node', () => ({})); + +import { ReplicatedVolumesProcessor } from './replicated-volumes.processor'; +import { OperationStatus } from '../../infrastructure/servers/entities/infrastructure-operations.entity'; + +describe('ReplicatedVolumesProcessor.enable', () => { + function build(voteState: { kind: string; kernel?: string }) { + const order: string[] = []; + const operations = { update: jest.fn().mockResolvedValue(undefined) }; + const preparer = { + prepare: jest.fn(async (_k: string, node: string) => { + order.push(`copies ${node}`); + return { kind: 'ready', sizeGiB: 20 }; + }), + prepareVote: jest.fn(async (_k: string, node: string) => { + order.push(`vote ${node}`); + return voteState; + }), + }; + const installer = { + ensureOperator: jest.fn(async () => order.push('engine')), + ensureCluster: jest.fn(), + restartStuckSatellites: jest.fn(), + moveControlOffCopies: jest.fn(), + ensureStorageClass: jest.fn(), + }; + const processor = new ReplicatedVolumesProcessor( + { + findOneOrFail: jest + .fn() + .mockResolvedValue({ name: 'ha-hz', kubeconfigEncrypted: 'x' }), + } as never, + operations as never, + { decrypt: () => 'kubeconfig' } as never, + preparer as never, + installer as never, + ); + const run = () => + processor.enable({ + data: { + operationId: 'op-1', + clusterId: 'c-1', + storageNodes: ['ha-hz-master', 'ha-hz-worker-1'], + sizeGiB: 20, + voteZones: ['hel1'], + tieBreakers: ['ha-hz-worker-2'], + }, + } as never); + return { run, order, operations, installer }; + } + + it('prepares the node that votes as well, before installing the engine', async () => { + const { run, order, operations, installer } = build({ kind: 'ready' }); + + await run(); + + expect(installer.ensureStorageClass).toHaveBeenCalledWith('kubeconfig', 2); + expect(installer.restartStuckSatellites).toHaveBeenCalledWith( + 'kubeconfig', + ['ha-hz-master', 'ha-hz-worker-1', 'ha-hz-worker-2'], + ); + + expect(order).toEqual([ + 'copies ha-hz-master', + 'copies ha-hz-worker-1', + 'vote ha-hz-worker-2', + 'engine', + ]); + expect(operations.update).toHaveBeenLastCalledWith( + 'op-1', + expect.objectContaining({ status: OperationStatus.COMPLETED }), + ); + }); + + it('stops before installing anything when the vote cannot load the engine module', async () => { + const { run, operations, installer } = build({ + kind: 'no-headers', + kernel: '6.8.0-138-generic', + }); + + await run(); + + expect(installer.ensureOperator).not.toHaveBeenCalled(); + expect(operations.update).toHaveBeenLastCalledWith( + 'op-1', + expect.objectContaining({ + status: OperationStatus.FAILED, + errorMessage: expect.stringContaining( + 'ha-hz-worker-2 has no kernel headers for 6.8.0-138-generic', + ), + }), + ); + }); +}); diff --git a/src/modules/replicated-volumes/processors/replicated-volumes.processor.ts b/src/modules/replicated-volumes/processors/replicated-volumes.processor.ts new file mode 100644 index 00000000..76d0419f --- /dev/null +++ b/src/modules/replicated-volumes/processors/replicated-volumes.processor.ts @@ -0,0 +1,131 @@ +import { Logger } from '@nestjs/common'; +import { Process, Processor } from '@nestjs/bull'; +import { Job } from 'bull'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { ClusterEntity } from '../../infrastructure/clusters/entities/cluster.entity'; +import { + InfrastructureOperationEntity, + OperationStatus, + OperationStep, +} from '../../infrastructure/servers/entities/infrastructure-operations.entity'; +import { EncryptionService } from '../../shared/encryption/services/encryption.service'; +import { + REPLICATED_VOLUMES_JOB_ENABLE, + REPLICATED_VOLUMES_QUEUE, +} from '../replicated-volumes.constants'; +import { EnableReplicatedVolumesJob } from '../services/replicated-volumes.service'; +import { PiraeusInstallerService } from '../services/piraeus-installer.service'; +import { ReplicatedPoolPreparer } from '../services/replicated-pool.preparer'; +import { ReplicatedPoolState } from '../utils/replicated-volumes.core'; + +function poolProblem(node: string, state: ReplicatedPoolState): string | null { + switch (state.kind) { + case 'ready': + return null; + case 'no-headers': + return `${node} has no kernel headers for ${state.kernel}, which the volume engine needs to load its module, and they could not be installed.`; + case 'no-space': + return `${node} has ${state.freeGiB} GiB free, not enough for the space asked plus a margin of 5 GiB.`; + case 'no-alloc': + return `${node} could not set aside the space for copies on its disk: its filesystem does not allow reserving it, or the disk filled up meanwhile.`; + default: + return `${node} could not be prepared: ${state.detail}`; + } +} + +/** + * Prepares the nodes first, so a node that cannot hold copies stops the work + * before anything is installed on the cluster; the storage class comes last, + * so nobody can ask for a replicated volume the cluster cannot place yet. + */ +@Processor(REPLICATED_VOLUMES_QUEUE) +export class ReplicatedVolumesProcessor { + private readonly logger = new Logger(ReplicatedVolumesProcessor.name); + + constructor( + @InjectRepository(ClusterEntity) + private readonly clusters: Repository, + @InjectRepository(InfrastructureOperationEntity) + private readonly operations: Repository, + private readonly encryption: EncryptionService, + private readonly preparer: ReplicatedPoolPreparer, + private readonly installer: PiraeusInstallerService, + ) {} + + @Process(REPLICATED_VOLUMES_JOB_ENABLE) + async enable(job: Job): Promise { + const { + operationId, + clusterId, + storageNodes, + sizeGiB, + voteZones = [], + tieBreakers = [], + } = job.data; + await this.operations.update(operationId, { + status: OperationStatus.IN_PROGRESS, + startedAt: new Date(), + }); + try { + const cluster = await this.clusters.findOneOrFail({ + where: { id: clusterId }, + }); + const kubeconfig = this.encryption.decrypt(cluster.kubeconfigEncrypted); + + await this.step(operationId, OperationStep.REPLICATED_PREPARE_NODES, 5); + for (const node of storageNodes) { + const problem = poolProblem( + node, + await this.preparer.prepare(kubeconfig, node, sizeGiB), + ); + if (problem) throw new Error(problem); + } + for (const node of tieBreakers) { + const problem = poolProblem( + node, + await this.preparer.prepareVote(kubeconfig, node), + ); + if (problem) + throw new Error(`${problem} It holds the tie-breaking vote.`); + } + + await this.step(operationId, OperationStep.REPLICATED_INSTALL_ENGINE, 40); + await this.installer.ensureOperator(kubeconfig); + await this.installer.ensureCluster(kubeconfig, voteZones); + await this.installer.restartStuckSatellites(kubeconfig, [ + ...storageNodes, + ...tieBreakers, + ]); + await this.installer.moveControlOffCopies(kubeconfig, storageNodes); + + await this.step(operationId, OperationStep.REPLICATED_STORAGE_CLASS, 90); + await this.installer.ensureStorageClass(kubeconfig, storageNodes.length); + + await this.operations.update(operationId, { + status: OperationStatus.COMPLETED, + progress: 100, + completedAt: new Date(), + }); + this.logger.log( + `[replicated-volumes] ${cluster.name}: enabled on ${storageNodes.join(', ')}`, + ); + } catch (error) { + const message = (error as Error).message; + await this.operations.update(operationId, { + status: OperationStatus.FAILED, + errorMessage: message, + completedAt: new Date(), + }); + this.logger.error(`[replicated-volumes] ${clusterId}: ${message}`); + } + } + + private async step( + operationId: string, + step: OperationStep, + progress: number, + ): Promise { + await this.operations.update(operationId, { currentStep: step, progress }); + } +} diff --git a/src/modules/replicated-volumes/replicated-volumes.constants.ts b/src/modules/replicated-volumes/replicated-volumes.constants.ts new file mode 100644 index 00000000..289b2a7e --- /dev/null +++ b/src/modules/replicated-volumes/replicated-volumes.constants.ts @@ -0,0 +1,21 @@ +export const PIRAEUS_VERSION = 'v2.12.0'; +export const PIRAEUS_MANIFEST_URL = `https://github.com/piraeusdatastore/piraeus-operator/releases/download/${PIRAEUS_VERSION}/manifest.yaml`; +export const PIRAEUS_NAMESPACE = 'piraeus-datastore'; +export const PIRAEUS_OPERATOR_DEPLOYMENT = + 'piraeus-operator-controller-manager'; +export const LINSTOR_CONTROLLER_DEPLOYMENT = 'linstor-controller'; +export const LINSTOR_CONTROLLER_SELECTOR = + 'app.kubernetes.io/component=linstor-controller'; +export const PIRAEUS_API_VERSION = 'piraeus.io/v1'; + +export const REPLICATED_STORAGE_CLASS = 'flui-replicated'; +export const REPLICATED_POOL = 'flui-replicated'; +export const REPLICATED_VOLUME_GROUP = 'flui-replicated'; +/** Nodes that hold a copy of the data; every other node can still cast the tie-breaking vote. */ +export const REPLICATED_STORAGE_NODE_LABEL = 'flui.cloud/replicated-storage'; +export const REPLICATED_POOL_FILE = '/var/lib/flui/replicated.img'; +export const REPLICATED_POOL_DEFAULT_GIB = 20; + +export const REPLICATED_VOLUMES_QUEUE = 'replicated-volumes'; +export const REPLICATED_VOLUMES_JOB_ENABLE = 'enable-replicated-volumes'; +export const REPLICATED_VOLUMES_JOB_NAMESPACE = 'flui-local-storage'; diff --git a/src/modules/replicated-volumes/replicated-volumes.module.ts b/src/modules/replicated-volumes/replicated-volumes.module.ts new file mode 100644 index 00000000..3c833eb5 --- /dev/null +++ b/src/modules/replicated-volumes/replicated-volumes.module.ts @@ -0,0 +1,33 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { BullModule } from '@nestjs/bull'; +import { ClusterEntity } from '../infrastructure/clusters/entities/cluster.entity'; +import { InfrastructureOperationEntity } from '../infrastructure/servers/entities/infrastructure-operations.entity'; +import { SharedInfrastructureModule } from '../infrastructure/shared/shared-infrastructure.module'; +import { EncryptionModule } from '../shared/encryption/encryption.module'; +import { ClusterHaModule } from '../cluster-ha/cluster-ha.module'; +import { PiraeusInstallerService } from './services/piraeus-installer.service'; +import { ReplicatedPoolPreparer } from './services/replicated-pool.preparer'; +import { ReplicatedVolumesService } from './services/replicated-volumes.service'; +import { ReplicatedVolumesProcessor } from './processors/replicated-volumes.processor'; +import { ReplicatedVolumesController } from './controllers/replicated-volumes.controller'; +import { REPLICATED_VOLUMES_QUEUE } from './replicated-volumes.constants'; + +@Module({ + imports: [ + TypeOrmModule.forFeature([ClusterEntity, InfrastructureOperationEntity]), + BullModule.registerQueue({ name: REPLICATED_VOLUMES_QUEUE }), + SharedInfrastructureModule, + EncryptionModule, + ClusterHaModule, + ], + controllers: [ReplicatedVolumesController], + providers: [ + PiraeusInstallerService, + ReplicatedPoolPreparer, + ReplicatedVolumesService, + ReplicatedVolumesProcessor, + ], + exports: [ReplicatedVolumesService, ReplicatedPoolPreparer], +}) +export class ReplicatedVolumesModule {} diff --git a/src/modules/replicated-volumes/services/piraeus-installer.service.ts b/src/modules/replicated-volumes/services/piraeus-installer.service.ts new file mode 100644 index 00000000..aa1c91e0 --- /dev/null +++ b/src/modules/replicated-volumes/services/piraeus-installer.service.ts @@ -0,0 +1,193 @@ +import { Injectable, Logger } from '@nestjs/common'; +import * as k8s from '@kubernetes/client-node'; +import { KubernetesService } from '../../infrastructure/shared/services/kubernetes.service'; +import { + LINSTOR_CONTROLLER_DEPLOYMENT, + PIRAEUS_MANIFEST_URL, + PIRAEUS_NAMESPACE, + PIRAEUS_OPERATOR_DEPLOYMENT, + PIRAEUS_VERSION, + REPLICATED_STORAGE_CLASS, +} from '../replicated-volumes.constants'; +import { + SATELLITE_SELECTOR, + controlPodsOnCopies, + stuckSatellites, + renderLinstorCluster, + renderReplicatedStorageClass, + renderSatelliteConfigurations, +} from '../utils/replicated-volumes.core'; +import { pinnedPiraeusManifest } from '../utils/piraeus-images'; + +const READY_TIMEOUT_MS = 10 * 60 * 1000; +const POLL_INTERVAL_MS = 5000; + +/** + * Installs the replicated volume engine (Piraeus, with LINSTOR and DRBD) on a + * cluster the first time someone enables replicated volumes there, and never + * before. Applied server-side, like CloudNativePG: the resource definitions + * are too large for a client-side apply. + */ +@Injectable() +export class PiraeusInstallerService { + private readonly logger = new Logger(PiraeusInstallerService.name); + private manifestCache: string | null = null; + + constructor(private readonly kubernetesService: KubernetesService) {} + + async isInstalled(kubeconfig: string): Promise { + const existing = await this.kubernetesService.getResource( + kubeconfig, + 'Deployment', + PIRAEUS_OPERATOR_DEPLOYMENT, + PIRAEUS_NAMESPACE, + ); + return Boolean(existing); + } + + async hasStorageClass(kubeconfig: string): Promise { + return !!(await this.readStorageClass(kubeconfig)); + } + + async readStorageClass( + kubeconfig: string, + ): Promise { + const storage = this.kubernetesService + .makeKubeConfig(kubeconfig) + .makeApiClient(k8s.StorageV1Api); + try { + return await storage.readStorageClass({ name: REPLICATED_STORAGE_CLASS }); + } catch (error) { + if ((error as { code?: number }).code === 404) return null; + throw error; + } + } + + async ensureOperator(kubeconfig: string): Promise { + if (!(await this.isInstalled(kubeconfig))) { + this.logger.log(`Installing Piraeus ${PIRAEUS_VERSION}`); + const applied = await this.kubernetesService.serverSideApply( + kubeconfig, + await this.fetchManifest(), + ); + this.logger.log(`Piraeus: ${applied} objects applied`); + } + await this.waitDeployment(kubeconfig, PIRAEUS_OPERATOR_DEPLOYMENT); + } + + async ensureCluster( + kubeconfig: string, + voteZones: string[] = [], + ): Promise { + const objects = [ + renderLinstorCluster(voteZones), + ...renderSatelliteConfigurations(), + ]; + await this.kubernetesService.serverSideApply( + kubeconfig, + objects.map((o) => k8s.dumpYaml(o)).join('---\n'), + ); + await this.waitDeployment(kubeconfig, LINSTOR_CONTROLLER_DEPLOYMENT); + } + + /** The satellites of `nodes` waiting out a crash backoff start again now, not minutes later. */ + async restartStuckSatellites( + kubeconfig: string, + nodes: string[], + ): Promise { + const core = this.kubernetesService + .makeKubeConfig(kubeconfig) + .makeApiClient(k8s.CoreV1Api); + const pods = await core.listNamespacedPod({ + namespace: PIRAEUS_NAMESPACE, + labelSelector: SATELLITE_SELECTOR, + }); + const stuck = stuckSatellites(pods.items ?? [], nodes); + for (const name of stuck) { + await core.deleteNamespacedPod({ name, namespace: PIRAEUS_NAMESPACE }); + } + return stuck; + } + + /** Deleting the pod, not patching the Deployment, leaves the operator nothing to reconcile back. */ + async moveControlOffCopies( + kubeconfig: string, + storageNodes: string[], + ): Promise { + const core = this.kubernetesService + .makeKubeConfig(kubeconfig) + .makeApiClient(k8s.CoreV1Api); + const pods = await core.listNamespacedPod({ namespace: PIRAEUS_NAMESPACE }); + const moved = controlPodsOnCopies(pods.items ?? [], storageNodes); + for (const name of moved) { + await core.deleteNamespacedPod({ name, namespace: PIRAEUS_NAMESPACE }); + } + if (moved.length) { + await this.waitDeployment(kubeconfig, LINSTOR_CONTROLLER_DEPLOYMENT); + } + return moved; + } + + /** + * A storage class's parameters cannot change, so a class made with other + * ones is replaced. Volumes already made keep their copies where they are. + */ + async ensureStorageClass(kubeconfig: string, copies: number): Promise { + const desired = renderReplicatedStorageClass(copies); + const current = await this.readStorageClass(kubeconfig); + if ( + current && + JSON.stringify(current.parameters ?? {}) !== + JSON.stringify(sortedKeys(desired.parameters as Record)) + ) { + await this.kubernetesService + .makeKubeConfig(kubeconfig) + .makeApiClient(k8s.StorageV1Api) + .deleteStorageClass({ name: REPLICATED_STORAGE_CLASS }); + } + await this.kubernetesService.serverSideApply( + kubeconfig, + k8s.dumpYaml(desired), + ); + } + + private async fetchManifest(): Promise { + if (this.manifestCache) return this.manifestCache; + const response = await fetch(PIRAEUS_MANIFEST_URL); + if (!response.ok) { + throw new Error( + `Failed to fetch the Piraeus manifest from ${PIRAEUS_MANIFEST_URL}: ${response.status} ${response.statusText}`, + ); + } + this.manifestCache = pinnedPiraeusManifest(await response.text()); + return this.manifestCache; + } + + private async waitDeployment( + kubeconfig: string, + name: string, + ): Promise { + const start = Date.now(); + while (Date.now() - start < READY_TIMEOUT_MS) { + const resource = await this.kubernetesService.getResource( + kubeconfig, + 'Deployment', + name, + PIRAEUS_NAMESPACE, + ); + const body = resource?.body ?? resource; + const desired = body?.spec?.replicas ?? 1; + if (body && (body?.status?.availableReplicas ?? 0) >= desired) return; + await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS)); + } + throw new Error( + `${name} did not become ready within ${READY_TIMEOUT_MS / 1000}s`, + ); + } +} + +function sortedKeys(record: Record): Record { + return Object.fromEntries( + Object.entries(record).sort(([a], [b]) => a.localeCompare(b)), + ); +} diff --git a/src/modules/replicated-volumes/services/replicated-pool.preparer.ts b/src/modules/replicated-volumes/services/replicated-pool.preparer.ts new file mode 100644 index 00000000..e9d3d3b2 --- /dev/null +++ b/src/modules/replicated-volumes/services/replicated-pool.preparer.ts @@ -0,0 +1,189 @@ +import { randomUUID } from 'node:crypto'; +import { Injectable } from '@nestjs/common'; +import * as k8s from '@kubernetes/client-node'; +import { KubernetesService } from '../../infrastructure/shared/services/kubernetes.service'; +import { + REPLICATED_STORAGE_NODE_LABEL, + REPLICATED_VOLUMES_JOB_NAMESPACE, +} from '../replicated-volumes.constants'; +import { + ReplicatedPoolState, + parseReplicatedPoolOutput, + replicatedPoolHostScript, +} from '../utils/replicated-volumes.core'; +import { replicatedVoteHostScript } from '../utils/replicated-votes.core'; + +const JOB_TIMEOUT_MS = 15 * 60 * 1000; +const JOB_POLL_MS = 3000; +const JOB_LABEL = 'flui-replicated-pool-job'; + +/** + * Prepares the nodes that will hold copies, through a privileged job on each + * one, and marks a node only once its pool is there: the engine puts data + * only on marked nodes. Running it again on a prepared node changes nothing. + */ +@Injectable() +export class ReplicatedPoolPreparer { + constructor(private readonly kubernetes: KubernetesService) {} + + async prepare( + kubeconfig: string, + node: string, + sizeGiB: number, + ): Promise { + const state = parseReplicatedPoolOutput( + await this.runHostScript( + kubeconfig, + node, + replicatedPoolHostScript(sizeGiB), + ), + ); + if (state.kind === 'ready') { + await this.kubernetes.addNodeLabels(kubeconfig, node, { + [REPLICATED_STORAGE_NODE_LABEL]: 'true', + }); + } + return state; + } + + /** A node that only votes gets the module's build headers, and no pool or copy label. */ + async prepareVote( + kubeconfig: string, + node: string, + ): Promise { + return parseReplicatedPoolOutput( + await this.runHostScript(kubeconfig, node, replicatedVoteHostScript()), + ); + } + + /** Runs a script on the node itself, through the same privileged job, and returns what it printed. */ + async runHostScript( + kubeconfig: string, + node: string, + hostScript: string, + ): Promise { + const jobName = `flui-replicated-pool-${Date.now()}-${randomUUID().slice(0, 5)}`; + await this.kubernetes.applyManifest( + kubeconfig, + hostScriptJobManifest(jobName, node, hostScript), + ); + try { + await this.awaitJob(kubeconfig, jobName); + return await this.readJobLogs(kubeconfig, jobName); + } finally { + await this.kubernetes + .deleteResource( + kubeconfig, + 'Job', + jobName, + REPLICATED_VOLUMES_JOB_NAMESPACE, + ) + .catch(() => undefined); + } + } + + private async awaitJob(kubeconfig: string, jobName: string): Promise { + const start = Date.now(); + while (Date.now() - start < JOB_TIMEOUT_MS) { + const job = await this.kubernetes.getResource( + kubeconfig, + 'Job', + jobName, + REPLICATED_VOLUMES_JOB_NAMESPACE, + ); + if ((job?.status?.succeeded ?? 0) > 0) return; + if ((job?.status?.failed ?? 0) > 0) { + throw new Error(`replicated pool job ${jobName} failed`); + } + await new Promise((r) => setTimeout(r, JOB_POLL_MS)); + } + throw new Error(`replicated pool job ${jobName} timed out`); + } + + private async readJobLogs( + kubeconfig: string, + jobName: string, + ): Promise { + const coreApi = this.kubernetes + .makeKubeConfig(kubeconfig) + .makeApiClient(k8s.CoreV1Api); + const pods = await coreApi.listNamespacedPod({ + namespace: REPLICATED_VOLUMES_JOB_NAMESPACE, + labelSelector: `${JOB_LABEL}=${jobName}`, + }); + const podName = pods.items?.[0]?.metadata?.name; + if (!podName) throw new Error(`no pod found for job ${jobName}`); + return this.kubernetes.getPodLogs( + kubeconfig, + podName, + REPLICATED_VOLUMES_JOB_NAMESPACE, + ); + } +} + +export function replicatedPoolJobManifest( + jobName: string, + node: string, + sizeGiB: number, +): string { + return hostScriptJobManifest( + jobName, + node, + replicatedPoolHostScript(sizeGiB), + ); +} + +export function hostScriptJobManifest( + jobName: string, + node: string, + hostScript: string, +): string { + const script = [ + 'apk add --no-cache util-linux >/dev/null 2>&1 || true', + "SCRIPT=$(cat <<'FLUIPOOL'", + hostScript, + 'FLUIPOOL', + ')', + 'nsenter -t 1 -m -u -i -n -p -- /bin/sh -c "$SCRIPT"', + ] + .join('\n') + .split('\n') + .map((line) => ` ${line}`) + .join('\n'); + return [ + 'apiVersion: batch/v1', + 'kind: Job', + 'metadata:', + ` name: ${jobName}`, + ` namespace: ${REPLICATED_VOLUMES_JOB_NAMESPACE}`, + ' labels:', + ' flui.cloud/managed-by: flui-cloud', + ' flui-resource-type: replicated-pool', + 'spec:', + ' ttlSecondsAfterFinished: 60', + ' backoffLimit: 0', + ' template:', + ' metadata:', + ' labels:', + ' flui.cloud/managed-by: flui-cloud', + ' flui-resource-type: replicated-pool', + ` ${JOB_LABEL}: ${jobName}`, + ' spec:', + ' restartPolicy: Never', + ' hostPID: true', + ' nodeSelector:', + ` kubernetes.io/hostname: ${node}`, + ' tolerations:', + ' - operator: Exists', + ' containers:', + ' - name: replicated-pool', + ' image: alpine:3.20', + ' securityContext:', + ' privileged: true', + ' command: ["sh","-c"]', + ' args:', + ' - |', + script, + '', + ].join('\n'); +} diff --git a/src/modules/replicated-volumes/services/replicated-volumes.service.ts b/src/modules/replicated-volumes/services/replicated-volumes.service.ts new file mode 100644 index 00000000..cd698683 --- /dev/null +++ b/src/modules/replicated-volumes/services/replicated-volumes.service.ts @@ -0,0 +1,286 @@ +import { + BadRequestException, + Injectable, + Logger, + NotFoundException, +} from '@nestjs/common'; +import { InjectQueue } from '@nestjs/bull'; +import { Queue } from 'bull'; +import { InjectRepository } from '@nestjs/typeorm'; +import { In, Repository } from 'typeorm'; +import { ClusterEntity } from '../../infrastructure/clusters/entities/cluster.entity'; +import { + InfrastructureOperationEntity, + OperationStatus, + OperationType, +} from '../../infrastructure/servers/entities/infrastructure-operations.entity'; +import { KubernetesService } from '../../infrastructure/shared/services/kubernetes.service'; +import { EncryptionService } from '../../shared/encryption/services/encryption.service'; +import { haCandidatesFrom } from '../../postgres-ha/utils/postgres-ha-feasibility.core'; +import { EnableReplicatedVolumesDto } from '../dto/enable-replicated-volumes.dto'; +import { + LINSTOR_CONTROLLER_DEPLOYMENT, + LINSTOR_CONTROLLER_SELECTOR, + PIRAEUS_NAMESPACE, + REPLICATED_POOL_DEFAULT_GIB, + REPLICATED_STORAGE_CLASS, + REPLICATED_STORAGE_NODE_LABEL, + REPLICATED_VOLUMES_JOB_ENABLE, + REPLICATED_VOLUMES_QUEUE, +} from '../replicated-volumes.constants'; +import { + ReplicatedVolumesFeasibility, + replicatedVolumesFeasibility, + staleCopyWarnings, + voteZonesOf, + storageNodeWarnings, + volumeCopiesFrom, +} from '../utils/replicated-volumes.core'; +import { missingVoteWarnings } from '../utils/replicated-votes.core'; +import { + ClusterHaService, + notThreeLocations, +} from '../../cluster-ha/services/cluster-ha.service'; +import { PiraeusInstallerService } from './piraeus-installer.service'; + +export interface EnableReplicatedVolumesJob { + operationId: string; + clusterId: string; + storageNodes: string[]; + sizeGiB: number; + /** Where the engine's single-copy components run; empty means anywhere away from copies. */ + voteZones?: string[]; + /** Nodes that hold the tie-breaking vote, with no data: they still need the engine's module. */ + tieBreakers?: string[]; +} + +export interface ReplicatedVolumesView { + clusterId: string; + enabled: boolean; + storageClass: string; + storageNodes: string[]; + warnings: string[]; + /** What enabling would do with the nodes the cluster has now. */ + plan: ReplicatedVolumesFeasibility; + operation: { id: string; status: OperationStatus } | null; +} + +const RUNNING = [OperationStatus.PENDING, OperationStatus.IN_PROGRESS]; + +function storageNodesOf( + nodes: Array<{ + metadata?: { name?: string; labels?: Record }; + }>, +): string[] { + return nodes + .filter( + (n) => n.metadata?.labels?.[REPLICATED_STORAGE_NODE_LABEL] === 'true', + ) + .map((n) => n.metadata!.name!); +} + +@Injectable() +export class ReplicatedVolumesService { + private readonly logger = new Logger(ReplicatedVolumesService.name); + + constructor( + @InjectRepository(ClusterEntity) + private readonly clusters: Repository, + @InjectRepository(InfrastructureOperationEntity) + private readonly operations: Repository, + @InjectQueue(REPLICATED_VOLUMES_QUEUE) private readonly queue: Queue, + private readonly kubernetes: KubernetesService, + private readonly encryption: EncryptionService, + private readonly installer: PiraeusInstallerService, + private readonly clusterHa: ClusterHaService, + ) {} + + async view(clusterId: string): Promise { + const ha = await this.clusterHa.inspect(clusterId); + const storageNodes = storageNodesOf(ha.nodes); + const storageClass = await this.installer.readStorageClass(ha.kubeconfig); + const candidates = haCandidatesFrom(ha.nodes); + const plan: ReplicatedVolumesFeasibility = + ha.state.level === 'three-locations' && ha.topology + ? replicatedVolumesFeasibility(candidates, ha.topology, { + prepared: storageNodes, + }) + : { + feasible: false, + reason: notThreeLocations(ha.cluster.name, ha.state), + }; + return { + clusterId, + enabled: !!storageClass, + storageClass: REPLICATED_STORAGE_CLASS, + storageNodes, + warnings: [ + ...storageNodeWarnings(candidates, storageNodes), + ...(storageClass ? await this.copyWarnings(ha.kubeconfig) : []), + ...(storageClass && 'tieBreakers' in plan + ? await this.voteWarnings(ha.kubeconfig, plan.tieBreakers) + : []), + ], + plan, + operation: await this.runningOperation(clusterId), + }; + } + + private async voteWarnings( + kubeconfig: string, + tieBreakers: string[], + ): Promise { + if (!tieBreakers.length) return []; + try { + return missingVoteWarnings( + tieBreakers, + await this.kubernetes.execInPod( + kubeconfig, + PIRAEUS_NAMESPACE, + LINSTOR_CONTROLLER_SELECTOR, + LINSTOR_CONTROLLER_DEPLOYMENT, + ['linstor', '-m', '--output-version', 'v1', 'node', 'list'], + ), + ); + } catch (err: any) { + this.logger.warn( + `[replicated-volumes] engine nodes not read (${err?.message ?? err})`, + ); + return []; + } + } + + private async copyWarnings(kubeconfig: string): Promise { + try { + const copies = volumeCopiesFrom( + await this.kubernetes.execInPod( + kubeconfig, + PIRAEUS_NAMESPACE, + LINSTOR_CONTROLLER_SELECTOR, + LINSTOR_CONTROLLER_DEPLOYMENT, + ['linstor', '-m', '--output-version', 'v1', 'resource', 'list'], + ), + ); + const claimOf = new Map(); + for (const volume of new Set( + copies.filter((c) => c.disk !== 'UpToDate').map((c) => c.volume), + )) { + const pv = await this.kubernetes + .readObject(kubeconfig, 'v1', 'PersistentVolume', volume) + .catch(() => null); + const claim = pv?.spec?.claimRef; + if (claim?.name) + claimOf.set(volume, `${claim.namespace}/${claim.name}`); + } + return staleCopyWarnings(copies, claimOf); + } catch (err: any) { + this.logger.warn( + `[replicated-volumes] copy states not read (${err?.message ?? err})`, + ); + return []; + } + } + + /** Null when the cluster could not be asked. */ + async isEnabled(clusterId: string): Promise { + try { + return await this.installer.hasStorageClass( + await this.kubeconfigOf(clusterId), + ); + } catch { + return null; + } + } + + /** + * Checks what can be checked before anything is installed, so a cluster + * that cannot hold a copy in each location for data is refused with the reason, + * then hands the work to the queue. + */ + async enable( + clusterId: string, + dto: EnableReplicatedVolumesDto, + userId?: string, + ): Promise<{ operationId: string; storageNodes: string[]; sizeGiB: number }> { + if (await this.runningOperation(clusterId)) { + throw new BadRequestException( + 'Replicated volumes are already being enabled on this cluster.', + ); + } + const ha = await this.clusterHa.requireThreeLocations(clusterId, dto.force); + const prepared = storageNodesOf(ha.nodes); + const plan = replicatedVolumesFeasibility( + haCandidatesFrom(ha.nodes), + ha.topology, + { requested: dto.nodes, prepared }, + ); + if (!plan.feasible) { + throw new BadRequestException((plan as { reason: string }).reason); + } + const left = prepared.filter((n) => !plan.storageNodes.includes(n)); + if (left.length) { + throw new BadRequestException( + `${left.join(', ')} already set aside space for copies, and new volumes could be placed there. Name the nodes that hold copies today, or take ${left.length === 1 ? 'that node' : 'those nodes'} out of replicated volumes first.`, + ); + } + const sizeGiB = dto.sizeGiB ?? REPLICATED_POOL_DEFAULT_GIB; + + const operation = await this.operations.save( + this.operations.create({ + operationType: OperationType.ENABLE_REPLICATED_VOLUMES, + status: OperationStatus.PENDING, + resourceType: 'cluster', + resourceId: clusterId, + resourceName: ha.cluster.name, + userId, + metadata: { + storageNodes: plan.storageNodes, + sizeGiB, + zones: plan.zones, + ...(!ha.topology.validated && { forced: true }), + } as InfrastructureOperationEntity['metadata'], + totalSteps: 3, + }), + ); + const job: EnableReplicatedVolumesJob = { + operationId: operation.id, + clusterId, + storageNodes: plan.storageNodes, + sizeGiB, + voteZones: voteZonesOf(ha.topology), + tieBreakers: 'tieBreakers' in plan ? plan.tieBreakers : [], + }; + await this.queue.add(REPLICATED_VOLUMES_JOB_ENABLE, job, { attempts: 1 }); + this.logger.log( + `[replicated-volumes] ${ha.cluster.name}: enabling on ${plan.storageNodes.join(', ')} (${plan.zones.join(', ')}, ${sizeGiB} GiB each)`, + ); + return { + operationId: operation.id, + storageNodes: plan.storageNodes, + sizeGiB, + }; + } + + private async runningOperation( + clusterId: string, + ): Promise<{ id: string; status: OperationStatus } | null> { + const op = await this.operations.findOne({ + where: { + operationType: OperationType.ENABLE_REPLICATED_VOLUMES, + resourceId: clusterId, + status: In(RUNNING), + }, + order: { createdAt: 'DESC' }, + }); + return op ? { id: op.id, status: op.status } : null; + } + + private async kubeconfigOf(clusterId: string): Promise { + const cluster = await this.clusters.findOne({ where: { id: clusterId } }); + if (!cluster) throw new NotFoundException(`Cluster ${clusterId} not found`); + if (!cluster.kubeconfigEncrypted) { + throw new BadRequestException('This cluster cannot be reached.'); + } + return this.encryption.decrypt(cluster.kubeconfigEncrypted); + } +} diff --git a/src/modules/replicated-volumes/utils/piraeus-images.spec.ts b/src/modules/replicated-volumes/utils/piraeus-images.spec.ts new file mode 100644 index 00000000..f871bd01 --- /dev/null +++ b/src/modules/replicated-volumes/utils/piraeus-images.spec.ts @@ -0,0 +1,82 @@ +import { createHash } from 'node:crypto'; +import * as yaml from 'js-yaml'; +import { PIRAEUS_IMAGE_DIGESTS, pinnedPiraeusManifest } from './piraeus-images'; + +const imageConfig = [ + '---', + 'base: quay.io/piraeusdatastore', + 'components:', + ' linstor-satellite:', + ' tag: v1.35.2', + ' image: piraeus-server', + ' drbd-module-loader:', + ' tag: v9.3.4', + ' image: drbd9-resolute', + ' match:', + ' - osImage: Ubuntu 24\\.04', + ' image: drbd9-noble', +].join('\n'); + +const manifest = [ + yaml.dump({ + apiVersion: 'v1', + kind: 'ConfigMap', + metadata: { name: 'piraeus-operator-image-config' }, + data: { '0_piraeus_datastore_images.yaml': imageConfig }, + }), + yaml.dump({ + apiVersion: 'apps/v1', + kind: 'Deployment', + metadata: { name: 'piraeus-operator-controller-manager' }, + spec: { + template: { + spec: { + containers: [ + { image: 'quay.io/piraeusdatastore/piraeus-operator:v2.12.0' }, + ], + }, + }, + }, + }), +].join('---\n'); +const sha = createHash('sha256').update(manifest).digest('hex'); + +describe('the volume engine pinned by digest', () => { + it('refuses a manifest that is not the one checked', () => { + expect(() => pinnedPiraeusManifest(manifest)).toThrow( + 'is not the one Flui checked', + ); + }); + + it('fixes every image of the image config, by operating system too, and the operator itself', () => { + const docs = yaml.loadAll(pinnedPiraeusManifest(manifest, sha)) as any[]; + const config = yaml.load( + docs[0].data['0_piraeus_datastore_images.yaml'], + ) as any; + + expect(config.components['linstor-satellite'].digest).toBe( + PIRAEUS_IMAGE_DIGESTS['quay.io/piraeusdatastore/piraeus-server:v1.35.2'], + ); + expect(config.components['drbd-module-loader'].digest).toBe( + PIRAEUS_IMAGE_DIGESTS['quay.io/piraeusdatastore/drbd9-resolute:v9.3.4'], + ); + expect(config.components['drbd-module-loader'].match[0].digest).toBe( + PIRAEUS_IMAGE_DIGESTS['quay.io/piraeusdatastore/drbd9-noble:v9.3.4'], + ); + expect(docs[1].spec.template.spec.containers[0].image).toBe( + `quay.io/piraeusdatastore/piraeus-operator:v2.12.0@${PIRAEUS_IMAGE_DIGESTS['quay.io/piraeusdatastore/piraeus-operator:v2.12.0']}`, + ); + }); + + it('refuses an image it has no digest for, rather than running it by tag', () => { + const other = manifest.replace('v1.35.2', 'v1.99.0'); + expect(() => + pinnedPiraeusManifest( + other, + createHash('sha256').update(other).digest('hex'), + ), + ).toThrow( + 'No pinned digest for quay.io/piraeusdatastore/piraeus-server:v1.99.0', + ); + }); +}); diff --git a/src/modules/replicated-volumes/utils/piraeus-images.ts b/src/modules/replicated-volumes/utils/piraeus-images.ts new file mode 100644 index 00000000..0171574d --- /dev/null +++ b/src/modules/replicated-volumes/utils/piraeus-images.ts @@ -0,0 +1,137 @@ +import { createHash } from 'node:crypto'; +import * as yaml from 'js-yaml'; + +/** The SHA-256 of the v2.12.0 release manifest: a manifest that differs is refused. */ +export const PIRAEUS_MANIFEST_SHA256 = + '1d3b8f6cee38a369acda013e126f896f1e2b29c1f86c1539e66ccd475868a161'; + +/** + * Every image that manifest runs, by the digest its registry served on + * 8 November 2026. A tag can be moved to other content; a digest cannot, and + * the DRBD module these images build goes into the host's kernel. + */ +export const PIRAEUS_IMAGE_DIGESTS: Record = { + 'quay.io/piraeusdatastore/drbd-reactor:v1.12.0': + 'sha256:a3d7a83f2eac5c5e8cd366f341b3c79cd6d0f2a3b4fcef9890e3b16140ce91ab', + 'quay.io/piraeusdatastore/drbd-shutdown-guard:v1.1.2': + 'sha256:a4adba73d07f6ba2605d0594676df47ab143e004991caadea3b7b556ead2db6e', + 'quay.io/piraeusdatastore/drbd9-almalinux10:v9.3.4': + 'sha256:4bd5a2a8c7309a3578d5f9d4385ecc4471b9ac5e4c3f209cece8a8cef1bcdfba', + 'quay.io/piraeusdatastore/drbd9-almalinux8:v9.3.4': + 'sha256:6954910569cf701adde2cf7f97d007b48ffaaea15cfa8fb4c5634086829007a3', + 'quay.io/piraeusdatastore/drbd9-almalinux9:v9.3.4': + 'sha256:e0a3ada199c28ff3d7f5ba5720a940a1b68764c05567103061c8c0d460160723', + 'quay.io/piraeusdatastore/drbd9-bookworm:v9.3.4': + 'sha256:fb35c74d0ad24b070131346c3ab1ca706c06bc691778d71591d4537dbb18b393', + 'quay.io/piraeusdatastore/drbd9-bullseye:v9.3.4': + 'sha256:4aa5011acefce274837910bb476e56de3572115f6d1ed1e728dc4abd5746751d', + 'quay.io/piraeusdatastore/drbd9-jammy:v9.3.4': + 'sha256:e58cc9342d45c0d629a10106b5f2e3ae6c1468fc5ebbe1c74be858c56ef18bb8', + 'quay.io/piraeusdatastore/drbd9-noble:v9.3.4': + 'sha256:34ce597f6107363035528ffe498be4c8c892fbdaaef93d558cfd9e8ec37cc83d', + 'quay.io/piraeusdatastore/drbd9-resolute:v9.3.4': + 'sha256:c2158adc51daa95688e5e37e3c50418d157af2c10abc0e8c78b559f485108fec', + 'quay.io/piraeusdatastore/drbd9-trixie:v9.3.4': + 'sha256:fc4abdb9f46455b765c841500d8f98fa0bbf71b3e391cab58764510ca5be9b94', + 'quay.io/piraeusdatastore/ktls-utils:v1.2.1': + 'sha256:d9ce9e1b31c4a19986abc9d5ce375dc4e407a198b28dd05e48ce18d647d3f968', + 'quay.io/piraeusdatastore/linstor-affinity-controller:v1.5.0': + 'sha256:27797e5c97d42d7b75af7a45877c6840cdc1be8c6e6d1624dc4a1a46aa79ecbf', + 'quay.io/piraeusdatastore/piraeus-csi-nfs-server:v1.13.1': + 'sha256:b9d91188e435fc9b2ddd8f2b9fba3147102faa08bcafcaba2d14d6df56f8e153', + 'quay.io/piraeusdatastore/piraeus-csi:v1.13.1': + 'sha256:900a9d34b8d2c4ed6479d1c0480b18aa5c003678532a12de71de54464b42d05e', + 'quay.io/piraeusdatastore/piraeus-ha-controller:v1.3.4': + 'sha256:ac13aebcbc161792373e181234d9660bbb54ac5425024d5d871992a7eb448b65', + 'quay.io/piraeusdatastore/piraeus-operator:v2.12.0': + 'sha256:2fee47e187c1c0f53335d888ffb151fbf1d02868fe8b9fdc18d32629c48cdf49', + 'quay.io/piraeusdatastore/piraeus-server:v1.35.2': + 'sha256:d8da06ae50c0fa9c36f2649a8073667b546d6e9dcacc123c203d3d47ab5bc0f0', + 'registry.k8s.io/sig-storage/csi-attacher:v4.13.0': + 'sha256:d1a26170efed45491074ca4545c81844eee2b9043fb885ac01fb4aed6f8b72d2', + 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.18.0': + 'sha256:b7fefd08651f00ac4df1a196ed4c621c2451fbb899297fe9fad9e36c667af0c0', + 'registry.k8s.io/sig-storage/csi-provisioner:v6.3.0': + 'sha256:a4b0b1a37605b7b04a293e136edf7006ec1786a8eb3f4e5a945f81d667dcc371', + 'registry.k8s.io/sig-storage/csi-resizer:v2.2.1': + 'sha256:ea1d25e23479000c7e8eeb92d827df66258df4e482ca054c5e7ce3fc0f5c41a5', + 'registry.k8s.io/sig-storage/csi-snapshotter:v8.6.0': + 'sha256:42af0929bcd60a43499825c078a60ff0534e08af8fbeb283aa391be40feb9f3e', + 'registry.k8s.io/sig-storage/livenessprobe:v2.20.0': + 'sha256:19f2cf2f40e1987c7943945deca5f65cd4e4bf5cc0657fc78a5df131861578e8', +}; + +interface ImageConfig { + base: string; + components: Record< + string, + { + tag: string; + image: string; + digest?: string; + match?: Array<{ image: string; digest?: string }>; + } + >; +} + +function digestOf(ref: string): string { + const digest = PIRAEUS_IMAGE_DIGESTS[ref]; + if (!digest) throw new Error(`No pinned digest for ${ref}`); + return digest; +} + +function pinImageConfig(text: string): string { + const config = yaml.load(text) as ImageConfig; + for (const component of Object.values(config.components ?? {})) { + const ref = (image: string) => `${config.base}/${image}:${component.tag}`; + component.digest = digestOf(ref(component.image)); + for (const match of component.match ?? []) { + match.digest = digestOf(ref(match.image)); + } + } + return yaml.dump(config, { lineWidth: -1 }); +} + +function pinContainers( + containers: Array<{ image?: string }> | undefined, +): void { + for (const c of containers ?? []) { + if (c.image && !c.image.includes('@')) { + c.image = `${c.image}@${digestOf(c.image)}`; + } + } +} + +/** + * The release manifest with every image fixed by digest, after checking it is + * the manifest these digests were read for. + */ +export function pinnedPiraeusManifest( + manifest: string, + expectedSha256 = PIRAEUS_MANIFEST_SHA256, +): string { + const sha = createHash('sha256').update(manifest).digest('hex'); + if (sha !== expectedSha256) { + throw new Error( + `The volume engine's manifest is not the one Flui checked (SHA-256 ${sha}): refusing to install it`, + ); + } + const docs = yaml.loadAll(manifest).filter(Boolean) as Array< + Record + >; + for (const doc of docs) { + if ( + doc.kind === 'ConfigMap' && + doc.metadata?.name === 'piraeus-operator-image-config' + ) { + for (const key of Object.keys(doc.data ?? {})) { + doc.data[key] = pinImageConfig(doc.data[key]); + } + } + if (doc.kind === 'Deployment') { + pinContainers(doc.spec?.template?.spec?.containers); + pinContainers(doc.spec?.template?.spec?.initContainers); + } + } + return docs.map((d) => yaml.dump(d, { lineWidth: -1 })).join('---\n'); +} diff --git a/src/modules/replicated-volumes/utils/replicated-volumes.core.spec.ts b/src/modules/replicated-volumes/utils/replicated-volumes.core.spec.ts new file mode 100644 index 00000000..7314a104 --- /dev/null +++ b/src/modules/replicated-volumes/utils/replicated-volumes.core.spec.ts @@ -0,0 +1,484 @@ +import { + controlPodsOnCopies, + parseReplicatedPoolOutput, + renderLinstorCluster, + renderReplicatedStorageClass, + renderSatelliteConfigurations, + replicatedPoolHostScript, + replicatedVolumesFeasibility, + volumesBlockingFailover, + storageNodeWarnings, + staleCopyWarnings, + volumeCopiesFrom, + voteZonesOf, + stuckSatellites, +} from './replicated-volumes.core'; +import { HaTopology } from '../../providers/core/ha-topology'; + +const hetzner: HaTopology = { + region: 'eu-central', + zones: ['fsn1', 'nbg1', 'hel1'], + dataZones: ['fsn1', 'nbg1'], + validated: true, + measured: true, +}; + +const node = (name: string, zone?: string, schedulable = true) => ({ + name, + zone, + schedulable, +}); + +describe('replicated volumes core', () => { + describe('renderReplicatedStorageClass', () => { + const sc = renderReplicatedStorageClass() as { + metadata: { name: string }; + volumeBindingMode: string; + parameters: Record; + }; + + it('keeps two copies on the prepared pool and the pod next to one', () => { + expect(sc.metadata.name).toBe('flui-replicated'); + expect(sc.volumeBindingMode).toBe('WaitForFirstConsumer'); + expect(sc.parameters['linstor.csi.linbit.com/storagePool']).toBe( + 'flui-replicated', + ); + expect(sc.parameters['linstor.csi.linbit.com/placementCount']).toBe('2'); + expect( + sc.parameters['linstor.csi.linbit.com/allowRemoteVolumeAccess'], + ).toBe('false'); + }); + + it('asks for quorum so a node cut off stops confirming writes', () => { + const p = 'property.linstor.csi.linbit.com/DrbdOptions'; + expect(sc.parameters[`${p}/auto-quorum`]).toBe('suspend-io'); + expect(sc.parameters[`${p}/Resource/on-no-data-accessible`]).toBe( + 'suspend-io', + ); + expect(sc.parameters[`${p}/Resource/on-suspended-primary-outdated`]).toBe( + 'force-secondary', + ); + }); + + it('keeps one copy per location for data', () => { + const three = renderReplicatedStorageClass(3) as { + parameters: Record; + }; + expect(three.parameters['linstor.csi.linbit.com/placementCount']).toBe( + '3', + ); + }); + + it('keeps each copy, and the tie-breaker, in a location of its own', () => { + expect(sc.parameters['linstor.csi.linbit.com/replicasOnDifferent']).toBe( + 'Aux/topology/topology.kubernetes.io/zone', + ); + }); + }); + + it('runs satellites on the host network and keeps data on prepared nodes only', () => { + const [network, pool] = renderSatelliteConfigurations() as Array<{ + spec: Record; + }>; + expect(network.spec.nodeSelector).toBeUndefined(); + expect(network.spec.podTemplate.spec.hostNetwork).toBe(true); + expect(network.spec.podTemplate.metadata.annotations).toEqual({ + 'prometheus.io/scrape': 'true', + 'prometheus.io/port': '9942', + 'prometheus.io/path': '/metrics', + }); + expect(pool.spec.nodeSelector).toEqual({ + 'flui.cloud/replicated-storage': 'true', + }); + expect(pool.spec.storagePools[0]).toEqual({ + name: 'flui-replicated', + lvmPool: { volumeGroup: 'flui-replicated' }, + }); + }); + + it('leaves the NFS server of the engine off', () => { + const cluster = renderLinstorCluster() as { spec: Record }; + expect(cluster.spec.nfsServer).toEqual({ enabled: false }); + expect(cluster.spec.highAvailabilityController).toEqual({ enabled: true }); + }); + + it('keeps the single-copy engine components off the nodes that hold data', () => { + const cluster = renderLinstorCluster(['hel1']) as { + spec: Record; + }; + for (const part of ['controller', 'csiController', 'affinityController']) { + const spec = cluster.spec[part].podTemplate.spec; + expect( + spec.affinity.nodeAffinity + .requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0] + .matchExpressions, + ).toContainEqual({ + key: 'flui.cloud/replicated-storage', + operator: 'DoesNotExist', + }); + expect(spec.tolerations).toEqual([ + { operator: 'Exists', effect: 'NoSchedule' }, + ]); + } + }); + + describe('replicatedPoolHostScript', () => { + const script = replicatedPoolHostScript(20); + + it('builds a plain volume group on a direct-I/O loop with 4K sectors', () => { + expect(script).toContain( + 'losetup --find --show --direct-io=on --sector-size 4096', + ); + expect(script).toContain('vgcreate -q "$VG"'); + expect(script).not.toContain('thin'); + expect(script).toContain('fallocate -l 20G'); + }); + + it('brings the pool back at boot before K3s starts', () => { + expect(script).toContain('Before=k3s.service k3s-agent.service'); + expect(script).toContain('systemctl enable flui-replicated-pool.service'); + }); + + it('refuses a size that is not a whole number of GiB', () => { + expect(() => replicatedPoolHostScript(0)).toThrow(); + expect(() => replicatedPoolHostScript(1.5)).toThrow(); + }); + }); + + describe('parseReplicatedPoolOutput', () => { + it('reads a ready pool', () => { + expect( + parseReplicatedPoolOutput('x\nPOOL flui-replicated 20.00\n'), + ).toEqual({ kind: 'ready', sizeGiB: 20 }); + }); + + it('reads the two refusals', () => { + expect(parseReplicatedPoolOutput('NOHEADERS 6.8.0-1-generic')).toEqual({ + kind: 'no-headers', + kernel: '6.8.0-1-generic', + }); + expect(parseReplicatedPoolOutput('NOSPACE 12')).toEqual({ + kind: 'no-space', + freeGiB: 12, + }); + }); + + it('keeps anything else as unknown', () => { + expect(parseReplicatedPoolOutput('boom').kind).toBe('unknown'); + }); + }); + + describe('replicatedVolumesFeasibility', () => { + const lab = [ + node('master', 'fsn1'), + node('w1', 'nbg1'), + node('w2', 'fsn1'), + node('w3', 'hel1'), + ]; + + it('puts one copy in each location for data and the vote in the third', () => { + expect(replicatedVolumesFeasibility(lab, hetzner)).toEqual({ + feasible: true, + zones: ['fsn1', 'nbg1'], + storageNodes: ['master', 'w1'], + tieBreakers: ['w3'], + }); + }); + + it('keeps nodes already prepared when they still fit', () => { + expect( + replicatedVolumesFeasibility(lab, hetzner, { prepared: ['w1', 'w2'] }), + ).toMatchObject({ storageNodes: ['w1', 'w2'] }); + }); + + it('never puts a copy in the voting location', () => { + const r = replicatedVolumesFeasibility(lab, hetzner, { + requested: ['w1', 'w3'], + }); + expect((r as { reason: string }).reason).toContain('fsn1 or nbg1'); + }); + + it('refuses two copies in one location', () => { + const r = replicatedVolumesFeasibility(lab, hetzner, { + requested: ['master', 'w2'], + }); + expect((r as { reason: string }).reason).toBe( + 'Name 2 nodes, one in each location for data: fsn1 and nbg1.', + ); + }); + + it('refuses a location for data with no node that takes applications', () => { + const r = replicatedVolumesFeasibility( + [node('m', 'fsn1'), node('w1', 'nbg1', false), node('w3', 'hel1')], + hetzner, + ); + expect((r as { reason: string }).reason).toContain('fsn1 and nbg1'); + }); + + it('refuses without a node in the third location', () => { + const r = replicatedVolumesFeasibility( + [node('m', 'fsn1'), node('w1', 'nbg1')], + hetzner, + ); + expect((r as { reason: string }).reason).toContain('third location'); + }); + + it('names a node that is not there', () => { + const r = replicatedVolumesFeasibility(lab, hetzner, { + requested: ['w1', 'nope'], + }); + expect((r as { reason: string }).reason).toContain('nope'); + }); + + describe('where every location holds data', () => { + const milan: HaTopology = { + region: 'eu-south', + zones: ['a', 'b', 'c'], + dataZones: ['a', 'b', 'c'], + validated: true, + measured: true, + }; + const servers = [node('s1', 'a'), node('s2', 'b'), node('s3', 'c')]; + + it('puts a copy in each of the three, with no vote to cast', () => { + expect(replicatedVolumesFeasibility(servers, milan)).toEqual({ + feasible: true, + zones: ['a', 'b', 'c'], + storageNodes: ['s1', 's2', 's3'], + tieBreakers: [], + }); + }); + + it('refuses when one of them has no node that takes applications', () => { + const r = replicatedVolumesFeasibility( + [node('s1', 'a'), node('s2', 'b'), node('s3', 'c', false)], + milan, + ); + expect((r as { reason: string }).reason).toBe( + 'Replicated volumes need a node that takes applications in a, b and c.', + ); + }); + + it('refuses naming fewer nodes than locations', () => { + const r = replicatedVolumesFeasibility(servers, milan, { + requested: ['s1', 's2'], + }); + expect((r as { reason: string }).reason).toBe( + 'Name 3 nodes, one in each location for data: a, b and c.', + ); + }); + }); + }); + + describe('volumesBlockingFailover', () => { + it('is empty for an app without replicated volumes', () => { + expect( + volumesBlockingFailover([{ name: 'data', storageClass: 'flui-local' }]), + ).toEqual([]); + }); + + it('names the volumes of another kind next to a replicated one', () => { + expect( + volumesBlockingFailover([ + { name: 'data', storageClass: 'flui-replicated' }, + { name: 'cache', storageClass: 'flui-local' }, + { name: 'uploads' }, + ]), + ).toEqual(['cache', 'uploads']); + }); + }); + + it('finds the single-copy engine components that sit on a node with copies', () => { + const pod = (name: string, nodeName: string) => ({ + metadata: { name }, + spec: { nodeName }, + }); + expect( + controlPodsOnCopies( + [ + pod('linstor-controller-694bf66d64-f9x9n', 'w1'), + pod('linstor-csi-controller-ccdfbd4c6-smtrl', 'w3'), + pod('linstor-affinity-controller-c95456f64-5l87p', 'w1'), + pod('linstor-satellite.w1-abcde', 'w1'), + pod('ha-controller-xdj2b', 'w1'), + ], + ['w1', 'w2'], + ), + ).toEqual([ + 'linstor-controller-694bf66d64-f9x9n', + 'linstor-affinity-controller-c95456f64-5l87p', + ]); + }); +}); + +describe('storage node warnings', () => { + it('names a node with copies that takes no applications', () => { + const nodes = [ + { name: 'master', zone: 'fsn1', schedulable: false }, + { name: 'w1', zone: 'fsn1', schedulable: true }, + { name: 'w2', zone: 'fsn1', schedulable: true }, + ]; + expect(storageNodeWarnings(nodes, ['w1', 'w2'])).toEqual([]); + expect(storageNodeWarnings(nodes, ['master', 'w2'])).toEqual([ + expect.stringContaining('master holds copies but takes no applications'), + ]); + }); +}); + +describe('copies that are not up to date', () => { + const pv = 'pvc-a044c935'; + const resource = (node: string, disk: string, inUse = false) => ({ + name: pv, + node_name: node, + state: { in_use: inUse }, + volumes: [{ volume_number: 0, state: { disk_state: disk } }], + }); + + it('reads each copy from LINSTOR, the way the lab printed it', () => { + const json = JSON.stringify([ + [ + resource('w1', 'UpToDate'), + resource('w2', 'Inconsistent', true), + resource('w3', 'Diskless'), + ], + ]); + expect(volumeCopiesFrom(json)).toEqual([ + { volume: pv, node: 'w1', disk: 'UpToDate', inUse: false }, + { volume: pv, node: 'w2', disk: 'Inconsistent', inUse: true }, + { volume: pv, node: 'w3', disk: 'Diskless', inUse: false }, + ]); + }); + + it('says which copy is behind, which loss would stop the volume, and where the application runs', () => { + const copies = volumeCopiesFrom( + JSON.stringify([ + [ + resource('w1', 'UpToDate'), + resource('w2', 'Inconsistent', true), + resource('w3', 'Diskless'), + ], + ]), + ); + expect( + staleCopyWarnings(copies, new Map([[pv, 'p-personal/data-writer']])), + ).toEqual([ + 'The copy of p-personal/data-writer on w2 is not up to date (Inconsistent): until it is, losing w1 stops the volume. The application runs on this copy.', + ]); + expect( + staleCopyWarnings( + copies.map((c) => ({ + ...c, + disk: c.node === 'w3' ? 'Diskless' : 'UpToDate', + })), + new Map(), + ), + ).toEqual([]); + }); +}); + +describe('where the engine runs its single-copy parts', () => { + const topology: HaTopology = { + region: 'eu-central', + zones: ['fsn1', 'nbg1', 'hel1'], + dataZones: ['fsn1', 'nbg1'], + validated: true, + measured: true, + }; + + it('keeps them in the voting location, away from copies', () => { + expect(voteZonesOf(topology)).toEqual(['hel1']); + const spec = renderLinstorCluster(['hel1']).spec as any; + for (const part of ['controller', 'csiController', 'affinityController']) { + expect( + spec[part].podTemplate.spec.affinity.nodeAffinity + .requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0] + .matchExpressions, + ).toEqual([ + { key: 'flui.cloud/replicated-storage', operator: 'DoesNotExist' }, + { + key: 'topology.kubernetes.io/zone', + operator: 'In', + values: ['hel1'], + }, + ]); + } + }); + + it('only prefers a node without copies where every location holds data', () => { + expect( + voteZonesOf({ + ...topology, + dataZones: topology.zones, + validated: false, + measured: false, + }), + ).toEqual([]); + const spec = renderLinstorCluster([]).spec as any; + for (const part of ['controller', 'csiController', 'affinityController']) { + const affinity = spec[part].podTemplate.spec.affinity.nodeAffinity; + expect( + affinity.requiredDuringSchedulingIgnoredDuringExecution, + ).toBeUndefined(); + expect(affinity.preferredDuringSchedulingIgnoredDuringExecution).toEqual([ + { + weight: 100, + preference: { + matchExpressions: [ + { + key: 'flui.cloud/replicated-storage', + operator: 'DoesNotExist', + }, + ], + }, + }, + ]); + } + }); +}); + +describe('the space for copies on a node', () => { + it('is allocated on the disk for real, never left sparse', () => { + const script = replicatedPoolHostScript(20); + expect(script).toContain('fallocate -l 20G "$FILE"'); + expect(script).not.toContain('truncate -s'); + }); + + it('allocates a pool made sparse before, without touching what it holds', () => { + const script = replicatedPoolHostScript(20); + expect(script).toContain( + 'fallocate -l "$(stat -c %s "$FILE")" "$FILE" || echo "NOALLOC"', + ); + }); + + it('says so when the space could not be set aside', () => { + expect( + parseReplicatedPoolOutput('NOALLOC\nPOOL flui-replicated 20'), + ).toEqual({ kind: 'no-alloc' }); + }); +}); + +describe('satellites that wait out a crash backoff', () => { + const pod = (name: string, node: string, waiting?: string) => ({ + metadata: { name }, + spec: { nodeName: node }, + status: { + initContainerStatuses: [ + { state: waiting ? { waiting: { reason: waiting } } : {} }, + ], + }, + }); + + it('names only the ones backing off on the nodes just prepared, never one still starting', () => { + expect( + stuckSatellites( + [ + pod('linstor-satellite.hel1-abc', 'w2', 'CrashLoopBackOff'), + pod('linstor-satellite.nbg1-xyz', 'w1', 'PodInitializing'), + pod('linstor-satellite.fsn1-def', 'master'), + pod('linstor-satellite.other-ghi', 'w9', 'CrashLoopBackOff'), + ], + ['master', 'w1', 'w2'], + ), + ).toEqual(['linstor-satellite.hel1-abc']); + }); +}); diff --git a/src/modules/replicated-volumes/utils/replicated-volumes.core.ts b/src/modules/replicated-volumes/utils/replicated-volumes.core.ts new file mode 100644 index 00000000..2ef0044f --- /dev/null +++ b/src/modules/replicated-volumes/utils/replicated-volumes.core.ts @@ -0,0 +1,500 @@ +import { HaCandidateNode } from '../../postgres-ha/utils/postgres-ha-feasibility.core'; +import { HEADERS_INSTALL } from './replicated-votes.core'; +import { HaTopology, locationList } from '../../providers/core/ha-topology'; +import { + PIRAEUS_API_VERSION, + REPLICATED_POOL, + REPLICATED_POOL_FILE, + REPLICATED_STORAGE_CLASS, + REPLICATED_STORAGE_NODE_LABEL, + REPLICATED_VOLUME_GROUP, +} from '../replicated-volumes.constants'; +import { TOPOLOGY_ZONE_LABEL } from '../../providers/core/node-topology'; + +const DRBD_OPTION = 'property.linstor.csi.linbit.com/DrbdOptions'; + +const OFF_COPIES = { + key: REPLICATED_STORAGE_NODE_LABEL, + operator: 'DoesNotExist', +}; + +/** + * The components that attach volumes and keep the engine's records run as a + * single copy; lost with a location that holds data, no volume could move + * until Kubernetes moved them minutes later. Where the layout has a voting + * location they stay there, off the nodes with copies: losing that location + * never asks a volume to move. Where every location holds data no such place + * exists, so they only prefer a node without copies. + */ +const awayFromCopies = (voteZones: string[]) => ({ + podTemplate: { + spec: { + affinity: { + nodeAffinity: voteZones.length + ? { + requiredDuringSchedulingIgnoredDuringExecution: { + nodeSelectorTerms: [ + { + matchExpressions: [ + OFF_COPIES, + { + key: TOPOLOGY_ZONE_LABEL, + operator: 'In', + values: voteZones, + }, + ], + }, + ], + }, + } + : { + preferredDuringSchedulingIgnoredDuringExecution: [ + { + weight: 100, + preference: { matchExpressions: [OFF_COPIES] }, + }, + ], + }, + }, + tolerations: [{ operator: 'Exists', effect: 'NoSchedule' }], + }, + }, +}); + +/** The locations that only vote; none where every location holds data. */ +export function voteZonesOf(topology: HaTopology): string[] { + return topology.zones.filter((z) => !topology.dataZones.includes(z)); +} + +/** + * The volume engine with the parts Flui does not use switched off: the NFS + * server on top of replicated volumes stays out until something asks for it. + */ +export function renderLinstorCluster( + voteZones: string[] = [], +): Record { + return { + apiVersion: PIRAEUS_API_VERSION, + kind: 'LinstorCluster', + metadata: { name: 'linstorcluster' }, + spec: { + nfsServer: { enabled: false }, + highAvailabilityController: { enabled: true }, + controller: awayFromCopies(voteZones), + csiController: awayFromCopies(voteZones), + affinityController: awayFromCopies(voteZones), + }, + }; +} + +/** + * Two satellite settings. Every node runs on the host network, so copies + * travel on the nodes' own network and not the slower pod network, and any + * node can cast the tie-breaking vote; its DRBD metrics are read like any + * other pod's, for the alerts on copies. Only the nodes Flui prepared hold data. + */ +export function renderSatelliteConfigurations(): Record[] { + return [ + { + apiVersion: PIRAEUS_API_VERSION, + kind: 'LinstorSatelliteConfiguration', + metadata: { name: 'flui-host-network' }, + spec: { + podTemplate: { + metadata: { + annotations: { + 'prometheus.io/scrape': 'true', + 'prometheus.io/port': '9942', + 'prometheus.io/path': '/metrics', + }, + }, + spec: { hostNetwork: true }, + }, + }, + }, + { + apiVersion: PIRAEUS_API_VERSION, + kind: 'LinstorSatelliteConfiguration', + metadata: { name: 'flui-replicated-pool' }, + spec: { + nodeSelector: { [REPLICATED_STORAGE_NODE_LABEL]: 'true' }, + storagePools: [ + { + name: REPLICATED_POOL, + lvmPool: { volumeGroup: REPLICATED_VOLUME_GROUP }, + }, + ], + }, + }, + ]; +} + +/** + * One synchronous copy per location for data, only on prepared nodes, and a + * pod only where a copy is. Quorum is asked for explicitly: without it a node + * cut off from the others keeps confirming writes only it holds, and nothing + * moves the pod. With two copies the rule on locations also holds for the + * tie-breaker, which needs no storage pool, so it lands in the third location; + * three copies are a majority on their own. + */ +export function renderReplicatedStorageClass( + copies = 2, +): Record { + return { + apiVersion: 'storage.k8s.io/v1', + kind: 'StorageClass', + metadata: { + name: REPLICATED_STORAGE_CLASS, + labels: { 'flui.cloud/managed-by': 'flui-cloud' }, + }, + provisioner: 'linstor.csi.linbit.com', + allowVolumeExpansion: true, + volumeBindingMode: 'WaitForFirstConsumer', + reclaimPolicy: 'Delete', + parameters: { + 'linstor.csi.linbit.com/storagePool': REPLICATED_POOL, + 'linstor.csi.linbit.com/placementCount': String(copies), + 'linstor.csi.linbit.com/allowRemoteVolumeAccess': 'false', + 'linstor.csi.linbit.com/replicasOnDifferent': + 'Aux/topology/topology.kubernetes.io/zone', + 'csi.storage.k8s.io/fstype': 'ext4', + [`${DRBD_OPTION}/auto-quorum`]: 'suspend-io', + [`${DRBD_OPTION}/Resource/on-no-data-accessible`]: 'suspend-io', + [`${DRBD_OPTION}/Resource/on-suspended-primary-outdated`]: + 'force-secondary', + [`${DRBD_OPTION}/Net/rr-conflict`]: 'retry-connect', + }, + }; +} + +/** + * Prepares one node to hold copies: kernel headers for the DRBD module, + * a plain (not thin) volume group on a direct-I/O loop file with 4K sectors, + * and a unit that brings the loop back at boot. Thin pools double the cost of + * a copy; a new volume is copied whole once, so the file is allocated on the + * disk for real, not left sparse. + */ +export function replicatedPoolHostScript(sizeGiB: number): string { + if (!Number.isInteger(sizeGiB) || sizeGiB < 1) { + throw new Error(`invalid pool size ${sizeGiB}`); + } + const dir = REPLICATED_POOL_FILE.slice( + 0, + REPLICATED_POOL_FILE.lastIndexOf('/'), + ); + return [ + 'set -e', + `FILE=${REPLICATED_POOL_FILE}`, + `VG=${REPLICATED_VOLUME_GROUP}`, + 'export DEBIAN_FRONTEND=noninteractive', + 'KREL=$(uname -r)', + ...HEADERS_INSTALL, + 'if [ ! -d "/lib/modules/$KREL/build" ]; then echo "NOHEADERS $KREL"; exit 0; fi', + 'command -v vgcreate >/dev/null 2>&1 || apt-get install -y -qq lvm2 >/dev/null 2>&1 || true', + 'if ! vgs "$VG" >/dev/null 2>&1; then', + ` mkdir -p ${dir}`, + ' if [ ! -f "$FILE" ]; then', + ` FREE=$(df -B1G --output=avail ${dir} | tail -1 | tr -d ' ')`, + ` if [ "$FREE" -lt $(( ${sizeGiB} + 5 )) ]; then echo "NOSPACE $FREE"; exit 0; fi`, + ` fallocate -l ${sizeGiB}G "$FILE" || { rm -f "$FILE"; echo "NOALLOC"; exit 0; }`, + ' fi', + ' DEV=$(losetup -j "$FILE" -O NAME -n | head -1)', + ' [ -n "$DEV" ] || DEV=$(losetup --find --show --direct-io=on --sector-size 4096 "$FILE")', + ' pvcreate -q "$DEV" >/dev/null', + ' vgcreate -q "$VG" "$DEV" >/dev/null', + 'fi', + // A pool made before the file was allocated is sparse: its blocks are + // taken only as copies are written, and a disk filled by anything else + // turns those writes into I/O errors. Allocating it is safe on a live + // loop device and changes nothing already written. + 'if [ -f "$FILE" ] && [ "$(du -B1 "$FILE" | cut -f1)" -lt "$(stat -c %s "$FILE")" ]; then', + ' fallocate -l "$(stat -c %s "$FILE")" "$FILE" || echo "NOALLOC"', + 'fi', + "cat > /usr/local/sbin/flui-replicated-pool <<'FLUIHELPER'", + '#!/bin/sh', + `f=${REPLICATED_POOL_FILE}`, + '[ -f "$f" ] || exit 0', + 'dev=$(losetup -j "$f" -O NAME -n | head -1)', + '[ -n "$dev" ] || losetup --find --direct-io=on --sector-size 4096 "$f"', + `exec vgchange -q -ay ${REPLICATED_VOLUME_GROUP}`, + 'FLUIHELPER', + 'chmod 755 /usr/local/sbin/flui-replicated-pool', + "cat > /etc/systemd/system/flui-replicated-pool.service <<'FLUIUNIT'", + '[Unit]', + 'Description=Loop device and volume group of Flui replicated volumes', + `RequiresMountsFor=${dir}`, + 'Before=k3s.service k3s-agent.service', + '', + '[Service]', + 'Type=oneshot', + 'ExecStart=/usr/local/sbin/flui-replicated-pool', + 'RemainAfterExit=yes', + '', + '[Install]', + 'WantedBy=multi-user.target', + 'FLUIUNIT', + 'systemctl daemon-reload >/dev/null 2>&1 || true', + 'systemctl enable flui-replicated-pool.service >/dev/null 2>&1 || true', + 'echo "POOL $VG $(vgs --noheadings --units g --nosuffix -o vg_size "$VG" | tr -d \' \')"', + ].join('\n'); +} + +export type ReplicatedPoolState = + | { kind: 'ready'; sizeGiB: number } + | { kind: 'no-headers'; kernel: string } + | { kind: 'no-space'; freeGiB: number } + | { kind: 'no-alloc' } + | { kind: 'unknown'; detail: string }; + +export function parseReplicatedPoolOutput(logs: string): ReplicatedPoolState { + for (const raw of logs.split('\n')) { + const line = raw.trim(); + const pool = /^POOL \S+ (\d+(?:\.\d+)?)$/.exec(line); + if (pool) return { kind: 'ready', sizeGiB: Math.floor(Number(pool[1])) }; + if (/^HEADERS \S+$/.test(line)) return { kind: 'ready', sizeGiB: 0 }; + const headers = /^NOHEADERS (\S+)$/.exec(line); + if (headers) return { kind: 'no-headers', kernel: headers[1] }; + const space = /^NOSPACE (\d+)$/.exec(line); + if (space) return { kind: 'no-space', freeGiB: Number(space[1]) }; + if (line === 'NOALLOC') return { kind: 'no-alloc' }; + } + return { kind: 'unknown', detail: logs.trim().slice(-200) }; +} + +export type ReplicatedVolumesFeasibility = + | { + feasible: true; + zones: string[]; + storageNodes: string[]; + tieBreakers: string[]; + } + | { feasible: false; reason: string }; + +/** + * Which nodes hold the copies: one that takes applications in each of the + * layout's locations for data and, with only two of them, the nodes of the + * remaining location for the tie-breaking vote. Nodes already prepared are + * kept when they still fit, so enabling again does not move copies around. + */ +export function replicatedVolumesFeasibility( + nodes: HaCandidateNode[], + topology: HaTopology, + options: { requested?: string[]; prepared?: string[] } = {}, +): ReplicatedVolumesFeasibility { + const copies = topology.dataZones.length; + const fits = (n: HaCandidateNode) => + n.schedulable && !!n.zone && topology.dataZones.includes(n.zone); + let storage: HaCandidateNode[]; + if (options.requested?.length) { + const unknown = options.requested.filter( + (r) => !nodes.some((n) => n.name === r), + ); + if (unknown.length) { + return { + feasible: false, + reason: `No node named ${unknown.join(', ')} in this cluster.`, + }; + } + storage = nodes.filter((n) => options.requested!.includes(n.name)); + const blocked = storage.filter((n) => !fits(n)); + if (blocked.length) { + return { + feasible: false, + reason: `${blocked.map((n) => n.name).join(', ')} cannot hold copies: a node that holds one must take applications and sit in ${topology.dataZones.join(' or ')}.`, + }; + } + if ( + new Set(storage.map((n) => n.zone)).size !== copies || + storage.length !== copies + ) { + return { + feasible: false, + reason: `Name ${copies} nodes, one in each location for data: ${locationList(topology.dataZones)}.`, + }; + } + } else { + const prepared = new Set(options.prepared ?? []); + const pick = (zone: string) => + nodes + .filter((n) => fits(n) && n.zone === zone) + .sort( + (a, b) => + Number(prepared.has(b.name)) - Number(prepared.has(a.name)) || + a.name.localeCompare(b.name), + )[0]; + storage = topology.dataZones + .map(pick) + .filter((n): n is HaCandidateNode => !!n); + if (storage.length < copies) { + return { + feasible: false, + reason: `Replicated volumes need a node that takes applications in ${locationList(topology.dataZones)}.`, + }; + } + } + const zones = storage.map((n) => n.zone!).sort((a, b) => a.localeCompare(b)); + const storageNames = storage + .map((n) => n.name) + .sort((a, b) => a.localeCompare(b)); + const tieBreakers = + copies < 3 + ? nodes + .filter((n) => !!n.zone && !zones.includes(n.zone)) + .map((n) => n.name) + : []; + if (copies < 3 && !tieBreakers.length) { + return { + feasible: false, + reason: `Copies in ${zones.join(' and ')} need a node in a third location for the tie-breaking vote.`, + }; + } + return { feasible: true, zones, storageNodes: storageNames, tieBreakers }; +} + +const CONTROL_COMPONENTS = [ + 'linstor-controller', + 'linstor-csi-controller', + 'linstor-affinity-controller', +]; + +/** + * The engine's single-copy components that sit on a node holding copies. The + * placement rule only applies when a pod is placed, so a node prepared after + * the engine was installed keeps whatever already ran there. + */ +export function controlPodsOnCopies( + pods: Array<{ metadata?: { name?: string }; spec?: { nodeName?: string } }>, + storageNodes: string[], +): string[] { + return pods + .filter((p) => { + const name = p.metadata?.name ?? ''; + return ( + CONTROL_COMPONENTS.some((c) => name.startsWith(`${c}-`)) && + storageNodes.includes(p.spec?.nodeName ?? '') + ); + }) + .map((p) => p.metadata!.name!); +} + +export const SATELLITE_SELECTOR = + 'app.kubernetes.io/component=linstor-satellite'; + +type ContainerStates = Array<{ + state?: { waiting?: { reason?: string } }; +}>; + +/** + * Satellites on the nodes just prepared that wait out a crash backoff: one + * that failed before its node had kernel headers tries again only after up to + * five minutes. One still starting, or serving volumes, is left alone. + */ +export function stuckSatellites( + pods: Array<{ + metadata?: { name?: string }; + spec?: { nodeName?: string }; + status?: { + initContainerStatuses?: ContainerStates; + containerStatuses?: ContainerStates; + }; + }>, + nodes: string[], +): string[] { + const backingOff = (states?: ContainerStates) => + (states ?? []).some((c) => c.state?.waiting?.reason === 'CrashLoopBackOff'); + return pods + .filter((p) => nodes.includes(p.spec?.nodeName ?? '')) + .filter( + (p) => + backingOff(p.status?.initContainerStatuses) || + backingOff(p.status?.containerStatuses), + ) + .map((p) => p.metadata?.name ?? '') + .filter(Boolean); +} + +export interface VolumeClassLike { + name: string; + storageClass?: string | null; +} + +/** + * A pod moves to the other copy only when every volume it mounts can follow + * it; one volume of another kind keeps it on the lost node. Returns the + * volumes that would hold it back, empty when the app is fine or uses no + * replicated volume at all. + */ +export function volumesBlockingFailover(volumes: VolumeClassLike[]): string[] { + const replicated = volumes.some( + (v) => v.storageClass === REPLICATED_STORAGE_CLASS, + ); + if (!replicated) return []; + return volumes + .filter((v) => v.storageClass !== REPLICATED_STORAGE_CLASS) + .map((v) => v.name); +} + +/** + * Copies on a node that takes no applications cannot be followed: when the + * other node is lost, the application has nowhere to start with its data. + */ +export function storageNodeWarnings( + nodes: HaCandidateNode[], + storageNodes: string[], +): string[] { + return storageNodes + .filter((name) => nodes.find((n) => n.name === name)?.schedulable === false) + .map( + (name) => + `${name} holds copies but takes no applications, so an application cannot move to it. Keep copies on nodes that run applications.`, + ); +} + +export interface VolumeCopyState { + volume: string; + node: string; + disk: string; + inUse: boolean; +} + +/** `linstor -m --output-version v1 resource list`: one list of resources, one per volume and node. */ +export function volumeCopiesFrom(linstorJson: string): VolumeCopyState[] { + const parsed = JSON.parse(linstorJson) as unknown[]; + const resources = (Array.isArray(parsed[0]) ? parsed[0] : parsed) as Array<{ + name: string; + node_name: string; + state?: { in_use?: boolean }; + volumes?: Array<{ state?: { disk_state?: string } }>; + }>; + return resources.map((r) => ({ + volume: r.name, + node: r.node_name, + disk: r.volumes?.[0]?.state?.disk_state ?? 'Unknown', + inUse: r.state?.in_use === true, + })); +} + +/** + * A copy that is not up to date leaves the volume on one real copy: the next + * loss stops it. DRBD 9.3.4 can leave a copy in that state with its data + * already complete after a short loss of the node the application ran on + * (LINBIT/drbd#153); bringing it back means stopping the application for a + * moment, so Flui says so instead of doing it. + */ +export function staleCopyWarnings( + copies: VolumeCopyState[], + claimOf: Map, +): string[] { + return copies + .filter((c) => c.disk !== 'UpToDate' && c.disk !== 'Diskless') + .map((c) => { + const others = copies + .filter( + (o) => + o.volume === c.volume && o.node !== c.node && o.disk === 'UpToDate', + ) + .map((o) => o.node); + const name = claimOf.get(c.volume) ?? c.volume; + return `The copy of ${name} on ${c.node} is not up to date (${c.disk}): until it is, losing ${others.join(' or ') || 'the other copy'} stops the volume.${c.inUse ? ' The application runs on this copy.' : ''}`; + }); +} diff --git a/src/modules/replicated-volumes/utils/replicated-votes.core.spec.ts b/src/modules/replicated-volumes/utils/replicated-votes.core.spec.ts new file mode 100644 index 00000000..d5bbedf2 --- /dev/null +++ b/src/modules/replicated-volumes/utils/replicated-votes.core.spec.ts @@ -0,0 +1,64 @@ +import { parseReplicatedPoolOutput } from './replicated-volumes.core'; +import { + missingVoteWarnings, + replicatedVoteHostScript, +} from './replicated-votes.core'; + +describe('replicated-votes.core', () => { + describe('missingVoteWarnings', () => { + const nodes = (list: object[]) => JSON.stringify([list]); + + it('names a tie-breaker the volume engine does not list', () => { + const warnings = missingVoteWarnings( + ['ha-hz-worker-2'], + nodes([ + { name: 'ha-hz-master', connection_status: 'ONLINE' }, + { name: 'ha-hz-worker-1', connection_status: 'ONLINE' }, + ]), + ); + expect(warnings).toEqual([ + expect.stringMatching( + /^ha-hz-worker-2 does not hold the tie-breaking vote yet/, + ), + ]); + }); + + it('names one listed but not online, and says nothing for one that is', () => { + expect( + missingVoteWarnings( + ['a', 'b'], + nodes([ + { name: 'a', connection_status: 'OFFLINE' }, + { name: 'b', connection_status: 'ONLINE' }, + ]), + ), + ).toHaveLength(1); + }); + + it('says nothing when it cannot read the answer', () => { + expect(missingVoteWarnings(['a'], 'not json')).toEqual([]); + }); + }); + + describe('replicatedVoteHostScript', () => { + it('installs the headers the vote needs, and nothing that holds data', () => { + const script = replicatedVoteHostScript(); + expect(script).toContain('linux-headers-$KREL'); + expect(script).toContain('KFLAVOUR="linux-headers-${KREL##*-}"'); + expect(script).toContain('! dpkg -s "$KFLAVOUR"'); + expect(script).toContain('echo "HEADERS $KREL"'); + expect(script).not.toMatch(/vgcreate|fallocate|losetup/); + }); + + it('is read as ready, or as missing headers', () => { + expect(parseReplicatedPoolOutput('HEADERS 6.8.0-138-generic')).toEqual({ + kind: 'ready', + sizeGiB: 0, + }); + expect(parseReplicatedPoolOutput('NOHEADERS 6.8.0-138-generic')).toEqual({ + kind: 'no-headers', + kernel: '6.8.0-138-generic', + }); + }); + }); +}); diff --git a/src/modules/replicated-volumes/utils/replicated-votes.core.ts b/src/modules/replicated-volumes/utils/replicated-votes.core.ts new file mode 100644 index 00000000..6f0d0d93 --- /dev/null +++ b/src/modules/replicated-volumes/utils/replicated-votes.core.ts @@ -0,0 +1,59 @@ +/** + * Installs the kernel headers the volume engine builds its module from, and + * the package that follows the kernel's flavour (`linux-headers-generic`), so + * the next kernel an upgrade boots into comes with its headers too. + */ +export const HEADERS_INSTALL = [ + 'KFLAVOUR="linux-headers-${KREL##*-}"', + 'if [ ! -d "/lib/modules/$KREL/build" ] || ! dpkg -s "$KFLAVOUR" >/dev/null 2>&1; then', + ' apt-get update -qq >/dev/null 2>&1 || true', + ' [ -d "/lib/modules/$KREL/build" ] || apt-get install -y -qq "linux-headers-$KREL" >/dev/null 2>&1 || true', + ' apt-get install -y -qq "$KFLAVOUR" >/dev/null 2>&1 || true', + 'fi', +]; + +/** + * Prepares a node that only votes: it holds no data, but the volume engine + * still loads its module there to count as the tie-breaker, and builds it from + * the kernel headers. Without them the vote never joins and a lost copy leaves + * the other one without a majority. + */ +export function replicatedVoteHostScript(): string { + return [ + 'set -e', + 'export DEBIAN_FRONTEND=noninteractive', + 'KREL=$(uname -r)', + ...HEADERS_INSTALL, + 'if [ ! -d "/lib/modules/$KREL/build" ]; then echo "NOHEADERS $KREL"; exit 0; fi', + 'echo "HEADERS $KREL"', + ].join('\n'); +} + +/** + * The tie-breakers the volume engine does not count. A node missing from its + * list, or not online there, never casts the deciding vote, so losing a copy + * leaves the other one without a majority and the application where it was. + * Unreadable output says nothing. + */ +export function missingVoteWarnings( + tieBreakers: string[], + linstorNodesJson: string, +): string[] { + let nodes: Array<{ name?: string; connection_status?: string }>; + try { + const parsed = JSON.parse(linstorNodesJson) as unknown[]; + nodes = (Array.isArray(parsed[0]) ? parsed[0] : parsed) as typeof nodes; + if (!Array.isArray(nodes)) return []; + } catch { + return []; + } + return tieBreakers + .filter((name) => { + const node = nodes.find((n) => n.name === name); + return !node || (node.connection_status ?? 'ONLINE') !== 'ONLINE'; + }) + .map( + (name) => + `${name} does not hold the tie-breaking vote yet: the volume engine is not running there, so losing a location with a copy leaves the other copy without a majority and applications stay where they are.`, + ); +} diff --git a/src/modules/sandbox/stand-in/sandbox-world-cluster.ts b/src/modules/sandbox/stand-in/sandbox-world-cluster.ts index da209709..9da15870 100644 --- a/src/modules/sandbox/stand-in/sandbox-world-cluster.ts +++ b/src/modules/sandbox/stand-in/sandbox-world-cluster.ts @@ -17,6 +17,7 @@ import { FLUI_SHARED_STORAGE_PATH, FLUI_SHARED_VOLUME_FS_LABEL, NFS_EXPORT_OPTIONS, + nfsDurability, NFS_MOUNT_OPTIONS, } from '../../infrastructure/clusters/constants/storage-conventions'; import { @@ -75,6 +76,7 @@ export function exampleClusterStorage( exportPath: FLUI_SHARED_STORAGE_PATH, exportOptions: NFS_EXPORT_OPTIONS, mountOptions: NFS_MOUNT_OPTIONS, + ...nfsDurability(NFS_EXPORT_OPTIONS), }), pvcs: mark({ bound: CLAIMS.length,