From 49a7ecfd00133b17ace1edaa1203f1f2acd77e92 Mon Sep 17 00:00:00 2001 From: Deep-Axe Date: Mon, 26 Oct 2026 17:45:38 +0530 Subject: [PATCH 1/4] Add per-store manifests and a namespace-guarded Kubernetes client store-manifests renders everything one store needs in its own namespace: quota and limit range, default-deny policies that admit only the gateway to the store's MCP server, MariaDB, WordPress with its Ingress, the bootstrap Job named by its spec, and the Woo MCP server. One function derives every host, URL and MCP origin from the slug. StoreCluster applies objects with server-side apply and refuses any namespace outside storelink-store-*, and deletes only namespaces labelled as managed. The upstream policy can approve exactly the provisioned store MCP origin form. The ServiceAccount HTTPS code is shared with the credential store. --- services/gateway/src/credentials.ts | 65 +-- services/gateway/src/k8s-client.ts | 181 ++++++++ services/gateway/src/store-manifests.ts | 544 ++++++++++++++++++++++++ services/gateway/src/upstream-policy.ts | 25 +- tests/stores.test.ts | 220 ++++++++++ 5 files changed, 982 insertions(+), 53 deletions(-) create mode 100644 services/gateway/src/k8s-client.ts create mode 100644 services/gateway/src/store-manifests.ts create mode 100644 tests/stores.test.ts diff --git a/services/gateway/src/credentials.ts b/services/gateway/src/credentials.ts index 3a908ae..599c7b0 100644 --- a/services/gateway/src/credentials.ts +++ b/services/gateway/src/credentials.ts @@ -1,6 +1,5 @@ import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'; -import https from 'node:https'; -import type { OutgoingHttpHeaders } from 'node:http'; +import { KubernetesClient } from './k8s-client.ts'; import { credentialKey, isSealed, @@ -58,54 +57,22 @@ export class KubernetesCredentials implements CredentialStore { method: string, body?: unknown, ): Promise> { - const directory = '/var/run/secrets/kubernetes.io/serviceaccount'; - const headers: OutgoingHttpHeaders = { - authorization: `Bearer ${readFileSync(`${directory}/token`, 'utf8').trim()}`, - }; - if (body) headers['content-type'] = 'application/merge-patch+json'; - return new Promise((resolve, reject) => { - const request = https.request( - { - hostname: process.env.KUBERNETES_SERVICE_HOST, - port: Number(process.env.KUBERNETES_SERVICE_PORT_HTTPS ?? 443), - path: this.path, - method, - headers, - ca: readFileSync(`${directory}/ca.crt`), - timeout: 3000, - }, - (response) => { - let data = ''; - response.on('data', (chunk) => { - data += chunk; - if (data.length > 1048576) - request.destroy(new Error('Secret exceeds limit')); - }); - response.on('end', () => { - if (response.statusCode === 409) { - reject(new SecretConflictError('Credential Secret changed')); - return; - } - if (response.statusCode !== 200) { - reject(new Error('Credential Secret is unavailable')); - return; - } - try { - resolve(JSON.parse(data)); - } catch { - reject(new Error('Invalid Secret response')); - } - }); - }, + let response; + try { + response = await new KubernetesClient().request( + method, + this.path, + body, + 'application/merge-patch+json', ); - request.on('timeout', () => - request.destroy(new Error('Secret request timed out')), - ); - request.on('error', () => - reject(new Error('Credential Secret is unavailable')), - ); - request.end(body ? JSON.stringify(body) : undefined); - }); + } catch { + throw new Error('Credential Secret is unavailable'); + } + if (response.status === 409) + throw new SecretConflictError('Credential Secret changed'); + if (response.status !== 200 || typeof response.body !== 'object') + throw new Error('Credential Secret is unavailable'); + return response.body as Record; } async get(id: string): Promise { const result = await this.request('GET'); diff --git a/services/gateway/src/k8s-client.ts b/services/gateway/src/k8s-client.ts new file mode 100644 index 0000000..8dbae71 --- /dev/null +++ b/services/gateway/src/k8s-client.ts @@ -0,0 +1,181 @@ +import { readFileSync } from 'node:fs'; +import https from 'node:https'; +import type { OutgoingHttpHeaders } from 'node:http'; + +export type KubeResponse = { status: number; body: unknown }; + +// Talks to the Kubernetes API with the pod's ServiceAccount. +export class KubernetesClient { + directory: string; + constructor(directory = '/var/run/secrets/kubernetes.io/serviceaccount') { + this.directory = directory; + } + request( + method: string, + path: string, + body?: unknown, + contentType = 'application/json', + ): Promise { + const headers: OutgoingHttpHeaders = { + authorization: `Bearer ${readFileSync(`${this.directory}/token`, 'utf8').trim()}`, + accept: 'application/json', + }; + if (body !== undefined) headers['content-type'] = contentType; + return new Promise((resolve, reject) => { + const request = https.request( + { + hostname: process.env.KUBERNETES_SERVICE_HOST, + port: Number(process.env.KUBERNETES_SERVICE_PORT_HTTPS ?? 443), + path, + method, + headers, + ca: readFileSync(`${this.directory}/ca.crt`), + timeout: 10000, + }, + (response) => { + let data = ''; + response.on('data', (chunk) => { + data += chunk; + if (data.length > 4194304) + request.destroy(new Error('Kubernetes response exceeds limit')); + }); + response.on('end', () => { + let parsed: unknown; + try { + parsed = data ? JSON.parse(data) : undefined; + } catch { + reject(new Error('Invalid Kubernetes API response')); + return; + } + resolve({ status: response.statusCode ?? 0, body: parsed }); + }); + }, + ); + request.on('timeout', () => + request.destroy(new Error('Kubernetes API request timed out')), + ); + request.on('error', () => + reject(new Error('Kubernetes API is unavailable')), + ); + request.end(body === undefined ? undefined : JSON.stringify(body)); + }); + } +} + +export const storeNamespacePattern = /^storelink-store-[a-z][a-z0-9-]{0,11}$/; +export const managedLabel = 'storelink/managed-store'; + +type KubeObject = { + apiVersion: string; + kind: string; + metadata: { + name: string; + namespace?: string; + labels?: Record; + }; +}; + +const resources: Record = { + Namespace: { base: '/api/v1', plural: 'namespaces' }, + ResourceQuota: { base: '/api/v1', plural: 'resourcequotas' }, + LimitRange: { base: '/api/v1', plural: 'limitranges' }, + Secret: { base: '/api/v1', plural: 'secrets' }, + ConfigMap: { base: '/api/v1', plural: 'configmaps' }, + PersistentVolumeClaim: { base: '/api/v1', plural: 'persistentvolumeclaims' }, + Service: { base: '/api/v1', plural: 'services' }, + NetworkPolicy: { + base: '/apis/networking.k8s.io/v1', + plural: 'networkpolicies', + }, + Ingress: { base: '/apis/networking.k8s.io/v1', plural: 'ingresses' }, + Deployment: { base: '/apis/apps/v1', plural: 'deployments' }, + Job: { base: '/apis/batch/v1', plural: 'jobs' }, +}; + +export class KubernetesApiError extends Error { + status: number; + constructor(message: string, status: number) { + super(message); + this.status = status; + } +} + +// The provisioner's only way into the cluster. RBAC cannot limit a ClusterRole +// to a name pattern, so every operation is checked against the store namespace +// pattern here, and namespaces are deleted only if they carry the managed label. +export class StoreCluster { + client: Pick; + fieldManager = 'storelink-provisioner'; + constructor(client: Pick) { + this.client = client; + } + private namespaceOf(object: KubeObject): string { + const namespace = + object.kind === 'Namespace' + ? object.metadata.name + : object.metadata.namespace; + if (!namespace || !storeNamespacePattern.test(namespace)) + throw new Error('Refusing to touch a namespace outside store namespaces'); + return namespace; + } + private path(object: KubeObject): string { + const resource = resources[object.kind]; + if (!resource) throw new Error(`Unsupported kind ${object.kind}`); + const namespace = this.namespaceOf(object); + const name = encodeURIComponent(object.metadata.name); + return object.kind === 'Namespace' + ? `${resource.base}/namespaces/${name}` + : `${resource.base}/namespaces/${namespace}/${resource.plural}/${name}`; + } + private check(response: KubeResponse, action: string) { + if (response.status >= 200 && response.status < 300) return response.body; + const message = + (response.body as { message?: string } | undefined)?.message ?? + `HTTP ${response.status}`; + throw new KubernetesApiError(`${action}: ${message}`, response.status); + } + // Server-side apply: creates or updates the object to match. + async apply(object: KubeObject): Promise { + const response = await this.client.request( + 'PATCH', + `${this.path(object)}?fieldManager=${this.fieldManager}&force=true`, + object, + 'application/apply-patch+yaml', + ); + return this.check(response, `Apply ${object.kind} ${object.metadata.name}`); + } + // Creates the object only if it does not exist; never overwrites it. + async create(object: KubeObject): Promise { + const resource = resources[object.kind]; + const namespace = this.namespaceOf(object); + const response = await this.client.request( + 'POST', + `${resource.base}/namespaces/${namespace}/${resource.plural}`, + object, + ); + if (response.status === 409) return; + this.check(response, `Create ${object.kind} ${object.metadata.name}`); + } + async get(object: KubeObject): Promise { + const response = await this.client.request('GET', this.path(object)); + if (response.status === 404) return undefined; + return this.check(response, `Read ${object.kind} ${object.metadata.name}`); + } + async deleteNamespace(name: string): Promise<'deleting' | 'gone'> { + const object = { + apiVersion: 'v1', + kind: 'Namespace', + metadata: { name }, + }; + const found = (await this.get(object)) as KubeObject | undefined; + if (!found) return 'gone'; + if (found.metadata.labels?.[managedLabel] !== 'true') + throw new Error( + 'Refusing to delete a namespace StoreLink does not manage', + ); + const response = await this.client.request('DELETE', this.path(object)); + if (response.status === 404) return 'gone'; + this.check(response, `Delete namespace ${name}`); + return 'deleting'; + } +} diff --git a/services/gateway/src/store-manifests.ts b/services/gateway/src/store-manifests.ts new file mode 100644 index 0000000..7ce77fa --- /dev/null +++ b/services/gateway/src/store-manifests.ts @@ -0,0 +1,544 @@ +import { createHash, randomBytes } from 'node:crypto'; +import { managedLabel, storeNamespacePattern } from './k8s-client.ts'; + +export const storeSlugPattern = /^[a-z][a-z0-9-]{0,11}$/; + +type Resources = { + requests: { cpu: string; memory: string }; + limits: { cpu: string; memory: string }; +}; +export type StoreSettings = { + images: { app: string; store: string; mariadb: string; pullPolicy: string }; + // Storefront hosts are .. + hostSuffix: string; + scheme: 'http' | 'https'; + tlsSecretName?: string; + ingressClass: string; + storageClass: string; + storage: { wordpress: string; mariadb: string }; + timezone: string; + woocommerceVersion: string; + // Where the gateway runs, so only it can reach each store's MCP server. + gateway: { namespace: string; instance: string }; + healthPhp: string; + resources: { wordpress: Resources; mariadb: Resources; wooMcp: Resources }; +}; + +// Every name and address a store uses comes from here, so the Ingress host, +// WordPress URLs, OAuth signing and the MCP origin always agree. +export function storeNames(slug: string, settings: StoreSettings) { + if (!storeSlugPattern.test(slug)) throw new Error('Invalid store slug'); + const namespace = `storelink-store-${slug}`; + if (!storeNamespacePattern.test(namespace)) + throw new Error('Invalid store namespace'); + const host = `${slug}.${settings.hostSuffix}`; + const mcpOrigin = `http://woo-mcp.${namespace}.svc.cluster.local:3001`; + return { + namespace, + host, + publicUrl: `${settings.scheme}://${host}`, + mcpOrigin, + mcpUrl: `${mcpOrigin}/mcp`, + upstreamSlug: `woo-${slug}`, + }; +} + +// Generated once per store; the provisioner creates it only if it is missing, +// since MariaDB and WooCommerce keep the first values they see. +export function storeSecret(slug: string, settings: StoreSettings) { + const { namespace } = storeNames(slug, settings); + const random = () => randomBytes(32).toString('base64url'); + return { + apiVersion: 'v1', + kind: 'Secret', + metadata: { name: 'store-credentials', namespace, labels: labels(slug) }, + type: 'Opaque', + stringData: { + 'mariadb-root-password': random(), + 'wordpress-db-password': random(), + 'wordpress-admin-password': random(), + 'woo-consumer-key': `ck_${randomBytes(20).toString('hex')}`, + 'woo-consumer-secret': `cs_${randomBytes(20).toString('hex')}`, + 'woo-mcp-token': random(), + }, + }; +} + +function labels(slug: string, component?: string) { + return { + 'app.kubernetes.io/managed-by': 'storelink-provisioner', + 'storelink/store': slug, + ...(component && { 'storelink/component': component }), + }; +} +const selector = (slug: string, component: string) => ({ + matchLabels: { 'storelink/store': slug, 'storelink/component': component }, +}); +const secretEnv = (name: string, key: string) => ({ + name, + valueFrom: { secretKeyRef: { name: 'store-credentials', key } }, +}); +const dropAll = { + allowPrivilegeEscalation: false, + capabilities: { drop: ['ALL'] }, +}; +const nonRoot = (id: number) => ({ + runAsNonRoot: true, + runAsUser: id, + runAsGroup: id, + fsGroup: id, + fsGroupChangePolicy: 'OnRootMismatch', +}); + +export function storeManifests(slug: string, settings: StoreSettings) { + const names = storeNames(slug, settings); + const { namespace } = names; + const meta = (name: string, component?: string) => ({ + name, + namespace, + labels: labels(slug, component), + }); + const deployment = ( + name: string, + podSpec: Record, + component = name, + ) => ({ + apiVersion: 'apps/v1', + kind: 'Deployment', + metadata: meta(name, component), + spec: { + replicas: 1, + strategy: { type: 'Recreate' }, + selector: selector(slug, component), + template: { + metadata: { labels: labels(slug, component) }, + spec: { automountServiceAccountToken: false, ...podSpec }, + }, + }, + }); + const service = ( + name: string, + port: number, + targetPort: string | number, + ) => ({ + apiVersion: 'v1', + kind: 'Service', + metadata: meta(name, name), + spec: { + selector: selector(slug, name).matchLabels, + ports: [{ port, targetPort }], + }, + }); + const claim = (name: string, size: string) => ({ + apiVersion: 'v1', + kind: 'PersistentVolumeClaim', + metadata: meta(name, name), + spec: { + accessModes: ['ReadWriteOnce'], + storageClassName: settings.storageClass, + resources: { requests: { storage: size } }, + }, + }); + const policy = ( + name: string, + component: string | undefined, + spec: Record, + ) => ({ + apiVersion: 'networking.k8s.io/v1', + kind: 'NetworkPolicy', + metadata: meta(name), + spec: { + podSelector: component ? selector(slug, component) : {}, + ...spec, + }, + }); + const from = (component: string) => ({ + podSelector: selector(slug, component), + }); + const traefik = { + namespaceSelector: { + matchLabels: { 'kubernetes.io/metadata.name': 'kube-system' }, + }, + podSelector: { matchLabels: { 'app.kubernetes.io/name': 'traefik' } }, + }; + const gateway = { + namespaceSelector: { + matchLabels: { + 'kubernetes.io/metadata.name': settings.gateway.namespace, + }, + }, + podSelector: { + matchLabels: { + 'storelink/component': 'gateway', + 'app.kubernetes.io/instance': settings.gateway.instance, + }, + }, + }; + const tcp = (port: number) => [{ protocol: 'TCP', port }]; + const wordpressEnv = [ + { name: 'WORDPRESS_DB_HOST', value: 'mariadb' }, + { name: 'WORDPRESS_DB_NAME', value: 'wordpress' }, + { name: 'WORDPRESS_DB_USER', value: 'wordpress' }, + secretEnv('WORDPRESS_DB_PASSWORD', 'wordpress-db-password'), + { name: 'STORE_PUBLIC_URL', value: names.publicUrl }, + ]; + const bootstrap = { + restartPolicy: 'OnFailure', + automountServiceAccountToken: false, + securityContext: nonRoot(33), + affinity: { + podAffinity: { + requiredDuringSchedulingIgnoredDuringExecution: [ + { + labelSelector: selector(slug, 'wordpress'), + topologyKey: 'kubernetes.io/hostname', + }, + ], + }, + }, + containers: [ + { + name: 'bootstrap', + image: settings.images.store, + imagePullPolicy: settings.images.pullPolicy, + command: ['bash', '/opt/storelink/bootstrap.sh'], + env: [ + ...wordpressEnv, + { name: 'STORE_TIMEZONE', value: settings.timezone }, + { name: 'WOOCOMMERCE_VERSION', value: settings.woocommerceVersion }, + { name: 'WORDPRESS_ADMIN_USER', value: 'admin' }, + secretEnv('WORDPRESS_ADMIN_PASSWORD', 'wordpress-admin-password'), + secretEnv('WOO_CONSUMER_KEY', 'woo-consumer-key'), + secretEnv('WOO_CONSUMER_SECRET', 'woo-consumer-secret'), + ], + resources: { + requests: { cpu: '100m', memory: '192Mi' }, + limits: { cpu: '1', memory: '512Mi' }, + }, + securityContext: dropAll, + volumeMounts: [ + { name: 'wordpress', mountPath: '/var/www/html' }, + { name: 'tmp', mountPath: '/tmp' }, + ], + }, + ], + volumes: [ + { name: 'wordpress', persistentVolumeClaim: { claimName: 'wordpress' } }, + { name: 'tmp', emptyDir: {} }, + ], + }; + // Jobs cannot be changed after creation, so the name follows the spec. + const jobName = `store-bootstrap-${createHash('sha256') + .update(JSON.stringify(bootstrap)) + .digest('hex') + .slice(0, 10)}`; + const ingress = { + apiVersion: 'networking.k8s.io/v1', + kind: 'Ingress', + metadata: meta('wordpress', 'wordpress'), + spec: { + ingressClassName: settings.ingressClass, + ...(settings.tlsSecretName && { + tls: [{ hosts: [names.host], secretName: settings.tlsSecretName }], + }), + rules: [ + { + host: names.host, + http: { + paths: [ + { + path: '/', + pathType: 'Prefix', + backend: { + service: { name: 'wordpress', port: { number: 80 } }, + }, + }, + ], + }, + }, + ], + }, + }; + return { + names, + jobName, + objects: [ + { + apiVersion: 'v1', + kind: 'Namespace', + metadata: { + name: namespace, + labels: { ...labels(slug), [managedLabel]: 'true' }, + }, + }, + { + apiVersion: 'v1', + kind: 'ResourceQuota', + metadata: meta('store'), + spec: { + hard: { + 'requests.cpu': '1', + 'requests.memory': '1Gi', + 'limits.cpu': '6', + 'limits.memory': '2Gi', + pods: '6', + persistentvolumeclaims: '2', + 'requests.storage': '8Gi', + services: '4', + }, + }, + }, + { + apiVersion: 'v1', + kind: 'LimitRange', + metadata: meta('store'), + spec: { + limits: [ + { + type: 'Container', + defaultRequest: { cpu: '50m', memory: '64Mi' }, + default: { cpu: '500m', memory: '256Mi' }, + max: { cpu: '2', memory: '1Gi' }, + }, + ], + }, + }, + policy('default-deny', undefined, { policyTypes: ['Ingress', 'Egress'] }), + policy('dns', undefined, { + policyTypes: ['Egress'], + egress: [ + { + to: [ + { + namespaceSelector: { + matchLabels: { 'kubernetes.io/metadata.name': 'kube-system' }, + }, + podSelector: { matchLabels: { 'k8s-app': 'kube-dns' } }, + }, + ], + ports: [ + { protocol: 'UDP', port: 53 }, + { protocol: 'TCP', port: 53 }, + ], + }, + ], + }), + policy('woo-mcp', 'woo-mcp', { + policyTypes: ['Ingress', 'Egress'], + ingress: [{ from: [gateway], ports: tcp(3001) }], + egress: [{ to: [from('wordpress')], ports: tcp(8080) }], + }), + policy('wordpress', 'wordpress', { + policyTypes: ['Ingress', 'Egress'], + ingress: [{ from: [from('woo-mcp'), traefik], ports: tcp(8080) }], + egress: [{ to: [from('mariadb')], ports: tcp(3306) }], + }), + policy('mariadb', 'mariadb', { + policyTypes: ['Ingress'], + ingress: [ + { + from: [from('wordpress'), from('store-bootstrap')], + ports: tcp(3306), + }, + ], + }), + policy('store-bootstrap', 'store-bootstrap', { + policyTypes: ['Egress'], + egress: [{ to: [from('mariadb')], ports: tcp(3306) }], + }), + { + apiVersion: 'v1', + kind: 'ConfigMap', + metadata: meta('store-health'), + data: { 'health.php': settings.healthPhp }, + }, + claim('mariadb', settings.storage.mariadb), + claim('wordpress', settings.storage.wordpress), + deployment('mariadb', { + securityContext: nonRoot(999), + containers: [ + { + name: 'mariadb', + image: settings.images.mariadb, + imagePullPolicy: settings.images.pullPolicy, + args: ['--innodb-buffer-pool-size=64M', '--max-connections=30'], + env: [ + { name: 'MARIADB_DATABASE', value: 'wordpress' }, + { name: 'MARIADB_USER', value: 'wordpress' }, + secretEnv('MARIADB_PASSWORD', 'wordpress-db-password'), + secretEnv('MARIADB_ROOT_PASSWORD', 'mariadb-root-password'), + ], + ports: [{ containerPort: 3306, name: 'mysql' }], + startupProbe: { + exec: { + command: [ + 'healthcheck.sh', + '--connect', + '--innodb_initialized', + ], + }, + periodSeconds: 2, + failureThreshold: 60, + }, + readinessProbe: { + exec: { + command: [ + 'healthcheck.sh', + '--connect', + '--innodb_initialized', + ], + }, + periodSeconds: 5, + }, + livenessProbe: { + exec: { command: ['healthcheck.sh', '--connect'] }, + periodSeconds: 10, + failureThreshold: 6, + }, + securityContext: dropAll, + resources: settings.resources.mariadb, + volumeMounts: [{ name: 'data', mountPath: '/var/lib/mysql' }], + }, + ], + volumes: [ + { name: 'data', persistentVolumeClaim: { claimName: 'mariadb' } }, + ], + }), + service('mariadb', 3306, 'mysql'), + deployment('wordpress', { + securityContext: nonRoot(33), + containers: [ + { + name: 'wordpress', + image: settings.images.store, + imagePullPolicy: settings.images.pullPolicy, + env: [ + ...wordpressEnv, + { + name: 'WORDPRESS_CONFIG_EXTRA', + value: [ + "define('WP_HOME', getenv('STORE_PUBLIC_URL'));", + "define('WP_SITEURL', getenv('STORE_PUBLIC_URL'));", + "define('DISALLOW_FILE_EDIT', true);", + "define('WP_MEMORY_LIMIT', '256M');", + ].join('\n'), + }, + ], + ports: [{ containerPort: 8080, name: 'http' }], + startupProbe: { + tcpSocket: { port: 'http' }, + periodSeconds: 2, + failureThreshold: 90, + }, + readinessProbe: { + httpGet: { path: '/storelink-health.php', port: 'http' }, + periodSeconds: 5, + timeoutSeconds: 5, + failureThreshold: 3, + }, + livenessProbe: { + tcpSocket: { port: 'http' }, + periodSeconds: 10, + failureThreshold: 6, + }, + securityContext: dropAll, + resources: settings.resources.wordpress, + volumeMounts: [ + { name: 'wordpress', mountPath: '/var/www/html' }, + { + name: 'health', + mountPath: '/var/www/html/storelink-health.php', + subPath: 'health.php', + readOnly: true, + }, + ], + }, + ], + volumes: [ + { + name: 'wordpress', + persistentVolumeClaim: { claimName: 'wordpress' }, + }, + { name: 'health', configMap: { name: 'store-health' } }, + ], + }), + service('wordpress', 80, 'http'), + ingress, + { + apiVersion: 'batch/v1', + kind: 'Job', + metadata: meta(jobName, 'store-bootstrap'), + spec: { + backoffLimit: 6, + activeDeadlineSeconds: 600, + template: { + metadata: { labels: labels(slug, 'store-bootstrap') }, + spec: bootstrap, + }, + }, + }, + deployment('woo-mcp', { + securityContext: { + runAsNonRoot: true, + runAsUser: 1000, + runAsGroup: 1000, + }, + containers: [ + { + name: 'woo-mcp', + image: settings.images.app, + imagePullPolicy: settings.images.pullPolicy, + command: ['node', 'services/woocommerce-mcp/src/main.ts'], + env: [ + { name: 'WOO_MCP_ORIGIN', value: names.mcpOrigin }, + { name: 'STORE_PUBLIC_URL', value: names.publicUrl }, + { name: 'STORE_CONNECT_SERVICE', value: 'wordpress' }, + { name: 'WOO_MCP_TOKEN_FILE', value: '/secrets/woo-mcp-token' }, + { + name: 'WOO_CONSUMER_KEY_FILE', + value: '/secrets/woo-consumer-key', + }, + { + name: 'WOO_CONSUMER_SECRET_FILE', + value: '/secrets/woo-consumer-secret', + }, + ], + ports: [{ name: 'http', containerPort: 3001 }], + startupProbe: { + httpGet: { path: '/health/live', port: 'http' }, + periodSeconds: 2, + failureThreshold: 60, + }, + readinessProbe: { + httpGet: { path: '/health/ready', port: 'http' }, + periodSeconds: 5, + }, + livenessProbe: { + httpGet: { path: '/health/live', port: 'http' }, + periodSeconds: 10, + }, + securityContext: { ...dropAll, readOnlyRootFilesystem: true }, + resources: settings.resources.wooMcp, + volumeMounts: [ + { name: 'secrets', mountPath: '/secrets', readOnly: true }, + ], + }, + ], + volumes: [ + { + name: 'secrets', + secret: { + secretName: 'store-credentials', + items: [ + 'woo-mcp-token', + 'woo-consumer-key', + 'woo-consumer-secret', + ].map((key) => ({ key, path: key })), + }, + }, + ], + }), + service('woo-mcp', 3001, 'http'), + ], + }; +} diff --git a/services/gateway/src/upstream-policy.ts b/services/gateway/src/upstream-policy.ts index f660ce6..cf36a15 100644 --- a/services/gateway/src/upstream-policy.ts +++ b/services/gateway/src/upstream-policy.ts @@ -15,12 +15,29 @@ export function publicAddress(address: string): boolean { return false; } } +// The MCP server of a store created by the provisioner (see store-manifests.ts). +export const managedStoreOrigin = + /^http:\/\/woo-mcp\.storelink-store-[a-z][a-z0-9-]{0,11}\.svc\.cluster\.local:3001$/; export class UpstreamPolicy { origins: Set; resolver: Resolver; - constructor(origins: string[], resolver: Resolver = resolve) { + managedStores: boolean; + constructor( + origins: string[], + resolver: Resolver = resolve, + managedStores = false, + ) { this.origins = new Set(origins); this.resolver = resolver; + this.managedStores = managedStores; + } + // Operator-approved private origins: the configured list and, when enabled, + // provisioned stores. + approved(origin: string): boolean { + return ( + this.origins.has(origin) || + (this.managedStores && managedStoreOrigin.test(origin)) + ); } validate(value: string): URL { let url: URL; @@ -39,7 +56,7 @@ export class UpstreamPolicy { throw new RegistryInputError( 'Upstream URL must not contain a query or fragment', ); - if (this.origins.has(url.origin)) return url; + if (this.approved(url.origin)) return url; if (url.protocol !== 'https:') throw new RegistryInputError( 'Public upstreams require HTTPS; private HTTP origins need operator approval', @@ -77,13 +94,13 @@ export class UpstreamPolicy { } async check(value: string): Promise { const url = this.validate(value); - if (!this.origins.has(url.origin)) + if (!this.approved(url.origin)) await this.addresses(url.hostname.replace(/^\[|\]$/g, '')); return url; } connectionOptions(value: string): { lookup?: LookupFunction } { const url = this.validate(value); - if (this.origins.has(url.origin)) return {}; + if (this.approved(url.origin)) return {}; // This resolver supplies the checked addresses directly to the socket. No second DNS lookup. const lookup: LookupFunction = (hostname, options, callback) => { void this.addresses(hostname).then( diff --git a/tests/stores.test.ts b/tests/stores.test.ts new file mode 100644 index 0000000..8dd2dd1 --- /dev/null +++ b/tests/stores.test.ts @@ -0,0 +1,220 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { + storeManifests, + storeNames, + storeSecret, +} from '../services/gateway/src/store-manifests.ts'; +import type { StoreSettings } from '../services/gateway/src/store-manifests.ts'; +import { StoreCluster } from '../services/gateway/src/k8s-client.ts'; +import type { KubeResponse } from '../services/gateway/src/k8s-client.ts'; +import { UpstreamPolicy } from '../services/gateway/src/upstream-policy.ts'; + +const resources = { + requests: { cpu: '50m', memory: '64Mi' }, + limits: { cpu: '1', memory: '256Mi' }, +}; +export const settings: StoreSettings = { + images: { + app: 'storelink-app:1', + store: 'storelink-wordpress:1', + mariadb: 'mariadb:11.8', + pullPolicy: 'IfNotPresent', + }, + hostSuffix: 'store.storelink.localhost', + scheme: 'http', + ingressClass: 'traefik', + storageClass: 'local-path', + storage: { wordpress: '2Gi', mariadb: '1Gi' }, + timezone: 'Asia/Kolkata', + woocommerceVersion: '11.2.1', + gateway: { namespace: 'storelink', instance: 'storelink' }, + healthPhp: '; + }; + spec?: any; +}; +const env = (container: any, name: string) => + container.env.find((e: { name: string }) => e.name === name)?.value; + +test('every address a store uses comes from its slug', () => { + const names = storeNames('north', settings); + assert.deepEqual(names, { + namespace: 'storelink-store-north', + host: 'north.store.storelink.localhost', + publicUrl: 'http://north.store.storelink.localhost', + mcpOrigin: 'http://woo-mcp.storelink-store-north.svc.cluster.local:3001', + mcpUrl: 'http://woo-mcp.storelink-store-north.svc.cluster.local:3001/mcp', + upstreamSlug: 'woo-north', + }); + for (const slug of ['North', '1st', 'a'.repeat(13), '../x', 'a_b', '']) + assert.throws(() => storeNames(slug, settings), /Invalid store slug/); + // The gateway approves exactly this origin form, and only when enabled. + assert.ok(new UpstreamPolicy([], undefined, true).approved(names.mcpOrigin)); + assert.ok(!new UpstreamPolicy([]).approved(names.mcpOrigin)); + for (const origin of [ + 'http://woo-mcp.storelink-store-north.svc.cluster.local:3002', + 'http://wordpress.storelink-store-north.svc.cluster.local:3001', + 'http://woo-mcp.storelink.svc.cluster.local:3001', + 'https://woo-mcp.storelink-store-north.svc.cluster.local:3001', + ]) + assert.ok( + !new UpstreamPolicy([], undefined, true).approved(origin), + origin, + ); +}); + +test('store manifests stay inside the store namespace and agree on addresses', () => { + const { names, objects, jobName } = storeManifests('north', settings); + const all = objects as Obj[]; + for (const object of all) + assert.equal( + object.kind === 'Namespace' + ? object.metadata.name + : object.metadata.namespace, + names.namespace, + ); + assert.ok( + !all.some((o) => o.kind === 'Secret'), + 'Secrets are created separately', + ); + assert.equal( + all.find((o) => o.kind === 'Namespace')!.metadata.labels![ + 'storelink/managed-store' + ], + 'true', + ); + const workloads = all.filter((o) => ['Deployment', 'Job'].includes(o.kind)); + const containers = workloads.flatMap((o) => + o.spec.template.spec.containers.map((c: any) => ({ + kind: o.kind, + c, + pod: o.spec.template.spec, + })), + ); + assert.equal(containers.length, 4); + for (const { kind, c, pod } of containers) { + assert.ok(c.resources.requests && c.resources.limits, c.name); + assert.equal(pod.securityContext.runAsNonRoot, true, c.name); + assert.equal(pod.automountServiceAccountToken, false, c.name); + assert.deepEqual(c.securityContext.capabilities.drop, ['ALL'], c.name); + if (kind === 'Deployment') + assert.ok(c.readinessProbe && c.livenessProbe, c.name); + } + const byName = (name: string) => + containers.find(({ c }) => c.name === name)!.c; + assert.equal(env(byName('woo-mcp'), 'WOO_MCP_ORIGIN'), names.mcpOrigin); + for (const name of ['woo-mcp', 'wordpress', 'bootstrap']) + assert.equal(env(byName(name), 'STORE_PUBLIC_URL'), names.publicUrl, name); + const ingress = all.find((o) => o.kind === 'Ingress')!; + assert.equal(ingress.spec.rules[0].host, names.host); + assert.ok(all.some((o) => o.kind === 'Job' && o.metadata.name === jobName)); + // A new image gives the immutable bootstrap Job a new name. + const updated = storeManifests('north', { + ...settings, + images: { ...settings.images, store: 'storelink-wordpress:2' }, + }); + assert.notEqual(updated.jobName, jobName); +}); + +test('store network policies admit only the gateway and the store itself', () => { + const policies = (storeManifests('north', settings).objects as Obj[]).filter( + (o) => o.kind === 'NetworkPolicy', + ); + const deny = policies.find((p) => p.metadata.name === 'default-deny')!; + assert.deepEqual(deny.spec.podSelector, {}); + assert.deepEqual(deny.spec.policyTypes, ['Ingress', 'Egress']); + const mcp = policies.find((p) => p.metadata.name === 'woo-mcp')!; + assert.deepEqual(mcp.spec.ingress[0].from, [ + { + namespaceSelector: { + matchLabels: { 'kubernetes.io/metadata.name': 'storelink' }, + }, + podSelector: { + matchLabels: { + 'storelink/component': 'gateway', + 'app.kubernetes.io/instance': 'storelink', + }, + }, + }, + ]); + // Peers within the store are selected by store and component, never by namespace. + for (const policy of policies) + for (const rule of [ + ...(policy.spec.ingress ?? []), + ...(policy.spec.egress ?? []), + ]) + for (const peer of rule.from ?? rule.to ?? []) + if (!peer.namespaceSelector) + assert.equal( + peer.podSelector.matchLabels['storelink/store'], + 'north', + ); +}); + +test('store secrets are random per store', () => { + const a = storeSecret('north', settings).stringData; + const b = storeSecret('north', settings).stringData; + assert.notEqual(a['woo-mcp-token'], b['woo-mcp-token']); + assert.match(a['woo-consumer-key'], /^ck_[0-9a-f]{40}$/); +}); + +test('the cluster client refuses namespaces outside the store pattern', async () => { + const calls: { method: string; path: string; type?: string }[] = []; + let namespaceLabels: Record | undefined; + const cluster = new StoreCluster({ + async request(method, path, _body, type): Promise { + calls.push({ method, path, type }); + if (method === 'GET') + return namespaceLabels + ? { + status: 200, + body: { metadata: { name: 'x', labels: namespaceLabels } }, + } + : { status: 404, body: {} }; + return { status: 200, body: {} }; + }, + }); + for (const namespace of ['storelink', 'kube-system', 'default']) + await assert.rejects( + cluster.apply({ + apiVersion: 'v1', + kind: 'ConfigMap', + metadata: { name: 'x', namespace }, + }), + /outside store namespaces/, + ); + await assert.rejects(cluster.deleteNamespace('kube-system'), /outside store/); + assert.equal(calls.length, 0); + + await cluster.apply(storeManifests('north', settings).objects[0] as Obj); + assert.equal( + calls[0].path, + '/api/v1/namespaces/storelink-store-north?fieldManager=storelink-provisioner&force=true', + ); + assert.equal(calls[0].type, 'application/apply-patch+yaml'); + + // A namespace with a store-like name but without the managed label is kept. + namespaceLabels = {}; + await assert.rejects( + cluster.deleteNamespace('storelink-store-north'), + /does not manage/, + ); + namespaceLabels = { 'storelink/managed-store': 'true' }; + assert.equal( + await cluster.deleteNamespace('storelink-store-north'), + 'deleting', + ); + assert.equal(calls.at(-1)!.method, 'DELETE'); + namespaceLabels = undefined; + assert.equal(await cluster.deleteNamespace('storelink-store-north'), 'gone'); +}); From 89ef9f3773c2767e6a9260d5c8dd2042b41450c3 Mon Sep 17 00:00:00 2001 From: Deep-Axe Date: Mon, 26 Oct 2026 18:34:37 +0530 Subject: [PATCH 2/4] Complete durable multi-store provisioning, retry attempts and atomic owner limits --- charts/storelink/templates/apps.yaml | 6 + .../storelink/templates/network-policies.yaml | 35 +- charts/storelink/templates/provisioner.yaml | 163 +++++++++ charts/storelink/values-prod.yaml | 7 + charts/storelink/values.yaml | 29 ++ images/app/Dockerfile | 2 + images/store/bootstrap.sh | 2 +- services/gateway/migrations/0008_stores.sql | 17 + .../migrations/0009_store_attempts.sql | 4 + services/gateway/src/dashboard-api.ts | 94 ++++- services/gateway/src/k8s-client.ts | 7 +- services/gateway/src/main.ts | 12 + services/gateway/src/provisioner-main.ts | 154 +++++++++ services/gateway/src/registry.ts | 10 +- services/gateway/src/store-manifests.ts | 66 +++- services/gateway/src/store-provisioner.ts | 323 ++++++++++++++++++ services/gateway/src/store-requests.ts | 83 +++++ tests/store-requests.test.ts | 123 +++++++ tests/stores.test.ts | 296 ++++++++++++++++ 19 files changed, 1420 insertions(+), 13 deletions(-) create mode 100644 charts/storelink/templates/provisioner.yaml create mode 100644 services/gateway/migrations/0008_stores.sql create mode 100644 services/gateway/migrations/0009_store_attempts.sql create mode 100644 services/gateway/src/provisioner-main.ts create mode 100644 services/gateway/src/store-provisioner.ts create mode 100644 services/gateway/src/store-requests.ts create mode 100644 tests/store-requests.test.ts diff --git a/charts/storelink/templates/apps.yaml b/charts/storelink/templates/apps.yaml index 3550573..6fffad6 100644 --- a/charts/storelink/templates/apps.yaml +++ b/charts/storelink/templates/apps.yaml @@ -54,6 +54,12 @@ spec: - { name: HEALTH_INTERVAL_MS, value: {{ .Values.gateway.healthIntervalMs | quote }} } - { name: SHUTDOWN_DELAY_MS, value: {{ .Values.gateway.shutdownDelayMs | quote }} } - { name: MCP_REQUESTS_PER_MINUTE, value: {{ .Values.gateway.mcpRequestsPerMinute | quote }} } + - { name: STORES_ENABLED, value: {{ .Values.stores.enabled | quote }} } + - { name: STORES_HOST_SUFFIX, value: {{ .Values.stores.hostSuffix | quote }} } + - { name: STORES_SCHEME, value: {{ .Values.stores.scheme | quote }} } + - { name: STORES_MAX_PER_TEAM, value: {{ .Values.stores.maxPerTeam | quote }} } + - { name: STORES_MAX_PER_USER, value: {{ .Values.stores.maxPerUser | quote }} } + - { name: STORES_TIMEOUT_MS, value: {{ mul .Values.stores.provisionTimeoutSeconds 1000 | quote }} } {{- if .Values.gateway.metrics.enabled }} - { name: METRICS_PORT, value: {{ .Values.gateway.metrics.port | quote }} } {{- end }} diff --git a/charts/storelink/templates/network-policies.yaml b/charts/storelink/templates/network-policies.yaml index 49deae4..69ef919 100644 --- a/charts/storelink/templates/network-policies.yaml +++ b/charts/storelink/templates/network-policies.yaml @@ -3,7 +3,7 @@ {{- $instance := .Release.Name }} {{- $ports := dict "gateway" 3000 "time-mcp" 3002 "operations-mcp" 3003 "woo-mcp" 3001 "postgres" 5432 "mariadb" 3306 "wordpress" 8080 }} # Only the listed components can connect to the MCP servers, databases and WordPress. -{{- range $destination, $sources := dict "time-mcp" (list "gateway") "operations-mcp" (list "gateway") "woo-mcp" (list "gateway") "postgres" (list "gateway") "mariadb" (list "wordpress" "store-bootstrap") "wordpress" (list "woo-mcp") }} +{{- range $destination, $sources := dict "time-mcp" (list "gateway") "operations-mcp" (list "gateway") "woo-mcp" (list "gateway") "postgres" (list "gateway" "provisioner") "mariadb" (list "wordpress" "store-bootstrap") "wordpress" (list "woo-mcp") }} --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy @@ -124,8 +124,41 @@ spec: except: [10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 100.64.0.0/10, 169.254.0.0/16, 127.0.0.0/8] ports: [{ protocol: TCP, port: 443 }] {{- end }} + {{- if .Values.stores.enabled }} + # MCP servers of provisioned stores. + - to: + - namespaceSelector: + matchLabels: { storelink/managed-store: "true" } + podSelector: + matchLabels: { storelink/component: woo-mcp } + ports: [{ protocol: TCP, port: 3001 }] + {{- end }} {{- with .Values.networkPolicies.gatewayEgress }} {{- toYaml . | nindent 4 }} {{- end }} +{{- if .Values.stores.enabled }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $name }}-provisioner +spec: + podSelector: + matchLabels: { storelink/component: provisioner, app.kubernetes.io/instance: {{ $instance }} } + policyTypes: [Ingress, Egress] + egress: + - to: + - podSelector: + matchLabels: { storelink/component: postgres, app.kubernetes.io/instance: {{ $instance }} } + ports: [{ protocol: TCP, port: 5432 }] + - to: + {{- range .Values.networkPolicies.kubernetesApi.cidrs }} + - ipBlock: { cidr: {{ . }} } + {{- end }} + ports: + {{- range .Values.networkPolicies.kubernetesApi.ports }} + - { protocol: TCP, port: {{ . }} } + {{- end }} +{{- end }} {{- end }} {{- end }} diff --git a/charts/storelink/templates/provisioner.yaml b/charts/storelink/templates/provisioner.yaml new file mode 100644 index 0000000..f936cb0 --- /dev/null +++ b/charts/storelink/templates/provisioner.yaml @@ -0,0 +1,163 @@ +{{- if .Values.stores.enabled }} +{{- if and (eq .Values.stores.scheme "https") (or (empty .Values.stores.tlsSecretName) (empty .Values.stores.tlsClusterIssuer)) }} +{{- fail "HTTPS stores require stores.tlsSecretName and stores.tlsClusterIssuer (cert-manager)" }} +{{- end }} +{{- $name := include "storelink.name" . }} +{{- $account := printf "system:serviceaccount:%s:%s-provisioner" .Release.Namespace $name }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }}-provisioner +--- +# Creating stores needs rights in namespaces that do not exist yet, which RBAC can +# only grant cluster-wide. The admission policy below limits them to store namespaces. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ $name }}-store-provisioner-{{ .Release.Namespace }} +rules: + - apiGroups: [""] + resources: [namespaces] + verbs: [get, create, patch, delete] + - apiGroups: [""] + resources: [resourcequotas, limitranges, configmaps, persistentvolumeclaims, services] + verbs: [get, create, patch] + # Store Secrets are created once and never read back. + - apiGroups: [""] + resources: [secrets] + verbs: [create] + - apiGroups: [networking.k8s.io] + resources: [networkpolicies, ingresses] + verbs: [get, create, patch] + - apiGroups: [apps] + resources: [deployments] + verbs: [get, create, patch] + - apiGroups: [batch] + resources: [jobs] + verbs: [get, create, patch] + - apiGroups: [cert-manager.io] + resources: [certificates] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $name }}-store-provisioner-{{ .Release.Namespace }} +subjects: + - kind: ServiceAccount + name: {{ $name }}-provisioner + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $name }}-store-provisioner-{{ .Release.Namespace }} + apiGroup: rbac.authorization.k8s.io +--- +# Registers each store's MCP credential in the gateway's credentials Secret. +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $name }}-provisioner-credentials +subjects: + - kind: ServiceAccount + name: {{ $name }}-provisioner +roleRef: + kind: Role + name: {{ $name }}-credentials + apiGroup: rbac.authorization.k8s.io +--- +# Enforced by the API server: the provisioner may change only namespaces labelled +# as managed stores and the objects inside them, plus the credentials Secret. +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: {{ $name }}-store-provisioner-{{ .Release.Namespace }} +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["", apps, batch, networking.k8s.io, rbac.authorization.k8s.io] + apiVersions: ["*"] + operations: [CREATE, UPDATE, DELETE] + resources: ["*"] + matchConditions: + - name: provisioner + expression: request.userInfo.username == {{ $account | quote }} + validations: + - message: The store provisioner may only change managed store namespaces + expression: >- + (request.kind.kind == 'Secret' && request.operation == 'UPDATE' + && request.namespace == {{ .Release.Namespace | quote }} + && request.name == {{ .Values.upstreamSecretName | quote }}) + || (request.kind.kind == 'Namespace' && request.name.startsWith('storelink-store-') + && (request.operation == 'DELETE' ? oldObject : object).metadata.?labels[?'storelink/managed-store'].orValue('') == 'true') + || (request.kind.kind != 'Namespace' && request.namespace.startsWith('storelink-store-') + && namespaceObject != null + && namespaceObject.metadata.?labels[?'storelink/managed-store'].orValue('') == 'true') +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: {{ $name }}-store-provisioner-{{ .Release.Namespace }} +spec: + policyName: {{ $name }}-store-provisioner-{{ .Release.Namespace }} + validationActions: [Deny] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $name }}-provisioner +spec: + replicas: 1 + strategy: { type: Recreate } + selector: + matchLabels: { storelink/component: provisioner, app.kubernetes.io/instance: {{ .Release.Name }} } + template: + metadata: + labels: { storelink/component: provisioner, app.kubernetes.io/instance: {{ .Release.Name }} } + spec: + serviceAccountName: {{ $name }}-provisioner + automountServiceAccountToken: true + securityContext: { runAsNonRoot: true, runAsUser: 1000, runAsGroup: 1000 } + containers: + - name: provisioner + image: {{ .Values.images.app | quote }} + imagePullPolicy: {{ .Values.images.pullPolicy }} + command: [node, services/gateway/src/provisioner-main.ts] + env: + - { name: DATABASE_URL_FILE, value: /secrets/database-url } + - { name: API_KEY_PEPPER_FILE, value: /secrets/api-key-pepper } + - name: POD_NAMESPACE + valueFrom: { fieldRef: { fieldPath: metadata.namespace } } + - { name: UPSTREAM_SECRET_NAME, value: {{ .Values.upstreamSecretName | quote }} } + - { name: GATEWAY_INSTANCE, value: {{ .Release.Name | quote }} } + - { name: STORE_IMAGE_APP, value: {{ .Values.images.app | quote }} } + - { name: STORE_IMAGE_STORE, value: {{ .Values.images.store | quote }} } + - { name: STORE_IMAGE_MARIADB, value: {{ .Values.images.mariadb | quote }} } + - { name: STORE_IMAGE_PULL_POLICY, value: {{ .Values.images.pullPolicy | quote }} } + - { name: STORES_HOST_SUFFIX, value: {{ .Values.stores.hostSuffix | quote }} } + - { name: STORES_SCHEME, value: {{ .Values.stores.scheme | quote }} } + - { name: STORES_TLS_SECRET, value: {{ .Values.stores.tlsSecretName | quote }} } + - { name: STORES_TLS_CLUSTER_ISSUER, value: {{ .Values.stores.tlsClusterIssuer | quote }} } + - { name: STORES_QUOTA, value: {{ toJson .Values.stores.quota | quote }} } + - { name: STORES_INGRESS_CLASS, value: {{ .Values.ingress.className | quote }} } + - { name: STORES_STORAGE_CLASS, value: {{ .Values.storage.className | quote }} } + - { name: STORES_WORDPRESS_STORAGE, value: {{ .Values.stores.storage.wordpress | quote }} } + - { name: STORES_MARIADB_STORAGE, value: {{ .Values.stores.storage.mariadb | quote }} } + - { name: STORES_RESOURCES, value: {{ toJson .Values.stores.resources | quote }} } + - { name: STORE_TIMEZONE, value: {{ .Values.store.timezone | quote }} } + - { name: WOOCOMMERCE_VERSION, value: {{ .Values.store.woocommerceVersion | quote }} } + ports: [{ name: http, containerPort: 3000 }] + startupProbe: { httpGet: { path: /health/live, port: http }, periodSeconds: 2, failureThreshold: 60 } + readinessProbe: { httpGet: { path: /health/ready, port: http }, periodSeconds: 10 } + livenessProbe: { httpGet: { path: /health/live, port: http }, periodSeconds: 10 } + securityContext: { allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: { drop: [ALL] } } + resources: {{ toYaml .Values.stores.provisioner.resources | nindent 12 }} + volumeMounts: [{ name: secrets, mountPath: /secrets, readOnly: true }] + volumes: + - name: secrets + secret: + secretName: {{ .Values.secretName }} + items: + - { key: database-url, path: database-url } + - { key: api-key-pepper, path: api-key-pepper } +{{- end }} diff --git a/charts/storelink/values-prod.yaml b/charts/storelink/values-prod.yaml index 21d54b7..d3cd7a9 100644 --- a/charts/storelink/values-prod.yaml +++ b/charts/storelink/values-prod.yaml @@ -6,6 +6,13 @@ images: store: host: store.example.com publicUrl: https://store.example.com +stores: + hostSuffix: store.example.com + scheme: https + tlsSecretName: storefront-tls + # Install cert-manager and configure this ClusterIssuer before enabling stores. + tlsClusterIssuer: letsencrypt-prod + provisionTimeoutSeconds: 600 gateway: host: dashboard.example.com publicUrl: https://dashboard.example.com diff --git a/charts/storelink/values.yaml b/charts/storelink/values.yaml index 013f227..53bdee9 100644 --- a/charts/storelink/values.yaml +++ b/charts/storelink/values.yaml @@ -57,6 +57,35 @@ mariadb: resources: requests: { cpu: 50m, memory: 96Mi } limits: { cpu: '1', memory: 384Mi } +# Additional stores created from the dashboard, each in its own namespace. +stores: + enabled: true + # Storefronts are served at .; production needs wildcard DNS. + hostSuffix: store.storelink.localhost + scheme: http + # cert-manager creates a separate TLS Secret in each store namespace for HTTPS. + tlsSecretName: '' + tlsClusterIssuer: '' + maxPerTeam: 3 + maxPerUser: 2 + quota: + requests.storage: 8Gi + provisionTimeoutSeconds: 240 + storage: { wordpress: 2Gi, mariadb: 1Gi } + resources: + wordpress: + requests: { cpu: 100m, memory: 256Mi } + limits: { cpu: '1', memory: 768Mi } + mariadb: + requests: { cpu: 50m, memory: 96Mi } + limits: { cpu: '1', memory: 384Mi } + wooMcp: + requests: { cpu: 25m, memory: 64Mi } + limits: { cpu: '1', memory: 192Mi } + provisioner: + resources: + requests: { cpu: 25m, memory: 64Mi } + limits: { cpu: 500m, memory: 192Mi } storage: className: local-path wordpress: 2Gi diff --git a/images/app/Dockerfile b/images/app/Dockerfile index 6d87ea9..0ebb2a5 100644 --- a/images/app/Dockerfile +++ b/images/app/Dockerfile @@ -20,6 +20,8 @@ RUN npm ci --omit=dev --ignore-scripts --cache=/tmp/npm-cache && rm -rf /tmp/npm COPY --chown=node:node packages packages COPY --chown=node:node services services COPY --chown=node:node scripts/bootstrap-cluster.ts scripts/bootstrap-cluster.ts +# The store provisioner mounts the same readiness probe into each store. +COPY --chown=node:node charts/storelink/files/health.php charts/storelink/files/health.php COPY --from=dashboard-build --chown=node:node /app/apps/dashboard/dist apps/dashboard/dist USER node ENV NODE_ENV=production BIND_ADDRESS=0.0.0.0 diff --git a/images/store/bootstrap.sh b/images/store/bootstrap.sh index 4792870..0544b83 100644 --- a/images/store/bootstrap.sh +++ b/images/store/bootstrap.sh @@ -8,7 +8,7 @@ for attempt in $(seq 1 120); do sleep 2 done if ! wp core is-installed --quiet; then - wp core install --url="$STORE_PUBLIC_URL" --title='StoreLink Demo Store' \ + wp core install --url="$STORE_PUBLIC_URL" --title="${STORE_TITLE:-StoreLink Demo Store}" \ --admin_user="$WORDPRESS_ADMIN_USER" --admin_password="$WORDPRESS_ADMIN_PASSWORD" \ --admin_email='admin@storelink.test' --skip-email --quiet fi diff --git a/services/gateway/migrations/0008_stores.sql b/services/gateway/migrations/0008_stores.sql new file mode 100644 index 0000000..0956b26 --- /dev/null +++ b/services/gateway/migrations/0008_stores.sql @@ -0,0 +1,17 @@ +CREATE TABLE IF NOT EXISTS stores ( + id TEXT PRIMARY KEY, + team_id TEXT NOT NULL, + created_by TEXT NOT NULL, + -- Slugs name a cluster-wide namespace, so they are unique across teams. The + -- row stays until the namespace is gone, which blocks reuse while it terminates. + slug TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + public_url TEXT NOT NULL, + upstream_slug TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('Provisioning','Ready','Failed','Deleting')), + failure_reason TEXT, + deadline_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS stores_team ON stores(team_id,created_at); diff --git a/services/gateway/migrations/0009_store_attempts.sql b/services/gateway/migrations/0009_store_attempts.sql new file mode 100644 index 0000000..2eeca51 --- /dev/null +++ b/services/gateway/migrations/0009_store_attempts.sql @@ -0,0 +1,4 @@ +-- Preserve existing stores; new requests use recoverable, derived MCP tokens. +ALTER TABLE stores ADD COLUMN IF NOT EXISTS attempt INTEGER NOT NULL DEFAULT 1; +ALTER TABLE stores ADD COLUMN IF NOT EXISTS token_version INTEGER NOT NULL DEFAULT 0; +CREATE INDEX IF NOT EXISTS stores_owner ON stores(team_id,created_by); diff --git a/services/gateway/src/dashboard-api.ts b/services/gateway/src/dashboard-api.ts index 4dddb40..0c1e7b4 100644 --- a/services/gateway/src/dashboard-api.ts +++ b/services/gateway/src/dashboard-api.ts @@ -19,6 +19,9 @@ import type { Gateway } from './gateway.ts'; import { runChat, ChatInputError, validateChatInput } from './chat.ts'; import type { Model } from './chat.ts'; import { summarizeCalls } from './metrics.ts'; +import { requestStore, StoreRequestError } from './store-requests.ts'; +import type { StoreRequestSettings } from './store-requests.ts'; +import { storeSlugPattern } from './store-manifests.ts'; class HttpError extends Error { status: number; @@ -53,9 +56,20 @@ type Options = { // Shared across replicas in the cluster; in memory for a single process. rateLimiter?: RateLimiter; chatAdmission?: Admission; + // Present when the store provisioner is deployed. + stores?: StoreRequestSettings; }; export function dashboardApi(options: Options) { const { store, registry, gateway, pepper, origin } = options; + if ( + options.stores && + [ + options.stores.maxPerTeam, + options.stores.maxPerUser, + options.stores.provisionTimeoutMs, + ].some((n) => !Number.isSafeInteger(n) || n < 1) + ) + throw new Error('Store limits must be positive integers'); const router = express.Router(); const rateLimiter = options.rateLimiter ?? new MemoryRateLimiter(); const chatAdmission = @@ -188,6 +202,13 @@ export function dashboardApi(options: Options) { user: actor(res), csrf: res.locals.csrf, chatConfigured: Boolean(options.model), + storesConfigured: Boolean(options.stores), + storeLimits: options.stores + ? { + perUser: options.stores.maxPerUser, + perTeam: options.stores.maxPerTeam, + } + : undefined, model: options.modelName, gatewayUrl: `${origin}/mcp`, }), @@ -388,6 +409,77 @@ export function dashboardApi(options: Options) { if (!result.rows[0]) throw new HttpError(404, 'Unknown conversation'); res.json(result.rows[0]); }); + // Stores are created and removed by the provisioner; the gateway only records + // requests and announces them. + function storesEnabled(_res: express.Response) { + if (!options.stores) + throw new HttpError(404, 'Store provisioning is not enabled'); + return options.stores; + } + const announceStores = () => + store.pool.query("SELECT pg_notify('storelink_stores', '')"); + router.get('/stores', async (_req, res) => { + storesEnabled(res); + const { rows } = await store.pool.query( + `SELECT id,slug,display_name AS "displayName",public_url AS "publicUrl", + upstream_slug AS "upstreamSlug",status,failure_reason AS "failureReason", + created_at AS "createdAt",created_by AS "createdBy",attempt + FROM stores WHERE team_id=$1 AND ($2='admin' OR created_by=$3) ORDER BY created_at`, + [actor(res).teamId, actor(res).role, actor(res).userId], + ); + res.json(rows); + }); + router.post('/stores', async (req, res) => { + const settings = storesEnabled(res); + const input = z + .object({ + slug: z + .string() + .regex( + storeSlugPattern, + 'Use 1–12 lowercase letters, digits or hyphens, starting with a letter', + ), + displayName: z.string().trim().min(1).max(60), + }) + .parse(req.body); + const id = await requestStore(store.pool, actor(res), input, settings); + res.status(201).json({ id }); + }); + router.post('/stores/:id/retry', async (req, res) => { + const settings = storesEnabled(res); + const updated = await store.pool.query( + `UPDATE stores SET status='Provisioning',attempt=attempt+1,failure_reason=NULL,updated_at=now(), + deadline_at=now()+$3*interval '1 millisecond' + WHERE id=$1 AND team_id=$2 AND status='Failed' AND ($4='admin' OR created_by=$5)`, + [ + String(req.params.id), + actor(res).teamId, + settings.provisionTimeoutMs, + actor(res).role, + actor(res).userId, + ], + ); + if (!updated.rowCount) + throw new HttpError(409, 'Only a failed store can be retried'); + await announceStores(); + res.json({ ok: true }); + }); + router.delete('/stores/:id', async (req, res) => { + storesEnabled(res); + const updated = await store.pool.query( + `UPDATE stores SET status='Deleting',failure_reason=NULL,updated_at=now() + WHERE id=$1 AND team_id=$2 AND ($3='admin' OR created_by=$4)`, + [ + String(req.params.id), + actor(res).teamId, + actor(res).role, + actor(res).userId, + ], + ); + if (!updated.rowCount) throw new HttpError(404, 'Unknown store'); + await announceStores(); + res.json({ ok: true }); + }); router.post('/chat', async (req, res) => { if (!options.model) throw new HttpError( @@ -489,7 +581,7 @@ export function dashboardApi(options: Options) { res.status(400).json({ error: error.message }); else if (error instanceof ChatBusyError) res.status(429).json({ error: error.message }); - else if (error instanceof HttpError) + else if (error instanceof HttpError || error instanceof StoreRequestError) res.status(error.status).json({ error: error.message }); else if (error instanceof z.ZodError) res.status(400).json({ diff --git a/services/gateway/src/k8s-client.ts b/services/gateway/src/k8s-client.ts index 8dbae71..762c907 100644 --- a/services/gateway/src/k8s-client.ts +++ b/services/gateway/src/k8s-client.ts @@ -88,6 +88,7 @@ const resources: Record = { plural: 'networkpolicies', }, Ingress: { base: '/apis/networking.k8s.io/v1', plural: 'ingresses' }, + Certificate: { base: '/apis/cert-manager.io/v1', plural: 'certificates' }, Deployment: { base: '/apis/apps/v1', plural: 'deployments' }, Job: { base: '/apis/batch/v1', plural: 'jobs' }, }; @@ -145,7 +146,8 @@ export class StoreCluster { return this.check(response, `Apply ${object.kind} ${object.metadata.name}`); } // Creates the object only if it does not exist; never overwrites it. - async create(object: KubeObject): Promise { + // Returns false when it already existed. + async create(object: KubeObject): Promise { const resource = resources[object.kind]; const namespace = this.namespaceOf(object); const response = await this.client.request( @@ -153,8 +155,9 @@ export class StoreCluster { `${resource.base}/namespaces/${namespace}/${resource.plural}`, object, ); - if (response.status === 409) return; + if (response.status === 409) return false; this.check(response, `Create ${object.kind} ${object.metadata.name}`); + return true; } async get(object: KubeObject): Promise { const response = await this.client.request('GET', this.path(object)); diff --git a/services/gateway/src/main.ts b/services/gateway/src/main.ts index ddc7540..2a45367 100644 --- a/services/gateway/src/main.ts +++ b/services/gateway/src/main.ts @@ -70,6 +70,7 @@ if (store instanceof PostgresStore) { url: process.env.OPERATIONS_MCP_URL ?? 'http://127.0.0.1:3003/mcp', }, ]; + const storesEnabled = process.env.STORES_ENABLED === 'true'; const credentials = process.env.UPSTREAM_SECRET_NAME ? new KubernetesCredentials( process.env.POD_NAMESPACE ?? 'storelink', @@ -95,6 +96,7 @@ if (store instanceof PostgresStore) { .split(',') .map((value) => value.trim()) .filter(Boolean), + storesEnabled, ); await registry.seed(teamId, defaults); await registry.sync(); @@ -124,6 +126,16 @@ if (store instanceof PostgresStore) { chatLimits, rateLimiter, chatAdmission: new PostgresChatAdmission(store.pool, chatLimits), + stores: storesEnabled + ? { + hostSuffix: + process.env.STORES_HOST_SUFFIX ?? 'store.storelink.localhost', + scheme: process.env.STORES_SCHEME === 'https' ? 'https' : 'http', + maxPerTeam: Number(process.env.STORES_MAX_PER_TEAM ?? 3), + maxPerUser: Number(process.env.STORES_MAX_PER_USER ?? 2), + provisionTimeoutMs: Number(process.env.STORES_TIMEOUT_MS ?? 240000), + } + : undefined, }), ); const dashboard = resolve('apps/dashboard/dist'); diff --git a/services/gateway/src/provisioner-main.ts b/services/gateway/src/provisioner-main.ts new file mode 100644 index 0000000..c0c696e --- /dev/null +++ b/services/gateway/src/provisioner-main.ts @@ -0,0 +1,154 @@ +import { createServer } from 'node:http'; +import { readFileSync } from 'node:fs'; +import { Client, Pool } from 'pg'; +import { apiKeyPepper, secret } from '../../../packages/shared/src/secrets.ts'; +import { KubernetesClient, StoreCluster } from './k8s-client.ts'; +import { KubernetesCredentials } from './credentials.ts'; +import { + PostgresStoreRegistrar, + PostgresStoreRepository, + StoreProvisioner, +} from './store-provisioner.ts'; +import type { StoreSettings } from './store-manifests.ts'; + +// Creates and removes stores requested through the dashboard. It runs as its own +// Deployment so the internet-facing gateway never holds cluster-wide rights. +const env = (name: string, fallback?: string) => { + const value = process.env[name] ?? fallback; + if (value === undefined) throw new Error(`Supply ${name}`); + return value; +}; +const namespace = env('POD_NAMESPACE', 'storelink'); +const settings: StoreSettings = { + images: { + app: env('STORE_IMAGE_APP'), + store: env('STORE_IMAGE_STORE'), + mariadb: env('STORE_IMAGE_MARIADB'), + pullPolicy: env('STORE_IMAGE_PULL_POLICY', 'IfNotPresent'), + }, + hostSuffix: env('STORES_HOST_SUFFIX'), + scheme: env('STORES_SCHEME', 'http') === 'https' ? 'https' : 'http', + tlsSecretName: process.env.STORES_TLS_SECRET || undefined, + tlsClusterIssuer: process.env.STORES_TLS_CLUSTER_ISSUER || undefined, + quota: JSON.parse(env('STORES_QUOTA', '{}')), + ingressClass: env('STORES_INGRESS_CLASS', 'traefik'), + storageClass: env('STORES_STORAGE_CLASS', 'local-path'), + storage: { + wordpress: env('STORES_WORDPRESS_STORAGE', '2Gi'), + mariadb: env('STORES_MARIADB_STORAGE', '1Gi'), + }, + timezone: env('STORE_TIMEZONE', 'Asia/Kolkata'), + woocommerceVersion: env('WOOCOMMERCE_VERSION'), + gateway: { namespace, instance: env('GATEWAY_INSTANCE') }, + // The same file the chart mounts for the main store's readiness probe. + healthPhp: readFileSync( + new URL('../../../charts/storelink/files/health.php', import.meta.url), + 'utf8', + ), + resources: JSON.parse(env('STORES_RESOURCES')), +}; + +const pool = new Pool({ + connectionString: secret('DATABASE_URL'), + max: 3, + connectionTimeoutMillis: 3000, + statement_timeout: 10000, +}); +pool.on('error', () => + console.error('Application database connection interrupted'), +); +const provisioner = new StoreProvisioner({ + tokenKey: apiKeyPepper(), + repository: new PostgresStoreRepository(pool), + cluster: new StoreCluster(new KubernetesClient()), + registrar: new PostgresStoreRegistrar( + pool, + new KubernetesCredentials( + namespace, + env('UPSTREAM_SECRET_NAME'), + apiKeyPepper(), + ), + ), + settings, +}); + +let running = false; +async function tick() { + if (running) return; + running = true; + try { + const client = await pool.connect(); + try { + // One pass at a time across overlapping provisioner pods. + const { rows } = await client.query<{ locked: boolean }>( + 'SELECT pg_try_advisory_lock(87264004) AS locked', + ); + if (!rows[0].locked) return; + try { + await provisioner.reconcile(); + } finally { + await client.query('SELECT pg_advisory_unlock(87264004)'); + } + } finally { + client.release(); + } + } catch (error) { + console.error( + `Store reconciliation failed: ${error instanceof Error ? error.message : 'unexpected error'}`, + ); + } finally { + running = false; + } +} +const timer = setInterval(() => void tick(), 5000); +void tick(); + +// Dashboard requests are announced so they start without waiting for the timer. +let listener: Client | undefined; +let closing = false; +async function listen() { + if (closing) return; + const client = new Client({ connectionString: secret('DATABASE_URL') }); + client.on('notification', () => void tick()); + client.on('error', () => { + void client.end().catch(() => {}); + if (!closing) setTimeout(() => void listen(), 2000); + }); + try { + await client.connect(); + await client.query('LISTEN storelink_stores'); + listener = client; + } catch { + void client.end().catch(() => {}); + setTimeout(() => void listen(), 2000); + } +} +void listen(); + +const health = createServer((req, res) => { + if (req.url === '/health/live') { + res.writeHead(200).end('ok'); + return; + } + if (req.url === '/health/ready') { + void pool.query('SELECT 1').then( + () => res.writeHead(200).end('ok'), + () => res.writeHead(503).end(), + ); + return; + } + res.writeHead(404).end(); +}).listen( + Number(process.env.PORT ?? 3000), + process.env.BIND_ADDRESS ?? '127.0.0.1', +); + +for (const signal of ['SIGINT', 'SIGTERM']) + process.once(signal, async () => { + closing = true; + clearInterval(timer); + health.close(); + await listener?.end().catch(() => {}); + while (running) await new Promise((resolve) => setTimeout(resolve, 100)); + await pool.end(); + }); diff --git a/services/gateway/src/registry.ts b/services/gateway/src/registry.ts index 20a40cc..01491ab 100644 --- a/services/gateway/src/registry.ts +++ b/services/gateway/src/registry.ts @@ -29,6 +29,11 @@ type Row = { // pg returns TIMESTAMPTZ columns as Date objects. const iso = (value: string | Date | null) => value ? new Date(value).toISOString() : undefined; +// Servers created by the store provisioner are changed only through Stores. +const storeOwned = () => + new RegistryInputError( + 'This server belongs to a store; manage it from Stores', + ); export class Registry { store: PostgresStore; gateway: Gateway; @@ -51,11 +56,12 @@ export class Registry { gateway: Gateway, secrets: CredentialStore, origins: string[], + managedStores = false, ) { this.store = store; this.gateway = gateway; this.secrets = secrets; - this.policy = new UpstreamPolicy(origins); + this.policy = new UpstreamPolicy(origins, undefined, managedStores); } validateUrl(value: string): string { return this.policy.validate(value).toString(); @@ -283,6 +289,7 @@ export class Registry { }, ) { if (actor.role !== 'admin') throw new Error('Administrator required'); + if (input.id?.startsWith('store-')) throw storeOwned(); if (!/^[a-z][a-z0-9_-]{0,15}$/.test(input.slug)) throw new RegistryInputError('Use a short lowercase server slug'); const url = (await this.policy.check(input.url)).toString(); @@ -350,6 +357,7 @@ export class Registry { } private async deleteUnlocked(actor: Identity, id: string): Promise { if (actor.role !== 'admin') throw new Error('Administrator required'); + if (id.startsWith('store-')) throw storeOwned(); const deleted = await this.store.pool.query( `WITH deleted AS ( DELETE FROM upstream_servers WHERE id=$1 AND team_id=$2 RETURNING id,team_id,slug,credential_ref diff --git a/services/gateway/src/store-manifests.ts b/services/gateway/src/store-manifests.ts index 7ce77fa..44a95c1 100644 --- a/services/gateway/src/store-manifests.ts +++ b/services/gateway/src/store-manifests.ts @@ -13,6 +13,8 @@ export type StoreSettings = { hostSuffix: string; scheme: 'http' | 'https'; tlsSecretName?: string; + tlsClusterIssuer?: string; + quota?: Record; ingressClass: string; storageClass: string; storage: { wordpress: string; mariadb: string }; @@ -26,7 +28,10 @@ export type StoreSettings = { // Every name and address a store uses comes from here, so the Ingress host, // WordPress URLs, OAuth signing and the MCP origin always agree. -export function storeNames(slug: string, settings: StoreSettings) { +export function storeNames( + slug: string, + settings: Pick, +) { if (!storeSlugPattern.test(slug)) throw new Error('Invalid store slug'); const namespace = `storelink-store-${slug}`; if (!storeNamespacePattern.test(namespace)) @@ -45,7 +50,11 @@ export function storeNames(slug: string, settings: StoreSettings) { // Generated once per store; the provisioner creates it only if it is missing, // since MariaDB and WooCommerce keep the first values they see. -export function storeSecret(slug: string, settings: StoreSettings) { +export function storeSecret( + slug: string, + settings: StoreSettings, + mcpToken?: string, +) { const { namespace } = storeNames(slug, settings); const random = () => randomBytes(32).toString('base64url'); return { @@ -59,7 +68,7 @@ export function storeSecret(slug: string, settings: StoreSettings) { 'wordpress-admin-password': random(), 'woo-consumer-key': `ck_${randomBytes(20).toString('hex')}`, 'woo-consumer-secret': `cs_${randomBytes(20).toString('hex')}`, - 'woo-mcp-token': random(), + 'woo-mcp-token': mcpToken ?? random(), }, }; } @@ -90,7 +99,16 @@ const nonRoot = (id: number) => ({ fsGroupChangePolicy: 'OnRootMismatch', }); -export function storeManifests(slug: string, settings: StoreSettings) { +export function storeManifests( + slug: string, + settings: StoreSettings, + title = 'StoreLink Store', + attempt = 1, +) { + if (!Number.isSafeInteger(attempt) || attempt < 1) + throw new Error('Invalid provisioning attempt'); + if (settings.scheme === 'https' && !settings.tlsSecretName) + throw new Error('HTTPS stores need a TLS Secret name'); const names = storeNames(slug, settings); const { namespace } = names; const meta = (name: string, component?: string) => ({ @@ -112,7 +130,11 @@ export function storeManifests(slug: string, settings: StoreSettings) { selector: selector(slug, component), template: { metadata: { labels: labels(slug, component) }, - spec: { automountServiceAccountToken: false, ...podSpec }, + spec: { + automountServiceAccountToken: false, + enableServiceLinks: false, + ...podSpec, + }, }, }, }); @@ -185,6 +207,8 @@ export function storeManifests(slug: string, settings: StoreSettings) { const bootstrap = { restartPolicy: 'OnFailure', automountServiceAccountToken: false, + // Service variables such as WOO_MCP_PORT would override the app's own settings. + enableServiceLinks: false, securityContext: nonRoot(33), affinity: { podAffinity: { @@ -204,6 +228,7 @@ export function storeManifests(slug: string, settings: StoreSettings) { command: ['bash', '/opt/storelink/bootstrap.sh'], env: [ ...wordpressEnv, + { name: 'STORE_TITLE', value: title }, { name: 'STORE_TIMEZONE', value: settings.timezone }, { name: 'WOOCOMMERCE_VERSION', value: settings.woocommerceVersion }, { name: 'WORDPRESS_ADMIN_USER', value: 'admin' }, @@ -231,11 +256,18 @@ export function storeManifests(slug: string, settings: StoreSettings) { const jobName = `store-bootstrap-${createHash('sha256') .update(JSON.stringify(bootstrap)) .digest('hex') - .slice(0, 10)}`; + .slice(0, 10)}-${attempt}`; const ingress = { apiVersion: 'networking.k8s.io/v1', kind: 'Ingress', - metadata: meta('wordpress', 'wordpress'), + metadata: { + ...meta('wordpress', 'wordpress'), + ...(settings.tlsClusterIssuer && { + annotations: { + 'cert-manager.io/cluster-issuer': settings.tlsClusterIssuer, + }, + }), + }, spec: { ingressClassName: settings.ingressClass, ...(settings.tlsSecretName && { @@ -285,6 +317,7 @@ export function storeManifests(slug: string, settings: StoreSettings) { persistentvolumeclaims: '2', 'requests.storage': '8Gi', services: '4', + ...settings.quota, }, }, }, @@ -304,6 +337,24 @@ export function storeManifests(slug: string, settings: StoreSettings) { }, }, policy('default-deny', undefined, { policyTypes: ['Ingress', 'Egress'] }), + // HTTP-01 solver pods are created by cert-manager, outside the store's + // application labels. Let Traefik reach only their challenge port. + ...(settings.tlsClusterIssuer + ? [ + { + apiVersion: 'networking.k8s.io/v1', + kind: 'NetworkPolicy', + metadata: meta('acme-http01-solver'), + spec: { + podSelector: { + matchLabels: { 'acme.cert-manager.io/http01-solver': 'true' }, + }, + policyTypes: ['Ingress'], + ingress: [{ from: [traefik], ports: tcp(8089) }], + }, + }, + ] + : []), policy('dns', undefined, { policyTypes: ['Egress'], egress: [ @@ -491,6 +542,7 @@ export function storeManifests(slug: string, settings: StoreSettings) { command: ['node', 'services/woocommerce-mcp/src/main.ts'], env: [ { name: 'WOO_MCP_ORIGIN', value: names.mcpOrigin }, + { name: 'WOO_MCP_PORT', value: '3001' }, { name: 'STORE_PUBLIC_URL', value: names.publicUrl }, { name: 'STORE_CONNECT_SERVICE', value: 'wordpress' }, { name: 'WOO_MCP_TOKEN_FILE', value: '/secrets/woo-mcp-token' }, diff --git a/services/gateway/src/store-provisioner.ts b/services/gateway/src/store-provisioner.ts new file mode 100644 index 0000000..903c30b --- /dev/null +++ b/services/gateway/src/store-provisioner.ts @@ -0,0 +1,323 @@ +import { createHmac } from 'node:crypto'; +import type { Pool } from 'pg'; +import { storeManifests, storeSecret } from './store-manifests.ts'; +import type { StoreSettings } from './store-manifests.ts'; +import type { StoreCluster } from './k8s-client.ts'; +import type { CredentialStore } from './credentials.ts'; + +export type StoreStatus = 'Provisioning' | 'Ready' | 'Failed' | 'Deleting'; +export type StoreRecord = { + id: string; + teamId: string; + slug: string; + displayName: string; + status: StoreStatus; + deadlineAt: number | null; + attempt?: number; + tokenVersion?: number; +}; +export interface StoreRepository { + // Stores the provisioner has work to do for. + active(): Promise; + // Changes status only if the store is still in `from`, so a concurrent + // delete is never overwritten. + update( + id: string, + from: StoreStatus, + to: StoreStatus, + reason: string | null, + ): Promise; + remove(id: string): Promise; +} +// Connects a store's MCP server to the gateway. +export interface StoreRegistrar { + hasCredential(store: StoreRecord): Promise; + saveCredential(store: StoreRecord, token: string): Promise; + register(store: StoreRecord, url: string, slug: string): Promise; + unregister(store: StoreRecord): Promise; +} +type Cluster = Pick< + StoreCluster, + 'apply' | 'create' | 'get' | 'deleteNamespace' +>; + +type Workload = { + status?: { + availableReplicas?: number; + succeeded?: number; + conditions?: { type: string; status: string; message?: string }[]; + }; +}; + +export class StoreProvisioner { + repository: StoreRepository; + cluster: Cluster; + registrar: StoreRegistrar; + settings: StoreSettings; + now: () => number; + tokenKey?: Uint8Array; + // Stores whose manifests this process has applied, keyed by store and Job name. + private applied = new Set(); + private pending = new Map(); + constructor(options: { + repository: StoreRepository; + cluster: Cluster; + registrar: StoreRegistrar; + settings: StoreSettings; + now?: () => number; + tokenKey?: Uint8Array; + }) { + this.repository = options.repository; + this.cluster = options.cluster; + this.registrar = options.registrar; + this.settings = options.settings; + this.now = options.now ?? Date.now; + this.tokenKey = options.tokenKey; + } + + async reconcile(): Promise { + for (const store of await this.repository.active()) { + try { + if (store.status === 'Provisioning') await this.provision(store); + else if (store.status === 'Deleting') await this.teardown(store); + } catch (error) { + // Recorded for the dashboard; the next pass tries again. + await this.repository + .update(store.id, store.status, store.status, message(error)) + .catch(() => {}); + } + } + } + + private async fail(store: StoreRecord, reason: string) { + for (const key of this.applied) + if (key.startsWith(`${store.id}:`)) this.applied.delete(key); + this.pending.delete(store.id); + await this.repository.update(store.id, 'Provisioning', 'Failed', reason); + } + + private async provision(store: StoreRecord) { + if (store.deadlineAt !== null && this.now() > store.deadlineAt) { + await this.fail( + store, + `Not ready before the provisioning deadline; ${this.pending.get(store.id) ?? 'nothing was created'}`, + ); + return; + } + const { objects, jobName, names } = storeManifests( + store.slug, + this.settings, + store.displayName, + store.attempt ?? 1, + ); + const key = `${store.id}:${jobName}`; + if (!this.applied.has(key)) { + await this.cluster.apply(objects[0] as never); + const token = + store.tokenVersion === 1 + ? storeMcpToken(this.tokenKey, store.id) + : undefined; + const secret = storeSecret(store.slug, this.settings, token); + const created = await this.cluster.create(secret as never); + // For new stores the token can be reconstructed after a crash between + // Secret creation and saving the gateway's credential. DB passwords stay random. + if (created || (token && !(await this.registrar.hasCredential(store)))) + await this.registrar.saveCredential( + store, + secret.stringData['woo-mcp-token'], + ); + else if (!(await this.registrar.hasCredential(store))) { + await this.fail( + store, + 'Legacy store credentials exist but the gateway has no copy; delete the store and create it again', + ); + return; + } + for (const object of objects.slice(1)) + await this.cluster.apply(object as never); + this.applied.add(key); + } + const waiting: string[] = []; + for (const name of ['mariadb', 'wordpress', 'woo-mcp']) { + const deployment = (await this.cluster.get({ + apiVersion: 'apps/v1', + kind: 'Deployment', + metadata: { name, namespace: names.namespace }, + })) as Workload | undefined; + if (!deployment?.status?.availableReplicas) waiting.push(name); + } + const job = (await this.cluster.get({ + apiVersion: 'batch/v1', + kind: 'Job', + metadata: { name: jobName, namespace: names.namespace }, + })) as Workload | undefined; + const failed = job?.status?.conditions?.find( + (c) => c.type === 'Failed' && c.status === 'True', + ); + if (failed) { + await this.fail( + store, + `Store setup failed${failed.message ? `: ${failed.message}` : ''}`, + ); + return; + } + if (!job?.status?.succeeded) waiting.push('store setup'); + if (this.settings.tlsClusterIssuer) { + const certificate = (await this.cluster.get({ + apiVersion: 'cert-manager.io/v1', + kind: 'Certificate', + metadata: { + name: this.settings.tlsSecretName!, + namespace: names.namespace, + }, + })) as Workload | undefined; + if ( + !certificate?.status?.conditions?.some( + (c) => c.type === 'Ready' && c.status === 'True', + ) + ) + waiting.push('TLS certificate'); + } + + if (waiting.length) { + this.pending.set(store.id, `waiting for ${waiting.join(', ')}`); + return; + } + await this.registrar.register(store, names.mcpUrl, names.upstreamSlug); + this.pending.delete(store.id); + await this.repository.update(store.id, 'Provisioning', 'Ready', null); + } + + private async teardown(store: StoreRecord) { + await this.registrar.unregister(store); + const { namespace } = storeManifests(store.slug, this.settings).names; + if ((await this.cluster.deleteNamespace(namespace)) === 'gone') { + for (const key of this.applied) + if (key.startsWith(`${store.id}:`)) this.applied.delete(key); + await this.repository.remove(store.id); + } + } +} + +export function storeMcpToken(key: Uint8Array | undefined, id: string): string { + if (!key || key.byteLength < 32) + throw new Error( + 'Store MCP token key must contain at least 32 random bytes', + ); + return createHmac('sha256', key) + .update('storelink/store-mcp/v1:' + id) + .digest('base64url'); +} + +const message = (error: unknown) => + error instanceof Error ? error.message.slice(0, 500) : 'Unexpected error'; + +export class PostgresStoreRepository implements StoreRepository { + pool: Pool; + constructor(pool: Pool) { + this.pool = pool; + } + async active(): Promise { + const { rows } = await this.pool.query<{ + id: string; + team_id: string; + slug: string; + display_name: string; + status: StoreStatus; + deadline_at: Date | null; + attempt: number; + token_version: number; + }>( + `SELECT id,team_id,slug,display_name,status,deadline_at,attempt,token_version FROM stores + WHERE status IN ('Provisioning','Deleting') ORDER BY created_at`, + ); + return rows.map((row) => ({ + id: row.id, + teamId: row.team_id, + slug: row.slug, + displayName: row.display_name, + status: row.status, + deadlineAt: row.deadline_at ? row.deadline_at.getTime() : null, + attempt: row.attempt, + tokenVersion: row.token_version, + })); + } + async update( + id: string, + from: StoreStatus, + to: StoreStatus, + reason: string | null, + ): Promise { + await this.pool.query( + `UPDATE stores SET status=$3,failure_reason=$4,updated_at=now() + WHERE id=$1 AND status=$2`, + [id, from, to, reason], + ); + } + async remove(id: string): Promise { + await this.pool.query( + "DELETE FROM stores WHERE id=$1 AND status='Deleting'", + [id], + ); + } +} + +// Registers a store's MCP server the way the dashboard registers any server: +// a credential entry in the gateway's Secret and a row in upstream_servers. +export class PostgresStoreRegistrar implements StoreRegistrar { + pool: Pool; + credentials: CredentialStore; + constructor(pool: Pool, credentials: CredentialStore) { + this.pool = pool; + this.credentials = credentials; + } + static upstreamId(store: { id: string }) { + return `store-${store.id}`; + } + async hasCredential(store: StoreRecord) { + return ( + (await this.credentials.get(PostgresStoreRegistrar.upstreamId(store))) !== + undefined + ); + } + async saveCredential(store: StoreRecord, token: string) { + await this.credentials.set(PostgresStoreRegistrar.upstreamId(store), { + header: 'authorization', + value: `Bearer ${token}`, + }); + } + async register(store: StoreRecord, url: string, slug: string) { + const id = PostgresStoreRegistrar.upstreamId(store); + try { + await this.pool.query( + `INSERT INTO upstream_servers (id,team_id,slug,url,enabled,has_credential,credential_ref) + VALUES ($1,$2,$3,$4,true,true,$1) + ON CONFLICT (id) DO UPDATE SET url=excluded.url,slug=excluded.slug, + revision=upstream_servers.revision+1`, + [id, store.teamId, slug, url], + ); + } catch (error) { + if ((error as { code?: string }).code === '23505') + throw new Error(`The MCP server name ${slug} is already in use`); + throw error; + } + await this.pool.query("SELECT pg_notify('storelink_registry', $1)", [ + store.teamId, + ]); + } + async unregister(store: StoreRecord) { + const id = PostgresStoreRegistrar.upstreamId(store); + const removed = await this.pool.query( + `WITH deleted AS ( + DELETE FROM upstream_servers WHERE id=$1 RETURNING id,team_id,slug + ) INSERT INTO upstream_events(team_id,server_id,slug,status) + SELECT team_id,id,slug,'Removed' FROM deleted`, + [id], + ); + await this.credentials.set(id, null); + if (removed.rowCount) + await this.pool.query("SELECT pg_notify('storelink_registry', $1)", [ + store.teamId, + ]); + } +} diff --git a/services/gateway/src/store-requests.ts b/services/gateway/src/store-requests.ts new file mode 100644 index 0000000..ff5be08 --- /dev/null +++ b/services/gateway/src/store-requests.ts @@ -0,0 +1,83 @@ +import { randomUUID } from 'node:crypto'; +import type { Pool } from 'pg'; +import type { Identity } from '../../../packages/shared/src/api-keys.ts'; +import { storeNames } from './store-manifests.ts'; + +export class StoreRequestError extends Error { + status: number; + constructor(status: number, message: string) { + super(message); + this.status = status; + } +} +export type StoreRequestSettings = { + hostSuffix: string; + scheme: 'http' | 'https'; + maxPerTeam: number; + maxPerUser: number; + provisionTimeoutMs: number; +}; + +export async function requestStore( + pool: Pick, + user: Identity, + input: { slug: string; displayName: string }, + settings: StoreRequestSettings, +): Promise { + const names = storeNames(input.slug, settings); + const id = randomUUID(); + const client = await pool.connect(); + try { + await client.query('BEGIN'); + // A short database admission transaction, independent of the long-running worker. + // The shared lock makes limits authoritative across gateway replicas. + await client.query('SELECT pg_advisory_xact_lock(87264005)'); + const { rows } = await client.query<{ total: string; mine: string }>( + 'SELECT count(*) AS total,count(*) FILTER (WHERE created_by=$2) AS mine FROM stores WHERE team_id=$1', + [user.teamId, user.userId], + ); + if (Number(rows[0].total) >= settings.maxPerTeam) + throw new StoreRequestError( + 409, + `A team can have ${settings.maxPerTeam} stores; delete one first`, + ); + if (Number(rows[0].mine) >= settings.maxPerUser) + throw new StoreRequestError( + 409, + `You can have ${settings.maxPerUser} stores; delete one first`, + ); + const conflict = await client.query( + 'SELECT 1 FROM upstream_servers WHERE team_id=$1 AND slug=$2', + [user.teamId, names.upstreamSlug], + ); + if (conflict.rowCount) + throw new StoreRequestError( + 409, + `The MCP server name ${names.upstreamSlug} is already in use`, + ); + await client.query( + `INSERT INTO stores (id,team_id,created_by,slug,display_name,public_url,upstream_slug,status,deadline_at,token_version) + VALUES ($1,$2,$3,$4,$5,$6,$7,'Provisioning',now()+$8*interval '1 millisecond',1)`, + [ + id, + user.teamId, + user.userId, + input.slug, + input.displayName, + names.publicUrl, + names.upstreamSlug, + settings.provisionTimeoutMs, + ], + ); + await client.query("SELECT pg_notify('storelink_stores', '')"); + await client.query('COMMIT'); + return id; + } catch (error) { + await client.query('ROLLBACK'); + if ((error as { code?: string }).code === '23505') + throw new StoreRequestError(409, 'That store name is in use'); + throw error; + } finally { + client.release(); + } +} diff --git a/tests/store-requests.test.ts b/tests/store-requests.test.ts new file mode 100644 index 0000000..71541ec --- /dev/null +++ b/tests/store-requests.test.ts @@ -0,0 +1,123 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import type { Pool } from 'pg'; +import { + requestStore, + StoreRequestError, +} from '../services/gateway/src/store-requests.ts'; +import type { Identity } from '../packages/shared/src/api-keys.ts'; + +const user: Identity = { + userId: 'alice', + teamId: 'team', + role: 'member', + keyId: 'key', +}; +const settings = { + hostSuffix: 'test.localhost', + scheme: 'http' as const, + maxPerUser: 3, + maxPerTeam: 3, + provisionTimeoutMs: 240000, +}; +function fixture(existing: { team: string; owner: string }[]) { + const records = [...existing]; + let tail = Promise.resolve(); + let rollbacks = 0; + const pool = { + async connect() { + let release: (() => void) | undefined; + let pending: { team: string; owner: string } | undefined; + return { + async query(sql: string, args: unknown[] = []) { + if (sql === 'BEGIN') return { rows: [] }; + if (sql.includes('pg_advisory_xact_lock')) { + const previous = tail; + tail = new Promise((resolve) => { + release = resolve; + }); + await previous; + return { rows: [] }; + } + if (sql.includes('count(*) AS total')) { + const team = records.filter((r) => r.team === args[0]); + return { + rows: [ + { + total: String(team.length), + mine: String(team.filter((r) => r.owner === args[1]).length), + }, + ], + }; + } + if (sql.includes('SELECT 1 FROM upstream_servers')) + return { rows: [], rowCount: 0 }; + if (sql.includes('INSERT INTO stores')) { + pending = { team: String(args[1]), owner: String(args[2]) }; + return { rows: [], rowCount: 1 }; + } + if (sql.includes('pg_notify')) return { rows: [] }; + if (sql === 'COMMIT') { + if (pending) records.push(pending); + release?.(); + return { rows: [] }; + } + if (sql === 'ROLLBACK') { + rollbacks++; + release?.(); + return { rows: [] }; + } + throw new Error('Unexpected query'); + }, + release() { + release?.(); + }, + }; + }, + } as unknown as Pick; + return { pool, records, rollbacks: () => rollbacks }; +} + +test('concurrent gateway requests admit exactly one store when a team has one slot', async () => { + const f = fixture([ + { team: 'team', owner: 'other' }, + { team: 'team', owner: 'other' }, + ]); + const results = await Promise.allSettled( + ['north', 'south'].map((slug) => + requestStore(f.pool, user, { slug, displayName: slug }, settings), + ), + ); + assert.equal(results.filter((r) => r.status === 'fulfilled').length, 1); + assert.equal(f.records.length, 3); + const rejected = results.find( + (r) => r.status === 'rejected', + ) as PromiseRejectedResult; + assert.ok(rejected.reason instanceof StoreRequestError); + assert.equal(rejected.reason.status, 409); + assert.equal(f.rollbacks(), 1); +}); +test('store admission enforces per-user limits independently of another team and releases the transaction', async () => { + const f = fixture([ + { team: 'team', owner: 'alice' }, + { team: 'other', owner: 'alice' }, + ]); + await assert.rejects( + () => + requestStore( + f.pool, + user, + { slug: 'north', displayName: 'North' }, + { ...settings, maxPerUser: 1 }, + ), + /You can have 1 stores/, + ); + await requestStore( + f.pool, + { ...user, userId: 'bob' }, + { slug: 'south', displayName: 'South' }, + { ...settings, maxPerUser: 1 }, + ); + assert.equal(f.records.length, 3); + assert.equal(f.rollbacks(), 1); +}); diff --git a/tests/stores.test.ts b/tests/stores.test.ts index 8dd2dd1..c6ca700 100644 --- a/tests/stores.test.ts +++ b/tests/stores.test.ts @@ -9,6 +9,11 @@ import type { StoreSettings } from '../services/gateway/src/store-manifests.ts'; import { StoreCluster } from '../services/gateway/src/k8s-client.ts'; import type { KubeResponse } from '../services/gateway/src/k8s-client.ts'; import { UpstreamPolicy } from '../services/gateway/src/upstream-policy.ts'; +import { + StoreProvisioner, + storeMcpToken, +} from '../services/gateway/src/store-provisioner.ts'; +import type { StoreRecord } from '../services/gateway/src/store-provisioner.ts'; const resources = { requests: { cpu: '50m', memory: '64Mi' }, @@ -106,6 +111,8 @@ test('store manifests stay inside the store namespace and agree on addresses', ( assert.ok(c.resources.requests && c.resources.limits, c.name); assert.equal(pod.securityContext.runAsNonRoot, true, c.name); assert.equal(pod.automountServiceAccountToken, false, c.name); + // A Service named woo-mcp would otherwise inject WOO_MCP_PORT=tcp://... + assert.equal(pod.enableServiceLinks, false, c.name); assert.deepEqual(c.securityContext.capabilities.drop, ['ALL'], c.name); if (kind === 'Deployment') assert.ok(c.readinessProbe && c.livenessProbe, c.name); @@ -218,3 +225,292 @@ test('the cluster client refuses namespaces outside the store pattern', async () namespaceLabels = undefined; assert.equal(await cluster.deleteNamespace('storelink-store-north'), 'gone'); }); + +// A provisioner wired to in-memory fakes of the cluster, database and gateway. +function provisionerFixture() { + let now = 0; + const store: StoreRecord = { + id: 's1', + teamId: 'team', + slug: 'north', + displayName: 'North', + status: 'Provisioning', + deadlineAt: 240000, + }; + const updates: [string, string, string | null][] = []; + const events: string[] = []; + const state = { + secretExists: false, + credential: false, + available: new Set(), + job: undefined as undefined | 'succeeded' | 'failed', + namespace: 'present' as 'present' | 'deleting' | 'gone', + removed: false, + }; + const provisioner = new StoreProvisioner({ + settings, + now: () => now, + repository: { + async active() { + return store.status === 'Provisioning' || store.status === 'Deleting' + ? [{ ...store }] + : []; + }, + async update(_id, from, to, reason) { + if (store.status !== from) return; + store.status = to; + updates.push([from, to, reason]); + }, + async remove() { + state.removed = true; + }, + }, + registrar: { + async hasCredential() { + return state.credential; + }, + async saveCredential() { + state.credential = true; + events.push('credential'); + }, + async register(_store, url, slug) { + events.push(`register ${slug} ${url}`); + }, + async unregister() { + events.push('unregister'); + }, + }, + cluster: { + async apply(object: { kind: string }) { + events.push(`apply ${object.kind}`); + return {}; + }, + async create() { + if (state.secretExists) return false; + state.secretExists = true; + return true; + }, + async get(object: { kind: string; metadata: { name: string } }) { + if (object.kind === 'Deployment') + return { + status: { + availableReplicas: state.available.has(object.metadata.name) + ? 1 + : 0, + }, + }; + if (!state.job) return { status: {} }; + return state.job === 'succeeded' + ? { status: { succeeded: 1 } } + : { + status: { + conditions: [ + { + type: 'Failed', + status: 'True', + message: 'BackoffLimitExceeded', + }, + ], + }, + }; + }, + async deleteNamespace() { + if (state.namespace === 'present') state.namespace = 'deleting'; + return state.namespace === 'gone' ? 'gone' : 'deleting'; + }, + } as never, + }); + return { + store, + updates, + events, + state, + provisioner, + advance: (ms: number) => { + now += ms; + }, + }; +} + +test('a store becomes Ready once its workloads and setup finish, then registers', async () => { + const f = provisionerFixture(); + await f.provisioner.reconcile(); + assert.equal(f.store.status, 'Provisioning'); + assert.ok(f.events.includes('credential')); + assert.equal(f.events.filter((e) => e === 'apply Namespace').length, 1); + // Later passes only check readiness; nothing is applied twice. + f.state.available = new Set(['mariadb', 'wordpress', 'woo-mcp']); + await f.provisioner.reconcile(); + assert.equal(f.events.filter((e) => e === 'apply Namespace').length, 1); + assert.equal(f.store.status, 'Provisioning'); + f.state.job = 'succeeded'; + await f.provisioner.reconcile(); + assert.equal(f.store.status, 'Ready'); + assert.ok( + f.events.includes( + 'register woo-north http://woo-mcp.storelink-store-north.svc.cluster.local:3001/mcp', + ), + ); +}); + +test('provisioning fails with a reason on a deadline, a failed setup or lost credentials', async () => { + const late = provisionerFixture(); + await late.provisioner.reconcile(); + late.advance(240001); + await late.provisioner.reconcile(); + assert.equal(late.store.status, 'Failed'); + assert.match( + late.updates.at(-1)![2]!, + /deadline; waiting for mariadb, wordpress, woo-mcp, store setup/, + ); + + const broken = provisionerFixture(); + broken.state.job = 'failed'; + await broken.provisioner.reconcile(); + assert.equal(broken.store.status, 'Failed'); + assert.match( + broken.updates.at(-1)![2]!, + /Store setup failed: BackoffLimitExceeded/, + ); + + // The Secret survived a restart but the gateway never received its copy. + const lost = provisionerFixture(); + lost.state.secretExists = true; + await lost.provisioner.reconcile(); + assert.equal(lost.store.status, 'Failed'); + assert.match(lost.updates.at(-1)![2]!, /no copy/); +}); + +test('deleting a store unregisters it and removes the row once the namespace is gone', async () => { + const f = provisionerFixture(); + f.store.status = 'Deleting'; + await f.provisioner.reconcile(); + assert.deepEqual(f.events, ['unregister']); + assert.equal(f.state.removed, false); + f.state.namespace = 'gone'; + await f.provisioner.reconcile(); + assert.equal(f.state.removed, true); +}); + +test('a status change never overwrites a concurrent delete', async () => { + const f = provisionerFixture(); + f.state.available = new Set(['mariadb', 'wordpress', 'woo-mcp']); + f.state.job = 'succeeded'; + // The pass read the store as Provisioning, then it was deleted before it finished. + const snapshot = { ...f.store }; + f.store.status = 'Deleting'; + f.provisioner.repository.active = async () => [snapshot]; + await f.provisioner.reconcile(); + assert.equal(f.store.status, 'Deleting'); +}); + +test('retry gives a failed bootstrap a new attempt without replacing database credentials', async () => { + const f = provisionerFixture(); + f.state.available = new Set(['mariadb', 'wordpress', 'woo-mcp']); + f.state.job = 'failed'; + const original = storeManifests( + f.store.slug, + settings, + f.store.displayName, + 1, + ).jobName; + const get = f.provisioner.cluster.get.bind(f.provisioner.cluster); + f.provisioner.cluster.get = async (object) => + object.kind === 'Job' && object.metadata.name === original + ? { status: { conditions: [{ type: 'Failed', status: 'True' }] } } + : get(object); + await f.provisioner.reconcile(); + assert.equal(f.store.status, 'Failed'); + f.store.status = 'Provisioning'; + f.store.attempt = 2; + f.state.job = 'succeeded'; + await f.provisioner.reconcile(); + assert.equal(f.store.status, 'Ready'); + assert.notEqual( + storeManifests(f.store.slug, settings, f.store.displayName, 2).jobName, + original, + ); + assert.equal(f.events.filter((e) => e === 'credential').length, 1); +}); + +test('new-store MCP tokens recover a crash after Secret creation without reading store passwords', async () => { + const f = provisionerFixture(); + const key = Buffer.alloc(32, 7); + f.store.tokenVersion = 1; + f.provisioner.tokenKey = key; + const tokens: string[] = []; + let interrupted = true; + f.provisioner.registrar.saveCredential = async (_store, token) => { + tokens.push(token); + if (interrupted) { + interrupted = false; + throw new Error('interrupted'); + } + f.state.credential = true; + }; + await f.provisioner.reconcile(); + assert.equal(f.state.secretExists, true); + assert.equal(f.state.credential, false); + const restarted = new StoreProvisioner({ + settings, + repository: f.provisioner.repository, + cluster: f.provisioner.cluster, + registrar: f.provisioner.registrar, + tokenKey: key, + now: () => 0, + }); + f.state.available = new Set(['mariadb', 'wordpress', 'woo-mcp']); + f.state.job = 'succeeded'; + await restarted.reconcile(); + assert.equal(f.store.status, 'Ready'); + assert.equal(tokens.length, 2); + assert.equal(tokens[0], tokens[1]); + assert.equal(tokens[0], storeMcpToken(key, f.store.id)); + assert.notEqual(tokens[0], storeMcpToken(key, 'other-store')); + assert.throws(() => storeMcpToken(undefined, 'id'), /32 random bytes/); +}); + +test('HTTPS stores request their own certificate and remain Provisioning until it is Ready', async () => { + const f = provisionerFixture(); + f.provisioner.settings = { + ...settings, + scheme: 'https', + tlsSecretName: 'storefront-tls', + tlsClusterIssuer: 'test-issuer', + }; + const objects = storeManifests('north', f.provisioner.settings) + .objects as Obj[]; + const ingress = objects.find((o) => o.kind === 'Ingress') as any; + assert.equal( + ingress.metadata.annotations['cert-manager.io/cluster-issuer'], + 'test-issuer', + ); + assert.equal(ingress.spec.tls[0].secretName, 'storefront-tls'); + const solver = objects.find((o) => o.metadata.name === 'acme-http01-solver')!; + assert.equal(solver.spec.ingress[0].ports[0].port, 8089); + assert.deepEqual(solver.spec.podSelector.matchLabels, { + 'acme.cert-manager.io/http01-solver': 'true', + }); + let ready = false; + const get = f.provisioner.cluster.get.bind(f.provisioner.cluster); + f.provisioner.cluster.get = async (o) => + o.kind === 'Certificate' + ? { + status: { + conditions: [{ type: 'Ready', status: ready ? 'True' : 'False' }], + }, + } + : get(o); + f.state.available = new Set(['mariadb', 'wordpress', 'woo-mcp']); + f.state.job = 'succeeded'; + await f.provisioner.reconcile(); + assert.equal(f.store.status, 'Provisioning'); + assert.ok(!f.events.some((e) => e.startsWith('register '))); + ready = true; + await f.provisioner.reconcile(); + assert.equal(f.store.status, 'Ready'); + assert.throws( + () => storeManifests('north', { ...settings, scheme: 'https' }), + /TLS Secret/, + ); +}); From a278a9d2f7860d767c23b079fb95fcd1dcaa3e5a Mon Sep 17 00:00:00 2001 From: Deep-Axe Date: Mon, 26 Oct 2026 18:37:26 +0530 Subject: [PATCH 3/4] Add Stores dashboard with ownership, progress, retry and deletion controls --- apps/dashboard/src/main.tsx | 23 +- apps/dashboard/src/stores.tsx | 304 ++++++++++++++++++++++++ apps/dashboard/src/style.css | 27 +++ services/gateway/src/k8s-client.ts | 3 +- services/gateway/src/store-manifests.ts | 2 +- services/gateway/src/upstream-policy.ts | 2 +- tests/stores.test.ts | 10 +- 7 files changed, 363 insertions(+), 8 deletions(-) create mode 100644 apps/dashboard/src/stores.tsx diff --git a/apps/dashboard/src/main.tsx b/apps/dashboard/src/main.tsx index 02dd8e1..6af04de 100644 --- a/apps/dashboard/src/main.tsx +++ b/apps/dashboard/src/main.tsx @@ -1,6 +1,7 @@ import React, { useEffect, useState } from 'react'; import { createRoot } from 'react-dom/client'; import './style.css'; +import { Stores } from './stores.tsx'; type User = { userId: string; @@ -14,6 +15,8 @@ type Me = { chatConfigured: boolean; model: string; gatewayUrl: string; + storesConfigured?: boolean; + storeLimits?: { perUser: number; perTeam: number }; }; type Server = { id: string; @@ -259,7 +262,13 @@ function App() {