From 21051eb38f7742f2bd61a174a57b386334dcbd22 Mon Sep 17 00:00:00 2001 From: Titouan Mathis Date: Thu, 5 Nov 2026 19:58:01 +0100 Subject: [PATCH 1/2] Add forge servers public-key to print the key of the forge user The local_public_key attribute of a server is ~/.ssh/id_rsa.pub of the forge user, the key that Forge clones repositories with. The GET /servers/{id}/key endpoint gives another key: the one that lets Forge connect to the server. Co-authored-by: Claude --- packages/cli/src/commands/completion.ts | 4 +- .../cli/src/commands/servers/command.test.ts | 7 ++ packages/cli/src/commands/servers/command.ts | 9 ++- .../cli/src/commands/servers/handlers.test.ts | 69 ++++++++++++++++++- packages/cli/src/commands/servers/handlers.ts | 31 +++++++++ packages/cli/src/commands/servers/help.ts | 23 +++++++ 6 files changed, 140 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/commands/completion.ts b/packages/cli/src/commands/completion.ts index 3ec4713..5d3b68c 100644 --- a/packages/cli/src/commands/completion.ts +++ b/packages/cli/src/commands/completion.ts @@ -22,7 +22,7 @@ _forge_completions() { # Subcommands for each command local config_cmds="set get delete" - local servers_cmds="list ls get create reboot" + local servers_cmds="list ls get public-key create reboot" local sites_cmds="list ls get apply" local deployments_cmds="list ls deploy logs log script update-script" local databases_cmds="list ls get" @@ -235,6 +235,7 @@ _forge() { 'list:List all servers' 'ls:List all servers (alias)' 'get:Get server details' + 'public-key:Print the public key of the forge user' 'create:Create a server' 'reboot:Reboot a server' ) @@ -594,6 +595,7 @@ complete -c forge -f -n "__fish_seen_subcommand_from config" -a "delete" -d "Del complete -c forge -f -n "__fish_seen_subcommand_from servers s" -a "list" -d "List all servers" complete -c forge -f -n "__fish_seen_subcommand_from servers s" -a "ls" -d "List all servers (alias)" complete -c forge -f -n "__fish_seen_subcommand_from servers s" -a "get" -d "Get server details" +complete -c forge -f -n "__fish_seen_subcommand_from servers s" -a "public-key" -d "Print the public key of the forge user" complete -c forge -f -n "__fish_seen_subcommand_from servers s" -a "create" -d "Create a server" complete -c forge -f -n "__fish_seen_subcommand_from servers s" -a "reboot" -d "Reboot a server" diff --git a/packages/cli/src/commands/servers/command.test.ts b/packages/cli/src/commands/servers/command.test.ts index 383e1fe..3bc6478 100644 --- a/packages/cli/src/commands/servers/command.test.ts +++ b/packages/cli/src/commands/servers/command.test.ts @@ -7,6 +7,7 @@ vi.mock("./handlers.ts", () => ({ serversGet: vi.fn().mockResolvedValue(), serversCreate: vi.fn().mockResolvedValue(), serversReboot: vi.fn().mockResolvedValue(), + serversPublicKey: vi.fn().mockResolvedValue(), })); vi.mock("../../context.ts", () => ({ @@ -61,6 +62,12 @@ describe("handleServersCommand routing", () => { expect(handlers.serversGet).toHaveBeenCalledWith(["123"], expect.anything()); }); + it("should route public-key subcommand with args", async () => { + const handlers = await import("./handlers.ts"); + await handleServersCommand("public-key", ["123"], {}); + expect(handlers.serversPublicKey).toHaveBeenCalledWith(["123"], expect.anything()); + }); + it("should route create subcommand", async () => { const handlers = await import("./handlers.ts"); await handleServersCommand("create", [], { name: "my-server" }); diff --git a/packages/cli/src/commands/servers/command.ts b/packages/cli/src/commands/servers/command.ts index 77c3985..610be8a 100644 --- a/packages/cli/src/commands/servers/command.ts +++ b/packages/cli/src/commands/servers/command.ts @@ -1,5 +1,11 @@ import { createCommandRouter } from "../../utils/command-router.ts"; -import { serversList, serversGet, serversCreate, serversReboot } from "./handlers.ts"; +import { + serversList, + serversGet, + serversCreate, + serversReboot, + serversPublicKey, +} from "./handlers.ts"; export const handleServersCommand = createCommandRouter({ resource: "servers", @@ -7,6 +13,7 @@ export const handleServersCommand = createCommandRouter({ list: serversList, ls: serversList, get: [serversGet, "args"], + "public-key": [serversPublicKey, "args"], create: serversCreate, reboot: [serversReboot, "args"], }, diff --git a/packages/cli/src/commands/servers/handlers.test.ts b/packages/cli/src/commands/servers/handlers.test.ts index 740695c..04be329 100644 --- a/packages/cli/src/commands/servers/handlers.test.ts +++ b/packages/cli/src/commands/servers/handlers.test.ts @@ -3,7 +3,13 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import type { ServerAttributes } from "@studiometa/forge-api"; import { createTestContext } from "../../context.ts"; -import { serversList, serversGet, serversCreate, serversReboot } from "./handlers.ts"; +import { + serversList, + serversGet, + serversCreate, + serversReboot, + serversPublicKey, +} from "./handlers.ts"; vi.mock("@studiometa/forge-core", () => ({ listServers: vi.fn(), @@ -448,3 +454,64 @@ describe("serversCreate", () => { expect(processExitSpy).toHaveBeenCalled(); }); }); + +describe("serversPublicKey", () => { + let processExitSpy: ReturnType; + const key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIExample root@web-1"; + + beforeEach(() => { + processExitSpy = vi.spyOn(process, "exit").mockImplementation(() => undefined as never); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("prints only the key in human format", async () => { + const { getServer } = await import("@studiometa/forge-core"); + vi.mocked(getServer).mockResolvedValue({ + data: { ...mockServer, local_public_key: `${key}\n` }, + }); + const ctx = createTestContext({ + token: "test", + mockClient: {} as never, + options: { format: "human" }, + }); + + await serversPublicKey(["1"], ctx); + expect(vi.mocked(console.log).mock.calls).toEqual([[key]]); + }); + + it("prints { public_key } in json format", async () => { + const { getServer } = await import("@studiometa/forge-core"); + vi.mocked(getServer).mockResolvedValue({ data: { ...mockServer, local_public_key: key } }); + const ctx = createTestContext({ + token: "test", + mockClient: {} as never, + options: { format: "json" }, + }); + + await serversPublicKey(["1"], ctx); + expect(JSON.parse(String(vi.mocked(console.log).mock.calls[0][0]))).toEqual({ + public_key: key, + }); + }); + + it("exits with not found when the server has no key yet", async () => { + const { getServer } = await import("@studiometa/forge-core"); + vi.mocked(getServer).mockResolvedValue({ data: { ...mockServer, local_public_key: null } }); + const ctx = createTestContext({ token: "test", mockClient: {} as never, options: {} }); + + await serversPublicKey(["1"], ctx); + expect(processExitSpy).toHaveBeenCalledWith(5); + }); + + it("exits with a validation error without a server", async () => { + const ctx = createTestContext({ token: "test", mockClient: {} as never, options: {} }); + + await serversPublicKey([], ctx).catch(() => {}); + expect(processExitSpy).toHaveBeenCalledWith(3); + }); +}); diff --git a/packages/cli/src/commands/servers/handlers.ts b/packages/cli/src/commands/servers/handlers.ts index 8f39f10..2301c9b 100644 --- a/packages/cli/src/commands/servers/handlers.ts +++ b/packages/cli/src/commands/servers/handlers.ts @@ -9,6 +9,7 @@ import { listServers, getServer, createServer, rebootServer } from "@studiometa/ import type { CommandContext } from "../../context.ts"; import { exitWithValidationError, runCommand } from "../../error-handler.ts"; +import { ApiError } from "../../errors.ts"; import { resolveServerId } from "../../utils/resolve.ts"; /** @@ -57,6 +58,36 @@ export async function serversGet(args: string[], ctx: CommandContext): Promise { + const [server] = args; + + if (!server) { + exitWithValidationError("server_id", "forge servers public-key ", ctx.formatter); + } + + await runCommand(async () => { + const token = ctx.getToken(); + const execCtx = ctx.createExecutorContext(token); + const server_id = await resolveServerId(server, execCtx); + const result = await getServer({ server_id }, execCtx); + const key = result.data.local_public_key?.trim(); + if (!key) { + throw new ApiError(`Server ${result.data.name} has no public key yet`, 404, [ + "Wait until Forge provisions the server", + ]); + } + if (ctx.formatter.isJson()) { + ctx.formatter.output({ public_key: key }); + } else { + console.log(key); + } + }, ctx.formatter); +} + const CREATE_USAGE = "forge servers create --name --ip-address [options]"; /** diff --git a/packages/cli/src/commands/servers/help.ts b/packages/cli/src/commands/servers/help.ts index 545f9a0..6b448b0 100644 --- a/packages/cli/src/commands/servers/help.ts +++ b/packages/cli/src/commands/servers/help.ts @@ -34,6 +34,28 @@ ${colors.bold("OPTIONS:")} ${colors.bold("EXAMPLES:")} forge servers get 123 forge servers get 123 --format json +`); + } else if (subcommand === "public-key") { + console.log(` +${colors.bold("forge servers public-key")} - Print the public key of the forge user + +${colors.bold("USAGE:")} + forge servers public-key [options] + +${colors.bold("ARGUMENTS:")} + Server ID or name (required) + +${colors.bold("OPTIONS:")} + -f, --format Output format: json ({ "public_key": "..." }), human (the key only) + +Forge clones repositories as the forge user, with this key (~/.ssh/id_rsa.pub). +Add it to the git host (e.g. as a deploy key) before you create a site with a +repository. It is not the key of GET /servers/{id}/key: that key lets Forge +connect to the server. + +${colors.bold("EXAMPLES:")} + forge servers public-key 123 + forge servers public-key 123 > server.pub `); } else if (subcommand === "create") { console.log(` @@ -91,6 +113,7 @@ ${colors.bold("ALIASES:")} ${colors.bold("SUBCOMMANDS:")} list, ls List all servers get Get server details + public-key Print the public key of the forge user create Create a server (requires --name, and --ip-address for custom) reboot Reboot a server From 76396f114de3d0f55547b57036ff1d866b43116d Mon Sep 17 00:00:00 2001 From: Titouan Mathis Date: Thu, 5 Nov 2026 19:58:02 +0100 Subject: [PATCH 2/2] Add an Nginx file and database .env keys to forge sites apply "nginx" points to an Nginx file with the variables of Forge Nginx templates. apply renders it locally for the site, shows the diff from Forge in the plan, and writes it, after the creation of a new site. An unknown variable is a validation error. "database.env" maps the database name, user and password of a new database user to other .env keys than the Laravel names. Co-authored-by: Claude --- .changelog/sites-apply-nginx-db-key.md | 5 + packages/cli/README.md | 37 +++- packages/cli/schemas/site.schema.json | 29 ++- .../src/commands/sites/apply/apply.test.ts | 202 ++++++++++++++++++ .../cli/src/commands/sites/apply/apply.ts | 81 ++++++- .../src/commands/sites/apply/handler.test.ts | 38 ++++ .../src/commands/sites/apply/nginx.test.ts | 142 ++++++++++++ .../cli/src/commands/sites/apply/nginx.ts | 118 ++++++++++ .../cli/src/commands/sites/apply/plan.test.ts | 97 ++++++++- packages/cli/src/commands/sites/apply/plan.ts | 51 ++++- .../commands/sites/apply/site-file.test.ts | 63 ++++++ .../cli/src/commands/sites/apply/site-file.ts | 55 ++++- .../src/commands/sites/apply/test-fixtures.ts | 19 ++ packages/cli/src/commands/sites/help.ts | 10 +- 14 files changed, 923 insertions(+), 24 deletions(-) create mode 100644 .changelog/sites-apply-nginx-db-key.md create mode 100644 packages/cli/src/commands/sites/apply/nginx.test.ts create mode 100644 packages/cli/src/commands/sites/apply/nginx.ts diff --git a/.changelog/sites-apply-nginx-db-key.md b/.changelog/sites-apply-nginx-db-key.md new file mode 100644 index 0000000..c46bfde --- /dev/null +++ b/.changelog/sites-apply-nginx-db-key.md @@ -0,0 +1,5 @@ +### Added + +- **CLI**: Add `nginx` to the site file of `forge sites apply`: an Nginx file with Forge template variables, rendered for the site, compared with Forge and written when it differs +- **CLI**: Add `database.env` to the site file of `forge sites apply` to write the database name, user and password into other `.env` keys +- **CLI**: Add `forge servers public-key ` to print the public key of the `forge` user, the key that the git host must accept diff --git a/packages/cli/README.md b/packages/cli/README.md index 072b233..b20b926 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -121,6 +121,7 @@ forge config delete # Delete all stored config ```bash forge servers list # List all servers forge servers get # Get server details +forge servers public-key # Print the public key of the forge user, for the git host forge servers reboot # Reboot a server forge servers create --name --ip-address [--php-version php84 --database mariadb1011] # Create a server (Custom VPS by default) @@ -155,6 +156,7 @@ forge sites apply --file site.json # Make the site match the file }, "database": { "name": "example", "user": "example" }, "env": { "APP_ENV": "production" }, + "nginx": "nginx.conf", "workers": [{ "command": "php artisan queue:work database --queue=default", "processes": 4 }], "schedule": [{ "command": "php artisan schedule:run", "frequency": "minutely" }] } @@ -165,11 +167,44 @@ forge sites apply --file site.json # Make the site match the file | `server`, `domain` | Find the server and the site by exact name (never a partial match, with or without `--exact`). `server` can also be an ID | | `type`, `php`, `webDirectory`, `repository` | Create the site when it is missing (`type` is then required) and wait until Forge installs it. On an existing site, update `php`, `webDirectory` and the branch | | `aliases` | Add the missing aliases. Never remove one | -| `database` | Create the database and the user when they are missing. A new user gets a random password, written only into the site `.env` (`DB_DATABASE`, `DB_USERNAME`, `DB_PASSWORD`) | +| `database` | Create the database and the user when they are missing. A new user gets a random password, written only into the site `.env` (`DB_DATABASE`, `DB_USERNAME`, `DB_PASSWORD`, see below) | | `env` | Add the missing keys to the site `.env`. A key with another value on Forge is reported, never overwritten | +| `nginx` | Nginx file, relative to the site file, with Forge template variables. Rendered for the site and written when it differs from Forge (see below) | | `workers` | Forge background processes of the site. Create the missing ones, delete the ones of the site that the file does not declare. The directory defaults to the site root | | `schedule` | Forge scheduled jobs of the site (`minutely`, `hourly`, `nightly`, `weekly`, `monthly`, `reboot`, or `custom` with `cron`). A relative `artisan` becomes the absolute path of the site | +For an application that does not read the Laravel names, `database.env` maps the database name, the user and the password to other `.env` keys. Each key is optional and defaults to the Laravel name. Other values, such as the host, go in `env`: + +```json +{ + "database": { + "name": "example", + "user": "example", + "env": { "name": "DB_NAME", "user": "DB_USER", "password": "DB_PASSWD" } + }, + "env": { "DB_SERVER": "127.0.0.1" } +} +``` + +`nginx` points to an Nginx file that uses the variables of [Forge Nginx templates](https://forge.laravel.com/docs/servers/nginx-templates). `apply` renders the file locally with the values of the site, so one file works on any server, without a template on the server: + +| Variable | Value | +| ----------------------------------------------- | -------------------------------------------------------------------------------------------------------- | +| `{{SITE}}`, `{{SITE_ID}}`, `{{USER}}` | Site name (`www.example.com`), site ID (`456`), site user (`forge`) | +| `{{DOMAINS}}` | Site name and aliases, separated by spaces | +| `{{ROOT_PATH}}`, `{{PATH}}`, `{{DIRECTORY}}` | Site root, web directory (`/home/forge/www.example.com/public`) and its relative path (`/public`) | +| `{{PROXY_PASS}}` | PHP-FPM socket, e.g. `unix:/var/run/php/php8.3-fpm.sock` (`php8.3-fpm-.sock` for an isolated site) | +| `{{PORT}}`, `{{PORT_V6}}` | `80` and `[::]:80` (with `default_server` for the site named `default`) | +| `{{SERVER_PUBLIC_IP}}`, `{{SERVER_PRIVATE_IP}}` | IP addresses of the server (empty when there is no private IP) | + +Spaces inside the braces are allowed (`{{ SITE_ID }}`). An unknown variable is an error (exit `3`). On a new site, `apply` writes the file after Forge installs the site. On an existing site, the plan shows the diff from Forge to the rendered file (CRLF and the final newline do not count), and `apply` writes it. When `apply` also changes the site or its aliases, it reads the site again and renders the file with the new values. Forge returns the full `server` block on old servers and only its content on recent ones: write the file in the same form as `forge nginx get` prints it. + +Forge clones the repository when it creates the site, with the key of the `forge` user of the server. Add this key to the git host (for example as a deploy key) before `apply` creates the site: + +```bash +forge servers public-key web-1 > server.pub # The key only; --format json gives { "public_key": "..." } +``` + A worker or a scheduled job matches the file when its command, directory, processes (or frequency) and user are the same. Commands are compared after normalization: quotes (`'database'` and `database`), repeated spaces and a relative `artisan` (`php artisan` and `php /home/forge/www.example.com/artisan`) do not count as differences. Forge cannot update a worker: a changed worker is created again, then the old one is deleted. `workers` and `schedule` delete what the file does not declare, so `apply` deletes only what surely belongs to the site, because a server often hosts several sites: diff --git a/packages/cli/schemas/site.schema.json b/packages/cli/schemas/site.schema.json index 31a4ee2..271e83f 100644 --- a/packages/cli/schemas/site.schema.json +++ b/packages/cli/schemas/site.schema.json @@ -68,7 +68,29 @@ "required": ["name", "user"], "properties": { "name": { "type": "string", "minLength": 1 }, - "user": { "type": "string", "minLength": 1 } + "user": { "type": "string", "minLength": 1 }, + "env": { + "description": ".env keys that get the database name, user and password of a new user. The keys must be different.", + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*$", + "default": "DB_DATABASE" + }, + "user": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*$", + "default": "DB_USERNAME" + }, + "password": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*$", + "default": "DB_PASSWORD" + } + } + } } }, "env": { @@ -77,6 +99,11 @@ "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" }, "additionalProperties": { "type": ["string", "number", "boolean"] } }, + "nginx": { + "description": "Nginx file, relative to the site file, with the Forge template variables {{DIRECTORY}}, {{DOMAINS}}, {{PATH}}, {{PORT}}, {{PORT_V6}}, {{PROXY_PASS}}, {{ROOT_PATH}}, {{SERVER_PUBLIC_IP}}, {{SERVER_PRIVATE_IP}}, {{SITE}}, {{SITE_ID}} and {{USER}}. Rendered for the site and written when it differs from Forge. An unknown variable is an error.", + "type": "string", + "minLength": 1 + }, "workers": { "description": "Forge background processes of the site. [] deletes all the workers of the site.", "type": "array", diff --git a/packages/cli/src/commands/sites/apply/apply.test.ts b/packages/cli/src/commands/sites/apply/apply.test.ts index ee6a314..528123d 100644 --- a/packages/cli/src/commands/sites/apply/apply.test.ts +++ b/packages/cli/src/commands/sites/apply/apply.test.ts @@ -56,6 +56,31 @@ describe("assertCanCreateSite", () => { }); describe("readRemoteState", () => { + it("reads and renders the Nginx file of an existing site", async () => { + const { ctx } = fakeForge({ sites: [site(456, "a.example.com")], nginx: "root /old;" }); + const state = await readRemoteState( + { server: 123, domain: "a.example.com", nginxTemplate: "root {{ PATH }};" }, + "123", + ctx, + ); + + expect(state.nginx).toEqual({ + remote: "root /old;", + rendered: "root /home/forge/a.example.com/public;", + }); + }); + + it("reads no Nginx configuration when the site does not exist", async () => { + const { ctx } = fakeForge(); + const state = await readRemoteState( + { server: 123, domain: "a.example.com", nginxTemplate: "root {{ PATH }};" }, + "123", + ctx, + ); + + expect(state.nginx).toBeNull(); + }); + it("reads only what the file manages", async () => { const { ctx, calls } = fakeForge({ sites: [site(456, "a.example.com")] }); const state = await readRemoteState({ server: 123, domain: "a.example.com" }, "123", ctx); @@ -384,6 +409,183 @@ describe("applySiteFile", () => { }); }); +describe("applySiteFile with an Nginx file", () => { + const template = "include forge-conf/{{ SITE_ID }}/server/*;\nroot {{ PATH }};\n"; + + it("renders the file for a new site and writes it once the site exists", async () => { + const { ctx, calls, state } = fakeForge({ newSiteStatuses: ["installing", "installed"] }); + const changes: Change[] = []; + const plan = await applySiteFile( + { server: 123, domain: "a.example.com", type: "php", nginxTemplate: template }, + "123", + ctx, + { sleep: async () => {}, pollIntervalMs: 1, onChange: (change) => changes.push(change) }, + ); + + expect(plan.changes).toEqual([ + expect.objectContaining({ action: "create-site" }), + { action: "update-nginx", diff: null }, + ]); + const writes = calls.filter(([method]) => method !== "GET").map(([m, url]) => `${m} ${url}`); + expect(writes).toEqual([ + "POST /orgs/org/servers/123/sites", + "PUT /orgs/org/servers/123/sites/999/nginx", + ]); + expect(state.nginx).toBe( + "include forge-conf/999/server/*;\nroot /home/forge/a.example.com/public;\n", + ); + expect(changes.at(-1)).toMatchObject({ + action: "update-nginx", + diff: expect.stringContaining("+root"), + }); + }); + + it("writes the rendered file when Forge differs", async () => { + const { ctx, calls, state } = fakeForge({ + sites: [site(456, "a.example.com")], + nginx: "include forge-conf/456/server/*;\nroot /home/forge/a.example.com/web;\n", + }); + const plan = await applySiteFile( + { server: 123, domain: "a.example.com", nginxTemplate: template }, + "123", + ctx, + ); + + expect(plan.changes).toEqual([ + { + action: "update-nginx", + diff: expect.stringContaining("+root /home/forge/a.example.com/public;"), + }, + ]); + expect(calls.filter(([method]) => method !== "GET")).toEqual([ + [ + "PUT", + "/orgs/org/servers/123/sites/456/nginx", + { config: "include forge-conf/456/server/*;\nroot /home/forge/a.example.com/public;\n" }, + ], + ]); + expect(state.nginx).toContain("/public;"); + }); + + it("changes nothing when Forge matches the rendered file", async () => { + const { ctx, calls } = fakeForge({ + sites: [site(456, "a.example.com")], + nginx: "include forge-conf/456/server/*;\nroot /home/forge/a.example.com/public;", + }); + const plan = await applySiteFile( + { server: 123, domain: "a.example.com", nginxTemplate: template }, + "123", + ctx, + ); + + expect(plan.changes).toEqual([]); + expect(calls.filter(([method]) => method !== "GET")).toEqual([]); + expect(calls.map(([, url]) => url)).not.toContain("/orgs/org/servers/123"); + }); + + it("reads the server only for the SERVER_ variables", async () => { + const { ctx, state } = fakeForge({ sites: [site(456, "a.example.com")] }); + await applySiteFile( + { + server: 123, + domain: "a.example.com", + nginxTemplate: "allow {{ SERVER_PRIVATE_IP }};\nset_real_ip_from {{ SERVER_PUBLIC_IP }};", + }, + "123", + ctx, + ); + + expect(state.nginx).toBe("allow 10.0.0.10;\nset_real_ip_from 203.0.113.10;"); + }); + + it("renders again after a change of the site", async () => { + const forge = fakeForge({ + sites: [site(456, "a.example.com")], + nginx: "include forge-conf/456/server/*;\nroot /home/forge/a.example.com/public;\n", + }); + const put = forge.client.put.getMockImplementation()!; + forge.client.put.mockImplementation(async (url: string, body: Record) => { + if (url.endsWith("/sites/456")) { + forge.state.sites[0] = { + ...forge.state.sites[0], + web_directory: "/home/forge/a.example.com/web", + }; + } + return put(url, body); + }); + + await applySiteFile( + { server: 123, domain: "a.example.com", webDirectory: "/web", nginxTemplate: template }, + "123", + forge.ctx, + ); + + const writes = forge.calls.filter(([method]) => method !== "GET"); + expect(writes.map(([m, url]) => `${m} ${url}`)).toEqual([ + "PUT /orgs/org/servers/123/sites/456", + "PUT /orgs/org/servers/123/sites/456/nginx", + ]); + expect(forge.state.nginx).toContain("root /home/forge/a.example.com/web;"); + }); + + it("does not read the site again when nothing changes", async () => { + const { ctx, calls } = fakeForge({ sites: [site(456, "a.example.com")] }); + await applySiteFile( + { server: 123, domain: "a.example.com", aliases: ["a.example.com"], nginxTemplate: "" }, + "123", + ctx, + ); + + expect(calls.map(([, url]) => url)).not.toContain("/orgs/org/sites/456"); + }); +}); + +describe("applySiteFile with database .env keys", () => { + it("writes the generated values into the keys of the file", async () => { + const { ctx, state } = fakeForge({ sites: [site(456, "a.example.com")] }); + const plan = await applySiteFile( + { + server: 123, + domain: "a.example.com", + database: { + name: "example", + user: "example_user", + env: { name: "DB_NAME", user: "DB_USER", password: "DB_PASS" }, + }, + }, + "123", + ctx, + { generatePassword: () => "s3cr3t" }, + ); + + expect(state.env).toBe("DB_NAME=example\nDB_USER=example_user\nDB_PASS=s3cr3t\n"); + expect(JSON.stringify(plan)).not.toContain("s3cr3t"); + }); +}); + +describe("applySiteFile repository hint", () => { + it("points to the public key of the server when a site with a repository is not installed", async () => { + const { ctx } = fakeForge({ newSiteStatuses: ["installing", "installing"] }); + const file: SiteFile = { + server: 123, + domain: "a.example.com", + type: "php", + repository: { provider: "custom", url: "git@git.example.com:group/a.git" }, + }; + + const error = await applySiteFile(file, "123", ctx, { + sleep: async () => {}, + pollIntervalMs: 10, + timeoutMs: 10, + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(ApplyError); + expect((error as ApplyError).hints).toContain( + "Check that the git host accepts the key of the server: forge servers public-key 123", + ); + }); +}); + describe("fakeForge", () => { it("rejects a route it does not know", async () => { const { client } = fakeForge(); diff --git a/packages/cli/src/commands/sites/apply/apply.ts b/packages/cli/src/commands/sites/apply/apply.ts index 8beadff..f264b03 100644 --- a/packages/cli/src/commands/sites/apply/apply.ts +++ b/packages/cli/src/commands/sites/apply/apply.ts @@ -14,6 +14,8 @@ import { deleteDaemon, deleteScheduledJob, getEnv, + getNginxConfig, + getServer, getSite, listDaemons, listDatabaseUsers, @@ -22,13 +24,23 @@ import { listServers, listSites, updateEnv, + updateNginxConfig, updateSite, } from "@studiometa/forge-core"; import type { CliExecutorContext } from "../../../context.ts"; import { CliError, ValidationError } from "../../../errors.ts"; +import { unifiedDiff } from "../../../utils/drift.ts"; import { matchServerId } from "../../../utils/resolve.ts"; -import { type Change, type Plan, type RemoteState, type Site, buildPlan } from "./plan.ts"; +import { needsServer, renderNginx } from "./nginx.ts"; +import { + type Change, + type Plan, + type RemoteState, + type Site, + buildPlan, + databaseEnvKeys, +} from "./plan.ts"; import type { SiteFile } from "./site-file.ts"; /** Error of `apply` that is not a validation error, e.g. a site that never gets installed. */ @@ -88,6 +100,10 @@ export async function readRemoteState( ); } + if (file.nginxTemplate !== undefined) { + state.nginx = site ? await readNginx(file.nginxTemplate, serverId, site, ctx) : null; + } + if (file.schedule) { const siteJobs = siteId ? (await listScheduledJobs({ server_id: serverId, site_id: siteId }, ctx)).data @@ -100,6 +116,23 @@ export async function readRemoteState( return state; } +/** + * Read the Nginx configuration of a site on Forge and render the Nginx file for the site. + */ +export async function readNginx( + template: string, + serverId: string, + site: Site, + ctx: CliExecutorContext, +): Promise<{ remote: string; rendered: string }> { + const remote = (await getNginxConfig({ server_id: serverId, site_id: String(site.id) }, ctx)) + .data; + const server = needsServer(template) + ? (await getServer({ server_id: serverId }, ctx)).data + : null; + return { remote, rendered: renderNginx(template, site, server) }; +} + /** * Exit with a validation error when the site does not exist and the site file cannot create it. */ @@ -168,12 +201,17 @@ export interface ApplyOptions { generatePassword?: () => string; } +/** Forge clones the repository with the key of the server: the git host must accept it. */ +const REPOSITORY_HINT = (serverId: string) => + `Check that the git host accepts the key of the server: forge servers public-key ${serverId}`; + const defaultSleep = (ms: number) => new Promise((resolve) => { setTimeout(resolve, ms); }); async function waitForSite( + file: SiteFile, serverId: string, siteId: string, ctx: CliExecutorContext, @@ -188,6 +226,7 @@ async function waitForSite( if (waited >= timeout) { throw new ApplyError(`Site ${site.name} is still ${site.status} on Forge`, [ "Run forge sites apply again once the site is installed", + ...(file.repository ? [REPOSITORY_HINT(serverId)] : []), ]); } await sleep(interval); @@ -232,15 +271,43 @@ async function applyDatabase( ctx, ); ids.database_user_id = String(user.data.id); - env.set("DB_DATABASE", change.database); - env.set("DB_USERNAME", change.name); - env.set("DB_PASSWORD", password); + const keys = databaseEnvKeys(file.database); + env.set(keys.name, change.database); + env.set(keys.user, change.name); + env.set(keys.password, password); } } if (databaseId !== undefined) ids.database_id = databaseId; return { ids, env }; } +/** + * Write the rendered Nginx file when it differs from Forge. After a change of the site or its + * aliases, read the site again: the web directory and the domains are template variables. + */ +async function applyNginx( + template: string, + plan: Plan, + serverId: string, + site: Site, + ctx: CliExecutorContext, + options: ApplyOptions, +): Promise { + const siteChanged = plan.changes.some( + (change) => change.action === "update-site" || change.action === "add-alias", + ); + const planned = plan.changes.some((change) => change.action === "update-nginx"); + if (!planned && !siteChanged) return; + + const params = { server_id: serverId, site_id: String(site.id) }; + const current = siteChanged ? (await getSite(params, ctx)).data : site; + const { remote, rendered } = await readNginx(template, serverId, current, ctx); + const diff = unifiedDiff(remote, rendered); + if (!diff) return; + options.onChange?.({ action: "update-nginx", diff }); + await updateNginxConfig({ ...params, content: rendered }, ctx); +} + /** * Run the changes of a plan for a site that exists. */ @@ -308,6 +375,10 @@ async function applyChanges( await updateEnv({ ...params, content: setEnvValues(content, env) }, ctx); } + if (file.nginxTemplate !== undefined) { + await applyNginx(file.nginxTemplate, plan, serverId, site, ctx, options); + } + // Create first, then delete: a changed worker runs again before the old one stops. for (const change of creations) { options.onChange?.(change); @@ -393,7 +464,7 @@ export async function applySiteFile( }, ctx, ); - await waitForSite(serverId, String(created.data.id), ctx, options); + await waitForSite(file, serverId, String(created.data.id), ctx, options); state = await readRemoteState(file, serverId, ctx); plan = buildPlan(file, state); } diff --git a/packages/cli/src/commands/sites/apply/handler.test.ts b/packages/cli/src/commands/sites/apply/handler.test.ts index fe5270d..5a6b3c8 100644 --- a/packages/cli/src/commands/sites/apply/handler.test.ts +++ b/packages/cli/src/commands/sites/apply/handler.test.ts @@ -71,6 +71,44 @@ describe("sitesApply", () => { expect(exitSpy).toHaveBeenCalledWith(3); }); + it("exits 3 on an unknown variable in the Nginx file, before any call", async () => { + vi.mocked(readFileSync).mockImplementation((path) => + String(path).endsWith("site.json") + ? JSON.stringify({ server: 123, domain: "a.example.com", nginx: "nginx.conf" }) + : "root {{ WEBROOT }};", + ); + const { ctx, forge } = context("json", { file: "site.json", check: true }); + + await sitesApply(ctx); + + expect(exitSpy).toHaveBeenCalledWith(3); + expect(forge.calls).toEqual([]); + }); + + it("prints the Nginx diff with --check and exits 1", async () => { + vi.mocked(readFileSync).mockImplementation((path) => + String(path).endsWith("site.json") + ? JSON.stringify({ server: 123, domain: "a.example.com", nginx: "nginx.conf" }) + : "root {{ PATH }};\n", + ); + const { ctx, forge } = context("json", { file: "site.json", check: true }); + + await sitesApply(ctx); + + expect(process.exitCode).toBe(1); + expect(JSON.parse(String(vi.mocked(console.log).mock.calls[0][0]))).toEqual({ + inSync: false, + changes: [ + { + action: "update-nginx", + diff: "--- remote\n+++ local\n@@ -0,0 +1,1 @@\n+root /home/forge/a.example.com/public;", + }, + ], + warnings: [], + }); + expect(forge.calls.filter(([method]) => method !== "GET")).toEqual([]); + }); + it("exits 0 with --check when Forge matches", async () => { siteFile({ server: 123, diff --git a/packages/cli/src/commands/sites/apply/nginx.test.ts b/packages/cli/src/commands/sites/apply/nginx.test.ts new file mode 100644 index 0000000..58341ec --- /dev/null +++ b/packages/cli/src/commands/sites/apply/nginx.test.ts @@ -0,0 +1,142 @@ +import { describe, expect, it } from "vitest"; + +import { ValidationError } from "../../../errors.ts"; +import { + NGINX_VARIABLES, + assertNginxTemplate, + needsServer, + nginxValues, + phpSocketVersion, + renderNginx, + templateVariables, +} from "./nginx.ts"; +import { site } from "./test-fixtures.ts"; + +const example = site(456, "www.example.com", { + aliases: ["example.com"], + php_version: "PHP 8.3", +}); + +describe("templateVariables", () => { + it("lists each variable once, with or without spaces in the braces", () => { + expect(templateVariables("{{SITE}} {{ SITE }} {{ PROXY_PASS }} $uri {x}")).toEqual([ + "SITE", + "PROXY_PASS", + ]); + }); +}); + +describe("assertNginxTemplate", () => { + it("accepts every variable of Forge", () => { + const template = NGINX_VARIABLES.map((name) => `{{ ${name} }}`).join("\n"); + expect(() => assertNginxTemplate(template, "nginx.conf")).not.toThrow(); + }); + + it("refuses an unknown variable and names it", () => { + let error: unknown; + try { + assertNginxTemplate("root {{PATH}};\nserver_name {{ DOMAIN }} {{site}};", "nginx.conf"); + } catch (caught) { + error = caught; + } + expect(error).toBeInstanceOf(ValidationError); + expect((error as ValidationError).message).toBe( + "Nginx file nginx.conf has unknown variables: {{DOMAIN}}, {{site}}", + ); + expect((error as ValidationError).field).toBe("nginx"); + }); +}); + +describe("needsServer", () => { + it("is true only with a SERVER_ variable", () => { + expect(needsServer("{{ SITE }}")).toBe(false); + expect(needsServer("allow {{ SERVER_PRIVATE_IP }};")).toBe(true); + }); +}); + +describe("phpSocketVersion", () => { + it("reads the versions of Forge", () => { + expect(phpSocketVersion("PHP 8.3")).toBe("8.3"); + expect(phpSocketVersion("php84")).toBe("8.4"); + expect(phpSocketVersion("php8.1")).toBe("8.1"); + }); + + it("refuses a version without minor", () => { + expect(() => phpSocketVersion("php")).toThrow(ValidationError); + }); +}); + +describe("nginxValues", () => { + it("gives the values of a site", () => { + expect(nginxValues(example, { ip_address: "203.0.113.10", private_ip_address: null })).toEqual({ + DIRECTORY: "/public", + DOMAINS: "www.example.com example.com", + PATH: "/home/forge/www.example.com/public", + PORT: "80", + PORT_V6: "[::]:80", + PROXY_PASS: "unix:/var/run/php/php8.3-fpm.sock", + ROOT_PATH: "/home/forge/www.example.com", + SERVER_PUBLIC_IP: "203.0.113.10", + SERVER_PRIVATE_IP: "", + SITE: "www.example.com", + SITE_ID: "456", + USER: "forge", + }); + }); + + it("uses the PHP pool of the user of an isolated site", () => { + const isolated = site(457, "app.example.com", { isolated: true, user: "app" }); + expect(nginxValues(isolated).PROXY_PASS).toBe("unix:/var/run/php/php8.3-fpm-app.sock"); + }); + + it("reads a relative web directory and the web directory at the root", () => { + expect(nginxValues(site(1, "a.example.com", { web_directory: "/web" })).PATH).toBe( + "/home/forge/a.example.com/web", + ); + const atRoot = nginxValues( + site(1, "a.example.com", { web_directory: "/home/forge/a.example.com" }), + ); + expect(atRoot.DIRECTORY).toBe("/"); + }); + + it("adds default_server to the ports of the default site", () => { + const values = nginxValues(site(1, "default", { aliases: null })); + expect(values.PORT).toBe("80 default_server"); + expect(values.PORT_V6).toBe("[::]:80 default_server"); + expect(values.DOMAINS).toBe("default"); + }); + + it("reads the PHP version only when PROXY_PASS is used", () => { + expect(() => + renderNginx("{{ SITE }}", site(1, "a.example.com", { php_version: "" })), + ).not.toThrow(); + expect(() => + renderNginx("{{ PROXY_PASS }}", site(1, "a.example.com", { php_version: "" })), + ).toThrow(ValidationError); + }); +}); + +describe("renderNginx", () => { + it("replaces each variable and keeps the rest of the file", () => { + const template = [ + "include forge-conf/{{ SITE_ID }}/server/*;", + "root {{PATH}};", + "location ~ \\.php$ { fastcgi_pass {{ PROXY_PASS }}; }", + "include /home/{{ USER }}/{{ SITE }}/nginx/*.conf;", + "set $x '{literal}';", + ].join("\n"); + expect(renderNginx(template, example)).toBe( + [ + "include forge-conf/456/server/*;", + "root /home/forge/www.example.com/public;", + "location ~ \\.php$ { fastcgi_pass unix:/var/run/php/php8.3-fpm.sock; }", + "include /home/forge/www.example.com/nginx/*.conf;", + "set $x '{literal}';", + ].join("\n"), + ); + }); + + it("keeps a text that is not a variable", () => { + expect(renderNginx("{{ NOT_A_VARIABLE }}", example)).toBe("{{ NOT_A_VARIABLE }}"); + }); +}); diff --git a/packages/cli/src/commands/sites/apply/nginx.ts b/packages/cli/src/commands/sites/apply/nginx.ts new file mode 100644 index 0000000..2965fdb --- /dev/null +++ b/packages/cli/src/commands/sites/apply/nginx.ts @@ -0,0 +1,118 @@ +/** + * The Nginx file of a site file: an Nginx configuration with the template variables of Forge, + * e.g. `{{ PROXY_PASS }}`, rendered locally with the values of the site. + * See https://forge.laravel.com/docs/servers/nginx-templates + */ + +import type { ServerAttributes } from "@studiometa/forge-api"; + +import { ValidationError } from "../../../errors.ts"; +import { isPathInside, normalizePath } from "./ownership.ts"; +import { type Site, siteWorkDirectory } from "./plan.ts"; + +/** The variables of Forge Nginx templates. Forge accepts spaces inside the braces. */ +export const NGINX_VARIABLES = [ + "DIRECTORY", + "DOMAINS", + "PATH", + "PORT", + "PORT_V6", + "PROXY_PASS", + "ROOT_PATH", + "SERVER_PUBLIC_IP", + "SERVER_PRIVATE_IP", + "SITE", + "SITE_ID", + "USER", +] as const; + +export type NginxVariable = (typeof NGINX_VARIABLES)[number]; + +/** The variables that need the server, not only the site. */ +const SERVER_VARIABLES: ReadonlySet = new Set(["SERVER_PUBLIC_IP", "SERVER_PRIVATE_IP"]); + +const VARIABLE = /\{\{\s*([^{}]*?)\s*\}\}/g; + +/** The names of the variables in a template, each once, in order. */ +export function templateVariables(template: string): string[] { + return [...new Set([...template.matchAll(VARIABLE)].map((match) => match[1]))]; +} + +/** + * Exit with a validation error when the template has a variable that Forge does not define. + */ +export function assertNginxTemplate(template: string, source: string): void { + const known: ReadonlySet = new Set(NGINX_VARIABLES); + const unknown = templateVariables(template).filter((name) => !known.has(name)); + if (unknown.length > 0) { + throw new ValidationError( + `Nginx file ${source} has unknown variables: ${unknown.map((name) => `{{${name}}}`).join(", ")}`, + "nginx", + [`Supported variables: ${NGINX_VARIABLES.map((name) => `{{${name}}}`).join(", ")}`], + ); + } +} + +/** True when rendering the template needs the attributes of the server. */ +export function needsServer(template: string): boolean { + return templateVariables(template).some((name) => SERVER_VARIABLES.has(name)); +} + +/** `PHP 8.3`, `php83` and `php8.3` all give `8.3`. */ +export function phpSocketVersion(version: string): string { + const digits = version.replaceAll(/\D/g, ""); + if (digits.length < 2) { + throw new ValidationError(`Cannot read the PHP version ${version} of the site`, "nginx"); + } + return `${digits[0]}.${digits.slice(1)}`; +} + +export type NginxServer = Pick; + +/** + * The value of each variable for a site, as Forge renders its templates. + * A server is needed only for the `SERVER_*` variables. + */ +export function nginxValues( + site: Site, + server: NginxServer | null = null, +): Record { + const root = siteWorkDirectory(site); + const path = isPathInside(site.web_directory, root) + ? normalizePath(site.web_directory) + : normalizePath(`${root}/${site.web_directory}`); + const directory = path === root ? "/" : path.slice(root.length); + const defaultServer = site.name === "default" ? " default_server" : ""; + const isolated = site.isolated ? `-${site.user}` : ""; + + return { + DIRECTORY: directory, + DOMAINS: [site.name, ...(site.aliases ?? [])].join(" "), + PATH: path, + PORT: `80${defaultServer}`, + PORT_V6: `[::]:80${defaultServer}`, + get PROXY_PASS() { + return `unix:/var/run/php/php${phpSocketVersion(site.php_version)}-fpm${isolated}.sock`; + }, + ROOT_PATH: root, + SERVER_PUBLIC_IP: server?.ip_address ?? "", + SERVER_PRIVATE_IP: server?.private_ip_address ?? "", + SITE: site.name, + SITE_ID: String(site.id), + USER: site.user, + }; +} + +/** + * Replace each variable of the template with its value for the site. + */ +export function renderNginx( + template: string, + site: Site, + server: NginxServer | null = null, +): string { + const values = nginxValues(site, server); + return template.replaceAll(VARIABLE, (match, name: string) => + name in values ? values[name as NginxVariable] : match, + ); +} diff --git a/packages/cli/src/commands/sites/apply/plan.test.ts b/packages/cli/src/commands/sites/apply/plan.test.ts index 76f1b62..e29483e 100644 --- a/packages/cli/src/commands/sites/apply/plan.test.ts +++ b/packages/cli/src/commands/sites/apply/plan.test.ts @@ -436,7 +436,12 @@ describe("buildPlan", () => { expect(plan.changes).toEqual([ { action: "create-database", name: "example" }, { action: "create-database-user", name: "example", database: "example" }, - { action: "add-env", keys: [], generated: ["DB_DATABASE", "DB_USERNAME", "DB_PASSWORD"] }, + { + action: "add-env", + keys: [], + generated: ["DB_DATABASE", "DB_USERNAME", "DB_PASSWORD"], + password: "DB_PASSWORD", + }, ]); expect(formatPlan(plan)).toContain("DB_PASSWORD="); }); @@ -607,6 +612,7 @@ describe("describeChange", () => { action: "add-env", keys: ["APP_ENV"], generated: ["DB_USERNAME", "DB_PASSWORD"], + password: "DB_PASSWORD", }), describeChange({ action: "create-worker", @@ -652,3 +658,92 @@ describe("describeChange", () => { ]); }); }); + +describe("buildPlan with database .env keys", () => { + it("writes the generated values into the keys of the file", () => { + const plan = buildPlan( + { + server: "web-1", + domain: "a.example.com", + database: { + name: "example", + user: "example", + env: { name: "DB_NAME", user: "DB_USER", password: "DB_PASS" }, + }, + env: { DB_PASS: "ignored" }, + }, + sharedServer({ databases: [], databaseUsers: [], env: "" }), + ); + + expect(plan.changes.at(-1)).toEqual({ + action: "add-env", + keys: [], + generated: ["DB_NAME", "DB_USER", "DB_PASS"], + password: "DB_PASS", + }); + expect(formatPlan(plan)).toContain("+ add .env keys: DB_NAME, DB_USER, DB_PASS="); + }); + + it("keeps the Laravel names for the keys that the file does not map", () => { + const plan = buildPlan( + { + server: "web-1", + domain: "a.example.com", + database: { name: "example", user: "example", env: { password: "DB_PASS" } }, + }, + sharedServer({ databases: [], databaseUsers: [], env: "" }), + ); + + expect(plan.changes.at(-1)).toMatchObject({ + generated: ["DB_DATABASE", "DB_USERNAME", "DB_PASS"], + password: "DB_PASS", + }); + }); +}); + +describe("buildPlan with an Nginx file", () => { + const file: SiteFile = { server: "web-1", domain: "a.example.com", nginxTemplate: "x" }; + + it("writes the file once a new site exists", () => { + const plan = buildPlan({ ...file, type: "php" }, { sites: [], site: null, nginx: null }); + expect(plan.changes).toEqual([ + expect.objectContaining({ action: "create-site" }), + { action: "update-nginx", diff: null }, + ]); + expect(formatPlan(plan)).toContain( + "~ write Nginx configuration (rendered once the site exists)", + ); + }); + + it("changes nothing when the rendered file matches, after normalization", () => { + const plan = buildPlan( + file, + sharedServer({ nginx: { remote: "root /a;\r\n", rendered: "root /a;" } }), + ); + expect(plan.changes).toEqual([]); + }); + + it("shows the diff from Forge to the rendered file", () => { + const plan = buildPlan( + file, + sharedServer({ nginx: { remote: "root /a;\nindex x;\n", rendered: "root /b;\nindex x;\n" } }), + ); + expect(plan.changes).toEqual([ + { + action: "update-nginx", + diff: "--- remote\n+++ local\n@@ -1,2 +1,2 @@\n-root /a;\n+root /b;\n index x;", + }, + ]); + expect(formatPlan(plan)).toBe( + "~ update Nginx configuration:\n--- remote\n+++ local\n@@ -1,2 +1,2 @@\n-root /a;\n+root /b;\n index x;", + ); + }); + + it("does not read Nginx when the file has no Nginx file", () => { + const plan = buildPlan( + { server: "web-1", domain: "a.example.com" }, + sharedServer({ nginx: { remote: "a", rendered: "b" } }), + ); + expect(plan.changes).toEqual([]); + }); +}); diff --git a/packages/cli/src/commands/sites/apply/plan.ts b/packages/cli/src/commands/sites/apply/plan.ts index 31d74da..eb06cb8 100644 --- a/packages/cli/src/commands/sites/apply/plan.ts +++ b/packages/cli/src/commands/sites/apply/plan.ts @@ -11,6 +11,7 @@ import type { SiteAttributes, } from "@studiometa/forge-api"; +import { unifiedDiff } from "../../../utils/drift.ts"; import { parseEnv } from "../../env/env-diff.ts"; import type { SiteFile } from "./site-file.ts"; import { @@ -51,6 +52,8 @@ export interface RemoteState { jobs?: ScheduledJob[]; /** IDs of the scheduled jobs that Forge lists on the site. */ siteJobIds?: ReadonlySet; + /** Nginx configuration on Forge and the Nginx file rendered for the site; null when the site does not exist. */ + nginx?: { remote: string; rendered: string } | null; } export interface SiteSettings { @@ -68,7 +71,14 @@ export type Change = | { action: "add-alias"; name: string } | { action: "create-database"; name: string } | { action: "create-database-user"; name: string; database: string } - | { action: "add-env"; keys: string[]; generated: string[] } + | { + action: "add-env"; + keys: string[]; + /** Keys that get the database name, user and password of a new database user. */ + generated: string[]; + /** The generated key that holds the password; its value never shows. */ + password?: string; + } | { action: "create-worker"; command: string; directory: string; processes: number } | { action: "delete-worker"; @@ -78,6 +88,8 @@ export type Change = processes: number; } | { action: "create-job"; command: string; frequency: string; cron?: string } + /** `diff` is null for a site that does not exist yet: the file is rendered once it exists. */ + | { action: "update-nginx"; diff: string | null } | { action: "delete-job"; id: number; @@ -232,6 +244,19 @@ function planAliases(file: SiteFile, site: Site | null, changes: Change[]): void } } +/** The .env keys of a new database user, from `database.env` or the Laravel names. */ +export function databaseEnvKeys(database: NonNullable): { + name: string; + user: string; + password: string; +} { + return { + name: database.env?.name ?? "DB_DATABASE", + user: database.env?.user ?? "DB_USERNAME", + password: database.env?.password ?? "DB_PASSWORD", + }; +} + /** * Plan the database and its user. Returns the names of the .env keys that get generated values. */ @@ -251,7 +276,8 @@ function planDatabase( } if (!userExists) { changes.push({ action: "create-database-user", name: user, database: name }); - return ["DB_DATABASE", "DB_USERNAME", "DB_PASSWORD"]; + const keys = databaseEnvKeys(file.database); + return [keys.name, keys.user, keys.password]; } if (!databaseExists) { warnings.push( @@ -278,8 +304,20 @@ function planEnv( } } if (keys.length > 0 || generated.length > 0) { - changes.push({ action: "add-env", keys, generated }); + const password = + file.database && generated.length > 0 ? databaseEnvKeys(file.database).password : undefined; + changes.push({ action: "add-env", keys, generated, ...(password ? { password } : {}) }); + } +} + +function planNginx(file: SiteFile, state: RemoteState, changes: Change[]): void { + if (file.nginxTemplate === undefined) return; + if (!state.nginx) { + changes.push({ action: "update-nginx", diff: null }); + return; } + const diff = unifiedDiff(state.nginx.remote, state.nginx.rendered); + if (diff) changes.push({ action: "update-nginx", diff }); } function describeOwnership(ownership: Ownership): string { @@ -458,6 +496,7 @@ export function buildPlan(file: SiteFile, state: RemoteState): Plan { if (file.env || generated.length > 0) { planEnv(file, state, generated, changes, warnings); } + planNginx(file, state, changes); planWorkers(file, state, changes, warnings); planSchedule(file, state, changes, warnings); @@ -492,8 +531,12 @@ export function describeChange(change: Change): string { case "add-env": return `+ add .env keys: ${[ ...change.keys, - ...change.generated.map((key) => (key === "DB_PASSWORD" ? `${key}=${GENERATED}` : key)), + ...change.generated.map((key) => (key === change.password ? `${key}=${GENERATED}` : key)), ].join(", ")}`; + case "update-nginx": + return change.diff === null + ? "~ write Nginx configuration (rendered once the site exists)" + : `~ update Nginx configuration:\n${change.diff}`; case "create-worker": return `+ create worker: ${change.command} (directory: ${change.directory}, processes: ${change.processes})`; case "delete-worker": diff --git a/packages/cli/src/commands/sites/apply/site-file.test.ts b/packages/cli/src/commands/sites/apply/site-file.test.ts index 64632e6..356b698 100644 --- a/packages/cli/src/commands/sites/apply/site-file.test.ts +++ b/packages/cli/src/commands/sites/apply/site-file.test.ts @@ -13,6 +13,7 @@ import { readFileSync } from "node:fs"; import { ValidationError } from "../../../errors.ts"; import { SITE_FILE_SCHEMA_URL, + DatabaseEnvSchema, SiteFileSchema, ScheduledJobSchema, WorkerSchema, @@ -99,6 +100,28 @@ describe("parseSiteFile", () => { ).toEqual(["schedule.0: cron is required with the custom frequency, and only with it"]); }); + it("accepts .env keys for the database, each one optional", () => { + const database = { + name: "example", + user: "example", + env: { name: "DB_NAME", password: "DB_PASS" }, + }; + expect(parseSiteFile({ ...example, database }).database).toEqual(database); + }); + + it("refuses invalid or identical .env keys for the database", () => { + const database = { name: "example", user: "example" }; + expect(issues({ ...example, database: { ...database, env: { name: "DB-NAME" } } })).toEqual([ + "database.env.name: invalid .env variable name", + ]); + expect(issues({ ...example, database: { ...database, env: { user: "DB_PASSWORD" } } })).toEqual( + ["database.env: the .env keys of the database must be different"], + ); + expect(issues({ ...example, database: { ...database, env: { host: "DB_HOST" } } })).toEqual([ + expect.stringMatching(/^database\.env\.host: /), + ]); + }); + it("names the root when the file is not an object", () => { expect(issues("site")).toEqual([expect.stringMatching(/^\(root\): /)]); }); @@ -127,6 +150,43 @@ describe("readSiteFile", () => { expect(() => readSiteFile("site.json")).toThrow(/is not valid JSON/); }); + it("reads the Nginx file next to the site file", () => { + vi.mocked(readFileSync).mockImplementation((path) => + String(path).endsWith("site.json") + ? JSON.stringify({ ...example, nginx: "nginx.conf" }) + : "root {{ PATH }};\n", + ); + const file = readSiteFile("config/forge/production/site.json"); + expect(file.nginx).toBe("nginx.conf"); + expect(file.nginxTemplate).toBe("root {{ PATH }};\n"); + expect(readFileSync).toHaveBeenLastCalledWith( + resolve("config/forge/production/nginx.conf"), + "utf8", + ); + }); + + it("reports a missing Nginx file", () => { + vi.mocked(readFileSync).mockImplementation((path) => { + if (String(path).endsWith("site.json")) + return JSON.stringify({ ...example, nginx: "x.conf" }); + throw new Error("ENOENT"); + }); + expect(() => readSiteFile("site.json")).toThrow( + `Cannot read Nginx file ${resolve("x.conf")}: ENOENT`, + ); + }); + + it("refuses an unknown variable in the Nginx file", () => { + vi.mocked(readFileSync).mockImplementation((path) => + String(path).endsWith("site.json") + ? JSON.stringify({ ...example, nginx: "/etc/forge/nginx.conf" }) + : "root {{ WEB_ROOT }};", + ); + expect(() => readSiteFile("site.json")).toThrow( + "Nginx file /etc/forge/nginx.conf has unknown variables: {{WEB_ROOT}}", + ); + }); + it("names the file in validation errors", () => { vi.mocked(readFileSync).mockReturnValue("{}"); expect(() => readSiteFile("site.json")).toThrow("Invalid site file site.json"); @@ -169,6 +229,9 @@ describe("site.schema.json", () => { expect(Object.keys(schema.properties.database.properties).toSorted()).toEqual( keys(SiteFileSchema.entries.database.wrapped), ); + expect(Object.keys(schema.properties.database.properties.env.properties).toSorted()).toEqual( + keys(DatabaseEnvSchema.pipe[0]), + ); }); it("has the same enums as the valibot schema", () => { diff --git a/packages/cli/src/commands/sites/apply/site-file.ts b/packages/cli/src/commands/sites/apply/site-file.ts index 85c2629..1ffe510 100644 --- a/packages/cli/src/commands/sites/apply/site-file.ts +++ b/packages/cli/src/commands/sites/apply/site-file.ts @@ -4,10 +4,12 @@ */ import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; import * as v from "valibot"; import { ValidationError } from "../../../errors.ts"; +import { assertNginxTemplate } from "./nginx.ts"; /** URL of the JSON Schema of the site file, published with the package. */ export const SITE_FILE_SCHEMA_URL = @@ -65,6 +67,26 @@ export const ScheduledJobSchema = v.pipe( ), ); +const envName = v.pipe( + v.string(), + v.regex(/^[A-Za-z_][A-Za-z0-9_]*$/, "invalid .env variable name"), +); + +/** The .env keys that a new database user gets. Default: the Laravel names. */ +export const DatabaseEnvSchema = v.pipe( + v.strictObject({ + name: v.optional(envName), + user: v.optional(envName), + password: v.optional(envName), + }), + v.check( + (env) => + new Set([env.name ?? "DB_DATABASE", env.user ?? "DB_USERNAME", env.password ?? "DB_PASSWORD"]) + .size === 3, + "the .env keys of the database must be different", + ), +); + export const SiteFileSchema = v.strictObject({ $schema: v.optional(v.string()), server: v.union([text, v.pipe(v.number(), v.integer(), v.minValue(1))]), @@ -80,25 +102,26 @@ export const SiteFileSchema = v.strictObject({ branch: v.optional(text), }), ), - database: v.optional(v.strictObject({ name: text, user: text })), - env: v.optional( - v.record( - v.pipe(v.string(), v.regex(/^[A-Za-z_][A-Za-z0-9_]*$/, "invalid .env variable name")), - v.union([v.string(), v.number(), v.boolean()]), - ), + database: v.optional( + v.strictObject({ name: text, user: text, env: v.optional(DatabaseEnvSchema) }), ), + env: v.optional(v.record(envName, v.union([v.string(), v.number(), v.boolean()]))), + nginx: v.optional(text), workers: v.optional(v.array(WorkerSchema)), schedule: v.optional(v.array(ScheduledJobSchema)), }); -export type SiteFile = v.InferOutput; +export type SiteFileData = v.InferOutput; + +/** A site file with the content of its Nginx file, when it declares one. */ +export type SiteFile = SiteFileData & { nginxTemplate?: string }; export type WorkerDeclaration = v.InferOutput; export type ScheduledJobDeclaration = v.InferOutput; /** * Validate parsed JSON as a site file. Throws a ValidationError that lists every issue. */ -export function parseSiteFile(data: unknown, source = "site file"): SiteFile { +export function parseSiteFile(data: unknown, source = "site file"): SiteFileData { const result = v.safeParse(SiteFileSchema, data); if (!result.success) { const issues = result.issues.map((issue) => { @@ -111,7 +134,8 @@ export function parseSiteFile(data: unknown, source = "site file"): SiteFile { } /** - * Read and validate a site file from disk. + * Read and validate a site file from disk, with its Nginx file: a relative `nginx` path is + * read against the folder of the site file. */ export function readSiteFile(path: string): SiteFile { let content: string; @@ -130,5 +154,16 @@ export function readSiteFile(path: string): SiteFile { throw new ValidationError(`Site file ${path} is not valid JSON: ${reason}`, "file"); } - return parseSiteFile(data, `site file ${path}`); + const file: SiteFile = parseSiteFile(data, `site file ${path}`); + if (file.nginx) { + const nginxPath = resolve(dirname(path), file.nginx); + try { + file.nginxTemplate = readFileSync(nginxPath, "utf8"); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + throw new ValidationError(`Cannot read Nginx file ${nginxPath}: ${reason}`, "nginx"); + } + assertNginxTemplate(file.nginxTemplate, nginxPath); + } + return file; } diff --git a/packages/cli/src/commands/sites/apply/test-fixtures.ts b/packages/cli/src/commands/sites/apply/test-fixtures.ts index 34d0e6f..6ff77df 100644 --- a/packages/cli/src/commands/sites/apply/test-fixtures.ts +++ b/packages/cli/src/commands/sites/apply/test-fixtures.ts @@ -131,6 +131,11 @@ function server(id: number, name: string) { }; } +/** Attributes of server 123, for the `SERVER_*` Nginx variables. */ +export function webServer() { + return { ...server(123, "web-1"), ip_address: "203.0.113.10", private_ip_address: "10.0.0.10" }; +} + /** Route on any organization: the CLI test context uses its own. */ function route(url: string): string { return decodeURIComponent(url).replace(/^\/orgs\/[^/]+/, "/orgs/org"); @@ -145,6 +150,8 @@ export interface FakeForge { databases: Array<{ id: number; name: string }>; users: Array<{ id: number; name: string }>; env: string; + /** Nginx configuration of every site. */ + nginx: string; /** Statuses a new site goes through, one per read. */ newSiteStatuses: string[]; } @@ -162,6 +169,7 @@ export function fakeForge(initial: Partial = {}) { databases: [], users: [], env: "", + nginx: "", newSiteStatuses: ["installed"], ...initial, }; @@ -181,7 +189,14 @@ export function fakeForge(initial: Partial = {}) { data: servers.map((item) => ({ id: String(item.id), type: "servers", attributes: item })), }; } + if (path === "/orgs/org/servers/123") { + const attributes = webServer(); + return { data: { id: "123", type: "servers", attributes } }; + } if (path === "/orgs/org/servers/123/sites") return list("sites", state.sites); + if ((match = /^\/orgs\/org\/servers\/123\/sites\/(\d+)\/nginx$/.exec(path))) { + return doc("nginxConfigs", { id: Number(match[1]), content: state.nginx } as never); + } if ((match = /^\/orgs\/org\/sites\/(\d+)$/.exec(path))) { const found = state.sites.find((s) => s.id === Number(match![1]))!; const status = state.newSiteStatuses.shift() ?? "installed"; @@ -263,6 +278,10 @@ export function fakeForge(initial: Partial = {}) { state.env = String(body.environment); return null; } + if (url.endsWith("/nginx")) { + state.nginx = String(body.config); + return null; + } return doc("sites", { ...state.sites[0] }); }), delete: vi.fn(async (url: string) => { diff --git a/packages/cli/src/commands/sites/help.ts b/packages/cli/src/commands/sites/help.ts index f55d117..526f8e1 100644 --- a/packages/cli/src/commands/sites/help.ts +++ b/packages/cli/src/commands/sites/help.ts @@ -48,9 +48,15 @@ ${colors.bold("SITE FILE:")} A key that is absent is not managed: apply does not read or change it. type, php, webDirectory, repository Create the site, or update php, web directory, branch aliases Add missing aliases, never remove one - database { name, user } Create when missing; a new user gets a random - password, written into the site .env only + database { name, user, env } Create when missing; a new user gets a random + password, written into the site .env only. + env maps { name, user, password } to .env keys + (default DB_DATABASE, DB_USERNAME, DB_PASSWORD) env Add missing keys, never overwrite a value + nginx Nginx file, relative to the site file, with Forge + template variables such as {{PATH}}, {{SITE_ID}}, + {{PROXY_PASS}}: rendered for the site, written + when it differs (an unknown variable exits 3) workers, schedule Create missing ones, delete the ones of the site that the file does not declare ([] deletes all)