diff --git a/.github/workflows/copy-production-db-to-dev.yml b/.github/workflows/copy-production-db-to-dev.yml new file mode 100644 index 0000000..2ee0cab --- /dev/null +++ b/.github/workflows/copy-production-db-to-dev.yml @@ -0,0 +1,54 @@ +name: Copy production database to dev + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + refresh-database: + name: Replace dev data with a production snapshot + if: vars.XEVENTS_DATABASE_REFRESH_ENABLED == 'true' + runs-on: ubuntu-latest + timeout-minutes: 45 + # This environment is restricted to main. It has its own private credentials. + environment: + name: development-refresh + concurrency: + group: xevents-development + cancel-in-progress: false + steps: + - name: Require the trusted workflow branch + env: + SELECTED_REF: ${{ github.ref }} + run: | + if [ "$SELECTED_REF" != 'refs/heads/main' ]; then + echo 'Run this database-only workflow from main.' >&2 + exit 1 + fi + - name: Check out the database-copy tools + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Read the current development migrations + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: dev + path: .wrangler/development-code + persist-credentials: false + - name: Set up Node.js + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version: 22 + cache: npm + - name: Install database tools + run: npm ci + - name: Copy production data into development only + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + WRANGLER_CONFIG_JSON: ${{ secrets.WRANGLER_CONFIG_JSON }} + PRODUCTION_CONFIG_JSON: ${{ secrets.PRODUCTION_CONFIG_JSON }} + run: node scripts/copy-production-database.mjs diff --git a/AGENTS.md b/AGENTS.md index 62f7571..e78368f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -65,9 +65,14 @@ existing invitations, and users' private event data when making changes. - Work on `dev` and verify the affected flows there before promoting to `main`. `main` deploys production; `dev` deploys the separate development Worker and D1. Keep all bot credentials, app URLs and database bindings isolated by environment. -- The production-to-dev workflow merges code without force-pushing. Optional D1 - copying is schema-only into an empty development database. Never copy real - guest records, media references, payment state or delivery queues into tests. +- Keep code synchronization and database refresh separate. The user explicitly + authorizes the manual production-to-dev database snapshot workflow to copy real + business records into the private development D1. Replace development data, + preserve its settings/whitelist, clear sessions and delivery/coordination state, + and require the development snapshot access/payment/notification safeguards. + Never merge code, deploy Workers, write production data or publish database + exports as part of this database-only workflow. Automated tests still use + fictional local fixtures, never actual production snapshots. - Run deployment automation from one canonical repository; mirrors run checks only. Store private configuration in GitHub environment secrets and ignored files. diff --git a/docs/development.md b/docs/development.md index e70ae1c..e3fa46f 100644 --- a/docs/development.md +++ b/docs/development.md @@ -37,7 +37,7 @@ Create GitHub environments named `production` and `development`. Configure these | `WRANGLER_CONFIG_JSON` | The complete private Wrangler configuration for that environment, as valid JSON. Use different Worker names, D1 IDs, bot usernames and app URLs. | | `PRODUCTION_CONFIG_JSON` | Development only: the private production configuration used to reject production resource collisions. It is a workflow reference, never the development Worker's runtime configuration. | -`PRODUCTION_CONFIG_JSON` must match the actual production target. Also create the `development-refresh` environment with the same four secrets as development; restrict this environment to `main`. It is used only for the optional schema-copy job. Production needs only the first three secrets. Keep private configuration in GitHub secrets, never repository variables, checked-in files or workflow text. +`PRODUCTION_CONFIG_JSON` must match the actual production target. Also create the `development-refresh` environment with the same four secrets as development; restrict this environment to `main`. It is used for manual database refreshes. Production needs only the first three secrets. Keep private configuration in GitHub secrets, never repository variables, checked-in files or workflow text. The deployment helper, `scripts/deploy-environment.mjs`, applies migrations to the selected environment and deploys with `--keep-vars`. The private configuration is materialized for that run and must not become a build artifact or committed file. Keep existing production bindings and secrets intact. @@ -59,7 +59,8 @@ Keep the real development configuration in an ignored local file, such as `wrang "preview_urls": false, "vars": { "BOT_USERNAME": "YOUR_DEVELOPMENT_BOT_USERNAME", - "APP_URL": "https://YOUR_DEVELOPMENT_WORKER.YOUR_SUBDOMAIN.workers.dev/app" + "APP_URL": "https://YOUR_DEVELOPMENT_WORKER.YOUR_SUBDOMAIN.workers.dev/app", + "TEST_WHITELIST_ENABLED": "false" }, "assets": { "directory": "./public", @@ -90,6 +91,7 @@ Store these secrets on the **development Cloudflare Worker**, using private inte - `TELEGRAM_BOT_TOKEN`: the development bot's token. - `TELEGRAM_WEBHOOK_SECRET`: a new random secret for the development webhook. - `SUPER_ADMIN_ID`: your administrator Telegram identity, if needed. +- `WHITELIST_USER_IDS`: optional initial allowed Telegram IDs for development, separated by commas or whitespace. Leave it blank to permit everyone. These bot secrets do not belong in GitHub source or `WRANGLER_CONFIG_JSON`, and the deployment workflow does not need copies of them in GitHub. Set `vars.BOT_USERNAME` to the development bot and `vars.APP_URL` to the development Mini App URL. The [self-hosting guide](self-hosting.md#3-set-worker-secrets) explains interactive secret setup and webhook registration. @@ -97,7 +99,44 @@ Do not connect production's bot to the development Worker. Telegram uses a singl Stars payments use Telegram's real Bot API endpoints in the current application. A separate regular bot does not create a payment sandbox. Keep payment/refund verification mocked unless deliberately performing a controlled real transaction; never reuse production charge IDs or payment records in development. -## Sync production to dev +## Restrict development access + +Set `vars.TEST_WHITELIST_ENABLED` to `"true"` in the private development configuration to enable the test whitelist. The deployment helper sets `APP_ENV` to `development` and defaults this flag to `"false"` when omitted. Production always ignores the whitelist, and the deployment helper writes `"false"` for production even if its input enables the flag. For local testing, set `APP_ENV=development` and `TEST_WHITELIST_ENABLED=true` in the ignored `.dev.vars` file. + +An empty whitelist permits everyone. Once it contains IDs, the development bot and authenticated Mini App APIs allow only those users and the superadmin configured by the `SUPER_ADMIN_ID` Worker secret. The superadmin always retains access so they can manage the list. + +Send these commands to the development bot as the configured superadmin: + +| Command | Effect | +| --- | --- | +| `/whitelist` | Show the current access setting and allowed IDs. | +| `/whitelist add ID...` | Add one or more numeric Telegram user IDs. | +| `/whitelist remove ID...` | Remove one or more IDs. | +| `/whitelist clear` | Empty the list and permit everyone. | + +Keep real IDs in private Worker secrets or bot commands. `WHITELIST_USER_IDS` supplies the initial list and is never serialized into generated deployment configuration. The first command that changes the list saves an override in that environment's D1 database; subsequent requests and commands use that override instead of the secret seed. `/whitelist clear` saves an empty override, so it keeps access open even if the seed secret still contains IDs. Development and production use separate D1 databases, so these settings do not cross environments. + +Removing the final tester also opens access to everyone; every change confirms whether access is restricted or open. Invalid configuration restricts access until the superadmin resets it. Payment and refund confirmations already issued by Telegram are still recorded after a tester is removed. Existing queued notifications are preserved. + +This restriction applies to the Cloudflare Worker, including its webhook and Mini App APIs. Use `wrangler dev` for local access-control testing; the standalone `npm start` polling process does not enforce this hosted feature. The whitelist command is advertised only in the configured superadmin's private chat. + +## Copy production data to dev + +Open **Actions → Copy production database to dev → Run workflow**, select `main`, and run it whenever you need the latest production records in development. This replaces the existing development database contents. It does not merge either branch, deploy a Worker, change bot credentials, or write to production. + +Enable this action only in the canonical repository by setting the repository variable `XEVENTS_DATABASE_REFRESH_ENABLED=true`. Leave it unset or `false` in mirrors. This switch is independent of `XEVENTS_DEPLOY_ENABLED`. The `development-refresh` environment needs all four secrets listed above, including its own `CLOUDFLARE_API_TOKEN` with D1 export and import access. A missing token must be configured before the action can run. + +The action exports production, prepares the snapshot privately, and imports one replacement SQL file into the verified development D1. It copies events, users, profiles, guest responses, invitation links, ticket/check-in records, payment audits and media references. Development settings, including the test whitelist, are retained; production settings are not copied. Sessions, queued deliveries, processed-update markers and coordination locks are cleared. Already-applied migrations are preserved, and pending migrations from the current `dev` checkout are applied offline before import. An incompatible migration set stops the refresh before writing development data. + +The development Worker must already include the database-refresh safeguards; the action checks its health capability before exporting. A copied database is restricted to the superadmin and explicitly allowed testers, even if the normal whitelist switch is disabled. An empty whitelist means **admin only** for a production snapshot; `/whitelist clear` cannot expose it to everyone. Add testers with `/whitelist add ID...` as usual. Copied reminders are paused, outbound messages are limited to these permitted accounts, and Stars invoices, checkout and refunds are disabled while the snapshot marker is present. Stored financial audit records remain intact. + +Telegram media references are copied, but their `file_id` values belong to the production bot. Some existing images or files will need to be uploaded again through the development bot. Never use the production bot token to work around this isolation. + +Exports and prepared SQL files are temporary, private files deleted after the run. They are never printed or uploaded as artifacts. D1 export can briefly block production database requests; import can briefly block development requests. A failed D1 file import rolls back the replacement. Both development deployment and database refresh use the same concurrency group. See Cloudflare's [D1 import and export guide](https://developers.cloudflare.com/d1/best-practices/import-export-data/). + +Automated tests continue to use fictional local fixtures and mocked Telegram calls. Production snapshots are private operational data, never test fixtures or public artifacts. + +## Sync production code to dev Run **Sync production code to development** manually from GitHub Actions on `main`. It merges production code into `dev` without resetting the branch or force-pushing. It preserves development work and stops if there is a merge conflict. Resolve that conflict in a normal checkout and push the reviewed result. After the merge, it explicitly starts the development deployment workflow because a push using GitHub's workflow token does not start another push workflow. That development run skips automated tests. @@ -105,7 +144,7 @@ The `copy_database_schema` input defaults to `false`. With that default, the wor Select `copy_database_schema` only when the development database is empty and you want to initialize it from production's schema. `scripts/refresh-development.mjs` verifies the separate targets and refuses a nonempty development database, preserving existing test data. It exports **schema only**, imports it into development, and applies reviewed `main` migrations; the subsequent development deployment applies any additional `dev` migrations. Schema export can briefly block production database requests, so use it deliberately. Cloudflare documents the supported [D1 schema export and import](https://developers.cloudflare.com/d1/best-practices/import-export-data/). -Only schema and applied migration filenames are transferred, so already-applied schema changes are not replayed. This workflow never copies production users, events, invitations, sessions, payment or refund records, queued notifications, reminders, configuration rows or media references. It does not promote development data to production. Create fictional test events in the development bot after initialization. A production database backup is private operational data and must never be uploaded as a public workflow artifact or committed as a test fixture. +Only schema and applied migration filenames are transferred by that optional initialization step, so already-applied schema changes are not replayed. For actual records, use the separate **Copy production database to dev** action above. Neither workflow promotes development data to production. ## Before promoting a change diff --git a/scripts/copy-production-database.mjs b/scripts/copy-production-database.mjs new file mode 100644 index 0000000..fcb23e0 --- /dev/null +++ b/scripts/copy-production-database.mjs @@ -0,0 +1,73 @@ +import { mkdir, mkdtemp, readFile, readdir, realpath, rm, writeFile } from 'node:fs/promises'; +import { resolve, sep } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseConfig, validateSeparation, buildConfig } from './deployment-config.mjs'; +import { temporaryConfig, wrangler, jsonOutput } from './deployment-tools.mjs'; +import { prepareSnapshot } from './database-snapshot.mjs'; + +const safeError = message => Object.assign(new Error(message), { safe: true }); +function rows(output) { + const result = jsonOutput(output); + if (!Array.isArray(result) || !result.length || result.some(statement => statement.success !== true || !Array.isArray(statement.results))) throw safeError('The database response could not be verified. Development was not changed.'); + return result.flatMap(statement => statement.results); +} + +export async function copyProductionDatabase({ env = process.env, run = wrangler, fetcher = fetch, migrationsDirectory = '.wrangler/development-code/migrations' } = {}) { + if (env.GITHUB_REF_NAME !== 'main') throw safeError('Run the database refresh workflow from main.'); + if (!env.CLOUDFLARE_API_TOKEN || !env.CLOUDFLARE_ACCOUNT_ID) throw safeError('Set CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID in the development-refresh GitHub environment.'); + const dev = parseConfig(env.WRANGLER_CONFIG_JSON), prod = parseConfig(env.PRODUCTION_CONFIG_JSON); + validateSeparation(dev, prod); + const health = await fetcher(new URL('/', dev.vars.APP_URL), { signal: AbortSignal.timeout(15000) }); + const status = health.ok && await health.json(); + if (status?.status !== 'running' || status.databaseRefreshProtection !== 1) throw safeError('Deploy the development database-refresh protection before copying production data. Development was not changed.'); + + const root = await realpath('.'); + const migrationPath = await realpath(migrationsDirectory); + if (!migrationPath.startsWith(root + sep)) throw safeError('Development migrations must come from the checked-out development source.'); + const names = (await readdir(migrationPath, { withFileTypes: true })).filter(entry => entry.isFile() && /^\d+_[A-Za-z0-9_-]+\.sql$/.test(entry.name)).map(entry => entry.name).sort(); + if (!names.length) throw safeError('The development migration set is missing. Development was not changed.'); + const migrations = await Promise.all(names.map(async name => ({ name, sql: await readFile(resolve(migrationPath, name), 'utf8') }))); + + await mkdir(resolve('.wrangler'), { recursive: true }); + const directory = await mkdtemp(resolve('.wrangler/database-copy-')); + let production, development; + try { + production = await temporaryConfig(await buildConfig(prod, 'production'), 'database-copy-production'); + development = await temporaryConfig(await buildConfig(dev, 'development'), 'database-copy-development'); + const query = command => run(['d1', 'execute', 'DB', '--remote', '--config', development.path, '--json', '--command', command]).then(rows); + const schemaQuery = "SELECT type,name FROM sqlite_schema WHERE type IN ('table','view') AND name NOT GLOB 'sqlite_*' AND name NOT GLOB '_cf_*' ORDER BY type,name"; + const exported = resolve(directory, 'production.sql'); + await run(['d1', 'export', 'DB', '--remote', '--config', production.path, '--output', exported, '--skip-confirmation']); + const developmentSchema = await query(schemaQuery); + const settingsQuery = 'SELECT key,value FROM app_settings ORDER BY key'; + const developmentSettings = developmentSchema.some(table => table.name === 'app_settings') ? await query(settingsQuery) : []; + const copiedAt = new Date().toISOString(); + const snapshot = prepareSnapshot({ sourceSql: await readFile(exported, 'utf8'), developmentSchema, developmentSettings, migrations, copiedAt }); + const prepared = resolve(directory, 'development.sql'); + await writeFile(prepared, snapshot.sql, { mode: 0o600 }); + const latestSchema = await query(schemaQuery); + const latestSettings = latestSchema.some(table => table.name === 'app_settings') ? await query(settingsQuery) : []; + if (JSON.stringify(latestSchema) !== JSON.stringify(developmentSchema) || JSON.stringify(latestSettings) !== JSON.stringify(developmentSettings)) throw safeError('Development schema or settings changed while preparing the snapshot. Run the refresh again; development was not changed.'); + // The only remote write targets the verified development binding. D1 imports + // the replacement as one file; failed imports roll back instead of leaving + // a separate reset operation committed ahead of the copied data. + await run(['d1', 'execute', 'DB', '--remote', '--config', development.path, '--file', prepared, '--yes']); + console.log('Production database snapshot copied to development. Development settings retained; delivery state cleared. Code branches and production data were not changed.'); + return snapshot.counts; + } finally { + try { + await Promise.all([production?.cleanup(), development?.cleanup()]); + } finally { + // mkdtemp creates this exact workspace-owned directory; never delete a + // caller-provided location when disposing of private database exports. + if (!directory.startsWith(root + sep) || !directory.startsWith(resolve('.wrangler/database-copy-'))) throw safeError('Private export cleanup path could not be verified.'); + await rm(directory, { recursive: true, force: true }); + } + } +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + process.umask(0o077); + try { await copyProductionDatabase(); } + catch (error) { console.error(error.safe ? error.message : 'Database refresh failed. Check the private environment configuration, Cloudflare permissions, migration compatibility and network connection. No database contents were logged.'); process.exitCode = 1; } +} diff --git a/scripts/database-snapshot.mjs b/scripts/database-snapshot.mjs new file mode 100644 index 0000000..44bb4f5 --- /dev/null +++ b/scripts/database-snapshot.mjs @@ -0,0 +1,201 @@ +import { DatabaseSync } from 'node:sqlite'; +import { randomUUID } from 'node:crypto'; + +const runtimeTables = ['outbox', 'processed', 'lease', 'delivery_lease', 'commits']; +const columnsRequired = { + records: ['kind', 'id', 'data'], + processed: ['id', 'at'], + lease: ['id', 'owner', 'expires'], + delivery_lease: ['id', 'owner', 'expires'], + commits: ['owner'], + outbox: ['id', 'method', 'params', 'due', 'attempts'], + app_settings: ['key', 'value'] +}; +const invalid = () => new Error('The database snapshot could not be prepared safely. Development was not changed.'); +const internal = name => /^(?:sqlite_|_cf_)/i.test(name); +const identifier = name => '"' + name.replaceAll('"', '""') + '"'; +const validName = name => typeof name === 'string' && name.length > 0 && !name.includes('\0'); + +function rows(db, sql) { + const statement = db.prepare(sql); + statement.setReadBigInts(true); + return statement.all(); +} + +function schema(db) { + const attached = rows(db, 'PRAGMA database_list'); + if (attached.some(row => row.name !== 'main' && row.name !== 'temp')) throw invalid(); + const tables = rows(db, 'PRAGMA table_list'); + // SQL dumps cannot faithfully restore virtual tables and their shadow state. + if (tables.some(row => !internal(row.name) && (row.schema !== 'main' || !['table', 'view'].includes(row.type)))) throw invalid(); + const objects = rows(db, "SELECT type,name,tbl_name,sql FROM sqlite_schema WHERE sql IS NOT NULL ORDER BY name").filter(row => !internal(row.name)); + if (objects.some(row => !['table', 'view', 'index', 'trigger'].includes(row.type) || !validName(row.name) || typeof row.sql !== 'string')) throw invalid(); + return objects; +} + +function validateRecords(db, objects) { + const tableNames = new Set(objects.filter(row => row.type === 'table').map(row => row.name)); + for (const [table, names] of Object.entries(columnsRequired)) { + if (!tableNames.has(table)) throw invalid(); + const columns = rows(db, `PRAGMA table_info(${identifier(table)})`); + if (names.some(name => !columns.some(column => column.name === name))) throw invalid(); + } + const columns = rows(db, 'PRAGMA table_info(records)'); + if (['kind', 'id', 'data'].some(name => !columns.some(column => column.name === name && column.type.toUpperCase() === 'TEXT'))) throw invalid(); + for (const record of rows(db, 'SELECT kind,id,data FROM records')) { + if (![record.kind, record.id, record.data].every(value => typeof value === 'string') || !record.kind || !record.id) throw invalid(); + if (['events', 'users', 'preferences'].includes(record.kind)) { + const value = JSON.parse(record.data); + if (!value || typeof value !== 'object' || Array.isArray(value)) throw invalid(); + } + } +} + +function applyPendingMigrations(db, available) { + if (!Array.isArray(available)) throw invalid(); + const byName = new Map(); + for (const migration of available) { + if (!migration || !validName(migration.name) || typeof migration.sql !== 'string' || !migration.sql.trim() || byName.has(migration.name)) throw invalid(); + byName.set(migration.name, migration.sql); + } + const exists = db.prepare("SELECT 1 FROM sqlite_schema WHERE type='table' AND name='d1_migrations'").get(); + const applied = exists ? rows(db, 'SELECT name FROM d1_migrations') : []; + if (applied.some(row => !byName.has(row.name))) throw new Error('Production uses migrations missing from development. Synchronize the reviewed migrations before copying the database.'); + if (!exists) db.exec('CREATE TABLE d1_migrations (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT, applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL)'); + const completed = new Set(applied.map(row => row.name)); + const pending = [...byName.keys()].sort().filter(name => !completed.has(name)); + const record = db.prepare('INSERT INTO d1_migrations(name) VALUES (?)'); + for (const name of pending) { + db.exec(byName.get(name)); + record.run(name); + } + return pending.length; +} + +function literal(value, storageType) { + // Read TEXT as bytes: Node 22's SQLite string conversion truncates embedded NULs. + if (storageType === 'text' && value instanceof Uint8Array) return "CAST(X'" + Buffer.from(value).toString('hex') + "' AS TEXT)"; + if (value === null) return 'NULL'; + if (typeof value === 'bigint') return value.toString(); + if (typeof value === 'number' && Number.isFinite(value)) { + const number = value.toString(); + // Integers are read as BigInt. A Number must retain SQLite's REAL storage class. + return /[.e]/i.test(number) ? number : number + '.0'; + } + if (typeof value === 'string') { + // SQLite's SQL parser cannot accept a literal containing a NUL character. + if (value.includes('\0')) return "CAST(X'" + Buffer.from(value, 'utf8').toString('hex') + "' AS TEXT)"; + return "'" + value.replaceAll("'", "''") + "'"; + } + if (value instanceof Uint8Array) return "X'" + Buffer.from(value).toString('hex') + "'"; + throw invalid(); +} + +function dump(db, objects) { + const sql = []; + const tables = objects.filter(row => row.type === 'table'); + for (const table of tables) sql.push(table.sql + ';'); + for (const table of tables) { + const allColumns = rows(db, `PRAGMA table_xinfo(${identifier(table.name)})`); + const columns = allColumns.filter(column => column.hidden === 0n).map(column => column.name); + if (!columns.length) throw invalid(); + const info = rows(db, 'PRAGMA table_list').find(row => row.schema === 'main' && row.name === table.name); + // Keep accessible rowids too; they may be referenced by existing code or indexes. + const rowid = info.wr === 0n ? ['_rowid_', 'rowid', 'oid'].find(name => !allColumns.some(column => column.name.toLowerCase() === name)) : undefined; + const selected = rowid ? [rowid, ...columns] : columns; + const names = selected.map(identifier).join(','); + const expressions = selected.map(name => { + const column = identifier(name); + return `typeof(${column}),CASE WHEN typeof(${column})='text' THEN CAST(${column} AS BLOB) ELSE ${column} END`; + }).join(','); + const statement = db.prepare(`SELECT ${expressions} FROM ${identifier(table.name)}`); + statement.setReadBigInts(true); + statement.setReturnArrays(true); + for (const row of statement.iterate()) { + const values = selected.map((_, index) => literal(row[index * 2 + 1], row[index * 2])); + sql.push(`INSERT INTO ${identifier(table.name)}(${names}) VALUES (${values.join(',')});`); + } + } + if (db.prepare("SELECT 1 FROM sqlite_schema WHERE name='sqlite_sequence' AND type='table'").get()) { + sql.push('DELETE FROM sqlite_sequence;'); + for (const row of rows(db, 'SELECT name,seq FROM sqlite_sequence ORDER BY name')) { + if (!internal(row.name)) sql.push(`INSERT INTO sqlite_sequence(name,seq) VALUES (${literal(row.name)},${literal(row.seq)});`); + } + } + // Guards, audit triggers, and indexes must be installed after copying table data. + for (const type of ['index', 'view', 'trigger']) for (const object of objects.filter(row => row.type === type)) sql.push(object.sql + ';'); + return sql.join('\n'); +} + +function resetSql(developmentSchema, settings) { + if (!Array.isArray(developmentSchema)) throw invalid(); + const objects = developmentSchema.filter(object => { + if (!object || !['table', 'view'].includes(object.type) || !validName(object.name)) throw invalid(); + return !internal(object.name); + }); + const unique = new Set(); + if (!objects.some(object => object.type === 'table' && object.name === 'app_settings')) throw invalid(); + const guard = identifier('_xevents_snapshot_guard_' + randomUUID().replaceAll('-', '')); + const bytes = value => "X'" + Buffer.from(value, 'utf8').toString('hex') + "'"; + const exactSettings = [...settings].map(([key, value]) => `EXISTS (SELECT 1 FROM "app_settings" WHERE typeof("key")='text' AND CAST("key" AS BLOB)=${bytes(key)} AND typeof("value")='text' AND CAST("value" AS BLOB)=${bytes(value)})`); + // Check again inside the same D1 import transaction, before resetting any data. + // An external whitelist/settings edit after the API re-read aborts the import. + const matches = [`(SELECT count(*) FROM "app_settings")=${settings.size}`, ...exactSettings].join(' AND '); + const result = [ + 'PRAGMA defer_foreign_keys=ON;', + `CREATE TABLE ${guard}(ok INTEGER NOT NULL CHECK(ok=1));`, + `INSERT INTO ${guard}(ok) SELECT CASE WHEN ${matches} THEN 1 ELSE 0 END;`, + `DROP TABLE ${guard};` + ]; + for (const type of ['view', 'table']) for (const object of objects.filter(row => row.type === type)) { + const key = object.name.toLowerCase(); + if (unique.has(key)) throw invalid(); + unique.add(key); + result.push(`DROP ${type.toUpperCase()} IF EXISTS ${identifier(object.name)};`); + } + return result.join('\n'); +} + +/** Build an atomic D1 replacement import entirely offline; never print SQL/data. */ +export function prepareSnapshot({ sourceSql, developmentSchema, developmentSettings, migrations, copiedAt }) { + let db; + try { + if (typeof sourceSql !== 'string' || !sourceSql.trim() || !Array.isArray(developmentSettings) || typeof copiedAt !== 'string' || !Number.isFinite(Date.parse(copiedAt))) throw invalid(); + const settings = new Map(); + for (const setting of developmentSettings) { + if (!setting || !validName(setting.key) || typeof setting.value !== 'string' || settings.has(setting.key)) throw invalid(); + settings.set(setting.key, setting.value); + } + const reset = resetSql(developmentSchema, settings); + db = new DatabaseSync(':memory:', { allowExtension: false }); + db.exec(sourceSql); + schema(db); + const appliedMigrations = applyPendingMigrations(db, migrations); + const objects = schema(db); + validateRecords(db, objects); + // Cleanup must not run production audit/business triggers in the private staging DB. + for (const trigger of objects.filter(row => row.type === 'trigger')) db.exec(`DROP TRIGGER ${identifier(trigger.name)}`); + const removedSessions = Number(rows(db, "SELECT count(*) AS n FROM records WHERE kind='sessions'")[0].n); + db.exec("DELETE FROM records WHERE kind='sessions'"); + for (const table of runtimeTables) db.exec(`DELETE FROM ${identifier(table)}`); + db.exec('DELETE FROM app_settings'); + const set = db.prepare('INSERT INTO app_settings(key,value) VALUES (?,?)'); + settings.set('production-snapshot', JSON.stringify({ version: 1, copiedAt })); + for (const [key, value] of settings) set.run(key, value); + if (rows(db, 'PRAGMA foreign_key_check').length || rows(db, 'PRAGMA integrity_check').some(row => row.integrity_check !== 'ok')) throw invalid(); + const grouped = rows(db, 'SELECT kind,count(*) AS n FROM records GROUP BY kind'); + const count = kind => Number(grouped.find(row => row.kind === kind)?.n || 0n); + const counts = { + records: grouped.reduce((total, row) => total + Number(row.n), 0), + events: count('events'), users: count('users'), preferences: count('preferences'), + removedSessions, tables: objects.filter(row => row.type === 'table').length, appliedMigrations + }; + return { sql: reset + '\n' + dump(db, objects) + '\n', counts }; + } catch (error) { + // SQLite errors can quote private values, SQL excerpts, or file paths. + if (error?.message === 'Production uses migrations missing from development. Synchronize the reviewed migrations before copying the database.') throw error; + throw invalid(); + } finally { + db?.close(); + } +} diff --git a/test/copy-production-database.test.js b/test/copy-production-database.test.js new file mode 100644 index 0000000..79d6472 --- /dev/null +++ b/test/copy-production-database.test.js @@ -0,0 +1,191 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { access, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { dirname, resolve, sep } from 'node:path'; +import { copyProductionDatabase } from '../scripts/copy-production-database.mjs'; + +const production = () => ({ name: 'fictional-production', vars: { BOT_USERNAME: 'FictionalProductionBot', APP_URL: 'https://production.test/app' }, d1_databases: [{ binding: 'DB', database_name: 'fictional-production-db', database_id: '11111111-1111-4111-8111-111111111111' }] }); +const development = () => ({ name: 'fictional-development', vars: { BOT_USERNAME: 'FictionalDevelopmentBot', APP_URL: 'https://development.test/app', TEST_WHITELIST_ENABLED: 'true' }, d1_databases: [{ binding: 'DB', database_name: 'fictional-development-db', database_id: '22222222-2222-4222-8222-222222222222' }] }); +const encoded = records => JSON.stringify([{ success: true, results: records }]); +const argument = (args, flag) => args[args.indexOf(flag) + 1]; +const copyFiles = names => names.filter(name => /^wrangler\.database-copy-.*\.jsonc$/.test(name)).sort(); +const copyDirectories = names => names.filter(name => name.startsWith('database-copy-')).sort(); + +async function fixture(t, { failAt, settingsRace = false, schemaRace = false, unhealthy } = {}) { + await mkdir(resolve('.wrangler'), { recursive: true }); + const directory = await mkdtemp(resolve('.wrangler/test-database-copy-')); + const migrationsDirectory = resolve(directory, 'migrations'); + await mkdir(migrationsDirectory); + t.after(async () => { + assert.ok(directory.startsWith(resolve('.wrangler') + sep)); + await rm(directory, { recursive: true, force: true }); + }); + const names = ['0001_initial.sql', '0002_delivery_lease.sql', '0003_mini_app.sql']; + const migrations = await Promise.all(names.map(async name => { + const sql = await readFile(new URL('../migrations/' + name, import.meta.url), 'utf8'); + await writeFile(resolve(migrationsDirectory, name), sql); + return { name, sql }; + })); + const sourceSql = migrations.map(migration => migration.sql).join('\n') + '\n' + + 'CREATE TABLE d1_migrations(id INTEGER PRIMARY KEY AUTOINCREMENT,name TEXT,applied_at TEXT DEFAULT CURRENT_TIMESTAMP);\n' + + names.map(name => `INSERT INTO d1_migrations(name) VALUES('${name}');`).join('\n') + '\n' + + "INSERT INTO records VALUES('events','fixture-event','{\"title\":\"Fictional fixture event\"}');\n" + + "INSERT INTO app_settings VALUES('mini-menu','App:https://production.test/app');"; + const settings = [{ key: 'test-whitelist', value: '["900000001"]' }, { key: 'mini-menu', value: 'App:https://development.test/app' }]; + const schema = ['records', 'outbox', 'processed', 'lease', 'commits', 'delivery_lease', 'app_settings', 'd1_migrations', 'dev_only'].map(name => ({ type: 'table', name })); + const calls = [], healthCalls = [], paths = new Set(); + let settingReads = 0, schemaReads = 0, imported; + const beforeFiles = copyFiles(await readdir('.')); + const beforeDirectories = copyDirectories(await readdir('.wrangler')); + const env = { + GITHUB_REF_NAME: 'main', + CLOUDFLARE_API_TOKEN: 'fictional-test-token', + CLOUDFLARE_ACCOUNT_ID: 'fictional-account', + WRANGLER_CONFIG_JSON: JSON.stringify(development()), + PRODUCTION_CONFIG_JSON: JSON.stringify(production()) + }; + const run = async args => { + const path = argument(args, '--config'); + paths.add(path); + const config = JSON.parse(await readFile(path, 'utf8')); + calls.push({ args, config }); + if (args[1] === 'export') { + const output = argument(args, '--output'); + paths.add(output); + paths.add(dirname(output)); + // Also write before the synthetic failure to verify private export cleanup. + await writeFile(output, sourceSql); + if (failAt === 'export') throw new Error('Mock export failure.'); + return 'Mock export completed.'; + } + if (args.includes('--file')) { + const path = argument(args, '--file'); + paths.add(path); + imported = await readFile(path, 'utf8'); + if (failAt === 'import') throw new Error('Mock import failure.'); + return 'Mock import completed.'; + } + const command = argument(args, '--command'); + if (command.includes('sqlite_schema')) { + schemaReads++; + if (failAt === 'schema') throw new Error('Mock schema query failure.'); + if (failAt === 'response') return encoded([]).replace('true', 'false'); + return encoded(schemaRace && schemaReads > 1 ? [...schema, { type: 'table', name: 'new_dev_table' }] : schema); + } + if (command.includes('app_settings')) { + settingReads++; + if (failAt === 'settings') throw new Error('Mock settings query failure.'); + return encoded(settingsRace && settingReads > 1 ? settings.map(setting => setting.key === 'test-whitelist' ? { ...setting, value: '[]' } : setting) : settings); + } + throw new Error('Unexpected mocked database command.'); + }; + const fetcher = async (url, options) => { + healthCalls.push({ url: String(url), options }); + if (unhealthy === 'network') throw new Error('Mock network failure.'); + if (unhealthy === 'invalid-json') return new Response('invalid JSON'); + return Response.json(unhealthy === 'missing-protection' ? { status: 'running' } : unhealthy === 'stopped' ? { status: 'stopped', databaseRefreshProtection: 1 } : { status: 'running', databaseRefreshProtection: 1 }, { status: unhealthy === 'http-error' ? 503 : 200 }); + }; + const assertCleaned = async () => { + for (const path of paths) await assert.rejects(access(path), { code: 'ENOENT' }); + assert.deepEqual(copyFiles(await readdir('.')), beforeFiles); + assert.deepEqual(copyDirectories(await readdir('.wrangler')), beforeDirectories); + }; + return { env, run, fetcher, migrationsDirectory, calls, healthCalls, assertCleaned, get imported() { return imported; } }; +} + +test('DB refresh exports production read-only and imports exactly one replacement file into dev', async t => { + const f = await fixture(t); + const counts = await copyProductionDatabase(f); + assert.equal(counts.events, 1); + assert.equal(counts.appliedMigrations, 0); + assert.equal(f.healthCalls.length, 1); + assert.equal(f.healthCalls[0].url, 'https://development.test/'); + assert.ok(f.healthCalls[0].options.signal instanceof AbortSignal); + const prodCalls = f.calls.filter(call => call.config.name === 'fictional-production'); + assert.equal(prodCalls.length, 1); + assert.equal(prodCalls[0].args[1], 'export'); + assert.equal(prodCalls[0].config.vars.APP_ENV, 'production'); + assert.ok(prodCalls[0].args.includes('--remote')); + assert.ok(!prodCalls[0].args.includes('--no-data')); + const imports = f.calls.filter(call => call.args.includes('--file')); + assert.equal(imports.length, 1); + assert.equal(imports[0].config.name, 'fictional-development'); + assert.equal(imports[0].config.vars.APP_ENV, 'development'); + assert.ok(imports[0].args.includes('--yes')); + assert.ok(f.calls.every(call => !['deploy', 'migrations'].includes(call.args[0]) && call.args[1] !== 'migrations')); + const queries = f.calls.filter(call => call.args.includes('--command')); + assert.equal(queries.length, 4); + assert.ok(queries.every(call => call.config.name === 'fictional-development' && /^SELECT\b/.test(argument(call.args, '--command')))); + assert.match(f.imported, /^PRAGMA defer_foreign_keys=ON;/); + assert.match(f.imported, /DROP TABLE IF EXISTS "dev_only"/); + await f.assertCleaned(); +}); + +test('DB refresh refuses non-main runs and missing credentials before network or database calls', async t => { + const f = await fixture(t); + for (const env of [{ ...f.env, GITHUB_REF_NAME: 'dev' }, { ...f.env, CLOUDFLARE_API_TOKEN: '' }, { ...f.env, CLOUDFLARE_ACCOUNT_ID: '' }]) { + await assert.rejects(copyProductionDatabase({ ...f, env }), /main|CLOUDFLARE/); + } + assert.equal(f.calls.length, 0); + assert.equal(f.healthCalls.length, 0); + await f.assertCleaned(); +}); + +test('DB refresh refuses shared production/dev resources before health checks or exports', async t => { + const f = await fixture(t); + for (const change of [ + (dev, prod) => { dev.d1_databases[0].database_id = prod.d1_databases[0].database_id; }, + (dev, prod) => { dev.name = prod.name; }, + (dev, prod) => { dev.vars.APP_URL = prod.vars.APP_URL; }, + (dev, prod) => { dev.vars.BOT_USERNAME = prod.vars.BOT_USERNAME; } + ]) { + const dev = development(), prod = production(); + change(dev, prod); + await assert.rejects(copyProductionDatabase({ ...f, env: { ...f.env, WRANGLER_CONFIG_JSON: JSON.stringify(dev) } }), /different Worker/); + } + assert.equal(f.calls.length, 0); + assert.equal(f.healthCalls.length, 0); + await f.assertCleaned(); +}); + +test('DB refresh requires reachable dev health and deployed snapshot protection', async t => { + for (const unhealthy of ['missing-protection', 'stopped', 'http-error', 'network', 'invalid-json']) { + const f = await fixture(t, { unhealthy }); + await assert.rejects(copyProductionDatabase(f)); + assert.equal(f.healthCalls.length, 1); + assert.equal(f.calls.length, 0); + await f.assertCleaned(); + } +}); + +test('DB refresh refuses a changing whitelist before the sole dev import', async t => { + const f = await fixture(t, { settingsRace: true }); + await assert.rejects(copyProductionDatabase(f), /settings changed/); + assert.equal(f.calls.filter(call => call.args[1] === 'export').length, 1); + assert.equal(f.calls.filter(call => call.args.includes('--file')).length, 0); + await f.assertCleaned(); +}); + +test('DB refresh refuses a changing dev schema before the sole dev import', async t => { + const f = await fixture(t, { schemaRace: true }); + await assert.rejects(copyProductionDatabase(f), /schema or settings changed/); + assert.equal(f.calls.filter(call => call.args.includes('--file')).length, 0); + await f.assertCleaned(); +}); + +test('DB refresh removes temporary configs and private snapshot files after operation failures', async t => { + for (const failAt of ['export', 'schema', 'settings', 'response', 'import']) { + const f = await fixture(t, { failAt }); + await assert.rejects(copyProductionDatabase(f)); + assert.equal(f.calls.filter(call => call.args.includes('--file')).length, failAt === 'import' ? 1 : 0); + assert.ok(f.calls.filter(call => call.config.name === 'fictional-production').every(call => call.args[1] === 'export')); + await f.assertCleaned(); + } +}); + +test('DB refresh refuses migration directories outside the checked-out workspace', async t => { + const f = await fixture(t); + await assert.rejects(copyProductionDatabase({ ...f, migrationsDirectory: resolve('..') }), /checked-out development source/); + assert.equal(f.calls.length, 0); + await f.assertCleaned(); +}); diff --git a/test/database-snapshot.test.js b/test/database-snapshot.test.js new file mode 100644 index 0000000..326688c --- /dev/null +++ b/test/database-snapshot.test.js @@ -0,0 +1,288 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { DatabaseSync } from 'node:sqlite'; +import { prepareSnapshot } from '../scripts/database-snapshot.mjs'; + +const copiedAt = '2026-10-08T04:30:00.000Z'; +const migrations = ['0001_initial.sql', '0002_delivery_lease.sql', '0003_mini_app.sql'].map(name => ({ name, sql: readFileSync(new URL('../migrations/' + name, import.meta.url), 'utf8') })); +const core = migrations.map(migration => migration.sql).join('\n'); +const metadata = 'CREATE TABLE d1_migrations(id INTEGER PRIMARY KEY AUTOINCREMENT,name TEXT,applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);\n' + migrations.map(({ name }) => `INSERT INTO d1_migrations(name,applied_at) VALUES ('${name}','2026-01-01');`).join('\n'); +const source = extra => core + '\n' + metadata + '\n' + (extra || ''); +const initialSchema = ['records', 'processed', 'lease', 'commits', 'outbox', 'delivery_lease', 'app_settings'].map(name => ({ type: 'table', name })); +const options = extra => ({ sourceSql: source(), developmentSchema: initialSchema, developmentSettings: [], migrations, copiedAt, ...extra }); + +function open(sql = '') { + const db = new DatabaseSync(':memory:'); + if (sql) db.exec(sql); + return db; +} + +function developmentSchema(db) { + return db.prepare("SELECT type,name FROM sqlite_schema WHERE type IN ('table','view')").all(); +} + +function apply(db, sql) { + try { db.exec('BEGIN;\n' + sql + '\nCOMMIT;'); } + catch (error) { db.exec('ROLLBACK'); throw error; } +} + +function restore(sql) { + const db = open(core); + apply(db, sql); + return db; +} + +function value(db, sql) { + const statement = db.prepare(sql); + statement.setReadBigInts(true); + return statement.get(); +} + +test('snapshot replaces nonempty development and can be copied repeatedly', () => { + const db = open(core + "INSERT INTO records VALUES('events','old','{\"title\":\"Old test event\"}'); CREATE TABLE dev_only(value TEXT); INSERT INTO dev_only VALUES('old'); CREATE VIEW old_view AS SELECT value FROM dev_only;"); + try { + const sourceSql = source("INSERT INTO records VALUES('events','source-event','{\"title\":\"Future gathering\",\"invitationMode\":\"legacy\"}');"); + for (let attempt = 0; attempt < 2; attempt++) { + const prepared = prepareSnapshot(options({ sourceSql, developmentSchema: developmentSchema(db), developmentSettings: db.prepare('SELECT key,value FROM app_settings ORDER BY key').all() })); + apply(db, prepared.sql); + assert.equal(db.prepare('SELECT count(*) AS n FROM records').get().n, 1); + assert.equal(db.prepare('SELECT id FROM records').get().id, 'source-event'); + assert.equal(db.prepare("SELECT count(*) AS n FROM sqlite_schema WHERE name IN ('dev_only','old_view')").get().n, 0); + assert.equal(prepared.counts.events, 1); + assert.equal(prepared.counts.appliedMigrations, 0); + assert.equal(db.prepare('SELECT count(*) AS n FROM d1_migrations').get().n, 3); + } + } finally { db.close(); } +}); + +test('snapshot clears sessions and delivery state while retaining the dev whitelist and settings', () => { + const sourceSql = source(` + INSERT INTO records VALUES('sessions','900000001','{"step":"upload","eventId":"source-event"}'); + INSERT INTO records VALUES('events','source-event','{"title":"Fixture event"}'); + INSERT INTO outbox(id,method,params) VALUES('queued','sendMessage','{"chat_id":900000001,"text":"Fixture"}'); + INSERT INTO processed VALUES(100,1); + INSERT INTO lease VALUES(1,'fixture-owner',9223372036854775807); + INSERT INTO delivery_lease VALUES(1,'fixture-owner',9223372036854775807); + INSERT INTO commits VALUES('fixture-owner'); + INSERT INTO app_settings VALUES('test-whitelist','["900000002"]'); + INSERT INTO app_settings VALUES('mini-menu','App:https://production.test/app'); + `); + const developmentSettings = [{ key: 'test-whitelist', value: '["900000003"]' }, { key: 'mini-menu', value: 'App:https://development.test/app' }, { key: 'empty-setting', value: '' }]; + const { sql, counts } = prepareSnapshot(options({ sourceSql, developmentSettings })); + const db = open(core); + try { + for (const { key, value } of developmentSettings) db.prepare('INSERT INTO app_settings VALUES(?,?)').run(key, value); + apply(db, sql); + for (const table of ['outbox', 'processed', 'lease', 'delivery_lease', 'commits']) assert.equal(db.prepare(`SELECT count(*) AS n FROM ${table}`).get().n, 0); + assert.equal(db.prepare("SELECT count(*) AS n FROM records WHERE kind='sessions'").get().n, 0); + for (const setting of developmentSettings) assert.equal(db.prepare('SELECT value FROM app_settings WHERE key=?').get(setting.key).value, setting.value); + assert.deepEqual(JSON.parse(db.prepare("SELECT value FROM app_settings WHERE key='production-snapshot'").get().value), { version: 1, copiedAt }); + assert.equal(counts.removedSessions, 1); + assert.equal(counts.records, 1); + assert.doesNotMatch(sql, /production\.test|fixture-owner|900000001|900000002/); + } finally { db.close(); } +}); + +test('empty development settings do not inherit a production whitelist or endpoint', () => { + const { sql } = prepareSnapshot(options({ sourceSql: source("INSERT INTO app_settings VALUES('test-whitelist','[\"900000002\"]'); INSERT INTO app_settings VALUES('stars-webhook','https://production.test/telegram');") })); + const db = restore(sql); + try { + assert.equal(db.prepare('SELECT count(*) AS n FROM app_settings').get().n, 1); + assert.equal(db.prepare('SELECT key FROM app_settings').get().key, 'production-snapshot'); + assert.doesNotMatch(sql, /900000002|production\.test/); + } finally { db.close(); } +}); + +test('event ownership, invitation, media, payment, refund and user JSON stay byte-for-byte unchanged', () => { + const raw = '{ "id":"event-one", "owner":900000001, "invitationMode":"named", "guests":{"900000002":{"status":"yes","participants":2,"ticketCode":"fixture-ticket","payment":{"status":"paid","chargeId":"fixture-charge","refund":{"status":"complete"}},"checkedInAt":"2026-10-07"}}, "invitations":[{"token":"fixture-token","claimedBy":900000002,"oneTime":true}], "media":[{"fileId":"fixture-file","name":"Photo.jpg"}], "location":"Guest location", "title":"L’été — میهمانی" }'; + const user = '{ "id":900000002,"username":"fixture_user","firstName":"O\'Brien","phone":"fictional optional phone" }'; + const preference = '{ "timezone":"Australia/Sydney","photo":"fixture-profile-file" }'; + const sourceSql = source() + [ + ['events', 'event-one', raw], ['users', '900000002', user], ['preferences', '900000002', preference], ['future-business', 'other', 'arbitrary legacy data'] + ].map(row => 'INSERT INTO records VALUES(' + row.map(text => "'" + text.replaceAll("'", "''") + "'").join(',') + ');').join('\n'); + const { sql, counts } = prepareSnapshot(options({ sourceSql })); + const db = restore(sql); + try { + for (const [kind, expected] of [['events', raw], ['users', user], ['preferences', preference], ['future-business', 'arbitrary legacy data']]) assert.equal(db.prepare('SELECT data FROM records WHERE kind=?').get(kind).data, expected); + assert.equal(counts.records, 4); + assert.equal(counts.events, 1); + assert.equal(counts.users, 1); + assert.equal(counts.preferences, 1); + } finally { db.close(); } +}); + +test('SQL literals preserve quoted Unicode, NUL text, blobs, REAL values and exact 64-bit integers', () => { + const sourceSql = source(`CREATE TABLE payload(id INTEGER PRIMARY KEY, text_value TEXT, bytes BLOB, exact INTEGER, floating, nullable); + INSERT INTO payload VALUES(7,'O''Brien — میهمانی\nnext line',X'00ff1027',9223372036854775807,1.0,NULL); + INSERT INTO payload VALUES(8,CAST(X'610062' AS TEXT),X'',-9223372036854775808,-1.25,NULL);`); + const { sql } = prepareSnapshot(options({ sourceSql })); + const db = restore(sql); + try { + const first = value(db, 'SELECT * FROM payload WHERE id=7'); + assert.equal(first.text_value, "O'Brien — میهمانی\nnext line"); + assert.deepEqual(first.bytes, new Uint8Array([0, 255, 16, 39])); + assert.equal(first.exact, 9223372036854775807n); + assert.equal(first.floating, 1); + assert.equal(db.prepare('SELECT typeof(floating) AS kind FROM payload WHERE id=7').get().kind, 'real'); + assert.equal(first.nullable, null); + const second = value(db, 'SELECT * FROM payload WHERE id=8'); + // Node 22's SQLite TEXT reader stops at NUL; compare the actual stored bytes. + assert.equal(db.prepare('SELECT hex(CAST(text_value AS BLOB)) AS bytes FROM payload WHERE id=8').get().bytes, '610062'); + assert.deepEqual(second.bytes, new Uint8Array()); + assert.equal(second.exact, -9223372036854775808n); + } finally { db.close(); } +}); + +test('pending development migrations run once and keep original applied metadata', () => { + const pending = { name: '0004_example.sql', sql: 'ALTER TABLE records ADD COLUMN extra TEXT DEFAULT \'example\'; CREATE INDEX records_extra ON records(extra);' }; + const { sql, counts } = prepareSnapshot(options({ migrations: [...migrations, pending], sourceSql: source("INSERT INTO records VALUES('events','fixture','{}');") })); + const db = restore(sql); + try { + assert.equal(db.prepare('SELECT extra FROM records').get().extra, 'example'); + assert.equal(db.prepare("SELECT applied_at FROM d1_migrations WHERE name='0001_initial.sql'").get().applied_at, '2026-01-01'); + assert.equal(db.prepare("SELECT count(*) AS n FROM d1_migrations WHERE name='0004_example.sql'").get().n, 1); + assert.equal(counts.appliedMigrations, 1); + // Treat the completed snapshot as the next source; ALTER must not run again. + const repeatSource = source("INSERT INTO records VALUES('events','fixture','{}');" + pending.sql + "INSERT INTO d1_migrations(name) VALUES('0004_example.sql');"); + const repeat = prepareSnapshot(options({ sourceSql: repeatSource, migrations: [...migrations, pending], developmentSchema: developmentSchema(db), developmentSettings: db.prepare('SELECT key,value FROM app_settings ORDER BY key').all() })); + apply(db, repeat.sql); + assert.equal(repeat.counts.appliedMigrations, 0); + assert.equal(db.prepare('SELECT count(*) AS n FROM d1_migrations').get().n, 4); + } finally { db.close(); } +}); + +test('missing migration fails before returning an import and keeps private names out of errors', () => { + assert.throws(() => prepareSnapshot(options({ sourceSql: source("INSERT INTO d1_migrations(name) VALUES('9999_private_example.sql');") })), error => /migrations missing/.test(error.message) && !error.message.includes('9999_private_example')); +}); + +test('unrecorded initial migrations can initialize metadata without duplicating application rows', () => { + const { sql, counts } = prepareSnapshot(options({ sourceSql: core + "INSERT INTO records VALUES('events','fixture','{}');" })); + const db = restore(sql); + try { + assert.equal(db.prepare('SELECT count(*) AS n FROM d1_migrations').get().n, 3); + assert.equal(db.prepare('SELECT count(*) AS n FROM records').get().n, 1); + assert.equal(counts.appliedMigrations, 3); + } finally { db.close(); } +}); + +test('development identifiers are quoted safely and platform internals are never dropped', () => { + const unusual = 'unusual"; DROP TABLE records;--'; + const db = open(core + 'CREATE TABLE "' + unusual.replaceAll('"', '""') + '"(value TEXT); CREATE TABLE _cf_KV(key TEXT,value TEXT); INSERT INTO _cf_KV VALUES(\'fixture\',\'platform state\');'); + try { + const { sql } = prepareSnapshot(options({ developmentSchema: developmentSchema(db) })); + assert.doesNotMatch(sql, /DROP TABLE IF EXISTS "(?:_cf_KV|sqlite_sequence)"/i); + apply(db, sql); + assert.equal(db.prepare('SELECT value FROM _cf_KV').get().value, 'platform state'); + assert.equal(db.prepare('SELECT count(*) AS n FROM sqlite_schema WHERE name=?').get(unusual).n, 0); + assert.equal(db.prepare('SELECT count(*) AS n FROM records').get().n, 0); + } finally { db.close(); } +}); + +test('auto-increment high-water marks and generated columns survive a snapshot', () => { + const { sql } = prepareSnapshot(options({ sourceSql: source(`CREATE TABLE sequence_example(id INTEGER PRIMARY KEY AUTOINCREMENT,value TEXT,length INTEGER GENERATED ALWAYS AS (length(value)) STORED); + INSERT INTO sequence_example(id,value) VALUES(5,'fixture'); INSERT INTO sequence_example(id,value) VALUES(99,'deleted'); DELETE FROM sequence_example WHERE id=99;`) })); + const db = restore(sql); + try { + assert.equal(db.prepare('SELECT length FROM sequence_example').get().length, 7); + db.prepare('INSERT INTO sequence_example(value) VALUES (?)').run('next'); + assert.equal(db.prepare("SELECT id FROM sequence_example WHERE value='next'").get().id, 100); + } finally { db.close(); } +}); + +test('rowids and WITHOUT ROWID tables survive while generated aliases remain read-only', () => { + const { sql } = prepareSnapshot(options({ sourceSql: source(` + CREATE TABLE rowid_example(value TEXT,"_rowid_" INTEGER GENERATED ALWAYS AS (length(value)) VIRTUAL); + INSERT INTO rowid_example(rowid,value) VALUES(45,'fixture'); + CREATE TABLE natural_key(a TEXT,b INTEGER,PRIMARY KEY(a,b)) WITHOUT ROWID; + INSERT INTO natural_key VALUES('fixture',9223372036854775807); + `) })); + const db = restore(sql); + try { + assert.equal(value(db, 'SELECT rowid FROM rowid_example').rowid, 45n); + assert.equal(db.prepare('SELECT "_rowid_" AS n FROM rowid_example').get().n, 7); + assert.equal(value(db, 'SELECT b FROM natural_key').b, 9223372036854775807n); + } finally { db.close(); } +}); + +test('indexes, views and lease guards are restored after data without replaying business triggers', () => { + const { sql } = prepareSnapshot(options({ sourceSql: source(`CREATE TABLE audit(value TEXT); CREATE INDEX audit_value ON audit(value); + CREATE TRIGGER record_audit AFTER INSERT ON records BEGIN INSERT INTO audit VALUES(NEW.id); END; + CREATE VIEW event_titles AS SELECT id FROM records WHERE kind='events'; + INSERT INTO records VALUES('events','fixture-event','{}'); + CREATE TRIGGER delete_audit AFTER DELETE ON records BEGIN INSERT INTO audit VALUES('deleted'); END; + INSERT INTO records VALUES('sessions','fixture-session','{}');`) })); + const db = restore(sql); + try { + assert.deepEqual(db.prepare('SELECT value FROM audit ORDER BY value').all().map(row => row.value), ['fixture-event', 'fixture-session']); + assert.equal(db.prepare('SELECT id FROM event_titles').get().id, 'fixture-event'); + assert.equal(db.prepare("SELECT count(*) AS n FROM sqlite_schema WHERE type='index' AND name='audit_value'").get().n, 1); + assert.throws(() => db.prepare("INSERT INTO commits VALUES('lost')").run(), /lease lost/); + db.prepare("INSERT INTO records VALUES('events','new-event','{}')").run(); + assert.equal(db.prepare("SELECT count(*) AS n FROM audit WHERE value='new-event'").get().n, 1); + assert.ok(sql.lastIndexOf('CREATE TRIGGER') > sql.lastIndexOf('INSERT INTO')); + } finally { db.close(); } +}); + +test('virtual tables and unsupported extensions fail safely without emitting SQL or private values', () => { + for (const extra of ["CREATE VIRTUAL TABLE private_search USING fts5(private_text);", "SELECT load_extension('private-extension-path');", "INSERT INTO missing_private_table VALUES('private-fixture-value');"]) { + assert.throws(() => prepareSnapshot(options({ sourceSql: source(extra) })), error => /could not be prepared safely/.test(error.message) && !/private|load_extension|missing_private_table/.test(error.message)); + } +}); + +test('malformed core records and schema fail before any replacement SQL is returned', () => { + for (const extra of [ + "INSERT INTO records VALUES('events','fixture','not valid JSON');", + "INSERT INTO records VALUES('users','fixture','null');", + "INSERT INTO records VALUES('preferences','fixture','[]');", + 'DROP TABLE outbox;', + 'DROP TABLE records; CREATE TABLE records(kind INTEGER,id TEXT,data TEXT);' + ]) assert.throws(() => prepareSnapshot(options({ sourceSql: source(extra) })), /could not be prepared safely/); +}); + +test('invalid development settings, schema and timestamps produce generic errors', () => { + for (const change of [ + { developmentSettings: [{ key: 'key', value: 'one' }, { key: 'key', value: 'two' }] }, + { developmentSettings: [{ key: 'key', value: null }] }, + { developmentSchema: [{ type: 'table', name: 'invalid\0name' }] }, + { developmentSchema: [{ type: 'trigger', name: 'invalid' }] }, + { developmentSchema: [{ type: 'table', name: 'records' }, { type: 'table', name: 'records' }] }, + { copiedAt: 'private invalid value' } + ]) assert.throws(() => prepareSnapshot(options(change)), error => /could not be prepared safely/.test(error.message) && !error.message.includes('private')); +}); + +test('atomic settings guard aborts a stale snapshot without resetting data or overwriting current settings', () => { + for (const change of [ + db => db.exec("UPDATE app_settings SET value='[]' WHERE key='test-whitelist'"), + db => db.exec("INSERT INTO app_settings VALUES('new-setting','current')"), + db => db.exec("DELETE FROM app_settings WHERE key='test-whitelist'"), + db => db.exec("UPDATE app_settings SET key='renamed-setting' WHERE key='test-whitelist'"), + db => db.exec("UPDATE app_settings SET value=CAST(value AS BLOB) WHERE key='test-whitelist'"), + db => db.exec("UPDATE app_settings SET key=CAST(key AS BLOB) WHERE key='test-whitelist'") + ]) { + const db = open(core + "INSERT INTO records VALUES('events','current-dev-event','{\"title\":\"Keep current dev data\"}'); INSERT INTO app_settings VALUES('test-whitelist','[\"900000001\"]');"); + try { + const developmentSettings = db.prepare('SELECT key,value FROM app_settings').all(); + const { sql } = prepareSnapshot(options({ sourceSql: source("INSERT INTO records VALUES('events','new-snapshot-event','{}');"), developmentSchema: developmentSchema(db), developmentSettings })); + change(db); + const current = db.prepare('SELECT hex(CAST(key AS BLOB)) AS keyBytes,hex(CAST(value AS BLOB)) AS valueBytes,typeof(key) AS keyType,typeof(value) AS valueType FROM app_settings ORDER BY key').all(); + assert.throws(() => apply(db, sql), /CHECK constraint failed/); + assert.equal(db.prepare('SELECT id FROM records').get().id, 'current-dev-event'); + assert.deepEqual(db.prepare('SELECT hex(CAST(key AS BLOB)) AS keyBytes,hex(CAST(value AS BLOB)) AS valueBytes,typeof(key) AS keyType,typeof(value) AS valueType FROM app_settings ORDER BY key').all(), current); + assert.equal(db.prepare("SELECT count(*) AS n FROM sqlite_schema WHERE name GLOB '_xevents_snapshot_guard_*'").get().n, 0); + } finally { db.close(); } + } +}); + +test('atomic settings guard compares exact text bytes and requires an initialized dev settings table', () => { + assert.throws(() => prepareSnapshot(options({ developmentSchema: [{ type: 'table', name: 'records' }] })), /could not be prepared safely/); + const db = open(core); + try { + const developmentSettings = [{ key: 'quote\' — نام', value: 'a\0b' }]; + db.prepare('INSERT INTO app_settings VALUES(?,?)').run(developmentSettings[0].key, developmentSettings[0].value); + const { sql } = prepareSnapshot(options({ developmentSettings })); + assert.ok(sql.indexOf('CHECK(ok=1)') < sql.indexOf('DROP TABLE IF EXISTS "records"')); + apply(db, sql); + assert.equal(db.prepare('SELECT hex(CAST(value AS BLOB)) AS bytes FROM app_settings WHERE key=?').get(developmentSettings[0].key).bytes, '610062'); + } finally { db.close(); } +});