From fa0d17c790be8638813102b781f35429a4301251 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 13:10:20 -0400 Subject: [PATCH 01/20] feat(migrate): add custom sources and migrate sources `clerk migrate import --source` now also takes the path to a source you wrote, loaded at runtime and validated before use. An import run records a custom source's content hash, so an edited source is a different one when a re-run decides whether to continue. `clerk migrate sources` lists the built-in sources with what each brings across, or shows one in full. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/completion/__complete.ts | 6 +- .../cli-core/src/commands/migrate/README.md | 111 +++++++- .../src/commands/migrate/index.test.ts | 15 +- .../cli-core/src/commands/migrate/index.ts | 29 +- .../commands/migrate/lib/transform.test.ts | 22 ++ .../cli-core/src/commands/migrate/run.test.ts | 136 ++++++++++ packages/cli-core/src/commands/migrate/run.ts | 38 ++- .../src/commands/migrate/runs.test.ts | 10 + .../cli-core/src/commands/migrate/runs.ts | 17 +- .../src/commands/migrate/sources/list.test.ts | 172 ++++++++++++ .../src/commands/migrate/sources/list.ts | 202 ++++++++++++++ .../migrate/sources/load-custom.test.ts | 250 ++++++++++++++++++ .../commands/migrate/sources/load-custom.ts | 184 +++++++++++++ .../src/commands/migrate/sources/registry.ts | 66 ++++- .../commands/migrate/sources/sources.test.ts | 15 +- .../src/test/integration/completion.test.ts | 17 ++ 16 files changed, 1250 insertions(+), 40 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/sources/list.test.ts create mode 100644 packages/cli-core/src/commands/migrate/sources/list.ts create mode 100644 packages/cli-core/src/commands/migrate/sources/load-custom.test.ts create mode 100644 packages/cli-core/src/commands/migrate/sources/load-custom.ts diff --git a/packages/cli-core/src/commands/completion/__complete.ts b/packages/cli-core/src/commands/completion/__complete.ts index 06f9d87f1..642281f9e 100644 --- a/packages/cli-core/src/commands/completion/__complete.ts +++ b/packages/cli-core/src/commands/completion/__complete.ts @@ -34,7 +34,10 @@ const HTTP_METHOD_COMPLETIONS: Completion[] = [ { name: "DELETE", description: "Delete resource" }, ]; -/** The built-in migrate sources. Kept off `.choices()` so an unknown key gets the source error. */ +/** + * The built-in migrate sources. `--source` also takes a path to a source you + * wrote, so it cannot use `.choices()`. + */ const SOURCE_COMPLETIONS: Completion[] = sources.map((entry) => ({ name: entry.key, description: entry.label, @@ -74,6 +77,7 @@ const KNOWN_OPTION_VALUES: Record = { * to root, e.g. "open dashboard". */ const KNOWN_POSITIONAL_COMPLETIONS: Record = { + "migrate sources": SOURCE_COMPLETIONS, "open dashboard": KNOWN_DASHBOARD_PATHS.map((path) => ({ name: path, description: "Dashboard subpath", diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 255e35d2e..3dcdb1c13 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -19,6 +19,7 @@ clerk migrate export [source] [-o ] [--json] clerk migrate import [--source ] [--dry-run] [--allow-partial] [--new-run] [--yes] [--json] clerk migrate runs [run-id] [--json] clerk migrate undo [--dry-run] [--yes] [--include-signed-in] [--json] +clerk migrate sources [source] [--json] clerk migrate help ``` @@ -653,7 +654,7 @@ clerk migrate import # a human is asked | Flag | Description | | --------------------------------------- | ------------------------------------------------------------------------------------------------- | | `[file\|export-run-id]` | The export file, or the ID of the export run that wrote it | -| `--source ` | Where the file came from: one of the [sources](#sources) | +| `--source ` | Where the file came from: a [source](#sources), or one you wrote | | `--dry-run` | Run the [checks](#checks) against the instance, and write nothing | | `--allow-partial` | Import the users that pass, and record the rest as skipped | | `--new-run` | Start a new run instead of [continuing](#re-running) an earlier one | @@ -724,8 +725,8 @@ which skip the checks too. #### Re-running Running the same import again continues where it left off. The match is the -file's sha256, the source, and the instance ID; the latest matching import run -decides what happens: +file's sha256, the source (and a custom source's content hash), and the +instance ID; the latest matching import run decides what happens: | Latest match | Re-running does | | ----------------------------------------- | ---------------------------------------------------------------------- | @@ -1067,6 +1068,26 @@ With `writeFailed` or `stopped`, `run` is the undo run as it stands on disk, once one was written: its counts come from its lines, and `state` says how it stopped, as in `clerk migrate runs`. +### `clerk migrate sources` + +```sh +clerk migrate sources # every source, with what it carries +clerk migrate sources betterauth # one source in full +clerk migrate sources ./my-source.ts # a source you wrote +clerk migrate sources --json +``` + +| Flag | Description | +| ---------- | ---------------------------------------------------------------- | +| `[source]` | A built-in key, or the path to a source you wrote, to show fully | +| `--json` | The same data, on stdout | + +`sources` alone prints the table under [Sources](#sources). `sources ` shows one source in +full: its export command, what it carries with a note for each, where each +field lands (`encrypted_password → password`), its fixed defaults, and any +caveats. An unknown key exits 2 and lists the valid ones. There is no +intro/outro gutter: this reads a static registry rather than running anything. + ### `clerk migrate help` `clerk migrate help` and `clerk migrate --help` print the help for the @@ -1089,13 +1110,85 @@ brings across. Adding a platform is one file in `sources/` plus one line in | `supabase` | Supabase `auth.users` export | yes | no | partial | | `workos` | WorkOS User Management API | no | no | yes | -An unknown `--source` exits 2 with the list of valid keys. - There is no column for social sign-ins, because no source copies them and none -needs to. Enable the same providers in Clerk, and a user who signs in with one -is linked to their imported account by verified email. See +needs to. Every source shows the same note instead: enable the same providers in +Clerk, and a user who signs in with one is linked to their imported account by +verified email. See [account linking](https://clerk.com/docs/guides/configure/auth-strategies/social-connections/account-linking). +### `--source` + +`clerk migrate import` takes `--source `: + +- A value starting with `./`, `../` or `/`, or ending in `.ts`, `.js` or + `.mjs`, is loaded as a [custom source](#custom-sources). +- Anything else must be a built-in key. An unknown key exits 2 with the list of + valid keys. + +A file `clerk migrate export` wrote names its own source, so it needs none. + +### Custom sources + +Migrating from a platform with no built-in, without recompiling the CLI: + +```sh +clerk migrate import users.json --source ./my-platform.ts +``` + +The file lives in **your** project, not in the CLI, and is imported at runtime. +It exports the same shape the built-ins use — plain data, no imports, since +there is nothing in a compiled binary for your file to import from: + +```ts +export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform's admin console.", + transformer: { + account_ref: "userId", // required: becomes the Clerk user's external_id + contact_email: "email", + given: "firstName", + family: "lastName", + pw_bcrypt: "password", + }, + carries: { + passwords: { level: "yes", note: "bcrypt hashes from the pw_bcrypt column." }, + mfa: { level: "no", note: "Not exported." }, + metadata: { level: "no", note: "Not exported." }, + }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { + if (!user.firstName) delete user.firstName; + }, +}; +``` + +TypeScript is fine — Bun's transpiler is part of the runtime, so `interface`, +`satisfies` and `as const` all work in a file the compiled binary imports. +Plain `.js` works too. + +An import run records a custom source's key and a hash of the file, so an +edited source counts as a different source. + +#### Validation + +The file is code the CLI executes, so its shape is checked before use and +rejected with the specific problem rather than crashing mid-pipeline: + +| Problem | Message | +| ---------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| Path does not exist | `No source file at /abs/path.ts.` | +| No default export, but a named one | ``has no default export. Found named export `myPlatform` — did you mean `export default`?`` | +| Does not parse | `Could not load ./f.ts: Expected identifier but found ","` | +| Nothing maps to `userId` | ``no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id`` | +| No `carries` | `` `carries` must say what the source brings across: { passwords, mfa, metadata }, each { level, note } `` | +| `key` clashes with a built-in | `key is "clerk", which is already a built-in source` | +| A hook is not a function | `postTransform must be a function when present` | + +The `userId` check is the load-bearing one: without it the import would run to +completion and create every user with no `external_id`, which is what makes a +migration re-runnable. + ### Metadata Metadata a user can edit on the source platform — Auth0's `user_metadata`, @@ -1128,6 +1221,7 @@ Better Auth's scrypt uses the hex salt string as the salt and a 64-byte key, which is what `scrypt_werkzeug` verifies once N, r and p are written inline. It also normalizes a password to NFKC before hashing, and Clerk does not, so a password whose NFKC form differs will not verify and that user resets it. +`clerk migrate sources betterauth` lists that caveat. A password Clerk cannot verify is **dropped, not rejected**: dropping it doesn't reject the user, who imports without it and can sign in another way or reset @@ -1194,7 +1288,8 @@ An export with no password hashes needs no parameters at all. What a source maps _onto_. Every user is validated against this schema before any request is made, so a field a source produces that is not listed here is dropped — Zod strips unknown keys — and never reaches Clerk. The checks -warn about each one (`Clerk won't store: …`). +warn about each one (`Clerk won't store: …`). Writing a custom source means +targeting these names exactly. The schema lives in `validator.ts`; adding a platform means adding a source, not editing it. diff --git a/packages/cli-core/src/commands/migrate/index.test.ts b/packages/cli-core/src/commands/migrate/index.test.ts index 449e4853c..b00ed9e60 100644 --- a/packages/cli-core/src/commands/migrate/index.test.ts +++ b/packages/cli-core/src/commands/migrate/index.test.ts @@ -32,12 +32,13 @@ describe("registerMigrate", () => { expect(migrate?.description()).toContain("Migrate users"); }); - test("registers import, export, undo and runs", () => { + test("registers import, export, undo, runs and sources", () => { expect(findCommand(["migrate"])?.commands.map((cmd) => cmd.name())).toEqual([ "import", "export", "undo", "runs", + "sources", ]); }); @@ -157,6 +158,12 @@ describe("registerMigrate", () => { expect(runsCommand?.options.map((option) => option.long)).toEqual(["--json", "--runs-dir"]); }); + test("registers sources with an optional source and --json", () => { + const sourcesCommand = findCommand(["migrate", "sources"]); + expect(sourcesCommand?.registeredArguments[0]?.required).toBe(false); + expect(sourcesCommand?.options.map((option) => option.long)).toEqual(["--json"]); + }); + test("registers undo with a required run ID and its flags", () => { const undoCommand = findCommand(["migrate", "undo"]); expect(undoCommand?.registeredArguments[0]?.required).toBe(true); @@ -172,9 +179,9 @@ describe("registerMigrate", () => { ]); }); - // No `.choices()`: an unknown key reaches resolveSource, whose error lists - // the valid ones. Completion offers them through `KNOWN_OPTION_VALUES`. - test("--source accepts any value", () => { + // It also takes a path, so it cannot use `.choices()`: completion offers the + // built-in keys through `KNOWN_OPTION_VALUES` instead. + test("--source accepts any value, so a path to a source you wrote gets through", () => { const option = findCommand(["migrate", "import"])?.options.find((o) => o.long === "--source"); expect(option?.argChoices).toBeUndefined(); }); diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index e0c1728c1..a4bb4afed 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -8,6 +8,7 @@ import { RUNS_DIR_DESCRIPTION, RUNS_DIR_FLAG } from "./lib/run-store.ts"; import { run } from "./run.ts"; import { runs } from "./runs.ts"; import { undo } from "./undo.ts"; +import { list as sources } from "./sources/list.ts"; export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = process.env): void { if (!isExperimentEnabled("migrate", env)) { @@ -51,6 +52,7 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce command: "clerk migrate undo 20260929-141502-a1b2", description: "Delete the users an import created", }, + { command: "clerk migrate sources", description: "What each source brings across" }, ]); // `-y` is read several layers down — by the credential-retry loop and the @@ -81,8 +83,8 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce .description("Import users from an exported JSON or CSV file") .argument("[file|export-run-id]", "The export file, or the ID of the export run that wrote it") .option( - "--source ", - "Where the file came from. Not needed for a file from `clerk migrate export`", + "--source ", + "Where the file came from: a built-in source, or a source you wrote. Not needed for a file from `clerk migrate export`", ) .option("--dry-run", "Check the file against the instance, report, and write nothing") .option("--allow-partial", "Import the users that pass the checks, and skip the rest") @@ -129,6 +131,10 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce command: "clerk migrate import users.json --source clerk --allow-partial --yes", description: "Import a Clerk Dashboard export, skipping users that would be rejected", }, + { + command: "clerk migrate import users.json --source ./my-source.ts --yes", + description: "Import with a source you wrote", + }, ]) .action(async (input, _opts, cmd) => run({ @@ -184,4 +190,23 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce .action(async (runId, _opts, cmd) => runs(runId, cmd.optsWithGlobals() as Parameters[1]), ); + + // A compiled binary has no source tree to grep, so the available mappings + // need a command rather than only appearing in the interactive picker. + migrateCommand + .command("sources") + .description("List the sources an import can read, or show one in full") + .argument("[source]", "A built-in source, or the path to a source you wrote") + .option("--json", "Output as JSON") + .setExamples([ + { command: "clerk migrate sources", description: "What each source brings across" }, + { + command: "clerk migrate sources betterauth", + description: "Where each field lands, how to export, and caveats", + }, + { command: "clerk migrate sources ./my-source.ts", description: "Check a source you wrote" }, + ]) + .action(async (source, _opts, cmd) => + sources(source, cmd.optsWithGlobals() as Parameters[1]), + ); } diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts index 561ad655e..766ae00e7 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -5,6 +5,7 @@ import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; import clerkSource from "../sources/clerk.ts"; import type { SourceEntry } from "../types.ts"; +import { __resetCustomSourcesForTesting, registerCustomSource } from "../sources/registry.ts"; import { consolidateClerkIdentifiers, flattenObjectSelectively, @@ -298,6 +299,27 @@ describe("loadUsersFromFile", () => { } }); + test("a custom preTransform's rows win over a CSV file", async () => { + registerCustomSource({ + ...clerkSource, + key: "rows-from-pretransform", + preTransform: (filePath) => ({ + filePath, + data: [{ id: "from-pretransform", primary_email_address: "p@x.dev" }], + }), + }); + try { + fs.writeFileSync( + path.join(workDir, "ignored.csv"), + "id,primary_email_address\nfile,f@x.dev\n", + ); + const { users } = await loadUsersFromFile("ignored.csv", "rows-from-pretransform"); + expect(users.map((user) => user.userId)).toEqual(["from-pretransform"]); + } finally { + __resetCustomSourcesForTesting(); + } + }); + test("rejects a JSON file that is not an array of users", async () => { fs.writeFileSync(path.join(workDir, "wrapped.json"), JSON.stringify({ users: [] })); await expect(loadUsersFromFile("wrapped.json", "clerk")).rejects.toThrow(CliError); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 383f693a7..94b43bd11 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -30,6 +30,7 @@ import { startRun, } from "./lib/run-store.ts"; import { explainErrors, run, validateRunOptions } from "./run.ts"; +import { __resetCustomSourcesForTesting } from "./sources/registry.ts"; // After run.ts: imported first, signals.ts loads version.ts ahead of its Bun // macro here, and the file fails to load. import { _resetInterruptState, abortInFlight, beginInterrupt } from "../../lib/signals.ts"; @@ -2007,6 +2008,141 @@ describe("run", () => { }); }); + describe("--source ", () => { + const CUSTOM = `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", contact_email: "email", given: "firstName", pw: "password" }, + carries: { + passwords: { level: "yes", note: "bcrypt." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { if (!user.firstName) delete user.firstName; }, + };`; + + let customFile: string; + let customCounter = 0; + + beforeEach(() => { + // A fresh filename each time: dynamic import() caches by URL, so reusing + // one would silently return a previous test's module. + customFile = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, customFile), CUSTOM); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { account_ref: "mp_1", contact_email: "a@x.dev", given: "Ada", pw: BCRYPT }, + { account_ref: "mp_2", contact_email: "b@x.dev", given: "", pw: BCRYPT }, + ]), + ); + }); + + afterEach(() => { + __resetCustomSourcesForTesting(); + }); + + const created = () => requests.filter((r) => r.url.endsWith("/v1/users")); + + // The registered key isn't something --source accepts; the path is. + test("the printed command names the source's path, not its key", async () => { + const error = (await run({ + input: "export.json", + source: customFile, + secretKey: "sk_test_x", + json: true, + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.examples?.[0]?.command).toBe( + `clerk migrate import export.json --source ${customFile} --secret-key --json --yes`, + ); + }); + + test("imports through a user-authored source", async () => { + await run({ + input: "export.json", + source: customFile, + yes: true, + secretKey: "sk_test_x", + }); + + expect(created().map((r) => (r.body as { external_id: string }).external_id)).toEqual([ + "mp_1", + "mp_2", + ]); + expect(captured.err).toContain("myplatform"); + expect(captured.err).toContain("source from"); + }); + + test("applies the custom source's defaults and postTransform", async () => { + await run({ + input: "export.json", + source: customFile, + yes: true, + secretKey: "sk_test_x", + }); + + const bodies = created().map((r) => r.body as Record); + expect(bodies[0]).toMatchObject({ first_name: "Ada", password_hasher: "bcrypt" }); + // postTransform dropped the empty given name rather than sending "". + expect("first_name" in (bodies[1] ?? {})).toBe(false); + }); + + // An edited source is a different source, so the run records which one. + test("records the custom source's content hash on the run", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + + const [record] = listRuns(runsDir()); + expect(record?.source).toBe("myplatform"); + expect(record?.sourceHash).toMatch(/^[0-9a-f]{64}$/); + }); + + test("an unknown built-in key is a usage error listing the valid ones", async () => { + await expect( + run({ input: "export.json", source: "okta", yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/Unknown source "okta". Valid sources: clerk, auth0/); + expect(created()).toHaveLength(0); + }); + + test("fails before any request when the file is not there", async () => { + await expect( + run({ + input: "export.json", + source: "./nope.ts", + yes: true, + secretKey: "sk_test_x", + }), + ).rejects.toThrow(/No source file at/); + expect(requests).toHaveLength(0); + }); + + test("fails before any request when the file is malformed", async () => { + const bad = `./bad-${customCounter++}.ts`; + fs.writeFileSync( + path.join(workDir, bad), + `export default { key: "x", label: "X", transformer: {} };`, + ); + + const error = (await run({ + input: "export.json", + source: bad, + yes: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + expect(error.message).toMatch(/no source field maps to `userId`/); + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(requests).toHaveLength(0); + }); + + test("still requires a file", async () => { + await expect(run({ source: customFile, yes: true, secretKey: "sk_test_x" })).rejects.toThrow( + /needs the file to import/, + ); + }); + }); + describe("per-platform imports", () => { /** One realistic record per platform, in that platform's export shape. */ const PLATFORMS: [string, unknown, string][] = [ diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 17943d32b..df31275fe 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -110,8 +110,12 @@ import { link } from "../link/index.ts"; export type MigrateRunOptions = { /** An export file, or the ID of the export run that wrote one. */ input?: string; - /** A built-in source key. */ + /** A built-in source key, or the path to a source you wrote. */ source?: string; + /** Content hash of a custom `--source`, set once it is loaded. */ + sourceHash?: string; + /** `--source` as typed, for printed commands: a custom source's path. Not a flag. */ + sourceArg?: string; /** The file to read, once `input` is resolved. Not a flag. */ file?: string; requirePassword?: boolean; @@ -139,7 +143,7 @@ export type MigrateRunOptions = { * Validates what a run needs before anything is read or sent. * * `--source` has already been resolved to a registered key by the time this - * runs. + * runs, custom sources included. * * @returns The source key and file path, both guaranteed present. */ @@ -149,7 +153,7 @@ export function validateRunOptions(options: MigrateRunOptions): { } { if (!options.source) { throwUsageError( - `Missing --source. Valid values: ${sourceKeys().join(", ")}.`, + `Missing --source. Valid values: ${sourceKeys().join(", ")}, or the path to a source you wrote.`, undefined, ERROR_CODE.USAGE_ERROR, [ @@ -371,14 +375,23 @@ function throwChangedExport(runId: string, file: string): never { } /** - * Resolves `--source` to a registered key. + * Resolves `--source` to a registered key, loading a custom source from its + * path so the rest of the run treats it exactly like a built-in. * * @throws UsageError for an unknown key. */ async function applySource(options: MigrateRunOptions): Promise { if (!options.source) return options; const resolved = await resolveSource(options.source); - return { ...options, source: resolved.key }; + if (resolved.path && !options.json) { + log.info(`Loaded the \`${resolved.key}\` source from ${options.source}.`); + } + return { + ...options, + source: resolved.key, + sourceArg: options.source, + ...(resolved.hash ? { sourceHash: resolved.hash } : {}), + }; } /** @@ -419,8 +432,8 @@ type ResumeCase = | { kind: "complete"; record: RunRecord }; /** - * Finds the latest import run of this file (by sha256), this source and this - * instance, and decides what a re-run does: + * Finds the latest import run of this file (by sha256), this source (and a + * custom source's hash) and this instance, and decides what a re-run does: * * - none, or undone: a new run * - interrupted: continue it, skipping the users it created @@ -437,6 +450,7 @@ function findResume( match: { sha256: string; source: string; + sourceHash?: string; instanceId: string; /** The key's stand-in ID, for a run recorded while Clerk could not name the instance. */ keyInstanceId: string; @@ -448,8 +462,9 @@ function findResume( const ofFile = importsOfFile(runs, match); const latest = ofFile.find( (record) => - record.target.instanceId === match.instanceId || - record.target.instanceId === match.keyInstanceId, + record.sourceHash === match.sourceHash && + (record.target.instanceId === match.instanceId || + record.target.instanceId === match.keyInstanceId), ); if (!latest) { // Clerk did not name the instance (GET /v1/instance failed), so a run of @@ -655,7 +670,8 @@ function commandFor( ) { const input = fromExport ?? options.input ?? options.file ?? ""; const parts = ["clerk migrate import", quoteArg(input)]; - if (!fromExport && options.source) parts.push("--source", quoteArg(options.source)); + const source = options.sourceArg ?? options.source; + if (!fromExport && source) parts.push("--source", quoteArg(source)); if (options.newRun) parts.push("--new-run"); parts.push(...importFlags(options)); // Names, not `<…>`: pasted as is, a shell reads `` as a redirect. @@ -860,6 +876,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const match = { sha256, source, + ...(options.sourceHash ? { sourceHash: options.sourceHash } : {}), instanceId: target.instanceId, keyInstanceId: keyInstanceId(secretKey), }; @@ -1218,6 +1235,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { kind: "import", target, source, + ...(options.sourceHash ? { sourceHash: options.sourceHash } : {}), file: { path: filePath, sha256 }, ...(input.fromExport ? { fromExport: input.fromExport } : {}), ...(firebaseHash ? { firebaseHash } : {}), diff --git a/packages/cli-core/src/commands/migrate/runs.test.ts b/packages/cli-core/src/commands/migrate/runs.test.ts index eb2c4b6cb..655537637 100644 --- a/packages/cli-core/src/commands/migrate/runs.test.ts +++ b/packages/cli-core/src/commands/migrate/runs.test.ts @@ -406,6 +406,16 @@ describe("runs ", () => { expect(next[0]).toContain("to retry the users it didn't finish"); }); + // The record has the custom source's hash, not its path. + test("a partial import from a source you wrote names the source file to pass", async () => { + const record = partialImport({ source: "acme", sourceHash: "abc123" }); + + await runs(record.id, { runsDir, json: true }); + + const { next } = JSON.parse(captured.out) as { next: string[] }; + expect(next[0]).toContain(`clerk migrate import ${quoteArg(usersFile())} --source SOURCE_FILE`); + }); + test("an unknown ID is a usage error", async () => { await expect(runs("nope", { runsDir })).rejects.toThrow(/No run `nope`/); }); diff --git a/packages/cli-core/src/commands/migrate/runs.ts b/packages/cli-core/src/commands/migrate/runs.ts index 70660eaa1..3e14c0934 100644 --- a/packages/cli-core/src/commands/migrate/runs.ts +++ b/packages/cli-core/src/commands/migrate/runs.ts @@ -124,16 +124,19 @@ function continueCommand(record: RunRecord, carry: string[]): string | undefined const input = record.fromExport ?? record.file?.path; if (!input) return undefined; const parts = ["clerk migrate import", quoteArg(input)]; - if (!record.fromExport && record.source) parts.push("--source", quoteArg(record.source)); + // A source you wrote is a file the record knows only by its hash. A name, + // not `<…>`: pasted as is, a shell reads `` as a redirect. + if (record.sourceHash) parts.push("--source", "SOURCE_FILE"); + else if (!record.fromExport && record.source) parts.push("--source", quoteArg(record.source)); return withFlags(parts.join(" "), [...importFlags(record.flags), ...carry, "--yes"]); } /** * Whether importing the record's file again continues this run, rather than - * starting another or refusing: no newer import of that file and source may - * be for the same instance, the file is unchanged, and no undo of it exists. - * An unfinished undo makes an import refuse; a finished one that never marked - * the run makes it start a new one. + * starting another or refusing: no newer import of that file and source (and + * version of a source you wrote) may be for the same instance, the file is + * unchanged, and no undo of it exists. An unfinished undo makes an import + * refuse; a finished one that never marked the run makes it start a new one. */ function continuesThisRun(runsDir: string, record: RunRecord): boolean { const file = record.file; @@ -146,8 +149,8 @@ function continuesThisRun(runsDir: string, record: RunRecord): boolean { id === record.target.instanceId || isStandInInstanceId(id) || isStandInInstanceId(record.target.instanceId); - const newest = importsOfFile(runs, { sha256: file.sha256, source: record.source }).find((other) => - mayBeSameInstance(other.target.instanceId), + const newest = importsOfFile(runs, { sha256: file.sha256, source: record.source }).find( + (other) => other.sourceHash === record.sourceHash && mayBeSameInstance(other.target.instanceId), ); if (newest?.id !== record.id) return false; if (undosOf(runs, record.id).length > 0) return false; diff --git a/packages/cli-core/src/commands/migrate/sources/list.test.ts b/packages/cli-core/src/commands/migrate/sources/list.test.ts new file mode 100644 index 000000000..a28a89cd1 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/list.test.ts @@ -0,0 +1,172 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError, EXIT_CODE } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { list, wrapText } from "./list.ts"; +import { ACCOUNT_LINKING_URL, sources } from "./registry.ts"; + +const captured = useCaptureLog(); + +/** `log.info` highlights backticked spans and the levels are coloured. */ +const plain = () => Bun.stripANSI(captured.err).replace(/\s+/g, " "); + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-sources-"))); + process.chdir(workDir); + fs.writeFileSync( + path.join(workDir, "custom.ts"), + `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", mail: "email" }, + carries: { + passwords: { level: "partial", note: "Only for accounts made after 2020." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "yes", note: "All of it." }, + }, + };`, + ); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("sources", () => { + test.each([...sources])("lists $key with what it carries", async (entry) => { + await list(undefined); + const row = plain().match(new RegExp(`${entry.key} (\\w+) (\\w+) (\\w+)`)); + expect(row?.slice(1)).toEqual([ + entry.carries.passwords.level, + entry.carries.mfa.level, + entry.carries.metadata.level, + ]); + }); + + // No social column: every source shares one note. + test("prints the account-linking note once", async () => { + await list(undefined); + expect(captured.err.split(ACCOUNT_LINKING_URL)).toHaveLength(2); + }); + + // A compiled binary has no source tree to grep, so the way to extend it has + // to be discoverable from the list itself. + test("says how to add one", async () => { + await list(undefined); + expect(captured.err).toContain("--source ./my-source.ts"); + }); + + test("--json lists every built-in with its carries on stdout", async () => { + await list(undefined, { json: true }); + + const parsed = JSON.parse(captured.out) as { + sources: { key: string; carries: unknown }[]; + account_linking: string; + }; + expect(parsed.sources.map((entry) => entry.key)).toEqual(sources.map((entry) => entry.key)); + expect(parsed.sources[0]?.carries).toEqual(sources[0]?.carries); + expect(parsed.account_linking).toContain(ACCOUNT_LINKING_URL); + }); +}); + +describe("sources ", () => { + test("shows where each field lands, how to export, and what comes across", async () => { + await list("supabase"); + + expect(plain()).toContain("Export with `clerk migrate export supabase`"); + expect(plain()).toContain("encrypted_password → password"); + expect(plain()).toContain("Passwords yes"); + expect(captured.err).toContain(ACCOUNT_LINKING_URL); + }); + + // Supabase detects its hasher per user, so a source with a fixed one shows it. + test("shows a source's fixed field values", async () => { + await list("firebase"); + expect(plain()).toContain('passwordHasher is always "scrypt_firebase"'); + }); + + test("shows a source you wrote, by its path", async () => { + await list("./custom.ts"); + + expect(plain()).toContain("myplatform My Platform (custom — ./custom.ts)"); + expect(plain()).toContain("mail → email"); + expect(plain()).toContain("Only for accounts made after 2020."); + }); + + test("--json returns the detail on stdout", async () => { + await list("auth0", { json: true }); + + expect(JSON.parse(captured.out)).toMatchObject({ + key: "auth0", + export_command: "clerk migrate export auth0", + fields: { user_id: "userId" }, + carries: { passwords: { level: "partial" } }, + }); + }); + + test("an unknown source is a usage error naming the valid ones", async () => { + const error = (await list("okta").catch((caught: unknown) => caught)) as CliError; + expect(error).toBeInstanceOf(CliError); + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("Valid sources: clerk, auth0"); + }); +}); + +describe("human-mode frame", () => { + let originalMode: string | undefined; + + beforeAll(() => { + originalMode = process.env.CLERK_MODE; + process.env.CLERK_MODE = "human"; + }); + + afterAll(() => { + if (originalMode === undefined) delete process.env.CLERK_MODE; + else process.env.CLERK_MODE = originalMode; + }); + + // Reading a static registry is not a run: there is no progress to bracket, + // and the gutter's `│` would sit in front of every wrapped line. + test("prints no intro/outro gutter", async () => { + await list(undefined); + + expect(captured.err).not.toContain("┌"); + expect(captured.err).not.toContain("└"); + }); + + test("--json stays on stdout only", async () => { + await list(undefined, { json: true }); + + expect(() => JSON.parse(captured.out)).not.toThrow(); + expect(captured.err).toBe(""); + }); +}); + +describe("wrapText", () => { + test("breaks on whitespace within the width", () => { + expect(wrapText("one two three four", 9)).toEqual(["one two", "three", "four"]); + }); + + // `log.info` pairs backticks per line, so a span split across two lines + // leaves one unmatched backtick on each and colours the wrong half of both. + test("never breaks inside a backticked span", () => { + const lines = wrapText("Assumes an export of `SELECT id, name FROM users`.", 30); + + expect(lines).toContain("`SELECT id, name FROM users`."); + for (const line of lines) { + expect((line.match(/`/g) ?? []).length % 2).toBe(0); + } + }); + + test("gives an over-long word its own line rather than dropping it", () => { + expect(wrapText("short supercalifragilistic", 8)).toEqual(["short", "supercalifragilistic"]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/sources/list.ts b/packages/cli-core/src/commands/migrate/sources/list.ts new file mode 100644 index 000000000..6955e37db --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/list.ts @@ -0,0 +1,202 @@ +/** + * `clerk migrate sources [source]` — which platforms an import can read, and + * what each one brings across. + * + * A compiled binary's users have no source tree to read the mappings in, so + * this is where they are shown. `sources` alone lists every source with its + * passwords, MFA and metadata at a glance. `sources ` shows one in + * full: what each field becomes, how to export from it, and its caveats. + */ + +import { bold, cyan, dim, green, red, yellow } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { exportPlatforms } from "../export/registry.ts"; +import type { Carry, CarryLevel, SourceEntry } from "../types.ts"; +import { ACCOUNT_LINKING_NOTE, resolveSource, sources as builtIns } from "./registry.ts"; + +export type SourcesOptions = { + json?: boolean; +}; + +const KINDS = [ + ["passwords", "Passwords"], + ["mfa", "MFA"], + ["metadata", "Metadata"], +] as const; + +/** + * Capped, not just measured: a description that rewrapped differently on every + * terminal makes two runs of the same command look like different output. 80 is + * the same width `--help` lays itself out at. + */ +const MAX_WIDTH = 80; + +function outputWidth(): number { + return Math.min(process.stderr.columns || MAX_WIDTH, MAX_WIDTH); +} + +/** + * A run of non-space characters, except that a backticked span counts as one + * character run even when it contains spaces. Keeps `SELECT a, b FROM users` + * whole: `log.info` pairs backticks per line, so a span broken across two lines + * leaves an unmatched backtick on each and colours the wrong half of both. + */ +const WORD = /(?:`[^`]*`|\S)+/g; + +/** + * Wraps on whitespace. Safe to measure raw because the backtick spans + * `log.info` highlights keep their backticks — the colour it adds is invisible + * to width, and nothing here is coloured before wrapping. + */ +export function wrapText(text: string, width: number): string[] { + const lines: string[] = []; + let line = ""; + + for (const word of text.match(WORD) ?? []) { + if (!line) line = word; + else if (line.length + 1 + word.length <= width) line += ` ${word}`; + else { + lines.push(line); + line = word; + } + } + if (line) lines.push(line); + + return lines; +} + +function levelMark(level: CarryLevel): string { + if (level === "yes") return green("yes"); + if (level === "partial") return yellow("partial"); + return red("no"); +} + +/** The command that writes a file this source reads, when there is one. */ +function exportCommand(entry: SourceEntry): string | undefined { + const platform = exportPlatforms.find((candidate) => candidate.sourceKey === entry.key); + return platform ? `clerk migrate export ${platform.key}` : undefined; +} + +function toJson(entry: SourceEntry, custom?: string) { + return { + key: entry.key, + label: entry.label, + description: entry.description, + ...(custom ? { path: custom } : {}), + carries: entry.carries, + export_command: exportCommand(entry) ?? null, + fields: entry.transformer, + caveats: entry.caveats ?? [], + }; +} + +function printList(width: number): void { + for (const line of wrapText( + "A source maps one platform's export onto the fields Clerk imports. A file from " + + "`clerk migrate export` names its own; anything else takes `--source `.", + width, + )) { + log.info(line); + } + log.blank(); + + const keyWidth = Math.max(...builtIns.map((entry) => entry.key.length)) + 2; + log.info( + bold( + `${"SOURCE".padEnd(keyWidth)}${"PASSWORDS".padEnd(11)}${"MFA".padEnd(9)}${"METADATA".padEnd(10)}`, + ), + ); + for (const entry of builtIns) { + const cell = (carry: Carry, pad: number) => + levelMark(carry.level) + " ".repeat(Math.max(1, pad - carry.level.length)); + log.info( + `${cyan(entry.key.padEnd(keyWidth))}${cell(entry.carries.passwords, 11)}` + + `${cell(entry.carries.mfa, 9)}${levelMark(entry.carries.metadata.level)}`, + ); + } + + log.blank(); + for (const line of wrapText(ACCOUNT_LINKING_NOTE, width)) log.info(dim(line)); + log.blank(); + log.info("Run `clerk migrate sources ` for what each field becomes."); + log.info("Migrating from something else? Write a source and pass --source ./my-source.ts."); +} + +function printDetail(entry: SourceEntry, width: number, custom?: string): void { + log.info(`${cyan(bold(entry.key))} ${entry.label}${custom ? dim(` (custom — ${custom})`) : ""}`); + for (const line of wrapText(entry.description, width)) log.info(line); + + const command = exportCommand(entry); + if (command) { + log.blank(); + log.info(`${bold("Export with")} \`${command}\``); + } + + log.blank(); + log.info(bold("What comes across")); + for (const [kind, label] of KINDS) { + const carry = entry.carries[kind]; + log.info(` ${label.padEnd(10)}${levelMark(carry.level)}`); + for (const line of wrapText(carry.note, width - 4)) log.info(` ${dim(line)}`); + } + + const fields = Object.entries(entry.transformer); + const fromWidth = Math.max(...fields.map(([from]) => from.length)) + 2; + log.blank(); + log.info(bold("Where each field lands")); + for (const [from, to] of fields) log.info(` ${from.padEnd(fromWidth)}→ ${to}`); + if (entry.defaults) { + for (const [field, value] of Object.entries(entry.defaults)) { + log.info(dim(` ${field} is always ${JSON.stringify(value)}`)); + } + } + + if (entry.caveats?.length) { + log.blank(); + log.info(bold("Caveats")); + for (const caveat of entry.caveats) { + const [first, ...rest] = wrapText(caveat, width - 4); + log.info(` - ${first ?? ""}`); + for (const line of rest) log.info(` ${line}`); + } + } + + log.blank(); + for (const line of wrapText(ACCOUNT_LINKING_NOTE, width)) log.info(dim(line)); +} + +export async function list(key: string | undefined, options: SourcesOptions = {}): Promise { + const width = outputWidth(); + + if (key === undefined) { + if (options.json) { + log.data( + JSON.stringify( + { + sources: builtIns.map((entry) => toJson(entry)), + account_linking: ACCOUNT_LINKING_NOTE, + }, + null, + 2, + ), + ); + return; + } + printList(width); + return; + } + + // No gutter: this reads a static registry, it does not run anything. + const resolved = await resolveSource(key); + if (options.json) { + log.data( + JSON.stringify( + { ...toJson(resolved.entry, resolved.path), account_linking: ACCOUNT_LINKING_NOTE }, + null, + 2, + ), + ); + return; + } + printDetail(resolved.entry, width, resolved.path ? key : undefined); +} diff --git a/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts new file mode 100644 index 000000000..513457375 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts @@ -0,0 +1,250 @@ +import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { loadCustomSource, validateSource } from "./load-custom.ts"; +import { __resetCustomSourcesForTesting } from "./registry.ts"; + +let workDir: string; +let originalCwd: string; +let counter = 0; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-custom-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +afterEach(() => { + __resetCustomSourcesForTesting(); +}); + +/** + * Writes a source file with a unique name. + * + * Names must not repeat: a dynamic `import()` caches by URL, so reusing one + * would silently return the previous test's module. + */ +function writeSource(source: string, ext = "ts"): string { + const name = `custom-${counter++}.${ext}`; + fs.writeFileSync(path.join(workDir, name), source); + return `./${name}`; +} + +const VALID = `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", contact_email: "email" }, + carries: { + passwords: { level: "no", note: "None." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, +};`; + +describe("loadCustomSource", () => { + test("loads a user-authored TypeScript source", async () => { + const entry = await loadCustomSource(writeSource(VALID)); + + expect(entry).toMatchObject({ + key: "myplatform", + label: "My Platform", + transformer: { account_ref: "userId", contact_email: "email" }, + }); + }); + + test("loads plain JavaScript too", async () => { + const entry = await loadCustomSource(writeSource(VALID, "js")); + expect(entry.key).toBe("myplatform"); + }); + + // The file is the user's own code; the CLI must transpile whatever they wrote. + test("transpiles TypeScript syntax the runtime has to strip", async () => { + const entry = await loadCustomSource( + writeSource(` + interface Entry { key: string; label: string; transformer: Record; carries: unknown } + const mapping = { my_id: "userId" } as const; + const custom: Entry = { key: "tsplatform", label: "TS", transformer: { ...mapping }, carries: { passwords: { level: "no", note: "-" }, mfa: { level: "no", note: "-" }, metadata: { level: "no", note: "-" } } }; + export default custom satisfies Entry; + `), + ); + expect(entry.key).toBe("tsplatform"); + }); + + test("carries the optional hooks through", async () => { + const entry = await loadCustomSource( + writeSource(`export default { + key: "hooked", label: "Hooked", + transformer: { id: "userId" }, + carries: { passwords: { level: "no", note: "-" }, mfa: { level: "no", note: "-" }, metadata: { level: "no", note: "-" } }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { user.firstName = "set"; }, + };`), + ); + + expect(entry.defaults).toEqual({ passwordHasher: "bcrypt" }); + const user: Record = {}; + entry.postTransform?.(user, {}); + expect(user.firstName).toBe("set"); + }); + + test("supplies a description when the author omitted one", async () => { + const entry = await loadCustomSource( + writeSource( + `export default { key: "bare", label: "Bare", transformer: { id: "userId" }, carries: { passwords: { level: "no", note: "-" }, mfa: { level: "no", note: "-" }, metadata: { level: "no", note: "-" } } };`, + ), + ); + expect(entry.description).toBe("Custom source"); + }); + + test("reports a path that is not there", async () => { + await expect(loadCustomSource("./nope.ts")).rejects.toThrow(/No source file at/); + }); + + test("reports a directory given instead of a file", async () => { + fs.mkdirSync(path.join(workDir, "adir"), { recursive: true }); + await expect(loadCustomSource("./adir")).rejects.toThrow(/is a directory/); + }); + + test("reports a file that does not parse, quoting the syntax error", async () => { + await expect(loadCustomSource(writeSource("export default { key: ,,, }"))).rejects.toThrow( + /Could not load/, + ); + }); + + test("reports a file that throws while loading", async () => { + await expect( + loadCustomSource(writeSource(`throw new Error("boom"); export default {};`)), + ).rejects.toThrow(/Could not load .*boom/s); + }); + + test("points at a named export when the default is missing", async () => { + const file = writeSource( + `export const myPlatform = { key: "x", label: "X", transformer: { a: "userId" } };`, + ); + + await expect(loadCustomSource(file)).rejects.toThrow( + /has no default export.*`myPlatform`.*did you mean `export default`/s, + ); + }); + + test("reports a missing default with no named exports to suggest", async () => { + await expect(loadCustomSource(writeSource("const unused = 1;"))).rejects.toThrow( + /has no default export\.$/m, + ); + }); +}); + +describe("validateSource", () => { + const valid = { + key: "myplatform", + label: "My Platform", + transformer: { account_ref: "userId" }, + carries: { + passwords: { level: "no", note: "None." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, + }; + + test("accepts a minimal valid entry", () => { + expect(validateSource(valid, "f.ts").key).toBe("myplatform"); + }); + + test.each([ + ["a null default export", null, /is null, not an object/], + ["a number default export", 42, /is number, not an object/], + ["a string default export", "nope", /is string, not an object/], + ])("rejects %s", (_label, value, expected) => { + expect(() => validateSource(value, "f.ts")).toThrow(expected); + }); + + test.each([ + ["key", { ...valid, key: undefined }], + ["key", { ...valid, key: "" }], + ["key", { ...valid, key: " " }], + ["key", { ...valid, key: 7 }], + ["label", { ...valid, label: undefined }], + ["label", { ...valid, label: "" }], + ])("rejects a bad %s naming the field", (field, value) => { + expect(() => validateSource(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); + }); + + test("rejects a non-string description", () => { + expect(() => validateSource({ ...valid, description: 7 }, "f.ts")).toThrow( + /`description` must be a string/, + ); + }); + + test.each([ + ["missing", { ...valid, transformer: undefined }], + ["null", { ...valid, transformer: null }], + ["an array", { ...valid, transformer: [] }], + ["a string", { ...valid, transformer: "id" }], + ])("rejects a transformer mapping that is %s", (_label, value) => { + expect(() => validateSource(value, "f.ts")).toThrow(/`transformer`|`transformer\./); + }); + + test("names the offending entry when a mapping target is not a field name", () => { + expect(() => + validateSource({ ...valid, transformer: { account_ref: "userId", bad: 7 } }, "f.ts"), + ).toThrow(/`transformer.bad` must map to a Clerk field name, got number/); + }); + + // Without it the import runs to completion and creates every user with no + // external_id — which is what makes a migration reversible. + test("rejects a mapping with no userId target", () => { + expect(() => validateSource({ ...valid, transformer: { a: "email" } }, "f.ts")).toThrow( + /no source field maps to `userId`/, + ); + }); + + test.each([ + ["defaults", { ...valid, defaults: "nope" }], + ["preTransform", { ...valid, preTransform: "nope" }], + ["postTransform", { ...valid, postTransform: 7 }], + ])("rejects a %s of the wrong type", (field, value) => { + expect(() => validateSource(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); + }); + + test.each([["clerk"], ["auth0"], ["supabase"]])( + "rejects %s, which would shadow a built-in", + (key) => { + expect(() => + validateSource({ ...valid, key }, "f.ts", ["clerk", "auth0", "supabase"]), + ).toThrow(/already a built-in source/); + }, + ); + + // What a source brings across is the first thing `sources` shows, and only + // its author knows it. + test("requires carries", () => { + expect(() => validateSource({ ...valid, carries: undefined }, "f.ts")).toThrow( + /`carries` must say what the source brings across/, + ); + }); + + test.each([ + ["a level that is not yes, no or partial", { level: "maybe", note: "x" }], + ["no note", { level: "yes" }], + ])("rejects a carries entry with %s", (_label, mfa) => { + expect(() => validateSource({ ...valid, carries: { ...valid.carries, mfa } }, "f.ts")).toThrow( + /`carries.mfa` must be/, + ); + }); + + test("names the file in every message, so the author knows which one", () => { + expect(() => validateSource({}, "./their-file.ts")).toThrow(/\.\/their-file\.ts/); + }); + + test("raises CliError, so the global handler formats it", () => { + expect(() => validateSource({}, "f.ts")).toThrow(CliError); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/sources/load-custom.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.ts new file mode 100644 index 000000000..03aacebbb --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.ts @@ -0,0 +1,184 @@ +/** + * Loading a user-authored source at runtime. + * + * In the standalone migration-tool, supporting a new platform meant adding a + * file to `src/transformers/` and one line to the registry — the user had the + * source tree. A compiled binary has neither a source tree to edit nor a way + * for an end user to rebuild it, so `--source ` restores that + * extensibility by importing a file from the user's own project instead. + * + * **Verified before this was built on:** a `bun build --compile` executable can + * `import()` an arbitrary external `.ts` file at runtime, including TypeScript + * that needs transpiling. Bun's transpiler is part of the runtime, not only the + * bundler. Confirmed with a throwaway compiled binary on darwin-arm64, + * linux-arm64 (glibc), linux-arm64-musl and linux-x64. + * + * The file is user-supplied code the CLI executes, so its shape is validated + * up front and rejected with a specific message rather than crashing deep in + * the transform pipeline on a missing field. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; +import type { CarryLevel, SourceEntry } from "../types.ts"; + +const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/overview"; + +const LEVELS: readonly CarryLevel[] = ["yes", "no", "partial"]; + +function invalid(problem: string, file: string): never { + throwUsageError(`${file} is not a valid source: ${problem}`, DOCS_URL); +} + +/** + * Checks a loaded value against the registry entry shape. + * + * Every failure names the specific field and what was wrong with it — the + * author is writing this file by hand against a shape they cannot see. + * + * @param file - Path as the user typed it, for the error message. + * @param reservedKeys - The built-in keys, which a custom source may not reuse. + */ +export function validateSource( + value: unknown, + file: string, + reservedKeys: readonly string[] = [], +): SourceEntry { + if (value === null || typeof value !== "object") { + invalid(`the default export is ${value === null ? "null" : typeof value}, not an object`, file); + } + + const entry = value as Record; + + for (const field of ["key", "label"] as const) { + if (typeof entry[field] !== "string" || entry[field].trim().length === 0) { + invalid(`\`${field}\` must be a non-empty string`, file); + } + } + + if (entry.description !== undefined && typeof entry.description !== "string") { + invalid("`description` must be a string when present", file); + } + + // Arrays are objects, and an author who wrote `transformer: []` should hear + // that rather than the downstream "no field maps to userId". + if ( + entry.transformer === null || + typeof entry.transformer !== "object" || + Array.isArray(entry.transformer) + ) { + invalid("`transformer` must be an object mapping source fields to Clerk fields", file); + } + + const mapping = entry.transformer as Record; + for (const [source, target] of Object.entries(mapping)) { + if (typeof target !== "string" || target.length === 0) { + invalid( + `\`transformer.${source}\` must map to a Clerk field name, got ${typeof target}`, + file, + ); + } + } + + // Without this the import runs to completion and creates every user with no + // external_id, which is what makes a migration re-runnable and reversible. + if (!Object.values(mapping).includes("userId")) { + invalid( + "no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id", + file, + ); + } + + if ( + entry.defaults !== undefined && + (entry.defaults === null || typeof entry.defaults !== "object" || Array.isArray(entry.defaults)) + ) { + invalid("`defaults` must be an object when present", file); + } + + // What a source brings across is the first thing `sources` shows, and the + // author is the only one who knows it. + const carries = entry.carries as Record | undefined; + if (!carries || typeof carries !== "object" || Array.isArray(carries)) { + invalid( + "`carries` must say what the source brings across: { passwords, mfa, metadata }, each { level, note }", + file, + ); + } + for (const kind of ["passwords", "mfa", "metadata"] as const) { + const carry = carries[kind] as { level?: unknown; note?: unknown } | undefined; + if (!carry || !LEVELS.includes(carry.level as CarryLevel) || typeof carry.note !== "string") { + invalid( + `\`carries.${kind}\` must be { level: "yes" | "no" | "partial", note: string }`, + file, + ); + } + } + + for (const hook of ["preTransform", "postTransform"] as const) { + if (entry[hook] !== undefined && typeof entry[hook] !== "function") { + invalid(`\`${hook}\` must be a function when present`, file); + } + } + + if (reservedKeys.includes(entry.key as string)) { + invalid( + `\`key\` is "${String(entry.key)}", which is already a built-in source. Choose another key`, + file, + ); + } + + return { + ...(entry as unknown as SourceEntry), + description: (entry.description as string | undefined) ?? "Custom source", + }; +} + +/** + * Imports and validates a user-authored source. + * + * @throws CliError when the path is missing, the module fails to load, or the + * exported value does not match the registry entry shape. + */ +export async function loadCustomSource( + file: string, + reservedKeys: readonly string[] = [], +): Promise { + const resolved = path.resolve(process.cwd(), file); + + if (!fs.existsSync(resolved)) { + throw new CliError(`No source file at ${resolved}.`, { + code: ERROR_CODE.FILE_NOT_FOUND, + docsUrl: DOCS_URL, + }); + } + if (fs.statSync(resolved).isDirectory()) { + throwUsageError(`${resolved} is a directory, not a source file.`); + } + + let module: Record; + try { + // A file URL rather than a bare path: an absolute POSIX path happens to + // work, but a Windows path (`C:\...`) is not a valid import specifier. + module = (await import(Bun.pathToFileURL(resolved).href)) as Record; + } catch (error) { + throwUsageError( + `Could not load ${file}: ${(error as Error).message}\n` + + "The file must be valid JavaScript or TypeScript that this CLI can import.", + DOCS_URL, + ); + } + + if (module.default === undefined) { + // Point at what they probably meant rather than just restating the rule. + const named = Object.keys(module).filter((key) => key !== "default"); + const hint = + named.length > 0 + ? ` Found named export${named.length === 1 ? "" : "s"} ${named.map((n) => `\`${n}\``).join(", ")} — did you mean \`export default\`?` + : ""; + throwUsageError(`${file} has no default export.${hint}`, DOCS_URL); + } + + return validateSource(module.default, file, reservedKeys); +} diff --git a/packages/cli-core/src/commands/migrate/sources/registry.ts b/packages/cli-core/src/commands/migrate/sources/registry.ts index 25d376e71..226ff7065 100644 --- a/packages/cli-core/src/commands/migrate/sources/registry.ts +++ b/packages/cli-core/src/commands/migrate/sources/registry.ts @@ -1,12 +1,16 @@ /** * Source registry. * - * `migrate import` resolves `--source` here. + * `migrate import` resolves `--source` here, and `migrate sources` lists what + * each one brings across. * * To add a platform: create `sources/.ts` exporting a `SourceEntry`, * then add it to the array below. */ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; import { throwUsageError } from "../../../lib/errors.ts"; import type { SourceEntry } from "../types.ts"; import auth0Source from "./auth0.ts"; @@ -14,6 +18,7 @@ import authjsSource from "./authjs.ts"; import betterAuthSource from "./betterauth.ts"; import clerkSource from "./clerk.ts"; import firebaseSource from "./firebase.ts"; +import { loadCustomSource } from "./load-custom.ts"; import supabaseSource from "./supabase.ts"; import workosSource from "./workos.ts"; @@ -41,36 +46,83 @@ export const ACCOUNT_LINKING_NOTE = "Social sign-ins are not copied. Enable the same providers in Clerk, and a user who signs in " + `with one is linked to their imported account by verified email. See ${ACCOUNT_LINKING_URL}`; +/** + * A source loaded from a user's `--source ` for this invocation. + * + * Kept beside the built-ins rather than pushed into them, so the shipped list + * is never mutated. One invocation loads at most one, so this holding a + * single entry is the normal case; the array shape just avoids a special case + * in the lookups. + */ +const customSources: SourceEntry[] = []; + +export function registerCustomSource(entry: SourceEntry): void { + customSources.push(entry); +} + +/** Test-only: drops anything a previous test registered. */ +export function __resetCustomSourcesForTesting(): void { + customSources.length = 0; +} + +/** Built-ins plus whatever `--source ` loaded. */ +export function allSources(): SourceEntry[] { + return [...sources, ...customSources]; +} + /** The built-in keys, for tab-completion and error messages. */ export function sourceKeys(): string[] { return sources.map((entry) => entry.key); } /** - * Looks up a source by key. + * Looks up a source by key, custom ones included. * * @throws Error when no source is registered under that key. */ export function getSource(key: string): SourceEntry { - const source = sources.find((entry) => entry.key === key); + const source = allSources().find((entry) => entry.key === key); if (!source) { throw new Error(`Source not found for key: ${key}`); } return source; } -/** A resolved `--source`. */ -export type ResolvedSource = { key: string; entry: SourceEntry }; +/** A `--source` value that names a file rather than a built-in key. */ +export function isSourcePath(value: string): boolean { + return /^(\.\.?\/|\/)/.test(value) || /\.(ts|js|mjs)$/.test(value); +} + +/** A resolved `--source`: its key, and for a custom one, the file's content hash. */ +export type ResolvedSource = { key: string; entry: SourceEntry; path?: string; hash?: string }; /** - * Resolves `--source` to a built-in source. + * Resolves `--source`: a built-in key, or a path to a source you wrote. + * + * A custom source is keyed by a hash of its file as well as its key, so an + * edited source is a different source when a re-run decides whether to + * continue an earlier import. * * @throws UsageError for an unknown key, listing the valid ones. */ export async function resolveSource(value: string): Promise { + if (isSourcePath(value)) { + const entry = await loadCustomSource(value, sourceKeys()); + registerCustomSource(entry); + const resolved = path.resolve(process.cwd(), value); + const hash = createHash("sha256").update(fs.readFileSync(resolved)).digest("hex"); + return { key: entry.key, entry, path: resolved, hash }; + } + const entry = sources.find((candidate) => candidate.key === value); if (!entry) { - throwUsageError(`Unknown source "${value}". Valid sources: ${sourceKeys().join(", ")}.`); + throwUsageError( + `Unknown source "${value}". Valid sources: ${sourceKeys().join(", ")}.\n` + + "For a platform with no built-in, pass the path to a source you wrote, e.g. --source ./my-source.ts.", + undefined, + undefined, + [{ command: "clerk migrate sources", description: "List the built-in sources" }], + ); } return { key: entry.key, entry }; } diff --git a/packages/cli-core/src/commands/migrate/sources/sources.test.ts b/packages/cli-core/src/commands/migrate/sources/sources.test.ts index 221babe5d..b8934283a 100644 --- a/packages/cli-core/src/commands/migrate/sources/sources.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/sources.test.ts @@ -5,7 +5,7 @@ import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; import { loadUsersFromFile, transformUsers } from "../lib/transform.ts"; import type { FirebaseHashConfig } from "../types.ts"; -import { getSource, sourceKeys, sources } from "./registry.ts"; +import { getSource, isSourcePath, sourceKeys, sources } from "./registry.ts"; import { isVerified } from "./shared.ts"; const FIREBASE_HASH: FirebaseHashConfig = { @@ -78,6 +78,19 @@ describe("registry", () => { }); }); +describe("isSourcePath", () => { + test.each([["./mine.ts"], ["../up/mine.js"], ["/abs/mine.mjs"], ["mine.ts"], ["dir/mine.js"]])( + "%s is a path", + (value) => { + expect(isSourcePath(value)).toBe(true); + }, + ); + + test.each([["clerk"], ["betterauth"], ["okta"]])("%s is a key", (value) => { + expect(isSourcePath(value)).toBe(false); + }); +}); + describe("isVerified", () => { // A CSV export stringifies everything, so the boolean style must read // "false" as false. Treating it as truthy would mark unconfirmed addresses diff --git a/packages/cli-core/src/test/integration/completion.test.ts b/packages/cli-core/src/test/integration/completion.test.ts index f0b1fc91c..11bc5e220 100644 --- a/packages/cli-core/src/test/integration/completion.test.ts +++ b/packages/cli-core/src/test/integration/completion.test.ts @@ -297,6 +297,23 @@ describe("generateCompletions", () => { expect(names).toContain("settings"); }); + test("migrate sources: suggests the built-in sources", () => { + const saved = process.env.CLERK_EXPERIMENTAL; + process.env.CLERK_EXPERIMENTAL = "migrate"; + try { + const names = generateCompletions(createProgram(), [ + "migrate", + "sources", + "", + ]).completions.map((c) => c.name); + expect(names).toContain("supabase"); + expect(names).toContain("workos"); + } finally { + if (saved === undefined) delete process.env.CLERK_EXPERIMENTAL; + else process.env.CLERK_EXPERIMENTAL = saved; + } + }); + test("open dashboard: filters subpaths by prefix", () => { const names = completionNames("open", "dashboard", "u"); expect(names).toContain("users"); From 0886155c22c79949a3432eb04abe1837d9a1b963 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 13:11:06 -0400 Subject: [PATCH 02/20] docs(changeset): Add clerk migrate sources and custom sources Co-Authored-By: Claude Opus 5.5 --- .changeset/migrate-custom-sources.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/migrate-custom-sources.md diff --git a/.changeset/migrate-custom-sources.md b/.changeset/migrate-custom-sources.md new file mode 100644 index 000000000..fb9542f26 --- /dev/null +++ b/.changeset/migrate-custom-sources.md @@ -0,0 +1,5 @@ +--- +"clerk": minor +--- + +Add `clerk migrate sources`, and let `clerk migrate import --source` take the path to a source you wrote. `clerk migrate sources` lists the built-in sources with the passwords, MFA and metadata each brings across, and `clerk migrate sources ` shows where each field lands, how to export, and any caveats. A source you wrote is a `.ts` or `.js` file in your project, checked before use, so a platform with no built-in source can be imported without rebuilding the CLI. `clerk migrate` is still experimental and needs `CLERK_EXPERIMENTAL=migrate`. From 38e9acf74b902b772fe53767d1d711be8b4aa0cd Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 18:51:22 -0400 Subject: [PATCH 03/20] docs(migrate): list both hook validation errors for a custom source The validation table quoted only the postTransform message, without the backticks the real one has, though preTransform is checked the same way. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 3dcdb1c13..c90507ec2 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -1175,15 +1175,15 @@ edited source counts as a different source. The file is code the CLI executes, so its shape is checked before use and rejected with the specific problem rather than crashing mid-pipeline: -| Problem | Message | -| ---------------------------------- | ---------------------------------------------------------------------------------------------------------- | -| Path does not exist | `No source file at /abs/path.ts.` | -| No default export, but a named one | ``has no default export. Found named export `myPlatform` — did you mean `export default`?`` | -| Does not parse | `Could not load ./f.ts: Expected identifier but found ","` | -| Nothing maps to `userId` | ``no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id`` | -| No `carries` | `` `carries` must say what the source brings across: { passwords, mfa, metadata }, each { level, note } `` | -| `key` clashes with a built-in | `key is "clerk", which is already a built-in source` | -| A hook is not a function | `postTransform must be a function when present` | +| Problem | Message | +| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| Path does not exist | `No source file at /abs/path.ts.` | +| No default export, but a named one | ``has no default export. Found named export `myPlatform` — did you mean `export default`?`` | +| Does not parse | `Could not load ./f.ts: Expected identifier but found ","` | +| Nothing maps to `userId` | ``no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id`` | +| No `carries` | `` `carries` must say what the source brings across: { passwords, mfa, metadata }, each { level, note } `` | +| `key` clashes with a built-in | `key is "clerk", which is already a built-in source` | +| `preTransform` or `postTransform` is not a function | `` `preTransform` must be a function when present `` (likewise `postTransform`) | The `userId` check is the load-bearing one: without it the import would run to completion and create every user with no `external_id`, which is what makes a From fec651ce436b97e979cbdd0c2617ec53986dc7a8 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 13:30:43 -0400 Subject: [PATCH 04/20] fix(migrate): refuse a second import while one of the same file is live The live-run check sat inside resume matching, after the `sourceHash` filter, so editing a custom source (or passing --new-run) skipped it: a second run could start beside the live one, and both would send the same external IDs. The check now runs first, for any import of the same file, source key and instance, whatever its source hash or --new-run. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/run.test.ts | 22 +++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 33 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 94b43bd11..0879b80c5 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -1948,6 +1948,8 @@ describe("run", () => { fs.writeFileSync(path.join(runsDir(), first!.id, "lock"), "1"); expect(await exitCodeOf(run(baseOptions))).toBe(EXIT_CODE.USAGE); + // A new run would send the same external IDs alongside it. + expect(await exitCodeOf(run({ ...baseOptions, newRun: true }))).toBe(EXIT_CODE.USAGE); }); // An edited file is a different job. @@ -2090,6 +2092,26 @@ describe("run", () => { expect("first_name" in (bodies[1] ?? {})).toBe(false); }); + // An edited source misses the live run's `sourceHash`, but both would send + // the same external IDs. + test("refuses while a run of this file with an earlier version of the source is live", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const [first] = listRuns(runsDir()); + const record = readRun(runsDir(), first!.id)!; + delete record.finishedAt; + fs.writeFileSync(path.join(runsDir(), first!.id, "run.json"), JSON.stringify(record)); + // PID 1 is always alive, and never this test. + fs.writeFileSync(path.join(runsDir(), first!.id, "lock"), "1"); + + const edited = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, edited), `${CUSTOM}\n// edited`); + requests = []; + await expect( + run({ input: "export.json", source: edited, yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/importing this file right now in another process/); + expect(created()).toHaveLength(0); + }); + // An edited source is a different source, so the run records which one. test("records the custom source's content hash on the run", async () => { await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index df31275fe..f95fce5ef 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -425,6 +425,33 @@ function applyEnvelope( // --- Continuing an earlier run --------------------------------------------- +/** + * Refuses while another process is importing this file, with this source key, + * into this instance. Checked before any resume matching, and under + * --new-run too: an edited custom source (another `sourceHash`) or a new run + * would otherwise send the same external IDs alongside it. + */ +function assertNoActiveImport( + runsDir: string, + match: { sha256: string; source: string; instanceId: string; keyInstanceId?: string }, +): void { + const active = listRuns(runsDir).find( + (record) => + record.kind === "import" && + record.file?.sha256 === match.sha256 && + record.source === match.source && + (record.target.instanceId === match.instanceId || + record.target.instanceId === match.keyInstanceId) && + runState(runsDir, record) === "running", + ); + if (active) { + throwUsageError( + `Run ${active.id} is importing this file right now in another process (PID ${liveLockPid(runsDir, active.id)}). ` + + `Wait for it to finish. If that process is not a migrate run, delete ${lockFile(runsDir, active.id)}.`, + ); + } +} + /** How this import relates to earlier runs of the same file, source and instance. */ type ResumeCase = | { kind: "new" } @@ -873,6 +900,12 @@ export async function run(rawOptions: MigrateRunOptions): Promise { if (expectedSha256 !== undefined && sha256 !== expectedSha256) throwChanged(filePath); const runsDir = await resolveRunsDir(options.runsDir); + assertNoActiveImport(runsDir, { + sha256, + source, + instanceId: target.instanceId, + keyInstanceId: keyInstanceId(secretKey), + }); const match = { sha256, source, From 27a7314593be07f9b133e14b328701e44c687234 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 14:25:51 -0400 Subject: [PATCH 05/20] fix(migrate): say --source takes a key or a path in the sources listing The listing said a file takes `--source `, but a custom source is loaded by its path, and its key reads as an unknown built-in. It now says `--source `, as the flag's own help does. Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/sources/list.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/sources/list.ts b/packages/cli-core/src/commands/migrate/sources/list.ts index 6955e37db..7991bb05a 100644 --- a/packages/cli-core/src/commands/migrate/sources/list.ts +++ b/packages/cli-core/src/commands/migrate/sources/list.ts @@ -93,7 +93,7 @@ function toJson(entry: SourceEntry, custom?: string) { function printList(width: number): void { for (const line of wrapText( "A source maps one platform's export onto the fields Clerk imports. A file from " + - "`clerk migrate export` names its own; anything else takes `--source `.", + "`clerk migrate export` names its own; anything else takes `--source `.", width, )) { log.info(line); From ef18ff09f52e57e5b3d6ae3cee20e8ada677f224 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 15:03:35 -0400 Subject: [PATCH 06/20] fix(migrate): hold an exclusive lock for an import of a file, from before its checks The check for a live import of the same file read the runs folder before the checks and prompts, but a run is locked only once it is written: two processes could both pass the check and start runs side by side. An import now takes an exclusive lock keyed by the runs folder, file hash, source key and instance before its checks and holds it to the end. It lives in the temp directory, since nothing may be written to the runs folder before consent; a dry run takes none. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/lib/run-store.ts | 74 ++++++++++++++++--- .../cli-core/src/commands/migrate/run.test.ts | 32 ++++++++ packages/cli-core/src/commands/migrate/run.ts | 22 ++++++ 3 files changed, 118 insertions(+), 10 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index 1ca85c585..a301d9249 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -20,6 +20,7 @@ import { createHash, randomBytes } from "node:crypto"; import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import { resolveProfile } from "../../../lib/config.ts"; import { throwUsageError } from "../../../lib/errors.ts"; @@ -213,9 +214,14 @@ function isPidAlive(pid: number): boolean { * the same PID every run, so a killed run's lock would otherwise read as live. */ export function liveLockPid(runsDir: string, id: string): number | undefined { + return livePidIn(lockFile(runsDir, id)); +} + +/** The live PID a lock file holds, by the same rules as {@link liveLockPid}. */ +function livePidIn(file: string): number | undefined { let raw: string; try { - raw = fs.readFileSync(lockFile(runsDir, id), "utf-8"); + raw = fs.readFileSync(file, "utf-8"); } catch { return undefined; } @@ -238,31 +244,38 @@ const isMissing = (error: unknown) => (error as NodeJS.ErrnoException).code === * both read "free" and both write; a stale lock is removed and taken once. */ function acquireLock(runsDir: string, id: string): void { - const file = lockFile(runsDir, id); - const refuse = (holder: number | undefined): never => + acquireLockFile(lockFile(runsDir, id), (holder, file) => throwUsageError( `Run ${id} is in use by another process${holder ? ` (PID ${holder})` : ""}. Wait for it to finish, then try again. ` + `If that process is not a migrate run, delete ${file}.`, - ); + ), + ); +} + +/** Takes the lock at `file`, or calls `refuse` with its live holder. */ +function acquireLockFile( + file: string, + refuse: (holder: number | undefined, file: string) => never, +): void { const take = () => fs.writeFileSync(file, String(process.pid), { flag: "wx", mode: 0o600 }); try { take(); - log.debug(`migrate: locked run ${id} (${file})`); + log.debug(`migrate: took the lock ${file}`); return; } catch (error) { if (!isExists(error)) throw error; } - const holder = liveLockPid(runsDir, id); - if (holder !== undefined) refuse(holder); - log.debug(`migrate: removing a stale lock on run ${id} (${file})`); + const holder = livePidIn(file); + if (holder !== undefined) refuse(holder, file); + log.debug(`migrate: removing a stale lock (${file})`); fs.rmSync(file, { force: true }); try { take(); - log.debug(`migrate: locked run ${id} (${file})`); + log.debug(`migrate: took the lock ${file}`); } catch (error) { // Another process took the stale lock between the remove and the write. - if (isExists(error)) refuse(liveLockPid(runsDir, id)); + if (isExists(error)) refuse(livePidIn(file), file); throw error; } } @@ -272,6 +285,47 @@ function releaseLock(runsDir: string, id: string): void { fs.rmSync(lockFile(runsDir, id), { force: true }); } +/** What makes two imports the same job: the file, the source and the instance. */ +export type ImportIdentity = { sha256: string; source: string; instanceId: string }; + +/** + * The lock one import of a file, by a source, into an instance holds. In the + * temp directory, keyed by the runs folder too: it is taken before consent, + * when nothing may be written to the runs folder yet. + */ +export function importLockFile(runsDir: string, identity: ImportIdentity): string { + const key = createHash("sha256") + .update( + JSON.stringify([ + path.resolve(runsDir), + identity.sha256, + identity.source, + identity.instanceId, + ]), + ) + .digest("hex") + .slice(0, 32); + return path.join(os.tmpdir(), `clerk-migrate-import-${key}.lock`); +} + +/** + * Takes the lock for one import of this file, source and instance, before + * its checks, so no second process can pass them and start a run beside it. + * + * @returns Releases the lock. + * @throws UsageError when a live process holds it. + */ +export function lockImport(runsDir: string, identity: ImportIdentity): () => void { + const file = importLockFile(runsDir, identity); + acquireLockFile(file, (holder) => + throwUsageError( + `Another process${holder ? ` (PID ${holder})` : ""} is importing this file into this instance right now. ` + + `Wait for it to finish. If that process is not a migrate run, delete ${file}.`, + ), + ); + return () => fs.rmSync(file, { force: true }); +} + // --- Reading --------------------------------------------------------------- /** One run's record, or `undefined` when the folder holds no readable run. */ diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 0879b80c5..be63409e4 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -28,6 +28,7 @@ import { readRun, sha256File, startRun, + importLockFile, } from "./lib/run-store.ts"; import { explainErrors, run, validateRunOptions } from "./run.ts"; import { __resetCustomSourcesForTesting } from "./sources/registry.ts"; @@ -1952,6 +1953,37 @@ describe("run", () => { expect(await exitCodeOf(run({ ...baseOptions, newRun: true }))).toBe(EXIT_CODE.USAGE); }); + // The scan of runs alone has a gap: a second process can pass it before + // either has written a run. The import's own lock closes it. + test("refuses while another process holds this import's lock, with no run yet", async () => { + const lock = importLockFile(runsDir(), { + sha256: sha256File(path.join(workDir, "export.json")), + source: "clerk", + instanceId: "ins_1", + }); + // PID 1 is always alive, and never this test. + fs.writeFileSync(lock, "1"); + try { + await expect(run(baseOptions)).rejects.toThrow( + /importing this file into this instance right now/, + ); + } finally { + fs.rmSync(lock, { force: true }); + } + expect(created()).toHaveLength(0); + expect(listRuns(runsDir())).toHaveLength(0); + }); + + test("lets go of the import's lock when the run ends", async () => { + await run(baseOptions); + const lock = importLockFile(runsDir(), { + sha256: sha256File(path.join(workDir, "export.json")), + source: "clerk", + instanceId: "ins_1", + }); + expect(fs.existsSync(lock)).toBe(false); + }); + // An edited file is a different job. // Rate limiting right after a large import is when this lookup fails, and // that is when someone re-runs after an interruption. diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index f95fce5ef..fcb13a306 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -61,6 +61,9 @@ import { continueRun, latestUserLines, listRuns, + liveLockPid, + lockFile, + lockImport, readRun, readUserLines, resolveRunsDir, @@ -810,6 +813,19 @@ function recordRejects(run: Run, checks: ImportChecks): void { } export async function run(rawOptions: MigrateRunOptions): Promise { + // Released however the import ends: a return, a refusal or a Ctrl-C. + const lock: ImportLock = {}; + try { + await runImport(rawOptions, lock); + } finally { + lock.release?.(); + } +} + +/** The import's identity lock, taken part-way through and released by {@link run}. */ +type ImportLock = { release?: () => void }; + +async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promise { // A mistyped --source fails before any sign-in or link. let options = await applySource(rawOptions); await ensureImportTarget(options); @@ -900,6 +916,12 @@ export async function run(rawOptions: MigrateRunOptions): Promise { if (expectedSha256 !== undefined && sha256 !== expectedSha256) throwChanged(filePath); const runsDir = await resolveRunsDir(options.runsDir); + // Held from before the checks to the end of the run: the scan below + // alone leaves a gap a second process can pass through before either + // one writes a run. + if (!options.dryRun) { + lock.release = lockImport(runsDir, { sha256, source, instanceId: target.instanceId }); + } assertNoActiveImport(runsDir, { sha256, source, From e3cb7401c54c7eb96256b12ef6957d81b9c19d6e Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 15:06:41 -0400 Subject: [PATCH 07/20] fix(migrate): load a custom source by the hash a run records for it An import hashed a custom source's file and recorded that, but the module loader caches by specifier, so loading an edited file at the same path again in one process returned the old module, and the registry kept the first entry registered under the key. A run could record the new hash beside the old mappings. The file is hashed first and imported by plain path with that hash as its query, which Bun loads afresh (a `file://` URL keeps its first module whatever the query), and a later load of a key replaces the earlier one. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/run.test.ts | 22 +++++++++++++++++++ .../commands/migrate/sources/load-custom.ts | 16 +++++++++++++- .../src/commands/migrate/sources/registry.ts | 13 ++++++++--- 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index be63409e4..4ea164e24 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -2144,6 +2144,28 @@ describe("run", () => { expect(created()).toHaveLength(0); }); + // The loader caches by URL: an edit at the same path must still load the + // edited mappings, the ones the run's hash names. + test("an edited source at the same path maps with its edited code", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + fs.writeFileSync( + path.join(workDir, customFile), + CUSTOM.replace('given: "firstName"', 'given: "lastName"'), + ); + requests = []; + await run({ + input: "export.json", + source: customFile, + yes: true, + secretKey: "sk_test_x", + newRun: true, + }); + + const bodies = created().map((r) => r.body as Record); + expect(bodies[0]).toMatchObject({ last_name: "Ada" }); + expect("first_name" in (bodies[0] ?? {})).toBe(false); + }); + // An edited source is a different source, so the run records which one. test("records the custom source's content hash on the run", async () => { await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); diff --git a/packages/cli-core/src/commands/migrate/sources/load-custom.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.ts index 03aacebbb..cec25c327 100644 --- a/packages/cli-core/src/commands/migrate/sources/load-custom.ts +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.ts @@ -144,6 +144,12 @@ export function validateSource( export async function loadCustomSource( file: string, reservedKeys: readonly string[] = [], + /** + * The file's content hash. The module loader caches by URL, so an edited + * file at the same path would load the old module; the hash in the URL + * loads the version that was hashed. + */ + version?: string, ): Promise { const resolved = path.resolve(process.cwd(), file); @@ -161,7 +167,15 @@ export async function loadCustomSource( try { // A file URL rather than a bare path: an absolute POSIX path happens to // work, but a Windows path (`C:\...`) is not a valid import specifier. - module = (await import(Bun.pathToFileURL(resolved).href)) as Record; + // Bun keys its module cache by specifier and keeps a `file://` URL's + // module whatever its query, but a plain path's query loads afresh. + // ponytail: Windows keeps the URL form, so an edit loaded twice in one + // process reuses the first; each CLI run is a process of its own. + const specifier = + version && process.platform !== "win32" + ? `${resolved}?v=${version}` + : Bun.pathToFileURL(resolved).href; + module = (await import(specifier)) as Record; } catch (error) { throwUsageError( `Could not load ${file}: ${(error as Error).message}\n` + diff --git a/packages/cli-core/src/commands/migrate/sources/registry.ts b/packages/cli-core/src/commands/migrate/sources/registry.ts index 226ff7065..257732d9a 100644 --- a/packages/cli-core/src/commands/migrate/sources/registry.ts +++ b/packages/cli-core/src/commands/migrate/sources/registry.ts @@ -57,6 +57,9 @@ export const ACCOUNT_LINKING_NOTE = const customSources: SourceEntry[] = []; export function registerCustomSource(entry: SourceEntry): void { + // The latest load of a key wins: an edited source replaces its old self. + const earlier = customSources.findIndex((source) => source.key === entry.key); + if (earlier >= 0) customSources.splice(earlier, 1); customSources.push(entry); } @@ -107,10 +110,14 @@ export type ResolvedSource = { key: string; entry: SourceEntry; path?: string; h */ export async function resolveSource(value: string): Promise { if (isSourcePath(value)) { - const entry = await loadCustomSource(value, sourceKeys()); - registerCustomSource(entry); const resolved = path.resolve(process.cwd(), value); - const hash = createHash("sha256").update(fs.readFileSync(resolved)).digest("hex"); + // Hashed first, and loaded by that hash: the record names the very code + // that maps the users, even if the file was edited since a last load. + const hash = fs.existsSync(resolved) + ? createHash("sha256").update(fs.readFileSync(resolved)).digest("hex") + : undefined; + const entry = await loadCustomSource(value, sourceKeys(), hash); + registerCustomSource(entry); return { key: entry.key, entry, path: resolved, hash }; } From e75d4abf9d4479870f09dad28e8e8418b4dbcf34 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 15:06:41 -0400 Subject: [PATCH 08/20] docs(migrate): say a custom source's hash covers its own file only A local helper a custom source imports is not hashed with it, so an edit to one does not mark the source as changed. The README now says to pass --new-run after editing a helper. Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index c90507ec2..0901f6213 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -1168,7 +1168,9 @@ TypeScript is fine — Bun's transpiler is part of the runtime, so `interface`, Plain `.js` works too. An import run records a custom source's key and a hash of the file, so an -edited source counts as a different source. +edited source counts as a different source. The hash covers that file only: +a local helper it imports is not part of it, so after editing one, pass +`--new-run` rather than continue a run made with the old helper. #### Validation From 1d652581ee882b4be372636189eec2c80281251d Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 17:19:40 -0400 Subject: [PATCH 09/20] fix(migrate): link a lock into place whole, so no reader sees it without its PID Co-Authored-By: Claude Opus 5.5 --- .../commands/migrate/lib/run-store.test.ts | 16 ++++++++++++++ .../src/commands/migrate/lib/run-store.ts | 21 ++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts index 974b28740..e49675b02 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts @@ -19,6 +19,7 @@ import { latestUserLines, listRuns, lockFile, + lockImport, newRunId, patchRun, readRun, @@ -204,6 +205,21 @@ describe("locks and interruptions", () => { expect(() => continueRun(runsDir, run.record)).not.toThrow(); }); + // A lock created empty and written after reads as stale to a second + // process in between, which would remove it and import beside the first. + test("a lock is linked into place with its PID already written", () => { + const identity = { sha256: "a".repeat(64), source: "supabase", instanceId: "ins_lock" }; + const write = spyOn(fs, "writeFileSync"); + try { + const release = lockImport(runsDir, identity); + const lockPaths = write.mock.calls.map(([target]) => String(target)); + expect(lockPaths.every((target) => target.endsWith(".tmp"))).toBe(true); + release(); + } finally { + write.mockRestore(); + } + }); + test("reads as running while another live process holds the lock", () => { const run = startRun(runsDir, init); fs.writeFileSync(path.join(run.dir, "lock"), "1"); diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index a301d9249..3eddeb5fa 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -257,7 +257,22 @@ function acquireLockFile( file: string, refuse: (holder: number | undefined, file: string) => never, ): void { - const take = () => fs.writeFileSync(file, String(process.pid), { flag: "wx", mode: 0o600 }); + // Written whole, then linked into place: the link fails on an existing lock + // as `wx` does, but the lock never exists without its PID, which a second + // process would read as stale and remove. + const take = () => { + const whole = `${file}.${process.pid}.tmp`; + fs.writeFileSync(whole, String(process.pid), { mode: 0o600 }); + try { + fs.linkSync(whole, file); + } catch (error) { + if (isExists(error)) throw error; + // A filesystem without hard links. + fs.writeFileSync(file, String(process.pid), { flag: "wx", mode: 0o600 }); + } finally { + fs.rmSync(whole, { force: true }); + } + }; try { take(); @@ -266,6 +281,10 @@ function acquireLockFile( } catch (error) { if (!isExists(error)) throw error; } + // ponytail: two processes that both read the same stale lock can both + // remove it, and the second can remove the first's new one. Clerk still + // refuses the second create of any email, phone or username, so the cost + // is failed lines, not duplicate users. A lock on the reclaim would close it. const holder = livePidIn(file); if (holder !== undefined) refuse(holder, file); log.debug(`migrate: removing a stale lock (${file})`); From 787e8cc2702f795ef5c9d6c029fe1664bdbd1b8a Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 17:19:40 -0400 Subject: [PATCH 10/20] docs(migrate): say a custom source can import its own files, but not the CLI Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 0901f6213..de4f61770 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -1136,8 +1136,9 @@ clerk migrate import users.json --source ./my-platform.ts ``` The file lives in **your** project, not in the CLI, and is imported at runtime. -It exports the same shape the built-ins use — plain data, no imports, since -there is nothing in a compiled binary for your file to import from: +It exports the same shape the built-ins use, as plain data. It can't import +from the CLI, since there is nothing in a compiled binary to import from, but +it can import your project's own files: ```ts export default { From 0f5926be31c760ec18e1105f5506214479fe3d7a Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 19:37:00 -0400 Subject: [PATCH 11/20] fix(migrate): key the import lock by the secret key, so a failed instance lookup takes the same one Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/lib/run-store.ts | 5 ++++- packages/cli-core/src/commands/migrate/run.test.ts | 7 +++++-- packages/cli-core/src/commands/migrate/run.ts | 9 ++++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index 3eddeb5fa..084792f2b 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -304,7 +304,10 @@ function releaseLock(runsDir: string, id: string): void { fs.rmSync(lockFile(runsDir, id), { force: true }); } -/** What makes two imports the same job: the file, the source and the instance. */ +/** + * What makes two imports the same job: the file, the source and the instance, + * named by its key's stand-in ID so it does not depend on `GET /v1/instance`. + */ export type ImportIdentity = { sha256: string; source: string; instanceId: string }; /** diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 4ea164e24..b2aac3345 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -30,6 +30,7 @@ import { startRun, importLockFile, } from "./lib/run-store.ts"; +import { keyInstanceId } from "./lib/target.ts"; import { explainErrors, run, validateRunOptions } from "./run.ts"; import { __resetCustomSourcesForTesting } from "./sources/registry.ts"; // After run.ts: imported first, signals.ts loads version.ts ahead of its Bun @@ -1956,10 +1957,11 @@ describe("run", () => { // The scan of runs alone has a gap: a second process can pass it before // either has written a run. The import's own lock closes it. test("refuses while another process holds this import's lock, with no run yet", async () => { + // Keyed by the key, so it holds whether or not `GET /v1/instance` answers. const lock = importLockFile(runsDir(), { sha256: sha256File(path.join(workDir, "export.json")), source: "clerk", - instanceId: "ins_1", + instanceId: keyInstanceId(baseOptions.secretKey), }); // PID 1 is always alive, and never this test. fs.writeFileSync(lock, "1"); @@ -1976,10 +1978,11 @@ describe("run", () => { test("lets go of the import's lock when the run ends", async () => { await run(baseOptions); + // Keyed by the key, so it holds whether or not `GET /v1/instance` answers. const lock = importLockFile(runsDir(), { sha256: sha256File(path.join(workDir, "export.json")), source: "clerk", - instanceId: "ins_1", + instanceId: keyInstanceId(baseOptions.secretKey), }); expect(fs.existsSync(lock)).toBe(false); }); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index fcb13a306..2d6397955 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -919,8 +919,15 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi // Held from before the checks to the end of the run: the scan below // alone leaves a gap a second process can pass through before either // one writes a run. + // Keyed by the secret key's stand-in ID, not `target.instanceId`: that is + // a `key_…` ID too when `GET /v1/instance` fails, so two processes on one + // instance would otherwise take different locks. if (!options.dryRun) { - lock.release = lockImport(runsDir, { sha256, source, instanceId: target.instanceId }); + lock.release = lockImport(runsDir, { + sha256, + source, + instanceId: keyInstanceId(secretKey), + }); } assertNoActiveImport(runsDir, { sha256, From b96984303ff960193f94c137e7ec18e0129f2b54 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 20:45:26 -0400 Subject: [PATCH 12/20] fix(migrate): lock an import by the instance too, so a second key for it takes the same lock Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/lib/run-store.ts | 2 +- .../cli-core/src/commands/migrate/run.test.ts | 34 ++++++++++++++++--- packages/cli-core/src/commands/migrate/run.ts | 25 +++++++++----- 3 files changed, 48 insertions(+), 13 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index 084792f2b..fcc1097a7 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -306,7 +306,7 @@ function releaseLock(runsDir: string, id: string): void { /** * What makes two imports the same job: the file, the source and the instance, - * named by its key's stand-in ID so it does not depend on `GET /v1/instance`. + * by its ID or its key's stand-in ID. */ export type ImportIdentity = { sha256: string; source: string; instanceId: string }; diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index b2aac3345..c1109a1e1 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -1976,15 +1976,41 @@ describe("run", () => { expect(listRuns(runsDir())).toHaveLength(0); }); - test("lets go of the import's lock when the run ends", async () => { - await run(baseOptions); - // Keyed by the key, so it holds whether or not `GET /v1/instance` answers. + // A second key for the same instance has its own stand-in ID. + test("refuses while another key's process holds this instance's lock", async () => { const lock = importLockFile(runsDir(), { + sha256: sha256File(path.join(workDir, "export.json")), + source: "clerk", + instanceId: "ins_1", + }); + fs.writeFileSync(lock, "1"); + try { + await expect(run(baseOptions)).rejects.toThrow( + /importing this file into this instance right now/, + ); + } finally { + fs.rmSync(lock, { force: true }); + } + expect(created()).toHaveLength(0); + // The key's own lock, taken first, is let go when the second is refused. + const own = importLockFile(runsDir(), { sha256: sha256File(path.join(workDir, "export.json")), source: "clerk", instanceId: keyInstanceId(baseOptions.secretKey), }); - expect(fs.existsSync(lock)).toBe(false); + expect(fs.existsSync(own)).toBe(false); + }); + + test("lets go of the import's locks when the run ends", async () => { + await run(baseOptions); + const lockFor = (instanceId: string) => + importLockFile(runsDir(), { + sha256: sha256File(path.join(workDir, "export.json")), + source: "clerk", + instanceId, + }); + expect(fs.existsSync(lockFor(keyInstanceId(baseOptions.secretKey)))).toBe(false); + expect(fs.existsSync(lockFor("ins_1"))).toBe(false); }); // An edited file is a different job. diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 2d6397955..07d5a5648 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -919,15 +919,24 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi // Held from before the checks to the end of the run: the scan below // alone leaves a gap a second process can pass through before either // one writes a run. - // Keyed by the secret key's stand-in ID, not `target.instanceId`: that is - // a `key_…` ID too when `GET /v1/instance` fails, so two processes on one - // instance would otherwise take different locks. + // Two locks: the key's stand-in ID, which the same key takes when + // `GET /v1/instance` fails, and the instance's ID when Clerk named it, + // which a second key for the instance takes. + // ponytail: a second key whose lookup failed takes neither; the run scan + // below is all that stops it. A lock on the key's instance needs Clerk. if (!options.dryRun) { - lock.release = lockImport(runsDir, { - sha256, - source, - instanceId: keyInstanceId(secretKey), - }); + const releases: (() => void)[] = []; + try { + for (const instanceId of new Set([keyInstanceId(secretKey), target.instanceId])) { + releases.push(lockImport(runsDir, { sha256, source, instanceId })); + } + } catch (error) { + for (const release of releases) release(); + throw error; + } + lock.release = () => { + for (const release of releases) release(); + }; } assertNoActiveImport(runsDir, { sha256, From a38bb9ba31146b8009996a8f6e16185b720ff7b1 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 14 Oct 2026 19:27:19 -0400 Subject: [PATCH 13/20] fix(migrate): refuse a custom source whose caveats are not a list of strings `migrate sources` wraps each caveat as text, so `caveats: [42]` crashed it with a raw TypeError and `caveats: "text"` printed one bullet per character. validateSource now checks it next to the hooks. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/sources/load-custom.test.ts | 11 +++++++++++ .../src/commands/migrate/sources/load-custom.ts | 7 +++++++ 2 files changed, 18 insertions(+) diff --git a/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts index 513457375..73d518d82 100644 --- a/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts @@ -214,6 +214,17 @@ describe("validateSource", () => { expect(() => validateSource(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); }); + // `migrate sources` wraps each caveat as text: a number crashed it, and a + // string printed one bullet per character. + test.each([ + ["a non-string caveat", [42]], + ["a string instead of a list", "text"], + ])("rejects caveats holding %s", (_label, caveats) => { + expect(() => validateSource({ ...valid, caveats }, "f.ts")).toThrow( + /`caveats` must be an array of strings/, + ); + }); + test.each([["clerk"], ["auth0"], ["supabase"]])( "rejects %s, which would shadow a built-in", (key) => { diff --git a/packages/cli-core/src/commands/migrate/sources/load-custom.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.ts index cec25c327..9c1774e6c 100644 --- a/packages/cli-core/src/commands/migrate/sources/load-custom.ts +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.ts @@ -122,6 +122,13 @@ export function validateSource( } } + if ( + entry.caveats !== undefined && + (!Array.isArray(entry.caveats) || entry.caveats.some((caveat) => typeof caveat !== "string")) + ) { + invalid("`caveats` must be an array of strings when present", file); + } + if (reservedKeys.includes(entry.key as string)) { invalid( `\`key\` is "${String(entry.key)}", which is already a built-in source. Choose another key`, From 09756f4d70d4dee42b4f91f7cf866643581fa55c Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 14 Oct 2026 19:27:19 -0400 Subject: [PATCH 14/20] fix(migrate): refuse an edited custom source beside its earlier run, and lock an import whatever its runs folder An edited custom source misses the earlier run's sourceHash, so a re-run started a new run without saying so, and the users the earlier run created came back as conflicts. findResume now refuses with exit 2 naming that run, unless it was undone; --new-run starts a new one anyway. Tests cover the resume decision: an unchanged source continues, an edited one is refused, and --new-run starts a new run. The import lock was keyed by the runs folder too, so two imports of the same file into the same instance with different --runs-dir values did not block each other. It is keyed by the file, source and instance alone. The README documents the lock file, and says `migrate sources` runs a source file when given its path. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 26 +++++- .../commands/migrate/lib/run-store.test.ts | 2 +- .../src/commands/migrate/lib/run-store.ts | 20 ++-- .../cli-core/src/commands/migrate/run.test.ts | 91 ++++++++++++++++++- packages/cli-core/src/commands/migrate/run.ts | 56 +++++++++--- 5 files changed, 162 insertions(+), 33 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index de4f61770..2bc62bebb 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -177,6 +177,19 @@ every run. Run folders are created owner-only (`0700`), and export files `0600`: they hold password hashes and user data. +#### The import lock + +Every import but a dry run also takes a lock outside the run store, before its +checks and prompts: `clerk-migrate-import-.lock` in the temp directory +(`os.tmpdir()`, usually `$TMPDIR` or `/tmp`). The hash covers the file's sha256, +the source and the instance, so a second import of the same file into the same +instance is refused with exit 2 even before either has written a run, and +whichever `--runs-dir` each uses. It lives outside the runs folder because +nothing may be written there before consent. It holds the importing process's +PID and is removed however the import ends. One left by a process that died is +reclaimed by the next import. The refusal names the file, which is safe to +delete when no `clerk migrate import` is running. + `users.ndjson` writes are synchronous appends, so a run interrupted with Ctrl-C still leaves a complete record of everything already processed. A line that cannot be written stops that user's create from going out. An export's file @@ -1086,7 +1099,9 @@ clerk migrate sources --json full: its export command, what it carries with a note for each, where each field lands (`encrypted_password → password`), its fixed defaults, and any caveats. An unknown key exits 2 and lists the valid ones. There is no -intro/outro gutter: this reads a static registry rather than running anything. +intro/outro gutter. `sources` alone, or with a built-in key, reads a static +registry and runs nothing. With a path, it loads and runs that file, as +`--source` does, to check it and show what it maps. ### `clerk migrate help` @@ -1169,8 +1184,12 @@ TypeScript is fine — Bun's transpiler is part of the runtime, so `interface`, Plain `.js` works too. An import run records a custom source's key and a hash of the file, so an -edited source counts as a different source. The hash covers that file only: -a local helper it imports is not part of it, so after editing one, pass +edited source counts as a different source. A re-run with an edited source +won't continue the earlier version's run, and won't start a new one beside it +unasked either: it refuses with exit 2 and names the earlier run, since the +users that run created would come back as conflicts. Pass `--new-run` to start +a new run anyway, or undo the earlier one first. The hash covers that file +only: a local helper it imports is not part of it, so after editing one, pass `--new-run` rather than continue a run made with the old helper. #### Validation @@ -1187,6 +1206,7 @@ rejected with the specific problem rather than crashing mid-pipeline: | No `carries` | `` `carries` must say what the source brings across: { passwords, mfa, metadata }, each { level, note } `` | | `key` clashes with a built-in | `key is "clerk", which is already a built-in source` | | `preTransform` or `postTransform` is not a function | `` `preTransform` must be a function when present `` (likewise `postTransform`) | +| `caveats` is not a list of strings | `` `caveats` must be an array of strings when present `` | The `userId` check is the load-bearing one: without it the import would run to completion and create every user with no `external_id`, which is what makes a diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts index e49675b02..30a2ad818 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts @@ -211,7 +211,7 @@ describe("locks and interruptions", () => { const identity = { sha256: "a".repeat(64), source: "supabase", instanceId: "ins_lock" }; const write = spyOn(fs, "writeFileSync"); try { - const release = lockImport(runsDir, identity); + const release = lockImport(identity); const lockPaths = write.mock.calls.map(([target]) => String(target)); expect(lockPaths.every((target) => target.endsWith(".tmp"))).toBe(true); release(); diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index fcc1097a7..26321ee08 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -312,19 +312,13 @@ export type ImportIdentity = { sha256: string; source: string; instanceId: strin /** * The lock one import of a file, by a source, into an instance holds. In the - * temp directory, keyed by the runs folder too: it is taken before consent, - * when nothing may be written to the runs folder yet. + * temp directory, as it is taken before consent, when nothing may be written + * to the runs folder yet. Not keyed by the runs folder: two imports with + * different `--runs-dir` values would still send the same users. */ -export function importLockFile(runsDir: string, identity: ImportIdentity): string { +export function importLockFile(identity: ImportIdentity): string { const key = createHash("sha256") - .update( - JSON.stringify([ - path.resolve(runsDir), - identity.sha256, - identity.source, - identity.instanceId, - ]), - ) + .update(JSON.stringify([identity.sha256, identity.source, identity.instanceId])) .digest("hex") .slice(0, 32); return path.join(os.tmpdir(), `clerk-migrate-import-${key}.lock`); @@ -337,8 +331,8 @@ export function importLockFile(runsDir: string, identity: ImportIdentity): strin * @returns Releases the lock. * @throws UsageError when a live process holds it. */ -export function lockImport(runsDir: string, identity: ImportIdentity): () => void { - const file = importLockFile(runsDir, identity); +export function lockImport(identity: ImportIdentity): () => void { + const file = importLockFile(identity); acquireLockFile(file, (holder) => throwUsageError( `Another process${holder ? ` (PID ${holder})` : ""} is importing this file into this instance right now. ` + diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index c1109a1e1..990a3cb38 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -1958,7 +1958,7 @@ describe("run", () => { // either has written a run. The import's own lock closes it. test("refuses while another process holds this import's lock, with no run yet", async () => { // Keyed by the key, so it holds whether or not `GET /v1/instance` answers. - const lock = importLockFile(runsDir(), { + const lock = importLockFile({ sha256: sha256File(path.join(workDir, "export.json")), source: "clerk", instanceId: keyInstanceId(baseOptions.secretKey), @@ -1976,9 +1976,27 @@ describe("run", () => { expect(listRuns(runsDir())).toHaveLength(0); }); + // A different --runs-dir is a different record of the same job. + test("refuses while another process holds this import's lock under another runs folder", async () => { + const lock = importLockFile({ + sha256: sha256File(path.join(workDir, "export.json")), + source: "clerk", + instanceId: keyInstanceId(baseOptions.secretKey), + }); + fs.writeFileSync(lock, "1"); + try { + await expect( + run({ ...baseOptions, runsDir: path.join(workDir, "other-runs") }), + ).rejects.toThrow(/importing this file into this instance right now/); + } finally { + fs.rmSync(lock, { force: true }); + } + expect(created()).toHaveLength(0); + }); + // A second key for the same instance has its own stand-in ID. test("refuses while another key's process holds this instance's lock", async () => { - const lock = importLockFile(runsDir(), { + const lock = importLockFile({ sha256: sha256File(path.join(workDir, "export.json")), source: "clerk", instanceId: "ins_1", @@ -1993,7 +2011,7 @@ describe("run", () => { } expect(created()).toHaveLength(0); // The key's own lock, taken first, is let go when the second is refused. - const own = importLockFile(runsDir(), { + const own = importLockFile({ sha256: sha256File(path.join(workDir, "export.json")), source: "clerk", instanceId: keyInstanceId(baseOptions.secretKey), @@ -2004,7 +2022,7 @@ describe("run", () => { test("lets go of the import's locks when the run ends", async () => { await run(baseOptions); const lockFor = (instanceId: string) => - importLockFile(runsDir(), { + importLockFile({ sha256: sha256File(path.join(workDir, "export.json")), source: "clerk", instanceId, @@ -2173,6 +2191,71 @@ describe("run", () => { expect(created()).toHaveLength(0); }); + /** Rewrites the only run as one a crash stopped, and returns its ID. */ + const interruptOnly = (): string => { + const [first] = listRuns(runsDir()); + const record = readRun(runsDir(), first!.id)!; + delete record.finishedAt; + fs.writeFileSync(path.join(runsDir(), first!.id, "run.json"), JSON.stringify(record)); + return first!.id; + }; + + test("continues an interrupted run when the source is unchanged", async () => { + stubClerk({ failing: new Set(["mp_2"]) }); + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const id = interruptOnly(); + requests = []; + process.exitCode = 0; + stubClerk(); + + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + + expect(created().map((r) => (r.body as { external_id: string }).external_id)).toEqual([ + "mp_2", + ]); + expect(captured.err).toContain(`Continuing run ${id}`); + expect(listRuns(runsDir())).toHaveLength(1); + }); + + // Starting over would bring the users the earlier run created back as conflicts. + test("refuses an edited source while an earlier version's run is interrupted, naming it", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const id = interruptOnly(); + const edited = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, edited), `${CUSTOM}\n// edited`); + requests = []; + + const error = (await run({ + input: "export.json", + source: edited, + yes: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain(`Run ${id} imported this file with an earlier version`); + expect(error.message).toContain("--new-run"); + expect(created()).toHaveLength(0); + }); + + test("an edited source with --new-run starts a new run beside the earlier one", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + interruptOnly(); + const edited = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, edited), `${CUSTOM}\n// edited`); + requests = []; + + await run({ + input: "export.json", + source: edited, + yes: true, + secretKey: "sk_test_x", + newRun: true, + }); + + expect(listRuns(runsDir())).toHaveLength(2); + }); + // The loader caches by URL: an edit at the same path must still load the // edited mappings, the ones the run's hash names. test("an edited source at the same path maps with its edited code", async () => { diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 07d5a5648..7146eebe6 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -455,6 +455,17 @@ function assertNoActiveImport( } } +/** True when a run's users are gone: it is marked undone, or an undo of it finished. */ +function undoneInFull(runsDir: string, record: RunRecord): boolean { + return ( + runState(runsDir, record) === "undone" || + listRuns(runsDir).some( + (undo) => + undo.kind === "undo" && undo.undoes === record.id && runState(runsDir, undo) === "complete", + ) + ); +} + /** How this import relates to earlier runs of the same file, source and instance. */ type ResumeCase = | { kind: "new" } @@ -473,7 +484,9 @@ type ResumeCase = * @throws UsageError when that run is still running in another process, or * has an undo that never finished: some of its users are gone and some are * not, so neither continuing it nor starting over is safe. Also when Clerk - * did not confirm the instance and a run of this file names a real one. + * did not confirm the instance and a run of this file names a real one, or + * when the latest run of this file was made with an earlier version of a + * custom source and was not undone. */ function findResume( runsDir: string, @@ -488,20 +501,37 @@ function findResume( /** The import's runs-folder and target flags, for the command a refusal suggests. */ carry: string[], ): ResumeCase { - const runs = listRuns(runsDir); - const ofFile = importsOfFile(runs, match); - const latest = ofFile.find( - (record) => - record.sourceHash === match.sourceHash && - (record.target.instanceId === match.instanceId || - record.target.instanceId === match.keyInstanceId), + const sameJob = (record: RunRecord): boolean => + record.kind === "import" && + record.file?.sha256 === match.sha256 && + record.source === match.source && + (record.target.instanceId === match.instanceId || + record.target.instanceId === match.keyInstanceId); + const latest = listRuns(runsDir).find( + (record) => sameJob(record) && record.sourceHash === match.sourceHash, ); if (!latest) { + // An edited custom source is a different source, but its earlier run sent + // the same users: starting over beside it would reject every user it + // created and leave it unfinished, so that takes --new-run. + const earlier = listRuns(runsDir).find(sameJob); + if (earlier && !undoneInFull(runsDir, earlier)) { + throwUsageError( + `Run ${earlier.id} imported this file with an earlier version of this source, so the users it created would come back as conflicts. Nothing was imported.\n` + + `Pass --new-run to start a new run beside it, or undo it first with \`clerk migrate undo ${earlier.id}\`.`, + ); + } // Clerk did not name the instance (GET /v1/instance failed), so a run of // this file recorded under a real ID can be neither matched nor ruled out. // Starting over would reject every user it created and leave it unfinished. if (isStandInInstanceId(match.instanceId)) { - const unconfirmed = ofFile.find((record) => isClerkInstanceId(record.target.instanceId)); + const unconfirmed = listRuns(runsDir).find( + (record) => + record.kind === "import" && + record.file?.sha256 === match.sha256 && + record.source === match.source && + isClerkInstanceId(record.target.instanceId), + ); if (unconfirmed) { throwUsageError( `Clerk did not confirm which instance this key addresses, so this import cannot tell whether run ${unconfirmed.id} is for it. ` + @@ -520,10 +550,12 @@ function findResume( } if (state === "undone") return { kind: "new" }; - const openUndo = openUndoOf(runsDir, latest.id, runs); + const openUndo = openUndoOf(runsDir, latest.id); if (openUndo) throwUsageError(unfinishedUndoMessage(latest.id, openUndo.id, carry)); // Undone in full, though the import run was never marked so. - if (undosOf(runs, latest.id).length > 0) return { kind: "new" }; + if (listRuns(runsDir).some((record) => record.kind === "undo" && record.undoes === latest.id)) { + return { kind: "new" }; + } if (state === "complete") return { kind: "complete", record: latest }; return { @@ -928,7 +960,7 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi const releases: (() => void)[] = []; try { for (const instanceId of new Set([keyInstanceId(secretKey), target.instanceId])) { - releases.push(lockImport(runsDir, { sha256, source, instanceId })); + releases.push(lockImport({ sha256, source, instanceId })); } } catch (error) { for (const release of releases) release(); From 3d47db08a6af12d62b18a9724ff5dd1ee2de82e0 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 14 Oct 2026 19:51:21 -0400 Subject: [PATCH 15/20] test(migrate): let the lock-linking test pass where hard links are refused On a filesystem without hard links, acquireLockFile writes the lock itself with `wx`, so "every write is a .tmp" failed there although the code was right. The check now holds only when the link went through, and a new test forces the link to fail and checks the fallback writes the lock exclusively. Co-Authored-By: Claude Opus 5.5 --- .../commands/migrate/lib/run-store.test.ts | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts index 30a2ad818..e40c58ad6 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts @@ -19,6 +19,7 @@ import { latestUserLines, listRuns, lockFile, + importLockFile, lockImport, newRunId, patchRun, @@ -210,13 +211,36 @@ describe("locks and interruptions", () => { test("a lock is linked into place with its PID already written", () => { const identity = { sha256: "a".repeat(64), source: "supabase", instanceId: "ins_lock" }; const write = spyOn(fs, "writeFileSync"); + const link = spyOn(fs, "linkSync"); try { const release = lockImport(identity); const lockPaths = write.mock.calls.map(([target]) => String(target)); - expect(lockPaths.every((target) => target.endsWith(".tmp"))).toBe(true); + // Where hard links are refused, the fallback below writes the lock itself. + if (link.mock.results.every((result) => result.type === "return")) { + expect(lockPaths.every((target) => target.endsWith(".tmp"))).toBe(true); + } release(); } finally { write.mockRestore(); + link.mockRestore(); + } + }); + + test("a filesystem without hard links gets the lock written exclusively", () => { + const identity = { sha256: "b".repeat(64), source: "supabase", instanceId: "ins_nolink" }; + const link = spyOn(fs, "linkSync").mockImplementation(() => { + throw Object.assign(new Error("not supported"), { code: "EPERM" }); + }); + const write = spyOn(fs, "writeFileSync"); + try { + const release = lockImport(identity); + const lock = write.mock.calls.find(([target]) => String(target).endsWith(".lock")); + expect(lock?.[2]).toMatchObject({ flag: "wx" }); + expect(fs.readFileSync(importLockFile(identity), "utf-8")).toBe(String(process.pid)); + release(); + } finally { + write.mockRestore(); + link.mockRestore(); } }); From 9d74c00eb6652765f89e8bebe28036dded5f0af4 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 14 Oct 2026 23:41:25 -0400 Subject: [PATCH 16/20] feat(migrate): let a source you wrote replace an export file's own `clerk migrate import --source ./custom.ts` was refused whenever the envelope named a built-in source, so bringing across a field the built-in mapping leaves out (a WorkOS metadata.username, an Auth.js name that is a username) meant stripping the envelope into a new file and losing the link to the export run. A source given by path now replaces the envelope's and reads its users in the platform's own shape. The import keeps fromExport, records the source the envelope named as envelopeSource, and its printed commands keep --source. One built-in name against another is still refused, and that message now points at a source you write. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 8 +++ .../cli-core/src/commands/migrate/index.ts | 2 +- .../src/commands/migrate/lib/run-store.ts | 2 + .../cli-core/src/commands/migrate/run.test.ts | 68 +++++++++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 27 ++++++-- 5 files changed, 101 insertions(+), 6 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 2bc62bebb..3d21dd664 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -287,6 +287,14 @@ The file is an envelope around the users: parameters, when it could read them: the import then needs no `--firebase-*` flags. When it couldn't, pass them to the import, or enter them at its prompt. +`--source` with a different built-in name is refused, since it would map the +users with the wrong fields. `--source` with the path to a +[source you wrote](#custom-sources) replaces the envelope's: it gets the same +users, in the platform's own shape, so it can bring across what the built-in +mapping leaves out, such as a WorkOS `metadata.username`. The import keeps its +link to the export run, and `run.json` records the source the envelope named +as `envelopeSource`. + `--json` prints the result on stdout instead — `{ target, run, output, users, skipped, coverage, next }` — and never prompts, so a missing credential exits 2 naming the flag to pass. `next`, and the commands printed after a human run, diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index a4bb4afed..0e27c76d2 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -84,7 +84,7 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce .argument("[file|export-run-id]", "The export file, or the ID of the export run that wrote it") .option( "--source ", - "Where the file came from: a built-in source, or a source you wrote. Not needed for a file from `clerk migrate export`", + "Where the file came from: a built-in source, or a source you wrote. Not needed for a file from `clerk migrate export`, though a source you wrote replaces its own", ) .option("--dry-run", "Check the file against the instance, report, and write nothing") .option("--allow-partial", "Import the users that pass the checks, and skip the rest") diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index 26321ee08..9df725d3a 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -90,6 +90,8 @@ export type RunRecord = { * continue for those. */ flags?: ImportFlags; + /** The source the export file named, when a custom source replaced it. */ + envelopeSource?: string; /** * A Firebase import's hash parameters, as SHA-256 of the four (never the * parameters themselves): every digest it builds carries them, so a continue diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 990a3cb38..5fb7c9287 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -599,6 +599,74 @@ describe("run", () => { expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(2); }); + test("says how to map the users differently when a built-in source contradicts it", async () => { + const { record } = exportRun("clerk", export2); + + await expect(run({ ...noSource, source: "auth0", input: record.id })).rejects.toThrow( + /pass the path to a source you wrote/, + ); + }); + + /** A source of your own, for an Auth0 export, that also brings `nickname` across. */ + const ownAuth0Source = () => { + const file = `./own-auth0-${Date.now()}-${Math.random().toString(36).slice(2)}.ts`; + fs.writeFileSync( + path.join(workDir, file), + `export default { + key: "myauth0", + label: "My Auth0", + transformer: { user_id: "userId", email: "email", nickname: "firstName" }, + carries: { + passwords: { level: "no", note: "None." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, + };`, + ); + return file; + }; + const auth0Rows = [{ user_id: "auth0|a1", email: "a@x.dev", nickname: "Ada" }]; + + // The built-in mapping leaves `nickname` out; the customer's source brings it. + test("a source you wrote replaces the envelope's, reading its users", async () => { + const { record } = exportRun("auth0", auth0Rows); + const own = ownAuth0Source(); + + await run({ ...noSource, source: own, input: record.id }); + + const created = requests.filter((r) => r.url.endsWith("/v1/users")); + expect(created).toHaveLength(1); + expect(created[0]?.body).toMatchObject({ external_id: "auth0|a1", first_name: "Ada" }); + expect(captured.err).toContain("in place of the built-in"); + }); + + test("an import through a source you wrote keeps its export run, and records what it replaced", async () => { + const { record } = exportRun("auth0", auth0Rows); + + await run({ ...noSource, source: ownAuth0Source(), input: record.id }); + + const imported = listRuns(runsDir()).find((r) => r.kind === "import"); + expect(imported).toMatchObject({ + source: "myauth0", + fromExport: record.id, + envelopeSource: "auth0", + }); + }); + + // A re-run without it would map the rest with the built-in source. + test("the printed command keeps a source you wrote in place of the envelope's", async () => { + const { record } = exportRun("auth0", auth0Rows); + const own = ownAuth0Source(); + + const error = (await run({ ...noSource, source: own, input: record.id, yes: false }).catch( + (caught: unknown) => caught, + )) as CliError; + + expect(error.examples?.[0]?.command).toContain( + `clerk migrate import ${record.id} --source ${own}`, + ); + }); + test("refuses a run ID that is not an export", async () => { await run(baseOptions); const [imported] = listRuns(runsDir()); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 7146eebe6..b2433f1e4 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -119,6 +119,8 @@ export type MigrateRunOptions = { sourceHash?: string; /** `--source` as typed, for printed commands: a custom source's path. Not a flag. */ sourceArg?: string; + /** The source an export file named, when a custom `--source` replaced it. Not a flag. */ + envelopeSource?: string; /** The file to read, once `input` is resolved. Not a flag. */ file?: string; requirePassword?: boolean; @@ -399,20 +401,33 @@ async function applySource(options: MigrateRunOptions): Promise"; const parts = ["clerk migrate import", quoteArg(input)]; const source = options.sourceArg ?? options.source; - if (!fromExport && source) parts.push("--source", quoteArg(source)); + // An export names its own source, unless a source you wrote replaced it. + if ((!fromExport || options.envelopeSource) && source) parts.push("--source", quoteArg(source)); if (options.newRun) parts.push("--new-run"); parts.push(...importFlags(options)); // Names, not `<…>`: pasted as is, a shell reads `` as a redirect. @@ -1341,6 +1357,7 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi ...(options.sourceHash ? { sourceHash: options.sourceHash } : {}), file: { path: filePath, sha256 }, ...(input.fromExport ? { fromExport: input.fromExport } : {}), + ...(options.envelopeSource ? { envelopeSource: options.envelopeSource } : {}), ...(firebaseHash ? { firebaseHash } : {}), flags, }); From 0f3dba88965b727f4516f478ef24bb2dc2ee19aa Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 15 Oct 2026 02:02:56 -0400 Subject: [PATCH 17/20] fix(migrate): refuse an edited custom source while any earlier version's run is unfinished, not just the newest The check took the newest run of the file, source and instance; when that one was undone, an older run that still held its users was never looked at, and the import started a new run beside it. Any run not undone in full now refuses, and the suggested undo keeps the runs folder and target flags. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/run.test.ts | 34 +++++++++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 10 ++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 5fb7c9287..75d8edb92 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -2306,6 +2306,40 @@ describe("run", () => { expect(created()).toHaveLength(0); }); + // The newest run being undone doesn't clear an older one that still holds users. + test("refuses an edited source while any earlier version's run is unfinished, not just the newest", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const older = interruptOnly(); + const second = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, second), `${CUSTOM}\n// second`); + await run({ + input: "export.json", + source: second, + yes: true, + secretKey: "sk_test_x", + newRun: true, + }); + const newer = listRuns(runsDir()).find((record) => record.id !== older)!; + fs.writeFileSync( + path.join(runsDir(), newer.id, "run.json"), + JSON.stringify({ ...readRun(runsDir(), newer.id), status: "undone" }), + ); + const third = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, third), `${CUSTOM}\n// third`); + requests = []; + + const error = (await run({ + input: "export.json", + source: third, + yes: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.message).toContain(`Run ${older} imported this file with an earlier version`); + expect(error.message).toContain(`clerk migrate undo ${older} --secret-key `); + expect(created()).toHaveLength(0); + }); + test("an edited source with --new-run starts a new run beside the earlier one", async () => { await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); interruptOnly(); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index b2433f1e4..9e7f749d5 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -529,11 +529,15 @@ function findResume( // An edited custom source is a different source, but its earlier run sent // the same users: starting over beside it would reject every user it // created and leave it unfinished, so that takes --new-run. - const earlier = listRuns(runsDir).find(sameJob); - if (earlier && !undoneInFull(runsDir, earlier)) { + // Any run of it not undone, not just the newest: an older interrupted run + // behind a newer undone one still holds its users. + const earlier = listRuns(runsDir).find( + (record) => sameJob(record) && !undoneInFull(runsDir, record), + ); + if (earlier) { throwUsageError( `Run ${earlier.id} imported this file with an earlier version of this source, so the users it created would come back as conflicts. Nothing was imported.\n` + - `Pass --new-run to start a new run beside it, or undo it first with \`clerk migrate undo ${earlier.id}\`.`, + `Pass --new-run to start a new run beside it, or undo it first with \`${withFlags(`clerk migrate undo ${earlier.id}`, carry)}\`.`, ); } // Clerk did not name the instance (GET /v1/instance failed), so a run of From 5829af53ab26146a315d1d098c78ba6f54ce11d1 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 15 Oct 2026 14:58:43 -0400 Subject: [PATCH 18/20] fix(migrate): match a file's earlier imports by file and instance, and pass Firebase's parameters to a source you wrote From Wyatt's review of #530: - A run is recorded under its source's key, and the import lock, the live-run scan and the earlier-run refusal all matched on it. So importing an export with the built-in, then re-running it through a source you wrote, started a second run beside the first, and two imports of one export through different sources took different locks. The three now match on the file and the instance only; only the run to continue needs the same source and version. The refusal names the other source. - A Firebase export through a source you wrote lost its hash parameters: every Firebase check keyed on `source === "firebase"`, and the source was the custom key. They now key on the envelope's source too, so the parameters reach `context.firebaseHashConfig` and the run records its firebaseHash. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 26 +++-- .../commands/migrate/lib/run-store.test.ts | 4 +- .../src/commands/migrate/lib/run-store.ts | 13 +-- .../cli-core/src/commands/migrate/run.test.ts | 100 ++++++++++++++++-- packages/cli-core/src/commands/migrate/run.ts | 88 ++++++++------- 5 files changed, 167 insertions(+), 64 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 3d21dd664..9e38aad22 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -181,10 +181,10 @@ password hashes and user data. Every import but a dry run also takes a lock outside the run store, before its checks and prompts: `clerk-migrate-import-.lock` in the temp directory -(`os.tmpdir()`, usually `$TMPDIR` or `/tmp`). The hash covers the file's sha256, -the source and the instance, so a second import of the same file into the same -instance is refused with exit 2 even before either has written a run, and -whichever `--runs-dir` each uses. It lives outside the runs folder because +(`os.tmpdir()`, usually `$TMPDIR` or `/tmp`). The hash covers the file's sha256 +and the instance, not the source, so a second import of the same file into the +same instance is refused with exit 2 even before either has written a run, +whichever source and `--runs-dir` each uses. It lives outside the runs folder because nothing may be written there before consent. It holds the importing process's PID and is removed however the import ends. One left by a process that died is reclaimed by the next import. The refusal names the file, which is safe to @@ -293,7 +293,14 @@ users with the wrong fields. `--source` with the path to a users, in the platform's own shape, so it can bring across what the built-in mapping leaves out, such as a WorkOS `metadata.username`. The import keeps its link to the export run, and `run.json` records the source the envelope named -as `envelopeSource`. +as `envelopeSource`. A Firebase export keeps its hash parameters through it: +the envelope's, the `--firebase-*` flags or the variables reach the source's +`postTransform` as `context.firebaseHashConfig`, as they do the built-in's. + +An earlier import of the same export into the same instance, through the +built-in or another source, holds the same users. While it isn't undone, a +re-run through your source refuses with exit 2 and names it, rather than +starting a new run beside it: pass `--new-run`, or undo it first. `--json` prints the result on stdout instead — `{ target, run, output, users, skipped, coverage, next }` — and never prompts, so a missing credential exits 2 @@ -1192,10 +1199,11 @@ TypeScript is fine — Bun's transpiler is part of the runtime, so `interface`, Plain `.js` works too. An import run records a custom source's key and a hash of the file, so an -edited source counts as a different source. A re-run with an edited source -won't continue the earlier version's run, and won't start a new one beside it -unasked either: it refuses with exit 2 and names the earlier run, since the -users that run created would come back as conflicts. Pass `--new-run` to start +edited source counts as a different source. A re-run with an edited source, +or with another source altogether, won't continue the earlier run of the file, +and won't start a new one beside it unasked either: while that run isn't +undone, it refuses with exit 2 and names it, since the users it created would +come back as conflicts. Pass `--new-run` to start a new run anyway, or undo the earlier one first. The hash covers that file only: a local helper it imports is not part of it, so after editing one, pass `--new-run` rather than continue a run made with the old helper. diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts index e40c58ad6..44b1f9997 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts @@ -209,7 +209,7 @@ describe("locks and interruptions", () => { // A lock created empty and written after reads as stale to a second // process in between, which would remove it and import beside the first. test("a lock is linked into place with its PID already written", () => { - const identity = { sha256: "a".repeat(64), source: "supabase", instanceId: "ins_lock" }; + const identity = { sha256: "a".repeat(64), instanceId: "ins_lock" }; const write = spyOn(fs, "writeFileSync"); const link = spyOn(fs, "linkSync"); try { @@ -227,7 +227,7 @@ describe("locks and interruptions", () => { }); test("a filesystem without hard links gets the lock written exclusively", () => { - const identity = { sha256: "b".repeat(64), source: "supabase", instanceId: "ins_nolink" }; + const identity = { sha256: "b".repeat(64), instanceId: "ins_nolink" }; const link = spyOn(fs, "linkSync").mockImplementation(() => { throw Object.assign(new Error("not supported"), { code: "EPERM" }); }); diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index 9df725d3a..fb32b6100 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -307,27 +307,28 @@ function releaseLock(runsDir: string, id: string): void { } /** - * What makes two imports the same job: the file, the source and the instance, - * by its ID or its key's stand-in ID. + * What makes two imports the same job: the file and the instance, by its ID + * or its key's stand-in ID. Not the source: whichever source maps the file, + * its users are the same people. */ -export type ImportIdentity = { sha256: string; source: string; instanceId: string }; +export type ImportIdentity = { sha256: string; instanceId: string }; /** - * The lock one import of a file, by a source, into an instance holds. In the + * The lock one import of a file into an instance holds. In the * temp directory, as it is taken before consent, when nothing may be written * to the runs folder yet. Not keyed by the runs folder: two imports with * different `--runs-dir` values would still send the same users. */ export function importLockFile(identity: ImportIdentity): string { const key = createHash("sha256") - .update(JSON.stringify([identity.sha256, identity.source, identity.instanceId])) + .update(JSON.stringify([identity.sha256, identity.instanceId])) .digest("hex") .slice(0, 32); return path.join(os.tmpdir(), `clerk-migrate-import-${key}.lock`); } /** - * Takes the lock for one import of this file, source and instance, before + * Takes the lock for one import of this file into this instance, before * its checks, so no second process can pass them and start a run beside it. * * @returns Releases the lock. diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 75d8edb92..3e71e04b7 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -654,6 +654,93 @@ describe("run", () => { }); // A re-run without it would map the rest with the built-in source. + // Import with the built-in, notice a field is missing, write a source, re-run: + // the first run's users are the same people, whichever source maps them. + test("refuses a source you wrote while the built-in's run of the same export holds its users", async () => { + const { record } = exportRun("auth0", auth0Rows); + await run({ ...noSource, input: record.id }); + const [first] = listRuns(runsDir()).filter((r) => r.kind === "import"); + requests = []; + + const error = (await run({ ...noSource, source: ownAuth0Source(), input: record.id }).catch( + (caught: unknown) => caught, + )) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain( + `Run ${first!.id} imported this file into this instance with the \`auth0\` source`, + ); + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(0); + + await run({ ...noSource, source: ownAuth0Source(), input: record.id, newRun: true }); + expect(listRuns(runsDir()).filter((r) => r.kind === "import")).toHaveLength(2); + }); + + // One lock for the file and instance, whichever source each import uses. + test("a source you wrote takes the same import lock as the built-in", async () => { + const { record, file } = exportRun("auth0", auth0Rows); + const lock = importLockFile({ + sha256: sha256File(file), + instanceId: keyInstanceId(baseOptions.secretKey), + }); + fs.writeFileSync(lock, "1"); + try { + await expect( + run({ ...noSource, source: ownAuth0Source(), input: record.id }), + ).rejects.toThrow(/importing this file into this instance right now/); + } finally { + fs.rmSync(lock, { force: true }); + } + }); + + // A Firebase export's digests need the project's parameters, whatever maps it. + test("a source you wrote over a Firebase export gets its hash parameters", async () => { + const { record } = exportRun( + "firebase", + [{ localId: "f1", email: "f@x.dev", passwordHash: "HASH", passwordSalt: "SALT" }], + { + firebase: { + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }, + }, + ); + const own = `./own-firebase-${Date.now()}.ts`; + fs.writeFileSync( + path.join(workDir, own), + `export default { + key: "myfirebase", + label: "My Firebase", + transformer: { localId: "userId", email: "email", passwordHash: "passwordHash", passwordSalt: "salt" }, + carries: { + passwords: { level: "yes", note: "scrypt." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, + defaults: { passwordHasher: "scrypt_firebase" }, + postTransform: (user, context) => { + const c = context.firebaseHashConfig; + if (!c) throw new Error("no hash parameters"); + user.password = [user.passwordHash, user.salt, c.base64_signer_key, c.base64_salt_separator, c.rounds, c.mem_cost].join("$"); + delete user.passwordHash; + delete user.salt; + }, + };`, + ); + + await run({ ...noSource, source: own, input: record.id }); + + const created = requests.find((r) => r.url.endsWith("/v1/users")); + expect(created?.body).toMatchObject({ + password_hasher: "scrypt_firebase", + password_digest: "HASH$SALT$SIGNER$Bw==$8$14", + }); + const imported = listRuns(runsDir()).find((r) => r.kind === "import"); + expect(imported?.firebaseHash).toMatch(/^[0-9a-f]{64}$/); + }); + test("the printed command keeps a source you wrote in place of the envelope's", async () => { const { record } = exportRun("auth0", auth0Rows); const own = ownAuth0Source(); @@ -2028,7 +2115,6 @@ describe("run", () => { // Keyed by the key, so it holds whether or not `GET /v1/instance` answers. const lock = importLockFile({ sha256: sha256File(path.join(workDir, "export.json")), - source: "clerk", instanceId: keyInstanceId(baseOptions.secretKey), }); // PID 1 is always alive, and never this test. @@ -2048,7 +2134,6 @@ describe("run", () => { test("refuses while another process holds this import's lock under another runs folder", async () => { const lock = importLockFile({ sha256: sha256File(path.join(workDir, "export.json")), - source: "clerk", instanceId: keyInstanceId(baseOptions.secretKey), }); fs.writeFileSync(lock, "1"); @@ -2066,7 +2151,6 @@ describe("run", () => { test("refuses while another key's process holds this instance's lock", async () => { const lock = importLockFile({ sha256: sha256File(path.join(workDir, "export.json")), - source: "clerk", instanceId: "ins_1", }); fs.writeFileSync(lock, "1"); @@ -2081,7 +2165,6 @@ describe("run", () => { // The key's own lock, taken first, is let go when the second is refused. const own = importLockFile({ sha256: sha256File(path.join(workDir, "export.json")), - source: "clerk", instanceId: keyInstanceId(baseOptions.secretKey), }); expect(fs.existsSync(own)).toBe(false); @@ -2092,7 +2175,6 @@ describe("run", () => { const lockFor = (instanceId: string) => importLockFile({ sha256: sha256File(path.join(workDir, "export.json")), - source: "clerk", instanceId, }); expect(fs.existsSync(lockFor(keyInstanceId(baseOptions.secretKey)))).toBe(false); @@ -2301,7 +2383,9 @@ describe("run", () => { }).catch((caught: unknown) => caught)) as CliError; expect(error.exitCode).toBe(EXIT_CODE.USAGE); - expect(error.message).toContain(`Run ${id} imported this file with an earlier version`); + expect(error.message).toContain( + `Run ${id} imported this file into this instance with an earlier version`, + ); expect(error.message).toContain("--new-run"); expect(created()).toHaveLength(0); }); @@ -2335,7 +2419,9 @@ describe("run", () => { secretKey: "sk_test_x", }).catch((caught: unknown) => caught)) as CliError; - expect(error.message).toContain(`Run ${older} imported this file with an earlier version`); + expect(error.message).toContain( + `Run ${older} imported this file into this instance with an earlier version`, + ); expect(error.message).toContain(`clerk migrate undo ${older} --secret-key `); expect(created()).toHaveLength(0); }); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 9e7f749d5..b9ea9ad7b 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -61,8 +61,6 @@ import { continueRun, latestUserLines, listRuns, - liveLockPid, - lockFile, lockImport, readRun, readUserLines, @@ -451,25 +449,35 @@ function applyEnvelope( */ function assertNoActiveImport( runsDir: string, - match: { sha256: string; source: string; instanceId: string; keyInstanceId?: string }, + match: { sha256: string; instanceId: string; keyInstanceId?: string }, ): void { const active = listRuns(runsDir).find( - (record) => - record.kind === "import" && - record.file?.sha256 === match.sha256 && - record.source === match.source && - (record.target.instanceId === match.instanceId || - record.target.instanceId === match.keyInstanceId) && - runState(runsDir, record) === "running", + (record) => sameFileAndInstance(record, match) && runState(runsDir, record) === "running", ); if (active) { throwUsageError( - `Run ${active.id} is importing this file right now in another process (PID ${liveLockPid(runsDir, active.id)}). ` + - `Wait for it to finish. If that process is not a migrate run, delete ${lockFile(runsDir, active.id)}.`, + inUseMessage(runsDir, active.id, "importing this file", "Wait for it to finish."), ); } } +/** + * An import of this file into this instance, by its ID or its key's stand-in + * ID, under any source. Whichever source maps them, its users are the same + * people, so another source's run, or another version of one, holds them too. + */ +function sameFileAndInstance( + record: RunRecord, + match: { sha256: string; instanceId: string; keyInstanceId?: string }, +): boolean { + return ( + record.kind === "import" && + record.file?.sha256 === match.sha256 && + (record.target.instanceId === match.instanceId || + record.target.instanceId === match.keyInstanceId) + ); +} + /** True when a run's users are gone: it is marked undone, or an undo of it finished. */ function undoneInFull(runsDir: string, record: RunRecord): boolean { return ( @@ -516,39 +524,39 @@ function findResume( /** The import's runs-folder and target flags, for the command a refusal suggests. */ carry: string[], ): ResumeCase { - const sameJob = (record: RunRecord): boolean => - record.kind === "import" && - record.file?.sha256 === match.sha256 && - record.source === match.source && - (record.target.instanceId === match.instanceId || - record.target.instanceId === match.keyInstanceId); - const latest = listRuns(runsDir).find( - (record) => sameJob(record) && record.sourceHash === match.sourceHash, + const runs = listRuns(runsDir); + // Only a run of this same source, and the same version of a custom one, + // is continued. + const latest = importsOfFile(runs, match).find( + (record) => sameFileAndInstance(record, match) && record.sourceHash === match.sourceHash, ); if (!latest) { - // An edited custom source is a different source, but its earlier run sent - // the same users: starting over beside it would reject every user it - // created and leave it unfinished, so that takes --new-run. - // Any run of it not undone, not just the newest: an older interrupted run - // behind a newer undone one still holds its users. - const earlier = listRuns(runsDir).find( - (record) => sameJob(record) && !undoneInFull(runsDir, record), + // Another source's run of this file, or an earlier version of this one, + // sent the same users: starting over beside it would reject every user it + // created and leave it unfinished, so that takes --new-run. Any such run + // not undone, not just the newest: an older interrupted run behind a + // newer undone one still holds its users. + const earlier = runs.find( + (record) => sameFileAndInstance(record, match) && !undoneInFull(runsDir, record), ); if (earlier) { + const how = + earlier.source === match.source + ? "with an earlier version of this source" + : `with the \`${earlier.source}\` source`; throwUsageError( - `Run ${earlier.id} imported this file with an earlier version of this source, so the users it created would come back as conflicts. Nothing was imported.\n` + - `Pass --new-run to start a new run beside it, or undo it first with \`${withFlags(`clerk migrate undo ${earlier.id}`, carry)}\`.`, + `Run ${earlier.id} imported this file into this instance ${how}, so the users it created would come back as conflicts. Nothing was imported.\n` + + `Pass --new-run to start a new run beside it, or undo it first with \`${undoCommand(earlier.id, carry)}\`.`, ); } // Clerk did not name the instance (GET /v1/instance failed), so a run of // this file recorded under a real ID can be neither matched nor ruled out. // Starting over would reject every user it created and leave it unfinished. if (isStandInInstanceId(match.instanceId)) { - const unconfirmed = listRuns(runsDir).find( + const unconfirmed = runs.find( (record) => record.kind === "import" && record.file?.sha256 === match.sha256 && - record.source === match.source && isClerkInstanceId(record.target.instanceId), ); if (unconfirmed) { @@ -569,12 +577,10 @@ function findResume( } if (state === "undone") return { kind: "new" }; - const openUndo = openUndoOf(runsDir, latest.id); + const openUndo = openUndoOf(runsDir, latest.id, runs); if (openUndo) throwUsageError(unfinishedUndoMessage(latest.id, openUndo.id, carry)); // Undone in full, though the import run was never marked so. - if (listRuns(runsDir).some((record) => record.kind === "undo" && record.undoes === latest.id)) { - return { kind: "new" }; - } + if (undosOf(runs, latest.id).length > 0) return { kind: "new" }; if (state === "complete") return { kind: "complete", record: latest }; return { @@ -908,15 +914,18 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi // The flags every suggested command needs to reach this runs folder and instance. const carry = carryFlags(options); + // A Firebase export through a source you wrote is still Firebase's: its + // digests need the project's parameters, from the envelope or the flags. + const hashSource = options.envelopeSource ?? source; // A flag is typed for this run, so it wins over the export file. A variable // is ambient, and may be left over from another project, so it only fills in // for an export file without the parameters: with one, a partial flag set is // refused rather than completed from the variables. - const fromEnvelope = source === "firebase" ? envelope?.firebase : undefined; + const fromEnvelope = hashSource === "firebase" ? envelope?.firebase : undefined; const fromFlagsAtAll = FIREBASE_FLAGS.some(([key]) => options[key] !== undefined); const fromFlags = fromFlagsAtAll || !fromEnvelope - ? resolveFirebaseHashConfig(options, source, fromEnvelope ? {} : process.env) + ? resolveFirebaseHashConfig(options, hashSource, fromEnvelope ? {} : process.env) : undefined; let firebaseHashConfig = fromFlags ?? fromEnvelope; let configFrom = fromFlags @@ -925,7 +934,7 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi : "the CLERK_FIREBASE_* variables" : "the export file"; // `-y` is "import without prompting": it gets the usage error naming the flags. - if (source === "firebase" && !firebaseHashConfig && !options.yes && canPrompt(options)) { + if (hashSource === "firebase" && !firebaseHashConfig && !options.yes && canPrompt(options)) { firebaseHashConfig = await promptForFirebaseHashConfig(); configFrom = "the parameters entered"; // A suggested command runs with --yes, which never asks: it names the flags. @@ -980,7 +989,7 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi const releases: (() => void)[] = []; try { for (const instanceId of new Set([keyInstanceId(secretKey), target.instanceId])) { - releases.push(lockImport({ sha256, source, instanceId })); + releases.push(lockImport({ sha256, instanceId })); } } catch (error) { for (const release of releases) release(); @@ -992,7 +1001,6 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi } assertNoActiveImport(runsDir, { sha256, - source, instanceId: target.instanceId, keyInstanceId: keyInstanceId(secretKey), }); From 93909f94fdded04e39430072b29e691ed6c4710d Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 16 Oct 2026 16:20:52 -0400 Subject: [PATCH 19/20] fix(migrate): check other unfinished runs behind an undone exact match, and create the lock's temp file exclusively - findResume started a new run when the exact-version run was undone, without checking the other runs of the file. An older version's interrupted run behind it then had its users come back as conflicts. It now checks them, as it does when there's no exact match. - The lock's temporary file gets a random name and `wx`: no planted link at a predictable name in a shared /tmp, and a crashed run's leftover under the same PID never reads as a held lock. - runs says SOURCE_FILE is to be replaced with the source's path. - The README says the export command shows only where one exists, and that a copy of a source beside different helpers also needs --new-run. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 20 ++++++----- .../commands/migrate/lib/run-store.test.ts | 16 +++++++++ .../src/commands/migrate/lib/run-store.ts | 6 ++-- .../cli-core/src/commands/migrate/run.test.ts | 35 +++++++++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 11 +++--- .../src/commands/migrate/runs.test.ts | 1 + .../cli-core/src/commands/migrate/runs.ts | 6 +++- 7 files changed, 79 insertions(+), 16 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 9e38aad22..a70b23dce 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -1110,13 +1110,14 @@ clerk migrate sources --json | `[source]` | A built-in key, or the path to a source you wrote, to show fully | | `--json` | The same data, on stdout | -`sources` alone prints the table under [Sources](#sources). `sources ` shows one source in -full: its export command, what it carries with a note for each, where each -field lands (`encrypted_password → password`), its fixed defaults, and any -caveats. An unknown key exits 2 and lists the valid ones. There is no -intro/outro gutter. `sources` alone, or with a built-in key, reads a static -registry and runs nothing. With a path, it loads and runs that file, as -`--source` does, to check it and show what it maps. +`sources` alone prints the table under [Sources](#sources). `sources ` +shows one source in full: its export command (when `clerk migrate export` has +one for it), what it carries with a note for each, where each field lands +(`encrypted_password → password`), its fixed defaults, and any caveats. An +unknown key exits 2 and lists the valid ones. There is no intro/outro gutter. +`sources` alone, or with a built-in key, reads a static registry and runs +nothing. With a path, it loads and runs that file, as `--source` does, to check +it and show what it maps. ### `clerk migrate help` @@ -1205,8 +1206,9 @@ and won't start a new one beside it unasked either: while that run isn't undone, it refuses with exit 2 and names it, since the users it created would come back as conflicts. Pass `--new-run` to start a new run anyway, or undo the earlier one first. The hash covers that file -only: a local helper it imports is not part of it, so after editing one, pass -`--new-run` rather than continue a run made with the old helper. +only: a local helper it imports is not part of it, so after editing one, or +when running a copy of the file beside different helpers, pass `--new-run` +rather than continue a run made with the old ones. #### Validation diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts index 44b1f9997..e6636720b 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts @@ -206,6 +206,22 @@ describe("locks and interruptions", () => { expect(() => continueRun(runsDir, run.record)).not.toThrow(); }); + // A predictable name in a shared /tmp could be a planted symlink, and a + // crashed run's leftover under the same PID would read as a held lock. + test("the lock's temporary file has a random name and is created exclusively", () => { + const identity = { sha256: "b".repeat(64), instanceId: "ins_tmp" }; + const write = spyOn(fs, "writeFileSync"); + try { + const release = lockImport(identity); + const tmp = write.mock.calls.find(([target]) => String(target).endsWith(".tmp")); + expect(tmp?.[2]).toMatchObject({ flag: "wx" }); + expect(String(tmp?.[0])).toMatch(new RegExp(`\\.${process.pid}\\.[0-9a-f]{16}\\.tmp$`)); + release(); + } finally { + write.mockRestore(); + } + }); + // A lock created empty and written after reads as stale to a second // process in between, which would remove it and import beside the first. test("a lock is linked into place with its PID already written", () => { diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index fb32b6100..4d3f60b77 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -263,8 +263,10 @@ function acquireLockFile( // as `wx` does, but the lock never exists without its PID, which a second // process would read as stale and remove. const take = () => { - const whole = `${file}.${process.pid}.tmp`; - fs.writeFileSync(whole, String(process.pid), { mode: 0o600 }); + // Random and exclusive: a name no one can plant a link at, and a crashed + // run's leftover (same PID, in a container) never reads as a held lock. + const whole = `${file}.${process.pid}.${randomBytes(8).toString("hex")}.tmp`; + fs.writeFileSync(whole, String(process.pid), { flag: "wx", mode: 0o600 }); try { fs.linkSync(whole, file); } catch (error) { diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 3e71e04b7..2a992c437 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -2426,6 +2426,41 @@ describe("run", () => { expect(created()).toHaveLength(0); }); + // The newer, undone run is of this same version: undone, it starts a new + // run, so the older interrupted one has to be checked too. + test("refuses while an earlier version's run is unfinished behind an undone run of this version", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const older = interruptOnly(); + const second = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, second), `${CUSTOM}\n// second`); + await run({ + input: "export.json", + source: second, + yes: true, + secretKey: "sk_test_x", + newRun: true, + }); + const newer = listRuns(runsDir()).find((record) => record.id !== older)!; + fs.writeFileSync( + path.join(runsDir(), newer.id, "run.json"), + JSON.stringify({ ...readRun(runsDir(), newer.id), status: "undone" }), + ); + requests = []; + + const error = (await run({ + input: "export.json", + source: second, + yes: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain( + `Run ${older} imported this file into this instance with an earlier version`, + ); + expect(created()).toHaveLength(0); + }); + test("an edited source with --new-run starts a new run beside the earlier one", async () => { await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); interruptOnly(); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index b9ea9ad7b..7896a2030 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -530,7 +530,8 @@ function findResume( const latest = importsOfFile(runs, match).find( (record) => sameFileAndInstance(record, match) && record.sourceHash === match.sourceHash, ); - if (!latest) { + // An undone exact match starts a new run, so it checks the others too. + if (!latest || undoneInFull(runsDir, latest)) { // Another source's run of this file, or an earlier version of this one, // sent the same users: starting over beside it would reject every user it // created and leave it unfinished, so that takes --new-run. Any such run @@ -541,9 +542,11 @@ function findResume( ); if (earlier) { const how = - earlier.source === match.source - ? "with an earlier version of this source" - : `with the \`${earlier.source}\` source`; + earlier.source !== match.source + ? `with the \`${earlier.source}\` source` + : earlier.sourceHash !== match.sourceHash + ? "with an earlier version of this source" + : "too, and was not undone"; throwUsageError( `Run ${earlier.id} imported this file into this instance ${how}, so the users it created would come back as conflicts. Nothing was imported.\n` + `Pass --new-run to start a new run beside it, or undo it first with \`${undoCommand(earlier.id, carry)}\`.`, diff --git a/packages/cli-core/src/commands/migrate/runs.test.ts b/packages/cli-core/src/commands/migrate/runs.test.ts index 655537637..d9b541333 100644 --- a/packages/cli-core/src/commands/migrate/runs.test.ts +++ b/packages/cli-core/src/commands/migrate/runs.test.ts @@ -414,6 +414,7 @@ describe("runs ", () => { const { next } = JSON.parse(captured.out) as { next: string[] }; expect(next[0]).toContain(`clerk migrate import ${quoteArg(usersFile())} --source SOURCE_FILE`); + expect(next[0]).toContain("with SOURCE_FILE replaced by the path to the source you wrote"); }); test("an unknown ID is a usage error", async () => { diff --git a/packages/cli-core/src/commands/migrate/runs.ts b/packages/cli-core/src/commands/migrate/runs.ts index 3e14c0934..427c7fe93 100644 --- a/packages/cli-core/src/commands/migrate/runs.ts +++ b/packages/cli-core/src/commands/migrate/runs.ts @@ -182,7 +182,11 @@ function nextCommands( // Continuing is what a stopped run is for: the same file, source and // target pick it up where it left off. const why = state === "interrupted" ? "continue it" : "retry the users it didn't finish"; - return [`Run \`${again}\` to ${why}`, undo]; + // The record knows a source you wrote only by its hash. + const placeholder = record.sourceHash + ? ", with SOURCE_FILE replaced by the path to the source you wrote" + : ""; + return [`Run \`${again}\` to ${why}${placeholder}`, undo]; } if (record.kind === "undo" && state !== "complete" && record.undoes) { return [`Run \`${undoCommand(record.undoes, carry)}\` to retry the users that failed`]; From 970f095963129a2ea9d3502e209057989b38beb0 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 16 Oct 2026 19:34:24 -0400 Subject: [PATCH 20/20] fix(migrate): name an earlier run of any source when a refusal meets its users earlierRunHolding still skipped runs of another source, though findResume and the import lock match on file and instance only. A custom-source --new-run beside a built-in run of the same file got the generic "Fix them" text instead of the run holding its users. It now matches any source. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/run.test.ts | 30 +++++++++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 8 ++--- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 2a992c437..08e5b660a 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -2291,6 +2291,36 @@ describe("run", () => { ); }); + // The checks reject the users a run under another source created, so the + // refusal names that run, as it does for one of the same source. + test("a refusal names the earlier run of another source that holds the users", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const [first] = listRuns(runsDir()); + const other = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync( + path.join(workDir, other), + CUSTOM.replace('key: "myplatform"', 'key: "otherplatform"'), + ); + stubClerk({ + existing: [ + { id: "user_1", external_id: "mp_1" }, + { id: "user_2", external_id: "mp_2" }, + ], + }); + + const error = (await run({ + input: "export.json", + source: other, + newRun: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.message).toContain(`2 of them were created by run ${first!.id}`); + expect(error.examples?.map((example) => example.description)).toContain( + `Undo run ${first!.id} first`, + ); + }); + test("imports through a user-authored source", async () => { await run({ input: "export.json", diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 7896a2030..fe3d0fb16 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -791,14 +791,13 @@ function rejectSummary(checks: ImportChecks, json = false): string { } /** - * The earlier import run, of this source into this instance, that created the + * The earlier import run into this instance, of any source, that created the * users the checks reject as already there: how many of them, and whether it * ran this same file. Fixing a skipped row changes the file, and the changed * file's run meets them; so does the same file run with `--new-run`. */ function earlierRunHolding( runsDir: string, - source: string, instanceId: string, checks: ImportChecks, sha256: string, @@ -808,7 +807,8 @@ function earlierRunHolding( ); if (already.size === 0) return undefined; for (const record of listRuns(runsDir)) { - if (record.kind !== "import" || record.source !== source) continue; + // Any source: findResume and the import lock match on file and instance. + if (record.kind !== "import") continue; if (record.target.instanceId !== instanceId || runState(runsDir, record) === "undone") continue; let count = 0; for (const line of latestUserLines(runsDir, record.id).values()) { @@ -1229,7 +1229,7 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi if (refused) { if (options.json) preview({ refused: true }); - const earlier = earlierRunHolding(runsDir, source, target.instanceId, checks, sha256); + const earlier = earlierRunHolding(runsDir, target.instanceId, checks, sha256); const undoEarlier = earlier ? undoCommand(earlier.id, carry) : undefined; const earlierRun = options.newRun && earlier