diff --git a/.changeset/migrate-custom-sources.md b/.changeset/migrate-custom-sources.md new file mode 100644 index 000000000..fb9542f26 --- /dev/null +++ b/.changeset/migrate-custom-sources.md @@ -0,0 +1,5 @@ +--- +"clerk": minor +--- + +Add `clerk migrate sources`, and let `clerk migrate import --source` take the path to a source you wrote. `clerk migrate sources` lists the built-in sources with the passwords, MFA and metadata each brings across, and `clerk migrate sources ` shows where each field lands, how to export, and any caveats. A source you wrote is a `.ts` or `.js` file in your project, checked before use, so a platform with no built-in source can be imported without rebuilding the CLI. `clerk migrate` is still experimental and needs `CLERK_EXPERIMENTAL=migrate`. diff --git a/packages/cli-core/src/commands/completion/__complete.ts b/packages/cli-core/src/commands/completion/__complete.ts index 06f9d87f1..642281f9e 100644 --- a/packages/cli-core/src/commands/completion/__complete.ts +++ b/packages/cli-core/src/commands/completion/__complete.ts @@ -34,7 +34,10 @@ const HTTP_METHOD_COMPLETIONS: Completion[] = [ { name: "DELETE", description: "Delete resource" }, ]; -/** The built-in migrate sources. Kept off `.choices()` so an unknown key gets the source error. */ +/** + * The built-in migrate sources. `--source` also takes a path to a source you + * wrote, so it cannot use `.choices()`. + */ const SOURCE_COMPLETIONS: Completion[] = sources.map((entry) => ({ name: entry.key, description: entry.label, @@ -74,6 +77,7 @@ const KNOWN_OPTION_VALUES: Record = { * to root, e.g. "open dashboard". */ const KNOWN_POSITIONAL_COMPLETIONS: Record = { + "migrate sources": SOURCE_COMPLETIONS, "open dashboard": KNOWN_DASHBOARD_PATHS.map((path) => ({ name: path, description: "Dashboard subpath", diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 255e35d2e..a70b23dce 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -19,6 +19,7 @@ clerk migrate export [source] [-o ] [--json] clerk migrate import [--source ] [--dry-run] [--allow-partial] [--new-run] [--yes] [--json] clerk migrate runs [run-id] [--json] clerk migrate undo [--dry-run] [--yes] [--include-signed-in] [--json] +clerk migrate sources [source] [--json] clerk migrate help ``` @@ -176,6 +177,19 @@ every run. Run folders are created owner-only (`0700`), and export files `0600`: they hold password hashes and user data. +#### The import lock + +Every import but a dry run also takes a lock outside the run store, before its +checks and prompts: `clerk-migrate-import-.lock` in the temp directory +(`os.tmpdir()`, usually `$TMPDIR` or `/tmp`). The hash covers the file's sha256 +and the instance, not the source, so a second import of the same file into the +same instance is refused with exit 2 even before either has written a run, +whichever source and `--runs-dir` each uses. It lives outside the runs folder because +nothing may be written there before consent. It holds the importing process's +PID and is removed however the import ends. One left by a process that died is +reclaimed by the next import. The refusal names the file, which is safe to +delete when no `clerk migrate import` is running. + `users.ndjson` writes are synchronous appends, so a run interrupted with Ctrl-C still leaves a complete record of everything already processed. A line that cannot be written stops that user's create from going out. An export's file @@ -273,6 +287,21 @@ The file is an envelope around the users: parameters, when it could read them: the import then needs no `--firebase-*` flags. When it couldn't, pass them to the import, or enter them at its prompt. +`--source` with a different built-in name is refused, since it would map the +users with the wrong fields. `--source` with the path to a +[source you wrote](#custom-sources) replaces the envelope's: it gets the same +users, in the platform's own shape, so it can bring across what the built-in +mapping leaves out, such as a WorkOS `metadata.username`. The import keeps its +link to the export run, and `run.json` records the source the envelope named +as `envelopeSource`. A Firebase export keeps its hash parameters through it: +the envelope's, the `--firebase-*` flags or the variables reach the source's +`postTransform` as `context.firebaseHashConfig`, as they do the built-in's. + +An earlier import of the same export into the same instance, through the +built-in or another source, holds the same users. While it isn't undone, a +re-run through your source refuses with exit 2 and names it, rather than +starting a new run beside it: pass `--new-run`, or undo it first. + `--json` prints the result on stdout instead — `{ target, run, output, users, skipped, coverage, next }` — and never prompts, so a missing credential exits 2 naming the flag to pass. `next`, and the commands printed after a human run, @@ -653,7 +682,7 @@ clerk migrate import # a human is asked | Flag | Description | | --------------------------------------- | ------------------------------------------------------------------------------------------------- | | `[file\|export-run-id]` | The export file, or the ID of the export run that wrote it | -| `--source ` | Where the file came from: one of the [sources](#sources) | +| `--source ` | Where the file came from: a [source](#sources), or one you wrote | | `--dry-run` | Run the [checks](#checks) against the instance, and write nothing | | `--allow-partial` | Import the users that pass, and record the rest as skipped | | `--new-run` | Start a new run instead of [continuing](#re-running) an earlier one | @@ -724,8 +753,8 @@ which skip the checks too. #### Re-running Running the same import again continues where it left off. The match is the -file's sha256, the source, and the instance ID; the latest matching import run -decides what happens: +file's sha256, the source (and a custom source's content hash), and the +instance ID; the latest matching import run decides what happens: | Latest match | Re-running does | | ----------------------------------------- | ---------------------------------------------------------------------- | @@ -1067,6 +1096,29 @@ With `writeFailed` or `stopped`, `run` is the undo run as it stands on disk, once one was written: its counts come from its lines, and `state` says how it stopped, as in `clerk migrate runs`. +### `clerk migrate sources` + +```sh +clerk migrate sources # every source, with what it carries +clerk migrate sources betterauth # one source in full +clerk migrate sources ./my-source.ts # a source you wrote +clerk migrate sources --json +``` + +| Flag | Description | +| ---------- | ---------------------------------------------------------------- | +| `[source]` | A built-in key, or the path to a source you wrote, to show fully | +| `--json` | The same data, on stdout | + +`sources` alone prints the table under [Sources](#sources). `sources ` +shows one source in full: its export command (when `clerk migrate export` has +one for it), what it carries with a note for each, where each field lands +(`encrypted_password → password`), its fixed defaults, and any caveats. An +unknown key exits 2 and lists the valid ones. There is no intro/outro gutter. +`sources` alone, or with a built-in key, reads a static registry and runs +nothing. With a path, it loads and runs that file, as `--source` does, to check +it and show what it maps. + ### `clerk migrate help` `clerk migrate help` and `clerk migrate --help` print the help for the @@ -1089,13 +1141,95 @@ brings across. Adding a platform is one file in `sources/` plus one line in | `supabase` | Supabase `auth.users` export | yes | no | partial | | `workos` | WorkOS User Management API | no | no | yes | -An unknown `--source` exits 2 with the list of valid keys. - There is no column for social sign-ins, because no source copies them and none -needs to. Enable the same providers in Clerk, and a user who signs in with one -is linked to their imported account by verified email. See +needs to. Every source shows the same note instead: enable the same providers in +Clerk, and a user who signs in with one is linked to their imported account by +verified email. See [account linking](https://clerk.com/docs/guides/configure/auth-strategies/social-connections/account-linking). +### `--source` + +`clerk migrate import` takes `--source `: + +- A value starting with `./`, `../` or `/`, or ending in `.ts`, `.js` or + `.mjs`, is loaded as a [custom source](#custom-sources). +- Anything else must be a built-in key. An unknown key exits 2 with the list of + valid keys. + +A file `clerk migrate export` wrote names its own source, so it needs none. + +### Custom sources + +Migrating from a platform with no built-in, without recompiling the CLI: + +```sh +clerk migrate import users.json --source ./my-platform.ts +``` + +The file lives in **your** project, not in the CLI, and is imported at runtime. +It exports the same shape the built-ins use, as plain data. It can't import +from the CLI, since there is nothing in a compiled binary to import from, but +it can import your project's own files: + +```ts +export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform's admin console.", + transformer: { + account_ref: "userId", // required: becomes the Clerk user's external_id + contact_email: "email", + given: "firstName", + family: "lastName", + pw_bcrypt: "password", + }, + carries: { + passwords: { level: "yes", note: "bcrypt hashes from the pw_bcrypt column." }, + mfa: { level: "no", note: "Not exported." }, + metadata: { level: "no", note: "Not exported." }, + }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { + if (!user.firstName) delete user.firstName; + }, +}; +``` + +TypeScript is fine — Bun's transpiler is part of the runtime, so `interface`, +`satisfies` and `as const` all work in a file the compiled binary imports. +Plain `.js` works too. + +An import run records a custom source's key and a hash of the file, so an +edited source counts as a different source. A re-run with an edited source, +or with another source altogether, won't continue the earlier run of the file, +and won't start a new one beside it unasked either: while that run isn't +undone, it refuses with exit 2 and names it, since the users it created would +come back as conflicts. Pass `--new-run` to start +a new run anyway, or undo the earlier one first. The hash covers that file +only: a local helper it imports is not part of it, so after editing one, or +when running a copy of the file beside different helpers, pass `--new-run` +rather than continue a run made with the old ones. + +#### Validation + +The file is code the CLI executes, so its shape is checked before use and +rejected with the specific problem rather than crashing mid-pipeline: + +| Problem | Message | +| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| Path does not exist | `No source file at /abs/path.ts.` | +| No default export, but a named one | ``has no default export. Found named export `myPlatform` — did you mean `export default`?`` | +| Does not parse | `Could not load ./f.ts: Expected identifier but found ","` | +| Nothing maps to `userId` | ``no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id`` | +| No `carries` | `` `carries` must say what the source brings across: { passwords, mfa, metadata }, each { level, note } `` | +| `key` clashes with a built-in | `key is "clerk", which is already a built-in source` | +| `preTransform` or `postTransform` is not a function | `` `preTransform` must be a function when present `` (likewise `postTransform`) | +| `caveats` is not a list of strings | `` `caveats` must be an array of strings when present `` | + +The `userId` check is the load-bearing one: without it the import would run to +completion and create every user with no `external_id`, which is what makes a +migration re-runnable. + ### Metadata Metadata a user can edit on the source platform — Auth0's `user_metadata`, @@ -1128,6 +1262,7 @@ Better Auth's scrypt uses the hex salt string as the salt and a 64-byte key, which is what `scrypt_werkzeug` verifies once N, r and p are written inline. It also normalizes a password to NFKC before hashing, and Clerk does not, so a password whose NFKC form differs will not verify and that user resets it. +`clerk migrate sources betterauth` lists that caveat. A password Clerk cannot verify is **dropped, not rejected**: dropping it doesn't reject the user, who imports without it and can sign in another way or reset @@ -1194,7 +1329,8 @@ An export with no password hashes needs no parameters at all. What a source maps _onto_. Every user is validated against this schema before any request is made, so a field a source produces that is not listed here is dropped — Zod strips unknown keys — and never reaches Clerk. The checks -warn about each one (`Clerk won't store: …`). +warn about each one (`Clerk won't store: …`). Writing a custom source means +targeting these names exactly. The schema lives in `validator.ts`; adding a platform means adding a source, not editing it. diff --git a/packages/cli-core/src/commands/migrate/index.test.ts b/packages/cli-core/src/commands/migrate/index.test.ts index 449e4853c..b00ed9e60 100644 --- a/packages/cli-core/src/commands/migrate/index.test.ts +++ b/packages/cli-core/src/commands/migrate/index.test.ts @@ -32,12 +32,13 @@ describe("registerMigrate", () => { expect(migrate?.description()).toContain("Migrate users"); }); - test("registers import, export, undo and runs", () => { + test("registers import, export, undo, runs and sources", () => { expect(findCommand(["migrate"])?.commands.map((cmd) => cmd.name())).toEqual([ "import", "export", "undo", "runs", + "sources", ]); }); @@ -157,6 +158,12 @@ describe("registerMigrate", () => { expect(runsCommand?.options.map((option) => option.long)).toEqual(["--json", "--runs-dir"]); }); + test("registers sources with an optional source and --json", () => { + const sourcesCommand = findCommand(["migrate", "sources"]); + expect(sourcesCommand?.registeredArguments[0]?.required).toBe(false); + expect(sourcesCommand?.options.map((option) => option.long)).toEqual(["--json"]); + }); + test("registers undo with a required run ID and its flags", () => { const undoCommand = findCommand(["migrate", "undo"]); expect(undoCommand?.registeredArguments[0]?.required).toBe(true); @@ -172,9 +179,9 @@ describe("registerMigrate", () => { ]); }); - // No `.choices()`: an unknown key reaches resolveSource, whose error lists - // the valid ones. Completion offers them through `KNOWN_OPTION_VALUES`. - test("--source accepts any value", () => { + // It also takes a path, so it cannot use `.choices()`: completion offers the + // built-in keys through `KNOWN_OPTION_VALUES` instead. + test("--source accepts any value, so a path to a source you wrote gets through", () => { const option = findCommand(["migrate", "import"])?.options.find((o) => o.long === "--source"); expect(option?.argChoices).toBeUndefined(); }); diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index e0c1728c1..0e27c76d2 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -8,6 +8,7 @@ import { RUNS_DIR_DESCRIPTION, RUNS_DIR_FLAG } from "./lib/run-store.ts"; import { run } from "./run.ts"; import { runs } from "./runs.ts"; import { undo } from "./undo.ts"; +import { list as sources } from "./sources/list.ts"; export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = process.env): void { if (!isExperimentEnabled("migrate", env)) { @@ -51,6 +52,7 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce command: "clerk migrate undo 20260929-141502-a1b2", description: "Delete the users an import created", }, + { command: "clerk migrate sources", description: "What each source brings across" }, ]); // `-y` is read several layers down — by the credential-retry loop and the @@ -81,8 +83,8 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce .description("Import users from an exported JSON or CSV file") .argument("[file|export-run-id]", "The export file, or the ID of the export run that wrote it") .option( - "--source ", - "Where the file came from. Not needed for a file from `clerk migrate export`", + "--source ", + "Where the file came from: a built-in source, or a source you wrote. Not needed for a file from `clerk migrate export`, though a source you wrote replaces its own", ) .option("--dry-run", "Check the file against the instance, report, and write nothing") .option("--allow-partial", "Import the users that pass the checks, and skip the rest") @@ -129,6 +131,10 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce command: "clerk migrate import users.json --source clerk --allow-partial --yes", description: "Import a Clerk Dashboard export, skipping users that would be rejected", }, + { + command: "clerk migrate import users.json --source ./my-source.ts --yes", + description: "Import with a source you wrote", + }, ]) .action(async (input, _opts, cmd) => run({ @@ -184,4 +190,23 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce .action(async (runId, _opts, cmd) => runs(runId, cmd.optsWithGlobals() as Parameters[1]), ); + + // A compiled binary has no source tree to grep, so the available mappings + // need a command rather than only appearing in the interactive picker. + migrateCommand + .command("sources") + .description("List the sources an import can read, or show one in full") + .argument("[source]", "A built-in source, or the path to a source you wrote") + .option("--json", "Output as JSON") + .setExamples([ + { command: "clerk migrate sources", description: "What each source brings across" }, + { + command: "clerk migrate sources betterauth", + description: "Where each field lands, how to export, and caveats", + }, + { command: "clerk migrate sources ./my-source.ts", description: "Check a source you wrote" }, + ]) + .action(async (source, _opts, cmd) => + sources(source, cmd.optsWithGlobals() as Parameters[1]), + ); } diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts index 974b28740..e6636720b 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts @@ -19,6 +19,8 @@ import { latestUserLines, listRuns, lockFile, + importLockFile, + lockImport, newRunId, patchRun, readRun, @@ -204,6 +206,60 @@ describe("locks and interruptions", () => { expect(() => continueRun(runsDir, run.record)).not.toThrow(); }); + // A predictable name in a shared /tmp could be a planted symlink, and a + // crashed run's leftover under the same PID would read as a held lock. + test("the lock's temporary file has a random name and is created exclusively", () => { + const identity = { sha256: "b".repeat(64), instanceId: "ins_tmp" }; + const write = spyOn(fs, "writeFileSync"); + try { + const release = lockImport(identity); + const tmp = write.mock.calls.find(([target]) => String(target).endsWith(".tmp")); + expect(tmp?.[2]).toMatchObject({ flag: "wx" }); + expect(String(tmp?.[0])).toMatch(new RegExp(`\\.${process.pid}\\.[0-9a-f]{16}\\.tmp$`)); + release(); + } finally { + write.mockRestore(); + } + }); + + // A lock created empty and written after reads as stale to a second + // process in between, which would remove it and import beside the first. + test("a lock is linked into place with its PID already written", () => { + const identity = { sha256: "a".repeat(64), instanceId: "ins_lock" }; + const write = spyOn(fs, "writeFileSync"); + const link = spyOn(fs, "linkSync"); + try { + const release = lockImport(identity); + const lockPaths = write.mock.calls.map(([target]) => String(target)); + // Where hard links are refused, the fallback below writes the lock itself. + if (link.mock.results.every((result) => result.type === "return")) { + expect(lockPaths.every((target) => target.endsWith(".tmp"))).toBe(true); + } + release(); + } finally { + write.mockRestore(); + link.mockRestore(); + } + }); + + test("a filesystem without hard links gets the lock written exclusively", () => { + const identity = { sha256: "b".repeat(64), instanceId: "ins_nolink" }; + const link = spyOn(fs, "linkSync").mockImplementation(() => { + throw Object.assign(new Error("not supported"), { code: "EPERM" }); + }); + const write = spyOn(fs, "writeFileSync"); + try { + const release = lockImport(identity); + const lock = write.mock.calls.find(([target]) => String(target).endsWith(".lock")); + expect(lock?.[2]).toMatchObject({ flag: "wx" }); + expect(fs.readFileSync(importLockFile(identity), "utf-8")).toBe(String(process.pid)); + release(); + } finally { + write.mockRestore(); + link.mockRestore(); + } + }); + test("reads as running while another live process holds the lock", () => { const run = startRun(runsDir, init); fs.writeFileSync(path.join(run.dir, "lock"), "1"); diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index 1ca85c585..4d3f60b77 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -20,6 +20,7 @@ import { createHash, randomBytes } from "node:crypto"; import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import { resolveProfile } from "../../../lib/config.ts"; import { throwUsageError } from "../../../lib/errors.ts"; @@ -89,6 +90,8 @@ export type RunRecord = { * continue for those. */ flags?: ImportFlags; + /** The source the export file named, when a custom source replaced it. */ + envelopeSource?: string; /** * A Firebase import's hash parameters, as SHA-256 of the four (never the * parameters themselves): every digest it builds carries them, so a continue @@ -213,9 +216,14 @@ function isPidAlive(pid: number): boolean { * the same PID every run, so a killed run's lock would otherwise read as live. */ export function liveLockPid(runsDir: string, id: string): number | undefined { + return livePidIn(lockFile(runsDir, id)); +} + +/** The live PID a lock file holds, by the same rules as {@link liveLockPid}. */ +function livePidIn(file: string): number | undefined { let raw: string; try { - raw = fs.readFileSync(lockFile(runsDir, id), "utf-8"); + raw = fs.readFileSync(file, "utf-8"); } catch { return undefined; } @@ -238,31 +246,59 @@ const isMissing = (error: unknown) => (error as NodeJS.ErrnoException).code === * both read "free" and both write; a stale lock is removed and taken once. */ function acquireLock(runsDir: string, id: string): void { - const file = lockFile(runsDir, id); - const refuse = (holder: number | undefined): never => + acquireLockFile(lockFile(runsDir, id), (holder, file) => throwUsageError( `Run ${id} is in use by another process${holder ? ` (PID ${holder})` : ""}. Wait for it to finish, then try again. ` + `If that process is not a migrate run, delete ${file}.`, - ); - const take = () => fs.writeFileSync(file, String(process.pid), { flag: "wx", mode: 0o600 }); + ), + ); +} + +/** Takes the lock at `file`, or calls `refuse` with its live holder. */ +function acquireLockFile( + file: string, + refuse: (holder: number | undefined, file: string) => never, +): void { + // Written whole, then linked into place: the link fails on an existing lock + // as `wx` does, but the lock never exists without its PID, which a second + // process would read as stale and remove. + const take = () => { + // Random and exclusive: a name no one can plant a link at, and a crashed + // run's leftover (same PID, in a container) never reads as a held lock. + const whole = `${file}.${process.pid}.${randomBytes(8).toString("hex")}.tmp`; + fs.writeFileSync(whole, String(process.pid), { flag: "wx", mode: 0o600 }); + try { + fs.linkSync(whole, file); + } catch (error) { + if (isExists(error)) throw error; + // A filesystem without hard links. + fs.writeFileSync(file, String(process.pid), { flag: "wx", mode: 0o600 }); + } finally { + fs.rmSync(whole, { force: true }); + } + }; try { take(); - log.debug(`migrate: locked run ${id} (${file})`); + log.debug(`migrate: took the lock ${file}`); return; } catch (error) { if (!isExists(error)) throw error; } - const holder = liveLockPid(runsDir, id); - if (holder !== undefined) refuse(holder); - log.debug(`migrate: removing a stale lock on run ${id} (${file})`); + // ponytail: two processes that both read the same stale lock can both + // remove it, and the second can remove the first's new one. Clerk still + // refuses the second create of any email, phone or username, so the cost + // is failed lines, not duplicate users. A lock on the reclaim would close it. + const holder = livePidIn(file); + if (holder !== undefined) refuse(holder, file); + log.debug(`migrate: removing a stale lock (${file})`); fs.rmSync(file, { force: true }); try { take(); - log.debug(`migrate: locked run ${id} (${file})`); + log.debug(`migrate: took the lock ${file}`); } catch (error) { // Another process took the stale lock between the remove and the write. - if (isExists(error)) refuse(liveLockPid(runsDir, id)); + if (isExists(error)) refuse(livePidIn(file), file); throw error; } } @@ -272,6 +308,45 @@ function releaseLock(runsDir: string, id: string): void { fs.rmSync(lockFile(runsDir, id), { force: true }); } +/** + * What makes two imports the same job: the file and the instance, by its ID + * or its key's stand-in ID. Not the source: whichever source maps the file, + * its users are the same people. + */ +export type ImportIdentity = { sha256: string; instanceId: string }; + +/** + * The lock one import of a file into an instance holds. In the + * temp directory, as it is taken before consent, when nothing may be written + * to the runs folder yet. Not keyed by the runs folder: two imports with + * different `--runs-dir` values would still send the same users. + */ +export function importLockFile(identity: ImportIdentity): string { + const key = createHash("sha256") + .update(JSON.stringify([identity.sha256, identity.instanceId])) + .digest("hex") + .slice(0, 32); + return path.join(os.tmpdir(), `clerk-migrate-import-${key}.lock`); +} + +/** + * Takes the lock for one import of this file into this instance, before + * its checks, so no second process can pass them and start a run beside it. + * + * @returns Releases the lock. + * @throws UsageError when a live process holds it. + */ +export function lockImport(identity: ImportIdentity): () => void { + const file = importLockFile(identity); + acquireLockFile(file, (holder) => + throwUsageError( + `Another process${holder ? ` (PID ${holder})` : ""} is importing this file into this instance right now. ` + + `Wait for it to finish. If that process is not a migrate run, delete ${file}.`, + ), + ); + return () => fs.rmSync(file, { force: true }); +} + // --- Reading --------------------------------------------------------------- /** One run's record, or `undefined` when the folder holds no readable run. */ diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts index 561ad655e..766ae00e7 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -5,6 +5,7 @@ import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; import clerkSource from "../sources/clerk.ts"; import type { SourceEntry } from "../types.ts"; +import { __resetCustomSourcesForTesting, registerCustomSource } from "../sources/registry.ts"; import { consolidateClerkIdentifiers, flattenObjectSelectively, @@ -298,6 +299,27 @@ describe("loadUsersFromFile", () => { } }); + test("a custom preTransform's rows win over a CSV file", async () => { + registerCustomSource({ + ...clerkSource, + key: "rows-from-pretransform", + preTransform: (filePath) => ({ + filePath, + data: [{ id: "from-pretransform", primary_email_address: "p@x.dev" }], + }), + }); + try { + fs.writeFileSync( + path.join(workDir, "ignored.csv"), + "id,primary_email_address\nfile,f@x.dev\n", + ); + const { users } = await loadUsersFromFile("ignored.csv", "rows-from-pretransform"); + expect(users.map((user) => user.userId)).toEqual(["from-pretransform"]); + } finally { + __resetCustomSourcesForTesting(); + } + }); + test("rejects a JSON file that is not an array of users", async () => { fs.writeFileSync(path.join(workDir, "wrapped.json"), JSON.stringify({ users: [] })); await expect(loadUsersFromFile("wrapped.json", "clerk")).rejects.toThrow(CliError); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 383f693a7..08e5b660a 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -28,8 +28,11 @@ import { readRun, sha256File, startRun, + importLockFile, } from "./lib/run-store.ts"; +import { keyInstanceId } from "./lib/target.ts"; import { explainErrors, run, validateRunOptions } from "./run.ts"; +import { __resetCustomSourcesForTesting } from "./sources/registry.ts"; // After run.ts: imported first, signals.ts loads version.ts ahead of its Bun // macro here, and the file fails to load. import { _resetInterruptState, abortInFlight, beginInterrupt } from "../../lib/signals.ts"; @@ -596,6 +599,161 @@ describe("run", () => { expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(2); }); + test("says how to map the users differently when a built-in source contradicts it", async () => { + const { record } = exportRun("clerk", export2); + + await expect(run({ ...noSource, source: "auth0", input: record.id })).rejects.toThrow( + /pass the path to a source you wrote/, + ); + }); + + /** A source of your own, for an Auth0 export, that also brings `nickname` across. */ + const ownAuth0Source = () => { + const file = `./own-auth0-${Date.now()}-${Math.random().toString(36).slice(2)}.ts`; + fs.writeFileSync( + path.join(workDir, file), + `export default { + key: "myauth0", + label: "My Auth0", + transformer: { user_id: "userId", email: "email", nickname: "firstName" }, + carries: { + passwords: { level: "no", note: "None." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, + };`, + ); + return file; + }; + const auth0Rows = [{ user_id: "auth0|a1", email: "a@x.dev", nickname: "Ada" }]; + + // The built-in mapping leaves `nickname` out; the customer's source brings it. + test("a source you wrote replaces the envelope's, reading its users", async () => { + const { record } = exportRun("auth0", auth0Rows); + const own = ownAuth0Source(); + + await run({ ...noSource, source: own, input: record.id }); + + const created = requests.filter((r) => r.url.endsWith("/v1/users")); + expect(created).toHaveLength(1); + expect(created[0]?.body).toMatchObject({ external_id: "auth0|a1", first_name: "Ada" }); + expect(captured.err).toContain("in place of the built-in"); + }); + + test("an import through a source you wrote keeps its export run, and records what it replaced", async () => { + const { record } = exportRun("auth0", auth0Rows); + + await run({ ...noSource, source: ownAuth0Source(), input: record.id }); + + const imported = listRuns(runsDir()).find((r) => r.kind === "import"); + expect(imported).toMatchObject({ + source: "myauth0", + fromExport: record.id, + envelopeSource: "auth0", + }); + }); + + // A re-run without it would map the rest with the built-in source. + // Import with the built-in, notice a field is missing, write a source, re-run: + // the first run's users are the same people, whichever source maps them. + test("refuses a source you wrote while the built-in's run of the same export holds its users", async () => { + const { record } = exportRun("auth0", auth0Rows); + await run({ ...noSource, input: record.id }); + const [first] = listRuns(runsDir()).filter((r) => r.kind === "import"); + requests = []; + + const error = (await run({ ...noSource, source: ownAuth0Source(), input: record.id }).catch( + (caught: unknown) => caught, + )) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain( + `Run ${first!.id} imported this file into this instance with the \`auth0\` source`, + ); + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(0); + + await run({ ...noSource, source: ownAuth0Source(), input: record.id, newRun: true }); + expect(listRuns(runsDir()).filter((r) => r.kind === "import")).toHaveLength(2); + }); + + // One lock for the file and instance, whichever source each import uses. + test("a source you wrote takes the same import lock as the built-in", async () => { + const { record, file } = exportRun("auth0", auth0Rows); + const lock = importLockFile({ + sha256: sha256File(file), + instanceId: keyInstanceId(baseOptions.secretKey), + }); + fs.writeFileSync(lock, "1"); + try { + await expect( + run({ ...noSource, source: ownAuth0Source(), input: record.id }), + ).rejects.toThrow(/importing this file into this instance right now/); + } finally { + fs.rmSync(lock, { force: true }); + } + }); + + // A Firebase export's digests need the project's parameters, whatever maps it. + test("a source you wrote over a Firebase export gets its hash parameters", async () => { + const { record } = exportRun( + "firebase", + [{ localId: "f1", email: "f@x.dev", passwordHash: "HASH", passwordSalt: "SALT" }], + { + firebase: { + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }, + }, + ); + const own = `./own-firebase-${Date.now()}.ts`; + fs.writeFileSync( + path.join(workDir, own), + `export default { + key: "myfirebase", + label: "My Firebase", + transformer: { localId: "userId", email: "email", passwordHash: "passwordHash", passwordSalt: "salt" }, + carries: { + passwords: { level: "yes", note: "scrypt." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, + defaults: { passwordHasher: "scrypt_firebase" }, + postTransform: (user, context) => { + const c = context.firebaseHashConfig; + if (!c) throw new Error("no hash parameters"); + user.password = [user.passwordHash, user.salt, c.base64_signer_key, c.base64_salt_separator, c.rounds, c.mem_cost].join("$"); + delete user.passwordHash; + delete user.salt; + }, + };`, + ); + + await run({ ...noSource, source: own, input: record.id }); + + const created = requests.find((r) => r.url.endsWith("/v1/users")); + expect(created?.body).toMatchObject({ + password_hasher: "scrypt_firebase", + password_digest: "HASH$SALT$SIGNER$Bw==$8$14", + }); + const imported = listRuns(runsDir()).find((r) => r.kind === "import"); + expect(imported?.firebaseHash).toMatch(/^[0-9a-f]{64}$/); + }); + + test("the printed command keeps a source you wrote in place of the envelope's", async () => { + const { record } = exportRun("auth0", auth0Rows); + const own = ownAuth0Source(); + + const error = (await run({ ...noSource, source: own, input: record.id, yes: false }).catch( + (caught: unknown) => caught, + )) as CliError; + + expect(error.examples?.[0]?.command).toContain( + `clerk migrate import ${record.id} --source ${own}`, + ); + }); + test("refuses a run ID that is not an export", async () => { await run(baseOptions); const [imported] = listRuns(runsDir()); @@ -1947,6 +2105,80 @@ describe("run", () => { fs.writeFileSync(path.join(runsDir(), first!.id, "lock"), "1"); expect(await exitCodeOf(run(baseOptions))).toBe(EXIT_CODE.USAGE); + // A new run would send the same external IDs alongside it. + expect(await exitCodeOf(run({ ...baseOptions, newRun: true }))).toBe(EXIT_CODE.USAGE); + }); + + // The scan of runs alone has a gap: a second process can pass it before + // either has written a run. The import's own lock closes it. + test("refuses while another process holds this import's lock, with no run yet", async () => { + // Keyed by the key, so it holds whether or not `GET /v1/instance` answers. + const lock = importLockFile({ + sha256: sha256File(path.join(workDir, "export.json")), + instanceId: keyInstanceId(baseOptions.secretKey), + }); + // PID 1 is always alive, and never this test. + fs.writeFileSync(lock, "1"); + try { + await expect(run(baseOptions)).rejects.toThrow( + /importing this file into this instance right now/, + ); + } finally { + fs.rmSync(lock, { force: true }); + } + expect(created()).toHaveLength(0); + expect(listRuns(runsDir())).toHaveLength(0); + }); + + // A different --runs-dir is a different record of the same job. + test("refuses while another process holds this import's lock under another runs folder", async () => { + const lock = importLockFile({ + sha256: sha256File(path.join(workDir, "export.json")), + instanceId: keyInstanceId(baseOptions.secretKey), + }); + fs.writeFileSync(lock, "1"); + try { + await expect( + run({ ...baseOptions, runsDir: path.join(workDir, "other-runs") }), + ).rejects.toThrow(/importing this file into this instance right now/); + } finally { + fs.rmSync(lock, { force: true }); + } + expect(created()).toHaveLength(0); + }); + + // A second key for the same instance has its own stand-in ID. + test("refuses while another key's process holds this instance's lock", async () => { + const lock = importLockFile({ + sha256: sha256File(path.join(workDir, "export.json")), + instanceId: "ins_1", + }); + fs.writeFileSync(lock, "1"); + try { + await expect(run(baseOptions)).rejects.toThrow( + /importing this file into this instance right now/, + ); + } finally { + fs.rmSync(lock, { force: true }); + } + expect(created()).toHaveLength(0); + // The key's own lock, taken first, is let go when the second is refused. + const own = importLockFile({ + sha256: sha256File(path.join(workDir, "export.json")), + instanceId: keyInstanceId(baseOptions.secretKey), + }); + expect(fs.existsSync(own)).toBe(false); + }); + + test("lets go of the import's locks when the run ends", async () => { + await run(baseOptions); + const lockFor = (instanceId: string) => + importLockFile({ + sha256: sha256File(path.join(workDir, "export.json")), + instanceId, + }); + expect(fs.existsSync(lockFor(keyInstanceId(baseOptions.secretKey)))).toBe(false); + expect(fs.existsSync(lockFor("ins_1"))).toBe(false); }); // An edited file is a different job. @@ -2007,6 +2239,351 @@ describe("run", () => { }); }); + describe("--source ", () => { + const CUSTOM = `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", contact_email: "email", given: "firstName", pw: "password" }, + carries: { + passwords: { level: "yes", note: "bcrypt." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { if (!user.firstName) delete user.firstName; }, + };`; + + let customFile: string; + let customCounter = 0; + + beforeEach(() => { + // A fresh filename each time: dynamic import() caches by URL, so reusing + // one would silently return a previous test's module. + customFile = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, customFile), CUSTOM); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { account_ref: "mp_1", contact_email: "a@x.dev", given: "Ada", pw: BCRYPT }, + { account_ref: "mp_2", contact_email: "b@x.dev", given: "", pw: BCRYPT }, + ]), + ); + }); + + afterEach(() => { + __resetCustomSourcesForTesting(); + }); + + const created = () => requests.filter((r) => r.url.endsWith("/v1/users")); + + // The registered key isn't something --source accepts; the path is. + test("the printed command names the source's path, not its key", async () => { + const error = (await run({ + input: "export.json", + source: customFile, + secretKey: "sk_test_x", + json: true, + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.examples?.[0]?.command).toBe( + `clerk migrate import export.json --source ${customFile} --secret-key --json --yes`, + ); + }); + + // The checks reject the users a run under another source created, so the + // refusal names that run, as it does for one of the same source. + test("a refusal names the earlier run of another source that holds the users", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const [first] = listRuns(runsDir()); + const other = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync( + path.join(workDir, other), + CUSTOM.replace('key: "myplatform"', 'key: "otherplatform"'), + ); + stubClerk({ + existing: [ + { id: "user_1", external_id: "mp_1" }, + { id: "user_2", external_id: "mp_2" }, + ], + }); + + const error = (await run({ + input: "export.json", + source: other, + newRun: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.message).toContain(`2 of them were created by run ${first!.id}`); + expect(error.examples?.map((example) => example.description)).toContain( + `Undo run ${first!.id} first`, + ); + }); + + test("imports through a user-authored source", async () => { + await run({ + input: "export.json", + source: customFile, + yes: true, + secretKey: "sk_test_x", + }); + + expect(created().map((r) => (r.body as { external_id: string }).external_id)).toEqual([ + "mp_1", + "mp_2", + ]); + expect(captured.err).toContain("myplatform"); + expect(captured.err).toContain("source from"); + }); + + test("applies the custom source's defaults and postTransform", async () => { + await run({ + input: "export.json", + source: customFile, + yes: true, + secretKey: "sk_test_x", + }); + + const bodies = created().map((r) => r.body as Record); + expect(bodies[0]).toMatchObject({ first_name: "Ada", password_hasher: "bcrypt" }); + // postTransform dropped the empty given name rather than sending "". + expect("first_name" in (bodies[1] ?? {})).toBe(false); + }); + + // An edited source misses the live run's `sourceHash`, but both would send + // the same external IDs. + test("refuses while a run of this file with an earlier version of the source is live", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const [first] = listRuns(runsDir()); + const record = readRun(runsDir(), first!.id)!; + delete record.finishedAt; + fs.writeFileSync(path.join(runsDir(), first!.id, "run.json"), JSON.stringify(record)); + // PID 1 is always alive, and never this test. + fs.writeFileSync(path.join(runsDir(), first!.id, "lock"), "1"); + + const edited = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, edited), `${CUSTOM}\n// edited`); + requests = []; + await expect( + run({ input: "export.json", source: edited, yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/importing this file right now in another process/); + expect(created()).toHaveLength(0); + }); + + /** Rewrites the only run as one a crash stopped, and returns its ID. */ + const interruptOnly = (): string => { + const [first] = listRuns(runsDir()); + const record = readRun(runsDir(), first!.id)!; + delete record.finishedAt; + fs.writeFileSync(path.join(runsDir(), first!.id, "run.json"), JSON.stringify(record)); + return first!.id; + }; + + test("continues an interrupted run when the source is unchanged", async () => { + stubClerk({ failing: new Set(["mp_2"]) }); + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const id = interruptOnly(); + requests = []; + process.exitCode = 0; + stubClerk(); + + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + + expect(created().map((r) => (r.body as { external_id: string }).external_id)).toEqual([ + "mp_2", + ]); + expect(captured.err).toContain(`Continuing run ${id}`); + expect(listRuns(runsDir())).toHaveLength(1); + }); + + // Starting over would bring the users the earlier run created back as conflicts. + test("refuses an edited source while an earlier version's run is interrupted, naming it", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const id = interruptOnly(); + const edited = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, edited), `${CUSTOM}\n// edited`); + requests = []; + + const error = (await run({ + input: "export.json", + source: edited, + yes: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain( + `Run ${id} imported this file into this instance with an earlier version`, + ); + expect(error.message).toContain("--new-run"); + expect(created()).toHaveLength(0); + }); + + // The newest run being undone doesn't clear an older one that still holds users. + test("refuses an edited source while any earlier version's run is unfinished, not just the newest", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const older = interruptOnly(); + const second = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, second), `${CUSTOM}\n// second`); + await run({ + input: "export.json", + source: second, + yes: true, + secretKey: "sk_test_x", + newRun: true, + }); + const newer = listRuns(runsDir()).find((record) => record.id !== older)!; + fs.writeFileSync( + path.join(runsDir(), newer.id, "run.json"), + JSON.stringify({ ...readRun(runsDir(), newer.id), status: "undone" }), + ); + const third = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, third), `${CUSTOM}\n// third`); + requests = []; + + const error = (await run({ + input: "export.json", + source: third, + yes: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.message).toContain( + `Run ${older} imported this file into this instance with an earlier version`, + ); + expect(error.message).toContain(`clerk migrate undo ${older} --secret-key `); + expect(created()).toHaveLength(0); + }); + + // The newer, undone run is of this same version: undone, it starts a new + // run, so the older interrupted one has to be checked too. + test("refuses while an earlier version's run is unfinished behind an undone run of this version", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const older = interruptOnly(); + const second = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, second), `${CUSTOM}\n// second`); + await run({ + input: "export.json", + source: second, + yes: true, + secretKey: "sk_test_x", + newRun: true, + }); + const newer = listRuns(runsDir()).find((record) => record.id !== older)!; + fs.writeFileSync( + path.join(runsDir(), newer.id, "run.json"), + JSON.stringify({ ...readRun(runsDir(), newer.id), status: "undone" }), + ); + requests = []; + + const error = (await run({ + input: "export.json", + source: second, + yes: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain( + `Run ${older} imported this file into this instance with an earlier version`, + ); + expect(created()).toHaveLength(0); + }); + + test("an edited source with --new-run starts a new run beside the earlier one", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + interruptOnly(); + const edited = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, edited), `${CUSTOM}\n// edited`); + requests = []; + + await run({ + input: "export.json", + source: edited, + yes: true, + secretKey: "sk_test_x", + newRun: true, + }); + + expect(listRuns(runsDir())).toHaveLength(2); + }); + + // The loader caches by URL: an edit at the same path must still load the + // edited mappings, the ones the run's hash names. + test("an edited source at the same path maps with its edited code", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + fs.writeFileSync( + path.join(workDir, customFile), + CUSTOM.replace('given: "firstName"', 'given: "lastName"'), + ); + requests = []; + await run({ + input: "export.json", + source: customFile, + yes: true, + secretKey: "sk_test_x", + newRun: true, + }); + + const bodies = created().map((r) => r.body as Record); + expect(bodies[0]).toMatchObject({ last_name: "Ada" }); + expect("first_name" in (bodies[0] ?? {})).toBe(false); + }); + + // An edited source is a different source, so the run records which one. + test("records the custom source's content hash on the run", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + + const [record] = listRuns(runsDir()); + expect(record?.source).toBe("myplatform"); + expect(record?.sourceHash).toMatch(/^[0-9a-f]{64}$/); + }); + + test("an unknown built-in key is a usage error listing the valid ones", async () => { + await expect( + run({ input: "export.json", source: "okta", yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/Unknown source "okta". Valid sources: clerk, auth0/); + expect(created()).toHaveLength(0); + }); + + test("fails before any request when the file is not there", async () => { + await expect( + run({ + input: "export.json", + source: "./nope.ts", + yes: true, + secretKey: "sk_test_x", + }), + ).rejects.toThrow(/No source file at/); + expect(requests).toHaveLength(0); + }); + + test("fails before any request when the file is malformed", async () => { + const bad = `./bad-${customCounter++}.ts`; + fs.writeFileSync( + path.join(workDir, bad), + `export default { key: "x", label: "X", transformer: {} };`, + ); + + const error = (await run({ + input: "export.json", + source: bad, + yes: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + expect(error.message).toMatch(/no source field maps to `userId`/); + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(requests).toHaveLength(0); + }); + + test("still requires a file", async () => { + await expect(run({ source: customFile, yes: true, secretKey: "sk_test_x" })).rejects.toThrow( + /needs the file to import/, + ); + }); + }); + describe("per-platform imports", () => { /** One realistic record per platform, in that platform's export shape. */ const PLATFORMS: [string, unknown, string][] = [ diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 17943d32b..fe3d0fb16 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -61,6 +61,7 @@ import { continueRun, latestUserLines, listRuns, + lockImport, readRun, readUserLines, resolveRunsDir, @@ -110,8 +111,14 @@ import { link } from "../link/index.ts"; export type MigrateRunOptions = { /** An export file, or the ID of the export run that wrote one. */ input?: string; - /** A built-in source key. */ + /** A built-in source key, or the path to a source you wrote. */ source?: string; + /** Content hash of a custom `--source`, set once it is loaded. */ + sourceHash?: string; + /** `--source` as typed, for printed commands: a custom source's path. Not a flag. */ + sourceArg?: string; + /** The source an export file named, when a custom `--source` replaced it. Not a flag. */ + envelopeSource?: string; /** The file to read, once `input` is resolved. Not a flag. */ file?: string; requirePassword?: boolean; @@ -139,7 +146,7 @@ export type MigrateRunOptions = { * Validates what a run needs before anything is read or sent. * * `--source` has already been resolved to a registered key by the time this - * runs. + * runs, custom sources included. * * @returns The source key and file path, both guaranteed present. */ @@ -149,7 +156,7 @@ export function validateRunOptions(options: MigrateRunOptions): { } { if (!options.source) { throwUsageError( - `Missing --source. Valid values: ${sourceKeys().join(", ")}.`, + `Missing --source. Valid values: ${sourceKeys().join(", ")}, or the path to a source you wrote.`, undefined, ERROR_CODE.USAGE_ERROR, [ @@ -371,32 +378,54 @@ function throwChangedExport(runId: string, file: string): never { } /** - * Resolves `--source` to a registered key. + * Resolves `--source` to a registered key, loading a custom source from its + * path so the rest of the run treats it exactly like a built-in. * * @throws UsageError for an unknown key. */ async function applySource(options: MigrateRunOptions): Promise { if (!options.source) return options; const resolved = await resolveSource(options.source); - return { ...options, source: resolved.key }; + if (resolved.path && !options.json) { + log.info(`Loaded the \`${resolved.key}\` source from ${options.source}.`); + } + return { + ...options, + source: resolved.key, + sourceArg: options.source, + ...(resolved.hash ? { sourceHash: resolved.hash } : {}), + }; } /** * The source an export file names for itself, checked against the one the - * flags name. + * flags name. A source you wrote replaces it: it reads the same users, in the + * platform's own shape, and maps them as it says. * - * @throws UsageError when the two disagree: importing an Auth0 export through - * the Supabase mapping would create users with the wrong fields. + * @throws UsageError when two built-in sources disagree: importing an Auth0 + * export through the Supabase mapping would create users with the wrong + * fields. */ function applyEnvelope( options: MigrateRunOptions, envelope: ExportEnvelope | undefined, ): MigrateRunOptions { if (!envelope) return options; + // A built-in name is a claim about where the file came from, and a wrong + // one is a mistake. A path is a choice to map it differently, say to bring + // across fields the built-in mapping leaves out. + if (options.sourceHash) { + if (!options.json) { + log.info( + `Mapping this ${envelope.source} export with ${options.sourceArg}, in place of the built-in \`${envelope.source}\` source.`, + ); + } + return { ...options, envelopeSource: envelope.source }; + } if (options.source && options.source !== envelope.source) { throwUsageError( `The file was exported from ${envelope.source}, but --source names ${options.source}. ` + - "Drop --source: the file already says where it came from.", + "Drop --source: the file already says where it came from. To map its users differently, pass the path to a source you wrote.", ); } // `--source` goes through the registry; the file's own word must too, or a @@ -412,6 +441,54 @@ function applyEnvelope( // --- Continuing an earlier run --------------------------------------------- +/** + * Refuses while another process is importing this file, with this source key, + * into this instance. Checked before any resume matching, and under + * --new-run too: an edited custom source (another `sourceHash`) or a new run + * would otherwise send the same external IDs alongside it. + */ +function assertNoActiveImport( + runsDir: string, + match: { sha256: string; instanceId: string; keyInstanceId?: string }, +): void { + const active = listRuns(runsDir).find( + (record) => sameFileAndInstance(record, match) && runState(runsDir, record) === "running", + ); + if (active) { + throwUsageError( + inUseMessage(runsDir, active.id, "importing this file", "Wait for it to finish."), + ); + } +} + +/** + * An import of this file into this instance, by its ID or its key's stand-in + * ID, under any source. Whichever source maps them, its users are the same + * people, so another source's run, or another version of one, holds them too. + */ +function sameFileAndInstance( + record: RunRecord, + match: { sha256: string; instanceId: string; keyInstanceId?: string }, +): boolean { + return ( + record.kind === "import" && + record.file?.sha256 === match.sha256 && + (record.target.instanceId === match.instanceId || + record.target.instanceId === match.keyInstanceId) + ); +} + +/** True when a run's users are gone: it is marked undone, or an undo of it finished. */ +function undoneInFull(runsDir: string, record: RunRecord): boolean { + return ( + runState(runsDir, record) === "undone" || + listRuns(runsDir).some( + (undo) => + undo.kind === "undo" && undo.undoes === record.id && runState(runsDir, undo) === "complete", + ) + ); +} + /** How this import relates to earlier runs of the same file, source and instance. */ type ResumeCase = | { kind: "new" } @@ -419,8 +496,8 @@ type ResumeCase = | { kind: "complete"; record: RunRecord }; /** - * Finds the latest import run of this file (by sha256), this source and this - * instance, and decides what a re-run does: + * Finds the latest import run of this file (by sha256), this source (and a + * custom source's hash) and this instance, and decides what a re-run does: * * - none, or undone: a new run * - interrupted: continue it, skipping the users it created @@ -430,13 +507,16 @@ type ResumeCase = * @throws UsageError when that run is still running in another process, or * has an undo that never finished: some of its users are gone and some are * not, so neither continuing it nor starting over is safe. Also when Clerk - * did not confirm the instance and a run of this file names a real one. + * did not confirm the instance and a run of this file names a real one, or + * when the latest run of this file was made with an earlier version of a + * custom source and was not undone. */ function findResume( runsDir: string, match: { sha256: string; source: string; + sourceHash?: string; instanceId: string; /** The key's stand-in ID, for a run recorded while Clerk could not name the instance. */ keyInstanceId: string; @@ -445,18 +525,43 @@ function findResume( carry: string[], ): ResumeCase { const runs = listRuns(runsDir); - const ofFile = importsOfFile(runs, match); - const latest = ofFile.find( - (record) => - record.target.instanceId === match.instanceId || - record.target.instanceId === match.keyInstanceId, + // Only a run of this same source, and the same version of a custom one, + // is continued. + const latest = importsOfFile(runs, match).find( + (record) => sameFileAndInstance(record, match) && record.sourceHash === match.sourceHash, ); - if (!latest) { + // An undone exact match starts a new run, so it checks the others too. + if (!latest || undoneInFull(runsDir, latest)) { + // Another source's run of this file, or an earlier version of this one, + // sent the same users: starting over beside it would reject every user it + // created and leave it unfinished, so that takes --new-run. Any such run + // not undone, not just the newest: an older interrupted run behind a + // newer undone one still holds its users. + const earlier = runs.find( + (record) => sameFileAndInstance(record, match) && !undoneInFull(runsDir, record), + ); + if (earlier) { + const how = + earlier.source !== match.source + ? `with the \`${earlier.source}\` source` + : earlier.sourceHash !== match.sourceHash + ? "with an earlier version of this source" + : "too, and was not undone"; + throwUsageError( + `Run ${earlier.id} imported this file into this instance ${how}, so the users it created would come back as conflicts. Nothing was imported.\n` + + `Pass --new-run to start a new run beside it, or undo it first with \`${undoCommand(earlier.id, carry)}\`.`, + ); + } // Clerk did not name the instance (GET /v1/instance failed), so a run of // this file recorded under a real ID can be neither matched nor ruled out. // Starting over would reject every user it created and leave it unfinished. if (isStandInInstanceId(match.instanceId)) { - const unconfirmed = ofFile.find((record) => isClerkInstanceId(record.target.instanceId)); + const unconfirmed = runs.find( + (record) => + record.kind === "import" && + record.file?.sha256 === match.sha256 && + isClerkInstanceId(record.target.instanceId), + ); if (unconfirmed) { throwUsageError( `Clerk did not confirm which instance this key addresses, so this import cannot tell whether run ${unconfirmed.id} is for it. ` + @@ -655,7 +760,9 @@ function commandFor( ) { const input = fromExport ?? options.input ?? options.file ?? ""; const parts = ["clerk migrate import", quoteArg(input)]; - if (!fromExport && options.source) parts.push("--source", quoteArg(options.source)); + const source = options.sourceArg ?? options.source; + // An export names its own source, unless a source you wrote replaced it. + if ((!fromExport || options.envelopeSource) && source) parts.push("--source", quoteArg(source)); if (options.newRun) parts.push("--new-run"); parts.push(...importFlags(options)); // Names, not `<…>`: pasted as is, a shell reads `` as a redirect. @@ -684,14 +791,13 @@ function rejectSummary(checks: ImportChecks, json = false): string { } /** - * The earlier import run, of this source into this instance, that created the + * The earlier import run into this instance, of any source, that created the * users the checks reject as already there: how many of them, and whether it * ran this same file. Fixing a skipped row changes the file, and the changed * file's run meets them; so does the same file run with `--new-run`. */ function earlierRunHolding( runsDir: string, - source: string, instanceId: string, checks: ImportChecks, sha256: string, @@ -701,7 +807,8 @@ function earlierRunHolding( ); if (already.size === 0) return undefined; for (const record of listRuns(runsDir)) { - if (record.kind !== "import" || record.source !== source) continue; + // Any source: findResume and the import lock match on file and instance. + if (record.kind !== "import") continue; if (record.target.instanceId !== instanceId || runState(runsDir, record) === "undone") continue; let count = 0; for (const line of latestUserLines(runsDir, record.id).values()) { @@ -767,6 +874,19 @@ function recordRejects(run: Run, checks: ImportChecks): void { } export async function run(rawOptions: MigrateRunOptions): Promise { + // Released however the import ends: a return, a refusal or a Ctrl-C. + const lock: ImportLock = {}; + try { + await runImport(rawOptions, lock); + } finally { + lock.release?.(); + } +} + +/** The import's identity lock, taken part-way through and released by {@link run}. */ +type ImportLock = { release?: () => void }; + +async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promise { // A mistyped --source fails before any sign-in or link. let options = await applySource(rawOptions); await ensureImportTarget(options); @@ -797,15 +917,18 @@ export async function run(rawOptions: MigrateRunOptions): Promise { // The flags every suggested command needs to reach this runs folder and instance. const carry = carryFlags(options); + // A Firebase export through a source you wrote is still Firebase's: its + // digests need the project's parameters, from the envelope or the flags. + const hashSource = options.envelopeSource ?? source; // A flag is typed for this run, so it wins over the export file. A variable // is ambient, and may be left over from another project, so it only fills in // for an export file without the parameters: with one, a partial flag set is // refused rather than completed from the variables. - const fromEnvelope = source === "firebase" ? envelope?.firebase : undefined; + const fromEnvelope = hashSource === "firebase" ? envelope?.firebase : undefined; const fromFlagsAtAll = FIREBASE_FLAGS.some(([key]) => options[key] !== undefined); const fromFlags = fromFlagsAtAll || !fromEnvelope - ? resolveFirebaseHashConfig(options, source, fromEnvelope ? {} : process.env) + ? resolveFirebaseHashConfig(options, hashSource, fromEnvelope ? {} : process.env) : undefined; let firebaseHashConfig = fromFlags ?? fromEnvelope; let configFrom = fromFlags @@ -814,7 +937,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { : "the CLERK_FIREBASE_* variables" : "the export file"; // `-y` is "import without prompting": it gets the usage error naming the flags. - if (source === "firebase" && !firebaseHashConfig && !options.yes && canPrompt(options)) { + if (hashSource === "firebase" && !firebaseHashConfig && !options.yes && canPrompt(options)) { firebaseHashConfig = await promptForFirebaseHashConfig(); configFrom = "the parameters entered"; // A suggested command runs with --yes, which never asks: it names the flags. @@ -857,9 +980,37 @@ export async function run(rawOptions: MigrateRunOptions): Promise { if (expectedSha256 !== undefined && sha256 !== expectedSha256) throwChanged(filePath); const runsDir = await resolveRunsDir(options.runsDir); + // Held from before the checks to the end of the run: the scan below + // alone leaves a gap a second process can pass through before either + // one writes a run. + // Two locks: the key's stand-in ID, which the same key takes when + // `GET /v1/instance` fails, and the instance's ID when Clerk named it, + // which a second key for the instance takes. + // ponytail: a second key whose lookup failed takes neither; the run scan + // below is all that stops it. A lock on the key's instance needs Clerk. + if (!options.dryRun) { + const releases: (() => void)[] = []; + try { + for (const instanceId of new Set([keyInstanceId(secretKey), target.instanceId])) { + releases.push(lockImport({ sha256, instanceId })); + } + } catch (error) { + for (const release of releases) release(); + throw error; + } + lock.release = () => { + for (const release of releases) release(); + }; + } + assertNoActiveImport(runsDir, { + sha256, + instanceId: target.instanceId, + keyInstanceId: keyInstanceId(secretKey), + }); const match = { sha256, source, + ...(options.sourceHash ? { sourceHash: options.sourceHash } : {}), instanceId: target.instanceId, keyInstanceId: keyInstanceId(secretKey), }; @@ -1078,7 +1229,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { if (refused) { if (options.json) preview({ refused: true }); - const earlier = earlierRunHolding(runsDir, source, target.instanceId, checks, sha256); + const earlier = earlierRunHolding(runsDir, target.instanceId, checks, sha256); const undoEarlier = earlier ? undoCommand(earlier.id, carry) : undefined; const earlierRun = options.newRun && earlier @@ -1218,8 +1369,10 @@ export async function run(rawOptions: MigrateRunOptions): Promise { kind: "import", target, source, + ...(options.sourceHash ? { sourceHash: options.sourceHash } : {}), file: { path: filePath, sha256 }, ...(input.fromExport ? { fromExport: input.fromExport } : {}), + ...(options.envelopeSource ? { envelopeSource: options.envelopeSource } : {}), ...(firebaseHash ? { firebaseHash } : {}), flags, }); diff --git a/packages/cli-core/src/commands/migrate/runs.test.ts b/packages/cli-core/src/commands/migrate/runs.test.ts index eb2c4b6cb..d9b541333 100644 --- a/packages/cli-core/src/commands/migrate/runs.test.ts +++ b/packages/cli-core/src/commands/migrate/runs.test.ts @@ -406,6 +406,17 @@ describe("runs ", () => { expect(next[0]).toContain("to retry the users it didn't finish"); }); + // The record has the custom source's hash, not its path. + test("a partial import from a source you wrote names the source file to pass", async () => { + const record = partialImport({ source: "acme", sourceHash: "abc123" }); + + await runs(record.id, { runsDir, json: true }); + + const { next } = JSON.parse(captured.out) as { next: string[] }; + expect(next[0]).toContain(`clerk migrate import ${quoteArg(usersFile())} --source SOURCE_FILE`); + expect(next[0]).toContain("with SOURCE_FILE replaced by the path to the source you wrote"); + }); + test("an unknown ID is a usage error", async () => { await expect(runs("nope", { runsDir })).rejects.toThrow(/No run `nope`/); }); diff --git a/packages/cli-core/src/commands/migrate/runs.ts b/packages/cli-core/src/commands/migrate/runs.ts index 70660eaa1..427c7fe93 100644 --- a/packages/cli-core/src/commands/migrate/runs.ts +++ b/packages/cli-core/src/commands/migrate/runs.ts @@ -124,16 +124,19 @@ function continueCommand(record: RunRecord, carry: string[]): string | undefined const input = record.fromExport ?? record.file?.path; if (!input) return undefined; const parts = ["clerk migrate import", quoteArg(input)]; - if (!record.fromExport && record.source) parts.push("--source", quoteArg(record.source)); + // A source you wrote is a file the record knows only by its hash. A name, + // not `<…>`: pasted as is, a shell reads `` as a redirect. + if (record.sourceHash) parts.push("--source", "SOURCE_FILE"); + else if (!record.fromExport && record.source) parts.push("--source", quoteArg(record.source)); return withFlags(parts.join(" "), [...importFlags(record.flags), ...carry, "--yes"]); } /** * Whether importing the record's file again continues this run, rather than - * starting another or refusing: no newer import of that file and source may - * be for the same instance, the file is unchanged, and no undo of it exists. - * An unfinished undo makes an import refuse; a finished one that never marked - * the run makes it start a new one. + * starting another or refusing: no newer import of that file and source (and + * version of a source you wrote) may be for the same instance, the file is + * unchanged, and no undo of it exists. An unfinished undo makes an import + * refuse; a finished one that never marked the run makes it start a new one. */ function continuesThisRun(runsDir: string, record: RunRecord): boolean { const file = record.file; @@ -146,8 +149,8 @@ function continuesThisRun(runsDir: string, record: RunRecord): boolean { id === record.target.instanceId || isStandInInstanceId(id) || isStandInInstanceId(record.target.instanceId); - const newest = importsOfFile(runs, { sha256: file.sha256, source: record.source }).find((other) => - mayBeSameInstance(other.target.instanceId), + const newest = importsOfFile(runs, { sha256: file.sha256, source: record.source }).find( + (other) => other.sourceHash === record.sourceHash && mayBeSameInstance(other.target.instanceId), ); if (newest?.id !== record.id) return false; if (undosOf(runs, record.id).length > 0) return false; @@ -179,7 +182,11 @@ function nextCommands( // Continuing is what a stopped run is for: the same file, source and // target pick it up where it left off. const why = state === "interrupted" ? "continue it" : "retry the users it didn't finish"; - return [`Run \`${again}\` to ${why}`, undo]; + // The record knows a source you wrote only by its hash. + const placeholder = record.sourceHash + ? ", with SOURCE_FILE replaced by the path to the source you wrote" + : ""; + return [`Run \`${again}\` to ${why}${placeholder}`, undo]; } if (record.kind === "undo" && state !== "complete" && record.undoes) { return [`Run \`${undoCommand(record.undoes, carry)}\` to retry the users that failed`]; diff --git a/packages/cli-core/src/commands/migrate/sources/list.test.ts b/packages/cli-core/src/commands/migrate/sources/list.test.ts new file mode 100644 index 000000000..a28a89cd1 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/list.test.ts @@ -0,0 +1,172 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError, EXIT_CODE } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { list, wrapText } from "./list.ts"; +import { ACCOUNT_LINKING_URL, sources } from "./registry.ts"; + +const captured = useCaptureLog(); + +/** `log.info` highlights backticked spans and the levels are coloured. */ +const plain = () => Bun.stripANSI(captured.err).replace(/\s+/g, " "); + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-sources-"))); + process.chdir(workDir); + fs.writeFileSync( + path.join(workDir, "custom.ts"), + `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", mail: "email" }, + carries: { + passwords: { level: "partial", note: "Only for accounts made after 2020." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "yes", note: "All of it." }, + }, + };`, + ); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("sources", () => { + test.each([...sources])("lists $key with what it carries", async (entry) => { + await list(undefined); + const row = plain().match(new RegExp(`${entry.key} (\\w+) (\\w+) (\\w+)`)); + expect(row?.slice(1)).toEqual([ + entry.carries.passwords.level, + entry.carries.mfa.level, + entry.carries.metadata.level, + ]); + }); + + // No social column: every source shares one note. + test("prints the account-linking note once", async () => { + await list(undefined); + expect(captured.err.split(ACCOUNT_LINKING_URL)).toHaveLength(2); + }); + + // A compiled binary has no source tree to grep, so the way to extend it has + // to be discoverable from the list itself. + test("says how to add one", async () => { + await list(undefined); + expect(captured.err).toContain("--source ./my-source.ts"); + }); + + test("--json lists every built-in with its carries on stdout", async () => { + await list(undefined, { json: true }); + + const parsed = JSON.parse(captured.out) as { + sources: { key: string; carries: unknown }[]; + account_linking: string; + }; + expect(parsed.sources.map((entry) => entry.key)).toEqual(sources.map((entry) => entry.key)); + expect(parsed.sources[0]?.carries).toEqual(sources[0]?.carries); + expect(parsed.account_linking).toContain(ACCOUNT_LINKING_URL); + }); +}); + +describe("sources ", () => { + test("shows where each field lands, how to export, and what comes across", async () => { + await list("supabase"); + + expect(plain()).toContain("Export with `clerk migrate export supabase`"); + expect(plain()).toContain("encrypted_password → password"); + expect(plain()).toContain("Passwords yes"); + expect(captured.err).toContain(ACCOUNT_LINKING_URL); + }); + + // Supabase detects its hasher per user, so a source with a fixed one shows it. + test("shows a source's fixed field values", async () => { + await list("firebase"); + expect(plain()).toContain('passwordHasher is always "scrypt_firebase"'); + }); + + test("shows a source you wrote, by its path", async () => { + await list("./custom.ts"); + + expect(plain()).toContain("myplatform My Platform (custom — ./custom.ts)"); + expect(plain()).toContain("mail → email"); + expect(plain()).toContain("Only for accounts made after 2020."); + }); + + test("--json returns the detail on stdout", async () => { + await list("auth0", { json: true }); + + expect(JSON.parse(captured.out)).toMatchObject({ + key: "auth0", + export_command: "clerk migrate export auth0", + fields: { user_id: "userId" }, + carries: { passwords: { level: "partial" } }, + }); + }); + + test("an unknown source is a usage error naming the valid ones", async () => { + const error = (await list("okta").catch((caught: unknown) => caught)) as CliError; + expect(error).toBeInstanceOf(CliError); + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("Valid sources: clerk, auth0"); + }); +}); + +describe("human-mode frame", () => { + let originalMode: string | undefined; + + beforeAll(() => { + originalMode = process.env.CLERK_MODE; + process.env.CLERK_MODE = "human"; + }); + + afterAll(() => { + if (originalMode === undefined) delete process.env.CLERK_MODE; + else process.env.CLERK_MODE = originalMode; + }); + + // Reading a static registry is not a run: there is no progress to bracket, + // and the gutter's `│` would sit in front of every wrapped line. + test("prints no intro/outro gutter", async () => { + await list(undefined); + + expect(captured.err).not.toContain("┌"); + expect(captured.err).not.toContain("└"); + }); + + test("--json stays on stdout only", async () => { + await list(undefined, { json: true }); + + expect(() => JSON.parse(captured.out)).not.toThrow(); + expect(captured.err).toBe(""); + }); +}); + +describe("wrapText", () => { + test("breaks on whitespace within the width", () => { + expect(wrapText("one two three four", 9)).toEqual(["one two", "three", "four"]); + }); + + // `log.info` pairs backticks per line, so a span split across two lines + // leaves one unmatched backtick on each and colours the wrong half of both. + test("never breaks inside a backticked span", () => { + const lines = wrapText("Assumes an export of `SELECT id, name FROM users`.", 30); + + expect(lines).toContain("`SELECT id, name FROM users`."); + for (const line of lines) { + expect((line.match(/`/g) ?? []).length % 2).toBe(0); + } + }); + + test("gives an over-long word its own line rather than dropping it", () => { + expect(wrapText("short supercalifragilistic", 8)).toEqual(["short", "supercalifragilistic"]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/sources/list.ts b/packages/cli-core/src/commands/migrate/sources/list.ts new file mode 100644 index 000000000..7991bb05a --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/list.ts @@ -0,0 +1,202 @@ +/** + * `clerk migrate sources [source]` — which platforms an import can read, and + * what each one brings across. + * + * A compiled binary's users have no source tree to read the mappings in, so + * this is where they are shown. `sources` alone lists every source with its + * passwords, MFA and metadata at a glance. `sources ` shows one in + * full: what each field becomes, how to export from it, and its caveats. + */ + +import { bold, cyan, dim, green, red, yellow } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { exportPlatforms } from "../export/registry.ts"; +import type { Carry, CarryLevel, SourceEntry } from "../types.ts"; +import { ACCOUNT_LINKING_NOTE, resolveSource, sources as builtIns } from "./registry.ts"; + +export type SourcesOptions = { + json?: boolean; +}; + +const KINDS = [ + ["passwords", "Passwords"], + ["mfa", "MFA"], + ["metadata", "Metadata"], +] as const; + +/** + * Capped, not just measured: a description that rewrapped differently on every + * terminal makes two runs of the same command look like different output. 80 is + * the same width `--help` lays itself out at. + */ +const MAX_WIDTH = 80; + +function outputWidth(): number { + return Math.min(process.stderr.columns || MAX_WIDTH, MAX_WIDTH); +} + +/** + * A run of non-space characters, except that a backticked span counts as one + * character run even when it contains spaces. Keeps `SELECT a, b FROM users` + * whole: `log.info` pairs backticks per line, so a span broken across two lines + * leaves an unmatched backtick on each and colours the wrong half of both. + */ +const WORD = /(?:`[^`]*`|\S)+/g; + +/** + * Wraps on whitespace. Safe to measure raw because the backtick spans + * `log.info` highlights keep their backticks — the colour it adds is invisible + * to width, and nothing here is coloured before wrapping. + */ +export function wrapText(text: string, width: number): string[] { + const lines: string[] = []; + let line = ""; + + for (const word of text.match(WORD) ?? []) { + if (!line) line = word; + else if (line.length + 1 + word.length <= width) line += ` ${word}`; + else { + lines.push(line); + line = word; + } + } + if (line) lines.push(line); + + return lines; +} + +function levelMark(level: CarryLevel): string { + if (level === "yes") return green("yes"); + if (level === "partial") return yellow("partial"); + return red("no"); +} + +/** The command that writes a file this source reads, when there is one. */ +function exportCommand(entry: SourceEntry): string | undefined { + const platform = exportPlatforms.find((candidate) => candidate.sourceKey === entry.key); + return platform ? `clerk migrate export ${platform.key}` : undefined; +} + +function toJson(entry: SourceEntry, custom?: string) { + return { + key: entry.key, + label: entry.label, + description: entry.description, + ...(custom ? { path: custom } : {}), + carries: entry.carries, + export_command: exportCommand(entry) ?? null, + fields: entry.transformer, + caveats: entry.caveats ?? [], + }; +} + +function printList(width: number): void { + for (const line of wrapText( + "A source maps one platform's export onto the fields Clerk imports. A file from " + + "`clerk migrate export` names its own; anything else takes `--source `.", + width, + )) { + log.info(line); + } + log.blank(); + + const keyWidth = Math.max(...builtIns.map((entry) => entry.key.length)) + 2; + log.info( + bold( + `${"SOURCE".padEnd(keyWidth)}${"PASSWORDS".padEnd(11)}${"MFA".padEnd(9)}${"METADATA".padEnd(10)}`, + ), + ); + for (const entry of builtIns) { + const cell = (carry: Carry, pad: number) => + levelMark(carry.level) + " ".repeat(Math.max(1, pad - carry.level.length)); + log.info( + `${cyan(entry.key.padEnd(keyWidth))}${cell(entry.carries.passwords, 11)}` + + `${cell(entry.carries.mfa, 9)}${levelMark(entry.carries.metadata.level)}`, + ); + } + + log.blank(); + for (const line of wrapText(ACCOUNT_LINKING_NOTE, width)) log.info(dim(line)); + log.blank(); + log.info("Run `clerk migrate sources ` for what each field becomes."); + log.info("Migrating from something else? Write a source and pass --source ./my-source.ts."); +} + +function printDetail(entry: SourceEntry, width: number, custom?: string): void { + log.info(`${cyan(bold(entry.key))} ${entry.label}${custom ? dim(` (custom — ${custom})`) : ""}`); + for (const line of wrapText(entry.description, width)) log.info(line); + + const command = exportCommand(entry); + if (command) { + log.blank(); + log.info(`${bold("Export with")} \`${command}\``); + } + + log.blank(); + log.info(bold("What comes across")); + for (const [kind, label] of KINDS) { + const carry = entry.carries[kind]; + log.info(` ${label.padEnd(10)}${levelMark(carry.level)}`); + for (const line of wrapText(carry.note, width - 4)) log.info(` ${dim(line)}`); + } + + const fields = Object.entries(entry.transformer); + const fromWidth = Math.max(...fields.map(([from]) => from.length)) + 2; + log.blank(); + log.info(bold("Where each field lands")); + for (const [from, to] of fields) log.info(` ${from.padEnd(fromWidth)}→ ${to}`); + if (entry.defaults) { + for (const [field, value] of Object.entries(entry.defaults)) { + log.info(dim(` ${field} is always ${JSON.stringify(value)}`)); + } + } + + if (entry.caveats?.length) { + log.blank(); + log.info(bold("Caveats")); + for (const caveat of entry.caveats) { + const [first, ...rest] = wrapText(caveat, width - 4); + log.info(` - ${first ?? ""}`); + for (const line of rest) log.info(` ${line}`); + } + } + + log.blank(); + for (const line of wrapText(ACCOUNT_LINKING_NOTE, width)) log.info(dim(line)); +} + +export async function list(key: string | undefined, options: SourcesOptions = {}): Promise { + const width = outputWidth(); + + if (key === undefined) { + if (options.json) { + log.data( + JSON.stringify( + { + sources: builtIns.map((entry) => toJson(entry)), + account_linking: ACCOUNT_LINKING_NOTE, + }, + null, + 2, + ), + ); + return; + } + printList(width); + return; + } + + // No gutter: this reads a static registry, it does not run anything. + const resolved = await resolveSource(key); + if (options.json) { + log.data( + JSON.stringify( + { ...toJson(resolved.entry, resolved.path), account_linking: ACCOUNT_LINKING_NOTE }, + null, + 2, + ), + ); + return; + } + printDetail(resolved.entry, width, resolved.path ? key : undefined); +} diff --git a/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts new file mode 100644 index 000000000..73d518d82 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts @@ -0,0 +1,261 @@ +import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { loadCustomSource, validateSource } from "./load-custom.ts"; +import { __resetCustomSourcesForTesting } from "./registry.ts"; + +let workDir: string; +let originalCwd: string; +let counter = 0; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-custom-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +afterEach(() => { + __resetCustomSourcesForTesting(); +}); + +/** + * Writes a source file with a unique name. + * + * Names must not repeat: a dynamic `import()` caches by URL, so reusing one + * would silently return the previous test's module. + */ +function writeSource(source: string, ext = "ts"): string { + const name = `custom-${counter++}.${ext}`; + fs.writeFileSync(path.join(workDir, name), source); + return `./${name}`; +} + +const VALID = `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", contact_email: "email" }, + carries: { + passwords: { level: "no", note: "None." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, +};`; + +describe("loadCustomSource", () => { + test("loads a user-authored TypeScript source", async () => { + const entry = await loadCustomSource(writeSource(VALID)); + + expect(entry).toMatchObject({ + key: "myplatform", + label: "My Platform", + transformer: { account_ref: "userId", contact_email: "email" }, + }); + }); + + test("loads plain JavaScript too", async () => { + const entry = await loadCustomSource(writeSource(VALID, "js")); + expect(entry.key).toBe("myplatform"); + }); + + // The file is the user's own code; the CLI must transpile whatever they wrote. + test("transpiles TypeScript syntax the runtime has to strip", async () => { + const entry = await loadCustomSource( + writeSource(` + interface Entry { key: string; label: string; transformer: Record; carries: unknown } + const mapping = { my_id: "userId" } as const; + const custom: Entry = { key: "tsplatform", label: "TS", transformer: { ...mapping }, carries: { passwords: { level: "no", note: "-" }, mfa: { level: "no", note: "-" }, metadata: { level: "no", note: "-" } } }; + export default custom satisfies Entry; + `), + ); + expect(entry.key).toBe("tsplatform"); + }); + + test("carries the optional hooks through", async () => { + const entry = await loadCustomSource( + writeSource(`export default { + key: "hooked", label: "Hooked", + transformer: { id: "userId" }, + carries: { passwords: { level: "no", note: "-" }, mfa: { level: "no", note: "-" }, metadata: { level: "no", note: "-" } }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { user.firstName = "set"; }, + };`), + ); + + expect(entry.defaults).toEqual({ passwordHasher: "bcrypt" }); + const user: Record = {}; + entry.postTransform?.(user, {}); + expect(user.firstName).toBe("set"); + }); + + test("supplies a description when the author omitted one", async () => { + const entry = await loadCustomSource( + writeSource( + `export default { key: "bare", label: "Bare", transformer: { id: "userId" }, carries: { passwords: { level: "no", note: "-" }, mfa: { level: "no", note: "-" }, metadata: { level: "no", note: "-" } } };`, + ), + ); + expect(entry.description).toBe("Custom source"); + }); + + test("reports a path that is not there", async () => { + await expect(loadCustomSource("./nope.ts")).rejects.toThrow(/No source file at/); + }); + + test("reports a directory given instead of a file", async () => { + fs.mkdirSync(path.join(workDir, "adir"), { recursive: true }); + await expect(loadCustomSource("./adir")).rejects.toThrow(/is a directory/); + }); + + test("reports a file that does not parse, quoting the syntax error", async () => { + await expect(loadCustomSource(writeSource("export default { key: ,,, }"))).rejects.toThrow( + /Could not load/, + ); + }); + + test("reports a file that throws while loading", async () => { + await expect( + loadCustomSource(writeSource(`throw new Error("boom"); export default {};`)), + ).rejects.toThrow(/Could not load .*boom/s); + }); + + test("points at a named export when the default is missing", async () => { + const file = writeSource( + `export const myPlatform = { key: "x", label: "X", transformer: { a: "userId" } };`, + ); + + await expect(loadCustomSource(file)).rejects.toThrow( + /has no default export.*`myPlatform`.*did you mean `export default`/s, + ); + }); + + test("reports a missing default with no named exports to suggest", async () => { + await expect(loadCustomSource(writeSource("const unused = 1;"))).rejects.toThrow( + /has no default export\.$/m, + ); + }); +}); + +describe("validateSource", () => { + const valid = { + key: "myplatform", + label: "My Platform", + transformer: { account_ref: "userId" }, + carries: { + passwords: { level: "no", note: "None." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, + }; + + test("accepts a minimal valid entry", () => { + expect(validateSource(valid, "f.ts").key).toBe("myplatform"); + }); + + test.each([ + ["a null default export", null, /is null, not an object/], + ["a number default export", 42, /is number, not an object/], + ["a string default export", "nope", /is string, not an object/], + ])("rejects %s", (_label, value, expected) => { + expect(() => validateSource(value, "f.ts")).toThrow(expected); + }); + + test.each([ + ["key", { ...valid, key: undefined }], + ["key", { ...valid, key: "" }], + ["key", { ...valid, key: " " }], + ["key", { ...valid, key: 7 }], + ["label", { ...valid, label: undefined }], + ["label", { ...valid, label: "" }], + ])("rejects a bad %s naming the field", (field, value) => { + expect(() => validateSource(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); + }); + + test("rejects a non-string description", () => { + expect(() => validateSource({ ...valid, description: 7 }, "f.ts")).toThrow( + /`description` must be a string/, + ); + }); + + test.each([ + ["missing", { ...valid, transformer: undefined }], + ["null", { ...valid, transformer: null }], + ["an array", { ...valid, transformer: [] }], + ["a string", { ...valid, transformer: "id" }], + ])("rejects a transformer mapping that is %s", (_label, value) => { + expect(() => validateSource(value, "f.ts")).toThrow(/`transformer`|`transformer\./); + }); + + test("names the offending entry when a mapping target is not a field name", () => { + expect(() => + validateSource({ ...valid, transformer: { account_ref: "userId", bad: 7 } }, "f.ts"), + ).toThrow(/`transformer.bad` must map to a Clerk field name, got number/); + }); + + // Without it the import runs to completion and creates every user with no + // external_id — which is what makes a migration reversible. + test("rejects a mapping with no userId target", () => { + expect(() => validateSource({ ...valid, transformer: { a: "email" } }, "f.ts")).toThrow( + /no source field maps to `userId`/, + ); + }); + + test.each([ + ["defaults", { ...valid, defaults: "nope" }], + ["preTransform", { ...valid, preTransform: "nope" }], + ["postTransform", { ...valid, postTransform: 7 }], + ])("rejects a %s of the wrong type", (field, value) => { + expect(() => validateSource(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); + }); + + // `migrate sources` wraps each caveat as text: a number crashed it, and a + // string printed one bullet per character. + test.each([ + ["a non-string caveat", [42]], + ["a string instead of a list", "text"], + ])("rejects caveats holding %s", (_label, caveats) => { + expect(() => validateSource({ ...valid, caveats }, "f.ts")).toThrow( + /`caveats` must be an array of strings/, + ); + }); + + test.each([["clerk"], ["auth0"], ["supabase"]])( + "rejects %s, which would shadow a built-in", + (key) => { + expect(() => + validateSource({ ...valid, key }, "f.ts", ["clerk", "auth0", "supabase"]), + ).toThrow(/already a built-in source/); + }, + ); + + // What a source brings across is the first thing `sources` shows, and only + // its author knows it. + test("requires carries", () => { + expect(() => validateSource({ ...valid, carries: undefined }, "f.ts")).toThrow( + /`carries` must say what the source brings across/, + ); + }); + + test.each([ + ["a level that is not yes, no or partial", { level: "maybe", note: "x" }], + ["no note", { level: "yes" }], + ])("rejects a carries entry with %s", (_label, mfa) => { + expect(() => validateSource({ ...valid, carries: { ...valid.carries, mfa } }, "f.ts")).toThrow( + /`carries.mfa` must be/, + ); + }); + + test("names the file in every message, so the author knows which one", () => { + expect(() => validateSource({}, "./their-file.ts")).toThrow(/\.\/their-file\.ts/); + }); + + test("raises CliError, so the global handler formats it", () => { + expect(() => validateSource({}, "f.ts")).toThrow(CliError); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/sources/load-custom.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.ts new file mode 100644 index 000000000..9c1774e6c --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.ts @@ -0,0 +1,205 @@ +/** + * Loading a user-authored source at runtime. + * + * In the standalone migration-tool, supporting a new platform meant adding a + * file to `src/transformers/` and one line to the registry — the user had the + * source tree. A compiled binary has neither a source tree to edit nor a way + * for an end user to rebuild it, so `--source ` restores that + * extensibility by importing a file from the user's own project instead. + * + * **Verified before this was built on:** a `bun build --compile` executable can + * `import()` an arbitrary external `.ts` file at runtime, including TypeScript + * that needs transpiling. Bun's transpiler is part of the runtime, not only the + * bundler. Confirmed with a throwaway compiled binary on darwin-arm64, + * linux-arm64 (glibc), linux-arm64-musl and linux-x64. + * + * The file is user-supplied code the CLI executes, so its shape is validated + * up front and rejected with a specific message rather than crashing deep in + * the transform pipeline on a missing field. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; +import type { CarryLevel, SourceEntry } from "../types.ts"; + +const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/overview"; + +const LEVELS: readonly CarryLevel[] = ["yes", "no", "partial"]; + +function invalid(problem: string, file: string): never { + throwUsageError(`${file} is not a valid source: ${problem}`, DOCS_URL); +} + +/** + * Checks a loaded value against the registry entry shape. + * + * Every failure names the specific field and what was wrong with it — the + * author is writing this file by hand against a shape they cannot see. + * + * @param file - Path as the user typed it, for the error message. + * @param reservedKeys - The built-in keys, which a custom source may not reuse. + */ +export function validateSource( + value: unknown, + file: string, + reservedKeys: readonly string[] = [], +): SourceEntry { + if (value === null || typeof value !== "object") { + invalid(`the default export is ${value === null ? "null" : typeof value}, not an object`, file); + } + + const entry = value as Record; + + for (const field of ["key", "label"] as const) { + if (typeof entry[field] !== "string" || entry[field].trim().length === 0) { + invalid(`\`${field}\` must be a non-empty string`, file); + } + } + + if (entry.description !== undefined && typeof entry.description !== "string") { + invalid("`description` must be a string when present", file); + } + + // Arrays are objects, and an author who wrote `transformer: []` should hear + // that rather than the downstream "no field maps to userId". + if ( + entry.transformer === null || + typeof entry.transformer !== "object" || + Array.isArray(entry.transformer) + ) { + invalid("`transformer` must be an object mapping source fields to Clerk fields", file); + } + + const mapping = entry.transformer as Record; + for (const [source, target] of Object.entries(mapping)) { + if (typeof target !== "string" || target.length === 0) { + invalid( + `\`transformer.${source}\` must map to a Clerk field name, got ${typeof target}`, + file, + ); + } + } + + // Without this the import runs to completion and creates every user with no + // external_id, which is what makes a migration re-runnable and reversible. + if (!Object.values(mapping).includes("userId")) { + invalid( + "no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id", + file, + ); + } + + if ( + entry.defaults !== undefined && + (entry.defaults === null || typeof entry.defaults !== "object" || Array.isArray(entry.defaults)) + ) { + invalid("`defaults` must be an object when present", file); + } + + // What a source brings across is the first thing `sources` shows, and the + // author is the only one who knows it. + const carries = entry.carries as Record | undefined; + if (!carries || typeof carries !== "object" || Array.isArray(carries)) { + invalid( + "`carries` must say what the source brings across: { passwords, mfa, metadata }, each { level, note }", + file, + ); + } + for (const kind of ["passwords", "mfa", "metadata"] as const) { + const carry = carries[kind] as { level?: unknown; note?: unknown } | undefined; + if (!carry || !LEVELS.includes(carry.level as CarryLevel) || typeof carry.note !== "string") { + invalid( + `\`carries.${kind}\` must be { level: "yes" | "no" | "partial", note: string }`, + file, + ); + } + } + + for (const hook of ["preTransform", "postTransform"] as const) { + if (entry[hook] !== undefined && typeof entry[hook] !== "function") { + invalid(`\`${hook}\` must be a function when present`, file); + } + } + + if ( + entry.caveats !== undefined && + (!Array.isArray(entry.caveats) || entry.caveats.some((caveat) => typeof caveat !== "string")) + ) { + invalid("`caveats` must be an array of strings when present", file); + } + + if (reservedKeys.includes(entry.key as string)) { + invalid( + `\`key\` is "${String(entry.key)}", which is already a built-in source. Choose another key`, + file, + ); + } + + return { + ...(entry as unknown as SourceEntry), + description: (entry.description as string | undefined) ?? "Custom source", + }; +} + +/** + * Imports and validates a user-authored source. + * + * @throws CliError when the path is missing, the module fails to load, or the + * exported value does not match the registry entry shape. + */ +export async function loadCustomSource( + file: string, + reservedKeys: readonly string[] = [], + /** + * The file's content hash. The module loader caches by URL, so an edited + * file at the same path would load the old module; the hash in the URL + * loads the version that was hashed. + */ + version?: string, +): Promise { + const resolved = path.resolve(process.cwd(), file); + + if (!fs.existsSync(resolved)) { + throw new CliError(`No source file at ${resolved}.`, { + code: ERROR_CODE.FILE_NOT_FOUND, + docsUrl: DOCS_URL, + }); + } + if (fs.statSync(resolved).isDirectory()) { + throwUsageError(`${resolved} is a directory, not a source file.`); + } + + let module: Record; + try { + // A file URL rather than a bare path: an absolute POSIX path happens to + // work, but a Windows path (`C:\...`) is not a valid import specifier. + // Bun keys its module cache by specifier and keeps a `file://` URL's + // module whatever its query, but a plain path's query loads afresh. + // ponytail: Windows keeps the URL form, so an edit loaded twice in one + // process reuses the first; each CLI run is a process of its own. + const specifier = + version && process.platform !== "win32" + ? `${resolved}?v=${version}` + : Bun.pathToFileURL(resolved).href; + module = (await import(specifier)) as Record; + } catch (error) { + throwUsageError( + `Could not load ${file}: ${(error as Error).message}\n` + + "The file must be valid JavaScript or TypeScript that this CLI can import.", + DOCS_URL, + ); + } + + if (module.default === undefined) { + // Point at what they probably meant rather than just restating the rule. + const named = Object.keys(module).filter((key) => key !== "default"); + const hint = + named.length > 0 + ? ` Found named export${named.length === 1 ? "" : "s"} ${named.map((n) => `\`${n}\``).join(", ")} — did you mean \`export default\`?` + : ""; + throwUsageError(`${file} has no default export.${hint}`, DOCS_URL); + } + + return validateSource(module.default, file, reservedKeys); +} diff --git a/packages/cli-core/src/commands/migrate/sources/registry.ts b/packages/cli-core/src/commands/migrate/sources/registry.ts index 25d376e71..257732d9a 100644 --- a/packages/cli-core/src/commands/migrate/sources/registry.ts +++ b/packages/cli-core/src/commands/migrate/sources/registry.ts @@ -1,12 +1,16 @@ /** * Source registry. * - * `migrate import` resolves `--source` here. + * `migrate import` resolves `--source` here, and `migrate sources` lists what + * each one brings across. * * To add a platform: create `sources/.ts` exporting a `SourceEntry`, * then add it to the array below. */ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; import { throwUsageError } from "../../../lib/errors.ts"; import type { SourceEntry } from "../types.ts"; import auth0Source from "./auth0.ts"; @@ -14,6 +18,7 @@ import authjsSource from "./authjs.ts"; import betterAuthSource from "./betterauth.ts"; import clerkSource from "./clerk.ts"; import firebaseSource from "./firebase.ts"; +import { loadCustomSource } from "./load-custom.ts"; import supabaseSource from "./supabase.ts"; import workosSource from "./workos.ts"; @@ -41,36 +46,90 @@ export const ACCOUNT_LINKING_NOTE = "Social sign-ins are not copied. Enable the same providers in Clerk, and a user who signs in " + `with one is linked to their imported account by verified email. See ${ACCOUNT_LINKING_URL}`; +/** + * A source loaded from a user's `--source ` for this invocation. + * + * Kept beside the built-ins rather than pushed into them, so the shipped list + * is never mutated. One invocation loads at most one, so this holding a + * single entry is the normal case; the array shape just avoids a special case + * in the lookups. + */ +const customSources: SourceEntry[] = []; + +export function registerCustomSource(entry: SourceEntry): void { + // The latest load of a key wins: an edited source replaces its old self. + const earlier = customSources.findIndex((source) => source.key === entry.key); + if (earlier >= 0) customSources.splice(earlier, 1); + customSources.push(entry); +} + +/** Test-only: drops anything a previous test registered. */ +export function __resetCustomSourcesForTesting(): void { + customSources.length = 0; +} + +/** Built-ins plus whatever `--source ` loaded. */ +export function allSources(): SourceEntry[] { + return [...sources, ...customSources]; +} + /** The built-in keys, for tab-completion and error messages. */ export function sourceKeys(): string[] { return sources.map((entry) => entry.key); } /** - * Looks up a source by key. + * Looks up a source by key, custom ones included. * * @throws Error when no source is registered under that key. */ export function getSource(key: string): SourceEntry { - const source = sources.find((entry) => entry.key === key); + const source = allSources().find((entry) => entry.key === key); if (!source) { throw new Error(`Source not found for key: ${key}`); } return source; } -/** A resolved `--source`. */ -export type ResolvedSource = { key: string; entry: SourceEntry }; +/** A `--source` value that names a file rather than a built-in key. */ +export function isSourcePath(value: string): boolean { + return /^(\.\.?\/|\/)/.test(value) || /\.(ts|js|mjs)$/.test(value); +} + +/** A resolved `--source`: its key, and for a custom one, the file's content hash. */ +export type ResolvedSource = { key: string; entry: SourceEntry; path?: string; hash?: string }; /** - * Resolves `--source` to a built-in source. + * Resolves `--source`: a built-in key, or a path to a source you wrote. + * + * A custom source is keyed by a hash of its file as well as its key, so an + * edited source is a different source when a re-run decides whether to + * continue an earlier import. * * @throws UsageError for an unknown key, listing the valid ones. */ export async function resolveSource(value: string): Promise { + if (isSourcePath(value)) { + const resolved = path.resolve(process.cwd(), value); + // Hashed first, and loaded by that hash: the record names the very code + // that maps the users, even if the file was edited since a last load. + const hash = fs.existsSync(resolved) + ? createHash("sha256").update(fs.readFileSync(resolved)).digest("hex") + : undefined; + const entry = await loadCustomSource(value, sourceKeys(), hash); + registerCustomSource(entry); + return { key: entry.key, entry, path: resolved, hash }; + } + const entry = sources.find((candidate) => candidate.key === value); if (!entry) { - throwUsageError(`Unknown source "${value}". Valid sources: ${sourceKeys().join(", ")}.`); + throwUsageError( + `Unknown source "${value}". Valid sources: ${sourceKeys().join(", ")}.\n` + + "For a platform with no built-in, pass the path to a source you wrote, e.g. --source ./my-source.ts.", + undefined, + undefined, + [{ command: "clerk migrate sources", description: "List the built-in sources" }], + ); } return { key: entry.key, entry }; } diff --git a/packages/cli-core/src/commands/migrate/sources/sources.test.ts b/packages/cli-core/src/commands/migrate/sources/sources.test.ts index 221babe5d..b8934283a 100644 --- a/packages/cli-core/src/commands/migrate/sources/sources.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/sources.test.ts @@ -5,7 +5,7 @@ import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; import { loadUsersFromFile, transformUsers } from "../lib/transform.ts"; import type { FirebaseHashConfig } from "../types.ts"; -import { getSource, sourceKeys, sources } from "./registry.ts"; +import { getSource, isSourcePath, sourceKeys, sources } from "./registry.ts"; import { isVerified } from "./shared.ts"; const FIREBASE_HASH: FirebaseHashConfig = { @@ -78,6 +78,19 @@ describe("registry", () => { }); }); +describe("isSourcePath", () => { + test.each([["./mine.ts"], ["../up/mine.js"], ["/abs/mine.mjs"], ["mine.ts"], ["dir/mine.js"]])( + "%s is a path", + (value) => { + expect(isSourcePath(value)).toBe(true); + }, + ); + + test.each([["clerk"], ["betterauth"], ["okta"]])("%s is a key", (value) => { + expect(isSourcePath(value)).toBe(false); + }); +}); + describe("isVerified", () => { // A CSV export stringifies everything, so the boolean style must read // "false" as false. Treating it as truthy would mark unconfirmed addresses diff --git a/packages/cli-core/src/test/integration/completion.test.ts b/packages/cli-core/src/test/integration/completion.test.ts index f0b1fc91c..11bc5e220 100644 --- a/packages/cli-core/src/test/integration/completion.test.ts +++ b/packages/cli-core/src/test/integration/completion.test.ts @@ -297,6 +297,23 @@ describe("generateCompletions", () => { expect(names).toContain("settings"); }); + test("migrate sources: suggests the built-in sources", () => { + const saved = process.env.CLERK_EXPERIMENTAL; + process.env.CLERK_EXPERIMENTAL = "migrate"; + try { + const names = generateCompletions(createProgram(), [ + "migrate", + "sources", + "", + ]).completions.map((c) => c.name); + expect(names).toContain("supabase"); + expect(names).toContain("workos"); + } finally { + if (saved === undefined) delete process.env.CLERK_EXPERIMENTAL; + else process.env.CLERK_EXPERIMENTAL = saved; + } + }); + test("open dashboard: filters subpaths by prefix", () => { const names = completionNames("open", "dashboard", "u"); expect(names).toContain("users");