diff --git a/.changeset/migrate-import.md b/.changeset/migrate-import.md new file mode 100644 index 000000000..daaf49a22 --- /dev/null +++ b/.changeset/migrate-import.md @@ -0,0 +1,5 @@ +--- +"clerk": minor +--- + +Add an experimental `clerk migrate import` that checks a Clerk or Supabase user export against your instance and imports it. It is hidden and refuses to run unless `CLERK_EXPERIMENTAL=migrate` is set. diff --git a/packages/cli-core/src/commands/completion/__complete.ts b/packages/cli-core/src/commands/completion/__complete.ts index 7921e307e..06f9d87f1 100644 --- a/packages/cli-core/src/commands/completion/__complete.ts +++ b/packages/cli-core/src/commands/completion/__complete.ts @@ -1,4 +1,5 @@ import type { CommandUnknownOpts, Option } from "@commander-js/extra-typings"; +import { sources } from "../migrate/sources/registry.ts"; import { KNOWN_DASHBOARD_PATHS } from "../open/dashboard-paths.ts"; const DIRECTIVE = { @@ -33,6 +34,12 @@ const HTTP_METHOD_COMPLETIONS: Completion[] = [ { name: "DELETE", description: "Delete resource" }, ]; +/** The built-in migrate sources. Kept off `.choices()` so an unknown key gets the source error. */ +const SOURCE_COMPLETIONS: Completion[] = sources.map((entry) => ({ + name: entry.key, + description: entry.label, +})); + /** * Hardcoded option-value completions for options that don't use Commander's `.choices()`. * Keys are the long or short flag (e.g., "--mode", "-X"). @@ -52,6 +59,7 @@ const KNOWN_OPTION_VALUES: Record = { { name: "latest", description: "Latest stable release" }, { name: "canary", description: "Latest canary (pre-release) build" }, ], + "--source": SOURCE_COMPLETIONS, "--for": [ { name: "orgs", description: "Organizations only" }, { name: "users", description: "Users only" }, diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 944fb7c50..f28593581 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -1,9 +1,551 @@ # `clerk migrate` -> **Experimental.** `clerk migrate` is hidden and refuses to run unless `CLERK_EXPERIMENTAL` includes `migrate`: +> **Experimental.** `clerk migrate` is hidden from help and completion, and +> refuses to run (exit 2, `experiment_disabled`), unless `CLERK_EXPERIMENTAL` +> includes `migrate`: > > ```sh -> CLERK_EXPERIMENTAL=migrate clerk migrate --help +> CLERK_EXPERIMENTAL=migrate clerk migrate import --help > ``` +> +> `CLERK_EXPERIMENTAL` is a comma-separated list. Names are trimmed and +> case-insensitive, and unknown names are ignored. + +Migrate users into a Clerk instance from another auth provider, or from another +Clerk instance. + +``` +clerk migrate import [--source ] [--dry-run] [--allow-partial] [--yes] [--json] +clerk migrate help +``` + +An import is usually two steps: + +```sh +clerk migrate import users.json --source supabase --dry-run # 1. check it against the instance +clerk migrate import users.json --source supabase --yes # 2. import it +``` + +`--runs-dir ` (or `CLERK_MIGRATE_DIR`) keeps runs somewhere else. +`clerk migrate` on its own is a group name, not a command: it prints its help. + +## The rules + +Every command follows these: + +1. **Nothing writes without consent.** Consent is a yes at a terminal prompt, or + `--yes`. Without either, `import` prints what it would do and exits 2 with + the command to run. `--json` means non-interactive: it never prompts. +2. **`--dry-run` checks against the real instance, and writes nothing.** An + import's [checks](#checks) run before anything is written. Predicted + rejects stop the import unless `--allow-partial` is passed; fields that would + be dropped are warnings. +3. **State lives in one place: the [run store](#the-run-store).** Each run + records its target, its file, and every source ID → Clerk ID outcome, + including the error for each user who failed. +4. **Every command prints its target first:** the environment, app and + instance, and where the key came from. +5. **Every subcommand takes `--json`.** Exit codes: `0` all good, `1` some users + failed, `2` a usage error or a refusal, and `130` (death by SIGINT) when + Ctrl-C stops an import partway. The UI goes to stderr and data to stdout. + + While users are created, a terminal shows a bar and the counts under it, not + a spinner: + + ``` + │ ██████████████████████████████████████████████████████░░░░░░░░░░░░░░░░░░ 75% + │ 7,500/10,000 users · ✓ 7,425 created · ✗ 75 failed · ~25s left + ``` + + A spinner takes the keyboard and exits 0 on Ctrl-C, so a script that deletes + the export once the import succeeds would delete it after an interrupted + one. Without a terminal, the counts are printed at each 10%. + +## Targeting and auth + +`clerk migrate import` resolves its Backend API key through the CLI's standard +chain: + +| Flag | Description | +| -------------------- | ------------------------------------------------------------- | +| `--secret-key ` | Use a specific Backend API secret key directly | +| `--app ` | Target an application directly, even outside a linked project | +| `--instance ` | Target `dev`, `prod`, or a full instance ID | + +Resolution order: `--secret-key` → `--app` + Platform API lookup → +`CLERK_SECRET_KEY` → the keyless project's own key → a linked project profile +from `clerk link`. + +With a key from `--secret-key` or `CLERK_SECRET_KEY`, the key alone picks the +instance, so `import` refuses (exit 2) an `--instance` that names a different +one. `--instance dev` next to an exported `sk_live_…` key would otherwise write +to production. + +The **instance type is read from the key**: `sk_live_…` is treated as +production, anything else as development. That choice drives the throughput +defaults and the development-instance user limit below. + +**Every command prints its target first.** `import` names the instance — its +environment, its app when the key came from one, and its ID from +`GET /v1/instance` — and where the key came from: `--secret-key`, `--app`, the +`CLERK_SECRET_KEY` env var, an accountless app's `.env.local`, or the linked +profile. `--json` carries the same facts as `target`. + +``` +Target: My App (app_2x9k…), production instance ins_2x9k… +Key from: linked profile +``` + +The instance ID is what a run records. + +## The run store + +Every import that gets as far as writing is a **run**, and the run store is +the one place `clerk migrate` keeps state. A dry run, a refusal, a run that +needs consent and an empty file write none (`run: null`). An import whose +users `--require-password` all leaves out still writes one, recording them +as skipped. + +### Where runs are kept + +The first of these that is set: + +1. `--runs-dir ` +2. `CLERK_MIGRATE_DIR` +3. `/.clerk/migrate/` + +The project root is the linked profile's directory, then the git toplevel, then +the current directory. The first run written to the default location adds +`.clerk/` to the project's `.gitignore`, because run files carry user data. That +happens only once there is consent to write: a dry run or a refused import +leaves `.gitignore` alone. + +### What a run holds + +Each run is a folder named for its ID, `YYYYMMDD-HHmmss-xxxx`: + +| File | Contents | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| `run.json` | Kind, status, start and finish times, the target, the source, the file and its sha256, and the counts | +| `users.ndjson` | One line per user outcome: `sourceId`, `clerkId`, `status`, and `reason`, `error`, `code`, `pending` or `passwordDropped` when present | +| `lock` | The PID of the process writing the run, while it runs | + +A user's status is `creating`, `created`, `failed` or `skipped`. The last line +for each `sourceId` wins. A `429` retry, an extra email or phone that did not +attach, a first phone Clerk refused (which the summary also counts), and a +validation failure all land in the line's `error` field. + +`creating` is written as a user's `POST /v1/users` goes out. It stays the +latest line when no answer says whether the create landed: an abort, a +network error, or a 5xx. A `created` line with `pending` lists the extra emails +and phones not yet attached. + +A run is `partial` when any user failed, was skipped, is still `creating` or +was never sent (`counts.notSent`), and `complete` otherwise. A run interrupted +with Ctrl-C stays `running`, with no `finishedAt`. Its run ID and folder are +printed as the run starts, so they are on screen however it ends. A lock +holding this process's own PID is stale: in a container the CLI often gets the +same PID every run. + +Run folders are created owner-only (`0700`), because they hold user data. + +`users.ndjson` writes are synchronous appends, so a run interrupted with Ctrl-C +still leaves a complete record of everything already processed. A line that +cannot be written stops that user's create from going out. + +### Why `users.ndjson` is NDJSON + +One JSON object per line, rather than one JSON array per file. A migration is a +long append-only stream, and that format is the one that survives it: + +- **Appendable.** Each entry is written as it happens, without rewriting the + file. A JSON array would have to be re-serialized on every user. +- **Crash-safe.** Kill the process at any point and every line already written + is still valid. A truncated array is not parseable at all. +- **Streamable.** `tail -f` shows a long import progressing live, and analysis + reads line by line instead of loading a million-user record into memory. + +Which is also why it greps usefully without any tooling: + +```sh +grep '"status":"created"' .clerk/migrate/20260929-141502-a1b2/users.ndjson | wc -l +grep '"sourceId":"user_123"' .clerk/migrate/20260929-141502-a1b2/users.ndjson +``` + +## Commands + +### `clerk migrate import` + +Reads an exported user file, maps it onto Clerk's user schema, checks every +user against the destination instance, and creates them through the Backend +API. + +```sh +clerk migrate import users.json --source supabase --dry-run # check, write nothing +clerk migrate import users.json --source supabase --yes # import +clerk migrate import users.json --source clerk --allow-partial --yes +clerk migrate import users.json --source clerk --json --yes +clerk migrate import users.json --source clerk --require-password --yes +clerk migrate import users.json --source clerk --skip-legal-checks --yes +clerk migrate import users.json --source clerk --runs-dir ./runs --yes +clerk migrate import users.json --source clerk --app app_123 --instance prod --yes +clerk migrate import users.json --source clerk --secret-key sk_test_... -y +clerk migrate import # a human is asked +``` + +| Flag | Description | +| --------------------- | ------------------------------------------------------------------- | +| `[file]` | A JSON or CSV export | +| `--source ` | Where the file came from: one of the [sources](#sources) | +| `--dry-run` | Run the [checks](#checks) against the instance, and write nothing | +| `--allow-partial` | Import the users that pass, and record the rest as skipped | +| `--require-password` | Import only users that carry a password digest | +| `--skip-legal-checks` | Import users with no legal acceptance into an instance requiring it | +| `-y, --yes` | Import without prompting | +| `--json` | Output as JSON. Never prompts, so importing needs `--yes` | +| `--runs-dir ` | Where runs are kept (see [the run store](#the-run-store)) | + +Plus the targeting flags from the table above: `--secret-key`, `--app` and +`--instance`. + +A file is a JSON array, a CSV, or NDJSON, one user per line (what Auth0's bulk +export job writes). NDJSON is read always for `.ndjson` and `.jsonl`, and for a +`.json` file that doesn't parse whole. A leading BOM is ignored in JSON and +CSV. A file that isn't valid JSON is named in the error. + +**What a human is asked, and what an agent is told.** A human at a terminal who +leaves out the file is asked for its path, and then for its source. An agent, a +non-TTY run, or `--json` without the file or `--source` exits 2 naming what to +pass. + +**Nothing is written without consent.** After the checks, a human is asked +`Import N users?`, and declining writes nothing. `--yes` skips the question. +Without either — an agent, a non-TTY run, `--json` — the run prints the checks +and exits 2 with the exact command to run. Printed commands shell-quote their +paths, keep `--json`, and put `` in place of a secret key. + +**Every run prints its target first**, then the checks. + +Failures do not stop the run: each user's outcome is written to the +[run](#the-run-store) and the import continues. A `429` backs off — +honouring `Retry-After` when the response carries it — and retries up to 5 +times before the user is recorded as failed. The command exits 1 if any user +failed. + +`--require-password` records each user it leaves out as `skipped`, so the run +ends `partial`. + +`--json` returns `{ target, run, checks, result }`. When a run stops before +importing, it carries one of `dryRun: true`, `refused: true`, +`consent: "required"` or `nothingToImport: true` in place of `result`, with +`run: null`. With `--require-password`, `withoutPassword` counts the users it +left out before the checks, so `checks.total` plus it is the file's size. + +#### Re-running + +Running the same import again starts a new run. The users the first run +created are rejected as [already in the instance](#checks), so +`--allow-partial --yes` imports the rest. A user an interrupted run left at +`creating` is rejected the same way when Clerk holds it, and imported when it +doesn't. + +#### Checks + +Every import runs the checks before writing anything, and `--dry-run` stops +after them. They sort the users three ways: + +- **Rejected** — users Clerk would refuse. Each gets the first reason that + applies: + - it failed schema validation + - its source requested a skip (Supabase: a soft-deleted user) + - its only emails are ones Clerk refuses: malformed, or a domain that can't + receive mail (`.local`, `.invalid`, `.test`, `.example`, `.arpa`, + `.internal`, `.lan`, `.corp` and the like). Such an email is dropped from + any other user, with a warning + - its only phones are not in E.164 form, and the instance has numeric + usernames on, which makes Clerk require E.164. Such a phone is dropped from + any other user, with a warning. Spaces around a phone don't count, as Clerk + trims them. Phones the instance has off are stripped anyway, so they are + not checked + - it lacks an identifier the instance requires. An email or phone counts + only when it is verified, because an unverified one is attached after the + user exists. A required phone dropped for not being in E.164 form is named + as such + - it has no identifier left once the emails and phones Clerk would refuse + are stripped: those of an instance that neither has them on nor signs in + or does MFA with them. A username is kept, as Clerk stores it with + usernames off, unless it is one Clerk refuses + - it lacks a first or last name the instance requires + - it has an authenticator app secret or backup codes, and the instance has + that turned off. Importing it without them would take away its second + factor, so the checks offer to turn the setting on instead + - it has no password, and password is the instance's only way to sign in + (no code, link, social or SSO sign-in; a passkey or a password reset does + not count, as in Clerk) + - it has no legal acceptance on record, and the instance requires legal + consent. `--skip-legal-checks`, or a yes at the prompt, imports these users + without it (`skip_legal_checks`), with a warning + - its username breaks the instance's username rules (length, letters, + the allowed special characters). The checks offer the setting that allows + it. With usernames off, such a username is dropped instead, with a warning. + Numeric usernames are not offered when the file has phones not in E.164 + form and the instance has phones on, since the setting would make Clerk + refuse them; the reject counts them + - its password is not the shape its hasher says (`bcrypt`, with a cost of 4 + to 15, `scrypt_firebase`, `argon2i`/`argon2id` and `scrypt_werkzeug` are + checked; other hashers are not, and Clerk refuses a bad one at create) + - Supabase: its only providers are ones Clerk has off, or doesn't offer at all + (Figma, Kakao, Keycloak, WorkOS, Zoom, Fly), and it has no verified email or + phone the instance signs in with by code or link. The checks offer to turn + on the first kind; nothing can turn on the second + - its source ID, primary email, primary phone or username repeats an + earlier user in the file that passes the checks above. Only what the + create sends counts: an extra email is attached after it, and a stripped + one never goes out. Usernames compare case-insensitively and + phones ignore punctuation. The first record in the file is kept, whatever + either holds, and the reject names it (`kept: …`) + - the instance already has a user with its source ID, email, phone or + username (a batched `GET /v1/users` lookup, 100 values a request, through + the scheduler) + - a development instance: it is past the 100-user headroom + (`CLERK_MIGRATE_DEV_USER_LIMIT` when Clerk raised it), counted in file + order +- **Imported, but not everything comes across** — emails or phones the + instance is not set up to store, fields Clerk has no place for (`Clerk won't +store: …`), passwords a source had to drop, emails Clerk refuses (malformed, + or a domain that can't receive mail), and names Clerk refuses (a phone + number, a URL, HTML, blank, or over 256 bytes). A password, username or name + whose setting is off is stored, and works or shows once it is turned on. +- **Imported** — everyone else. + +Any reject stops the import, and it exits 2 with the command that adds +`--allow-partial`. With `--allow-partial`, the rest import and each reject is +recorded as `skipped` with its reason. `--dry-run` exits 2 when the real run +would be refused, and 0 otherwise. + +``` +Checks + 120 users checked + ✗ 12 users rejected + 12: only has an unverified email, and this instance requires an email + u_17, u_22, u_40, u_51, u_88, and 7 more + ⚠ Imported, but not everything comes across + 6 users have a username, which this instance does not use: it is stored, and works only once usernames are turned on + Clerk won't store: department (120 users) + ✓ 108 users to import + +Or change the instance instead + Make Email optional at sign-up + clerk config patch --app app_… --instance ins_… --json '{"auth_email":{"required_for_sign_up":false}}' + Enable Username + clerk config patch --app app_… --instance ins_… --json '{"auth_username":{"used_for_sign_up":true}}' +``` + +Each fix names its instance with `--instance`, so it changes the instance the +import targets, whatever the key's source. A key from `--secret-key` or +`CLERK_SECRET_KEY` names no app, so its fix reads `--app APP_ID`: fill in the +app that owns the instance. When Clerk could not name the +instance (a `key_…` fallback ID), the fix points at the Dashboard instead: in +`--json` it carries `url` in place of `command`. + +The fixes are offers, not corrections: an instance that requires an email is +configured as its owner intended, and fixing the export may be the answer. When +the instance settings cannot be read (BAPI `/v1/domains` → the instance's +Frontend API `/v1/environment`), required fields are not checked and the run +says so. When a development instance's user count cannot be read, the run says +so too, rather than checking the headroom against zero. + +#### Sign-up restrictions + +Every create sends `skip_restriction_checks: true`. The instance's allowlist, +blocklist, disposable-email and subaddress rules police new sign-ups, and these +users already signed up on the source platform. + +#### Additional identifiers + +Only the first verified email and phone go on `POST /v1/users`. Every +additional verified identifier, and every unverified one, is attached +afterwards with its own request, ahead of any create still queued, and backs +off on a `429` like the create. A refusal there is logged and the user still +counts as imported — a duplicate secondary email should not undo an otherwise +successful user. An attach with no answer stays `pending` on the user's line. + +The first phone gets the same treatment when Clerk refuses it — a country the +instance does not support, or a number that is not E.164 — and the user has an +email: the create is retried without the phone. The user counts as imported, +and the summary lists them under "Imported without their phone", by Clerk's +reason (`result.warnings` in `--json`). + +#### Throughput + +Defaults follow Clerk's documented `POST /v1/users` limits: 100 req/s for +production instances, 10 req/s for development. Concurrency defaults to ~95% of +that, assuming ~100ms of API latency. Both are overridable: + +| Variable | Effect | +| --------------------------------- | ------------------------------------------------------------ | +| `CLERK_MIGRATE_RATE_LIMIT` | Requests per second | +| `CLERK_MIGRATE_CONCURRENCY_LIMIT` | Concurrent in-flight requests | +| `CLERK_MIGRATE_DEV_USER_LIMIT` | Development-instance user limit the checks use (default 100) | + +A non-numeric or non-positive value is ignored in favour of the default. + +A development instance's user limit is checked with the other +[checks](#checks): new development instances are created with a 100-user limit, +production instances have none by default (a plan can set one; the import +stops at the first refusal), and the run reads the live count +(`GET /v1/users/count`). The limit itself is not served by any API, so for a +development instance Clerk has raised, set `CLERK_MIGRATE_DEV_USER_LIMIT` to +the raised limit; `--allow-partial` imports up to the headroom. + +Users that do exceed the limit come back in the error breakdown as +`You have reached your limit of N users`, annotated with what a development +instance can do about it. The first refusal stops the import: the users it +never sent are counted under "Not sent" (`result.notSent` in `--json`, and +`counts.notSent` in `run.json`). Once the limit is raised, +[run the import again](#re-running) with `--allow-partial --yes` to send them. + +### `clerk migrate help` + +`clerk migrate help` and `clerk migrate import --help` print the help for the +group or the command, with examples. + +## Sources + +A source maps one platform's export onto Clerk's user schema, and says what it +brings across. Adding a platform is one file in `sources/` plus one line in +`sources/registry.ts`; `--source`'s tab-completion reads from that array. + +| Key | Reads | Passwords | MFA | Metadata | +| ---------- | ---------------------------- | --------- | ------- | -------- | +| `clerk` | Clerk Dashboard export | partial | partial | partial | +| `supabase` | Supabase `auth.users` export | yes | no | partial | + +An unknown `--source` exits 2 with the list of valid keys. + +There is no column for social sign-ins, because no source copies them and none +needs to. Enable the same providers in Clerk, and a user who signs in with one +is linked to their imported account by verified email. See +[account linking](https://clerk.com/docs/guides/configure/auth-strategies/social-connections/account-linking). + +### Metadata + +Supabase's `raw_user_meta_data`, which a user can edit on Supabase, goes to +Clerk's `unsafe_metadata`, which is the user-editable one. Public metadata is +read-only to the user, so putting it there would take away an edit the user +had. A Clerk Dashboard CSV carries no metadata. + +### Verified vs unverified identifiers + +Every platform records verification differently, and each source declares +which style it uses. An identifier the source never confirmed is routed to +`unverifiedEmailAddresses` / `unverifiedPhoneNumbers` rather than the primary +field, because Clerk creates primary identifiers **verified** — sending an +unconfirmed address there would silently promote it. + +- **Boolean**: `true`/`false`. A CSV export stringifies these, so `"false"` is + read as false, not as a non-empty string. `TRUE`, `FALSE`, `t` and `f` read + too, as a spreadsheet or psql writes them. +- **Timestamp** (`supabase`): a nullable confirmation time. Any real value + means verified; `""`, `null` and `\N` do not. + +A Clerk export keeps an unverified primary email or phone unverified. + +## Schema fields + +What a source maps _onto_. Every user is validated against this schema +before any request is made, so a field a source produces that is not listed +here is dropped — Zod strips unknown keys — and never reaches Clerk. The checks +warn about each one (`Clerk won't store: …`). + +The schema lives in `validator.ts`; adding a platform means adding a source, +not editing it. + +**Required:** `userId` (`string`). It becomes the Clerk user's `external_id`. + +**Identifiers.** At least one of these must be present, or the import's checks +reject the user as invalid. Each accepts a single value or an array. + +| Field | Type | Description | +| -------------------------- | -------------------- | ---------------------------------- | +| `email` | `string \| string[]` | Primary verified email address(es) | +| `emailAddresses` | `string \| string[]` | Additional verified emails | +| `unverifiedEmailAddresses` | `string \| string[]` | Unverified emails | +| `phone` | `string \| string[]` | Primary verified phone number(s) | +| `phoneNumbers` | `string \| string[]` | Additional verified phones | +| `unverifiedPhoneNumbers` | `string \| string[]` | Unverified phones | +| `username` | `string` | Username | + +**Profile, password and 2FA.** + +| Field | Type | Description | +| -------------------- | ---------- | --------------------------------------------------- | +| `firstName` | `string` | First name | +| `lastName` | `string` | Last name | +| `password` | `string` | The hashed password from the source platform | +| `passwordHasher` | `enum` | **Required whenever `password` is set** (see below) | +| `totpSecret` | `string` | TOTP secret | +| `backupCodesEnabled` | `boolean` | Whether backup codes are enabled | +| `backupCodes` | `string[]` | Backup codes | + +Clerk verifies the digest as-is, so `passwordHasher` must name the algorithm the +source actually used: + +`argon2i`, `argon2id`, `awscognito`, `bcrypt`, `bcrypt_peppered`, +`bcrypt_sha256_django`, `hmac_sha256_utf16_b64`, `ldap_ssha`, `md5`, +`md5_phpass`, `md5_salted`, `phpass`, `pbkdf2_sha1`, `pbkdf2_sha256`, +`pbkdf2_sha256_django`, `pbkdf2_sha512`, `pbkdf2_sha512_hex`, `scrypt_firebase`, +`scrypt_werkzeug`, `sha256`, `sha256_salted`, `sha512_symfony` + +A user with an unrecognized hasher is rejected by the checks, naming the +hasher, like any other invalid user. + +**Metadata.** + +| Field | Type | Description | +| ----------------- | -------- | -------------------------------------------------------- | +| `unsafeMetadata` | `object` | Readable **and writable** by the client — never trust it | +| `publicMetadata` | `object` | Readable by the client, writable only server-side | +| `privateMetadata` | `object` | Server-side only | + +**Account state.** These are passed straight through to `POST /v1/users`, and +are how a Clerk-to-Clerk migration keeps original signup dates instead of +stamping every user with today's. + +| Field | Type | Description | +| --------------------------- | --------- | ------------------------------------------------------------------------------------ | +| `createdAt` | `string` | Original creation timestamp. A number is epoch milliseconds, or seconds below `1e11` | +| `legalAcceptedAt` | `string` | When legal terms were accepted | +| `banned` | `boolean` | Whether the user is banned | +| `bypassClientTrust` | `boolean` | Skip client trust verification | +| `createOrganizationEnabled` | `boolean` | Whether the user can create orgs | +| `createOrganizationsLimit` | `number` | Maximum orgs the user can create | +| `deleteSelfEnabled` | `boolean` | Whether the user can delete their account | +| `skipLegalChecks` | `boolean` | Skip legal acceptance checks | +| `skipPasswordChecks` | `boolean` | Skip password requirements on import | + +## API endpoints + +| Method | Path | Used by | +| ------ | ------------------------- | ------------------------------------------------------------------------------ | +| `POST` | `/v1/users` | `migrate import` — creates each user | +| `POST` | `/v1/email_addresses` | `migrate import` — attaches additional emails | +| `POST` | `/v1/phone_numbers` | `migrate import` — attaches additional phones | +| `GET` | `/v1/users/count` | `migrate import` — headroom against a development instance's user limit | +| `GET` | `/v1/users?external_id=…` | `migrate import` — checks for users already in the instance, 100 values a call | +| `GET` | `/v1/instance` | `migrate import` — names the instance behind the key | +| `GET` | `/v1/domains` | `migrate import` checks — resolves the Frontend API host | + +The checks also read the instance's Frontend API `GET /v1/environment` +(bootstrapping a dev browser first on development instances) for its +attributes and enabled social providers. Nothing in `clerk migrate` writes +instance settings: the checks print the `clerk config patch` to run instead. + +## Notes -Moves users into Clerk from another auth provider or another Clerk instance. +- `userId` in the source file becomes the Clerk user's `external_id`. +- CSV input is coerced before validation: `a@x.dev,b@x.dev` and `["a@x.dev"]` + both become arrays, `"true"`/`1` become booleans, and JSON metadata columns + are parsed. An empty column is dropped rather than sent as null. +- A user must end up with at least one identifier (email, phone or username). + Users that do not are rejected by the checks as invalid. diff --git a/packages/cli-core/src/commands/migrate/index.test.ts b/packages/cli-core/src/commands/migrate/index.test.ts new file mode 100644 index 000000000..d26c5556f --- /dev/null +++ b/packages/cli-core/src/commands/migrate/index.test.ts @@ -0,0 +1,147 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { getMode, setMode } from "../../mode.ts"; +import { createProgram } from "../../cli-program.ts"; +import { isAssumeYes, setAssumeYes } from "./lib/assume-yes.ts"; + +let saved: string | undefined; +beforeAll(() => { + saved = process.env.CLERK_EXPERIMENTAL; + process.env.CLERK_EXPERIMENTAL = "migrate"; +}); +afterAll(() => { + if (saved === undefined) delete process.env.CLERK_EXPERIMENTAL; + else process.env.CLERK_EXPERIMENTAL = saved; +}); + +function findCommand(names: string[]) { + let current = createProgram().commands.find((cmd) => cmd.name() === names[0]); + for (const name of names.slice(1)) { + current = current?.commands.find((cmd) => cmd.name() === name); + } + return current; +} + +describe("registerMigrate", () => { + test("registers migrate as a top-level command group", () => { + const migrate = findCommand(["migrate"]); + expect(migrate).toBeDefined(); + expect(migrate?.description()).toContain("Migrate users"); + }); + + test("registers import as its only subcommand", () => { + expect(findCommand(["migrate"])?.commands.map((cmd) => cmd.name())).toEqual(["import"]); + }); + + test("leaves migrate with no default subcommand", () => { + const migrate = findCommand(["migrate"]) as unknown as { _defaultCommandName?: string }; + expect(migrate._defaultCommandName).toBeFalsy(); + }); + + test("migrate import takes exactly its flags", () => { + expect(findCommand(["migrate", "import"])?.options.map((option) => option.long)).toEqual([ + "--source", + "--dry-run", + "--allow-partial", + "--require-password", + "--skip-legal-checks", + "--yes", + "--json", + "--secret-key", + "--app", + "--instance", + "--runs-dir", + ]); + }); + + // No `.choices()`: an unknown key reaches resolveSource, whose error lists + // the valid ones. Completion offers them through `KNOWN_OPTION_VALUES`. + test("--source accepts any value", () => { + const option = findCommand(["migrate", "import"])?.options.find((o) => o.long === "--source"); + expect(option?.argChoices).toBeUndefined(); + }); + + test("takes the file as an optional argument", () => { + const [argument] = findCommand(["migrate", "import"])?.registeredArguments ?? []; + expect(argument?.name()).toBe("file"); + expect(argument?.required).toBe(false); + }); + + test("exposes -y as the short form of --yes", () => { + const option = findCommand(["migrate", "import"])?.options.find((o) => o.long === "--yes"); + expect(option?.short).toBe("-y"); + }); +}); + +/** + * The hook is the only link between the parsed flag and the code that reads + * it below the action. If it stopped firing — a Commander upgrade that dropped + * hook inheritance, an action registered outside the group — the behaviour + * would silently revert and every unit test around it would still pass, + * because they set the flag directly. + */ +describe("the migrate group's -y hook", () => { + let missing: string; + beforeAll(() => { + missing = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-hook-")), "none.json"); + }); + + async function parse(argv: string[]) { + const program = createProgram(); + // `exitOverride` so a usage error inside the action throws here instead of + // taking the test runner down with it; the hook has already run by then. + program.exitOverride(); + try { + await program.parseAsync(["node", "clerk", ...argv]); + } catch { + // The action is allowed to fail — only the hook's effect is under test. + } + return isAssumeYes(); + } + + test("records -y on import", async () => { + expect(await parse(["migrate", "import", missing, "-y", "--secret-key", "sk_test_x"])).toBe( + true, + ); + }); + + // `--json` means nobody reads a prompt, and agent mode is how every prompt in + // this tree already knows to stand down. + test("--json runs the command in agent mode", async () => { + const original = getMode(); + try { + setMode("human"); + await parse(["migrate", "import", missing, "--json", "--secret-key", "sk_test_x"]); + expect(getMode()).toBe("agent"); + } finally { + setMode(original); + } + }); + + test("an explicit --mode wins over --json", async () => { + const original = getMode(); + try { + setMode("agent"); + await parse([ + "--mode", + "human", + "migrate", + "import", + missing, + "--json", + "--secret-key", + "sk_test_x", + ]); + expect(getMode()).toBe("human"); + } finally { + setMode(original); + } + }); + + test("records its absence, so a previous run cannot leak into this one", async () => { + setAssumeYes(true); + expect(await parse(["migrate", "import", missing, "--secret-key", "sk_test_x"])).toBe(false); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index 34847075c..4323a6893 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -1,5 +1,9 @@ import type { Program } from "../../cli-program.ts"; import { isExperimentEnabled, requireExperiment } from "../../lib/experimental.ts"; +import { setMode } from "../../mode.ts"; +import { setAssumeYes } from "./lib/assume-yes.ts"; +import { RUNS_DIR_DESCRIPTION, RUNS_DIR_FLAG } from "./lib/run-store.ts"; +import { run } from "./run.ts"; export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = process.env): void { if (!isExperimentEnabled("migrate", env)) { @@ -22,7 +26,70 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce return; } - program + const migrateCommand = program .command("migrate") - .description("Migrate users into Clerk from another auth provider or another Clerk instance"); + .description("Migrate users into Clerk from another auth provider or another Clerk instance") + .setExamples([ + { + command: "clerk migrate import users.json --source supabase --dry-run", + description: "Check a Supabase export against the instance, and write nothing", + }, + { + command: "clerk migrate import users.json --source supabase --yes", + description: "Import it", + }, + ]); + + // `-y` is resolved once here, for code below a command's options to read + // rather than having it passed down. Hooks are inherited, so this fires for + // every subcommand under `migrate`; one that declares no `-y` resolves to false. + // + // `--json` means nobody is reading a prompt, so it runs the command in agent + // mode: every prompt in this tree already stands down for an agent, with the + // usage error naming what to pass instead. An explicit `--mode` still wins. + migrateCommand.hook("preAction", (_thisCommand, actionCommand) => { + const opts = actionCommand.optsWithGlobals(); + setAssumeYes(Boolean(opts.yes)); + if (opts.json && !opts.mode) setMode("agent"); + }); + + // Named, not `isDefault`: bare `clerk migrate` prints help. + // + // The flags stay here rather than on `migrate`, matching how `config` keeps + // its own on `pull`/`patch`/`put` — a group's help is a list of subcommands + // and examples, not a merge of everything underneath it. + migrateCommand + .command("import") + .description("Import users from an exported JSON or CSV file") + .argument("[file]", "A JSON or CSV export") + .option("--source ", "Where the file came from") + .option("--dry-run", "Check the file against the instance, report, and write nothing") + .option("--allow-partial", "Import the users that pass the checks, and skip the rest") + .option("--require-password", "Import only users that have a password") + .option( + "--skip-legal-checks", + "Import users with no legal acceptance on record into an instance that requires it", + ) + .option("-y, --yes", "Import without prompting") + .option("--json", "Output as JSON; never prompts, so pair it with --yes to import") + .option("--secret-key ", "Backend API secret key to use") + .option("--app ", "Application ID to target (works from any directory)") + .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") + .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) + .setExamples([ + { + command: "clerk migrate import users.json --source supabase --dry-run", + description: "Check the file against the instance, and write nothing", + }, + { + command: "clerk migrate import users.json --source clerk --allow-partial --yes", + description: "Import a Clerk Dashboard export, skipping users that would be rejected", + }, + ]) + .action(async (input, _opts, cmd) => + run({ + ...(cmd.optsWithGlobals() as Parameters[0]), + ...(input ? { input } : {}), + }), + ); } diff --git a/packages/cli-core/src/commands/migrate/lib/assume-yes.ts b/packages/cli-core/src/commands/migrate/lib/assume-yes.ts new file mode 100644 index 000000000..c61c7ef86 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/assume-yes.ts @@ -0,0 +1,24 @@ +/** + * Whether this run was given `-y`. + * + * Not the same question as {@link isAgent}: agent mode says the CLI *cannot* + * prompt; `-y` says the operator does not want it to. + * + * Set once by the `migrate` group's `preAction` hook, so every subcommand under + * it is covered whether or not it declares the flag — one that does not simply + * resolves to `false`. Held per run, the way `mode.ts` holds `--mode`, so code + * below a command's options can read it without the flag being passed down. + * Nothing in `import` needs that: it reads `options.yes` directly. + */ + +let assumeYes = false; + +/** Records this run's `-y`. Called once per invocation, before the action. */ +export function setAssumeYes(value: boolean): void { + assumeYes = value; +} + +/** Whether `-y` was passed to the command now running. */ +export function isAssumeYes(): boolean { + return assumeYes; +} diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts index 48fab60e2..049cffb8f 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts @@ -193,6 +193,27 @@ describe("locks", () => { }); // In case a folder's mode is ever looser than the run store sets it. + // Users never sent have no line to count, so the caller says how many. + test("a finish with users never sent is partial", () => { + const sent = startRun(runsDir, init); + sent.append({ sourceId: "a", status: "created", clerkId: "user_a" }); + expect(sent.finish().status).toBe("complete"); + + const stopped = startRun(runsDir, init); + stopped.append({ sourceId: "a", status: "created", clerkId: "user_a" }); + const record = stopped.finish({ notSent: 2 }); + expect(record.status).toBe("partial"); + // Saved, so a later reader can tell how many were never sent. + expect(readRun(runsDir, record.id)?.counts).toEqual({ total: 1, created: 1, notSent: 2 }); + }); + + test("release leaves the run unfinished and unlocked", () => { + const run = startRun(runsDir, init); + run.release(); + expect(readRun(runsDir, run.record.id)?.finishedAt).toBeUndefined(); + expect(fs.existsSync(path.join(run.dir, "lock"))).toBe(false); + }); + test.skipIf(process.platform === "win32")("run files are owner-only", () => { const run = startRun(runsDir, init); run.append({ sourceId: "u1", status: "creating" }); diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index acc0017ef..f39b5150a 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -60,7 +60,8 @@ export type RunTarget = { export type RunFile = { path: string; sha256: string }; -export type RunCounts = Partial> & { total: number }; +/** `notSent`: users a stopped import never sent, so they have no line. */ +export type RunCounts = Partial> & { total: number; notSent?: number }; export type RunRecord = { id: string; @@ -312,10 +313,16 @@ export type Run = { /** * Counts the outcomes, settles the status and releases the lock. * - * `partial` when any user failed, was skipped, may not have been created or - * has identifiers still pending, `complete` otherwise. + * `partial` when any user failed, was skipped, may not have been created, + * was never sent (`notSent`: they have no line to count) or has identifiers + * still pending, `complete` otherwise. */ - finish(): RunRecord; + finish(options?: { notSent?: number }): RunRecord; + /** + * Releases the lock and leaves the run unfinished: with no `finishedAt` and + * no live holder, it reads as interrupted. + */ + release(): void; }; function openRun(runsDir: string, record: RunRecord): Run { @@ -336,23 +343,31 @@ function openRun(runsDir: string, record: RunRecord): Run { run.record = { ...run.record, ...patch }; writeRecord(runsDir, run.record); }, - finish() { + finish(options) { const lines = [...latestUserLines(runsDir, run.record.id).values()]; const counts = countLines(lines); + if (options?.notSent) counts.notSent = options.notSent; // A created user whose extra identifiers never attached is left for a continue. const pendingAttaches = lines.filter( (line) => line.status === "created" && (line.pending?.length ?? 0) > 0, ).length; const unfinished = - (counts.failed ?? 0) + (counts.skipped ?? 0) + (counts.creating ?? 0) + pendingAttaches; + (counts.failed ?? 0) + + (counts.skipped ?? 0) + + (counts.creating ?? 0) + + (counts.notSent ?? 0) + + pendingAttaches; run.update({ counts, status: unfinished > 0 ? "partial" : "complete", finishedAt: new Date().toISOString(), }); - fs.rmSync(path.join(dir, LOCK_FILE), { force: true }); + run.release(); return run.record; }, + release() { + fs.rmSync(path.join(dir, LOCK_FILE), { force: true }); + }, }; return run; } diff --git a/packages/cli-core/src/commands/migrate/readme.test.ts b/packages/cli-core/src/commands/migrate/readme.test.ts new file mode 100644 index 000000000..05faefc31 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/readme.test.ts @@ -0,0 +1,186 @@ +/** + * Keeps README.md and the command tree honest about each other. + * + * Checking the README by eye at review time does not scale, and a doc that + * names a flag the binary rejects is worse than no doc: the reader trusts it + * and gets a usage error. + * + * Both directions are checked — every example must resolve, and every flag must + * be written down — so neither renaming a flag nor adding one passes silently. + */ + +import { afterAll, describe, expect, test } from "bun:test"; +import type { Command } from "commander"; +import { createProgram } from "../../cli-program.ts"; + +// Without it `migrate` is the gate's stub, with no subcommands to check. Set at +// load, because the cases below are built from the command tree. +const savedExperimental = process.env.CLERK_EXPERIMENTAL; +process.env.CLERK_EXPERIMENTAL = "migrate"; +afterAll(() => { + if (savedExperimental === undefined) delete process.env.CLERK_EXPERIMENTAL; + else process.env.CLERK_EXPERIMENTAL = savedExperimental; +}); + +const README = await Bun.file(new URL("./README.md", import.meta.url)).text(); + +/** Fenced blocks only, so prose that merely mentions a flag is not parsed. */ +function fencedBlocks(markdown: string): string[] { + return [...markdown.matchAll(/^```[a-z]*\n([\s\S]*?)^```/gm)].map((match) => match[1] ?? ""); +} + +/** + * Every `clerk migrate …` invocation the README puts in front of a reader, from + * fenced blocks and inline backticks alike — both get copied. + */ +function documentedCommands(markdown: string): string[] { + const found = new Set(); + + for (const block of fencedBlocks(markdown)) { + // Line continuations first, so a multi-line example reads as one. + for (const line of block.replace(/\\\n\s*/g, " ").split("\n")) { + const start = line.indexOf("clerk migrate"); + // A command never contains a backtick, a `#`, or a run of two spaces; + // the sample error output that quotes `clerk migrate` mid-sentence does, + // and so does a pasted listing whose descriptions sit in a padded column. + if (start !== -1) + found.add( + line + .slice(start) + .split(/[`#]|\s{2,}/)[0]! + .trim(), + ); + } + } + + for (const match of markdown.matchAll(/`(clerk migrate[^`]*)`/g)) { + found.add(match[1]!.trim()); + } + + return [...found]; +} + +/** Walks as deep as the tree allows; the first flag or positional stops it. */ +function resolve(tokens: string[]): { command: Command; rest: string[] } { + let command = createProgram() as Command; + let index = 0; + for (; index < tokens.length; index++) { + const child = command.commands.find( + (candidate) => + candidate.name() === tokens[index] || candidate.aliases().includes(tokens[index]!), + ); + if (!child) break; + command = child; + } + return { command, rest: tokens.slice(index) }; +} + +/** + * The flags a command accepts, including those of a default subcommand. + * + * A group that registers a subcommand `isDefault` hands it everything after + * the group name, so this has to see the same flags Commander does or every + * such example reads as unsupported. + */ +function flagsOf(command: Command): string[] { + const own = command.options.flatMap( + (option) => [option.short, option.long].filter(Boolean) as string[], + ); + + const defaultChild = command.commands.find( + // Commander records the default subcommand on the parent, not the child. + (child) => child.name() === (command as any)._defaultCommandName, + ); + + return defaultChild ? [...own, ...flagsOf(defaultChild)] : own; +} + +/** Every command under `migrate`, so no subcommand escapes the flag sweep. */ +function migrateTree(): { path: string; command: Command }[] { + const collected: { path: string; command: Command }[] = []; + const visit = (command: Command, path: string) => { + collected.push({ path, command }); + for (const child of command.commands) { + if (child.name() !== "help") visit(child, `${path} ${child.name()}`); + } + }; + visit(resolve(["migrate"]).command, "migrate"); + return collected; +} + +const EXAMPLES = documentedCommands(README); + +/** + * The words of an example, minus grammar: a synopsis writes `` and + * `[--json]` to say what goes there, and those are not values to resolve. + */ +function words(example: string): string[] { + return example + .split(/\s+/) + .slice(1) + .filter((token) => !/^[<[]/.test(token) && !/[>\]]$/.test(token)); +} + +/** One case per (example, flag) pair, so a failure names the exact flag. */ +const FLAG_USES: [string, string][] = EXAMPLES.flatMap((example) => + words(example) + // Every command accepts `--help`; Commander does not list it as an option. + .filter((token) => token.startsWith("-") && token !== "--help") + .map((token) => [example, token.split("=")[0]!] as [string, string]), +); + +describe("migrate README", () => { + // Guards the extractor: a regex that silently matched nothing would make + // every check below pass vacuously. + test("finds the documented examples", () => { + expect(EXAMPLES.length).toBeGreaterThan(10); + expect(FLAG_USES.length).toBeGreaterThan(10); + }); + + test.each(EXAMPLES)("`%s` resolves to a real command", (example) => { + // `help` is Commander's own subcommand on every group, not a registered one. + const tokens = words(example).filter((token) => token !== "help"); + const { command, rest } = resolve(tokens); + const firstFlag = rest.findIndex((token) => token.startsWith("-")); + const positionals = firstFlag === -1 ? rest : rest.slice(0, firstFlag); + // Leftover words before any flag are positionals — only some commands take + // them, and a subcommand that does not exist lands here too. + if (positionals.length > 0) expect(command.registeredArguments.length).toBeGreaterThan(0); + // A parent means at least `migrate` resolved. Checking the name instead + // would be wrong: `migrate export clerk` is itself named `clerk`. + expect(command.parent).not.toBeNull(); + }); + + test.each(FLAG_USES)("`%s` uses %s, which the command accepts", (example, flag) => { + const { command } = resolve(words(example)); + expect(flagsOf(command)).toContain(flag); + }); + + // The README is organised around the commands, so each one gets a heading. + test.each(resolve(["migrate"]).command.commands.map((child) => child.name()))( + "has a section for `clerk migrate %s`", + (name) => { + expect(README).toContain(`### \`clerk migrate ${name}\``); + }, + ); + + test("states the rules every command follows", () => { + expect(README).toContain("## The rules"); + for (const rule of [ + "Nothing writes without consent", + "`--dry-run` checks against the real instance", + "State lives in one place", + "Every command prints its target first", + "Every subcommand takes `--json`", + ]) { + expect(README).toContain(rule); + } + }); + + test.each(migrateTree())("$path documents every flag it accepts", ({ command }) => { + const undocumented = flagsOf(command).filter( + (flag) => flag.startsWith("--") && flag !== "--help" && !README.includes(flag), + ); + expect(undocumented).toEqual([]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts new file mode 100644 index 000000000..c4c201526 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -0,0 +1,254 @@ +/** + * The human-mode half of `migrate import`: the prompts fill in what was not + * passed, and nothing is written until the operator says yes. + * + * Kept in its own file because `mock.module` replaces `prompts.ts` for the whole + * file; `bun test --parallel` isolates each file, so the mock ends with it. + * Human mode is set through `CLERK_MODE` rather than `setMode`, because + * `mode.ts` has no way to clear a forced mode once the file is done. + */ + +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { listageStubs, useCaptureLog } from "../../test/lib/stubs.ts"; + +const mockSelect = mock(async () => "clerk" as unknown); +const mockText = mock(async () => "export.json" as unknown); +let confirmAnswer = true; +/** User settings the instance serves, or none (the settings read fails). */ +let instanceSettings: Record | undefined; +/** Every confirmation the run put up, in order — the wording is the assertion. */ +let confirmMessages: string[] = []; +let originalMode: string | undefined; + +mock.module("../../lib/listage.ts", () => ({ + ...listageStubs, + select: (...args: unknown[]) => mockSelect(...(args as [])), +})); + +// Every export of the real module must appear here — a missing one is a link +// error at import time, which takes down the whole file rather than one prompt. +mock.module("../../lib/prompts.ts", () => ({ + confirm: async ({ message }: { message: string }) => { + confirmMessages.push(message); + return confirmAnswer; + }, + multiselect: async () => [], + text: (...args: unknown[]) => mockText(...(args as [])), + password: async () => "", + editor: async () => "{}", +})); + +const { run } = await import("./run.ts"); +const { UserAbortError } = await import("../../lib/errors.ts"); +const { _setConfigDir } = await import("../../lib/config.ts"); + +const captured = useCaptureLog(); + +let workDir: string; +let configDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: { method: string; url: string }[]; + +const EXPORT = [ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", primary_email_address: "b@x.dev" }, +]; + +const runsDir = () => path.join(workDir, ".clerk", "migrate"); + +beforeAll(() => { + originalMode = process.env.CLERK_MODE; + process.env.CLERK_MODE = "human"; + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-interactive-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-interactive-config-")); + _setConfigDir(configDir); + process.chdir(workDir); +}); + +afterAll(() => { + if (originalMode === undefined) delete process.env.CLERK_MODE; + else process.env.CLERK_MODE = originalMode; + globalThis.fetch = originalFetch; + _setConfigDir(undefined); + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + requests = []; + confirmAnswer = true; + instanceSettings = undefined; + confirmMessages = []; + mockSelect.mockReset(); + mockText.mockReset(); + mockSelect.mockResolvedValue("clerk"); + mockText.mockResolvedValue("export.json"); + fs.rmSync(path.join(workDir, ".clerk"), { recursive: true, force: true }); + fs.rmSync(path.join(configDir, "config.json"), { force: true }); + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(EXPORT)); + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(input.toString()); + const method = init?.method ?? "GET"; + requests.push({ method, url: url.toString() }); + if (url.pathname === "/v1/instance") { + return Response.json({ object: "instance", id: "ins_1", environment_type: "development" }); + } + if (url.pathname === "/v1/users" && method === "GET") return Response.json([]); + if (url.pathname === "/v1/users/count") return Response.json({ total_count: 0 }); + if (url.pathname === "/v1/domains") { + if (!instanceSettings) return new Response("nope", { status: 500 }); + return Response.json({ + data: [{ is_satellite: false, frontend_api_url: "https://fapi.example.com" }], + }); + } + if (url.pathname.includes("/v1/dev_browser")) return Response.json({ token: "jwt" }); + if (url.pathname.includes("/v1/environment")) { + return Response.json({ user_settings: instanceSettings }); + } + return Response.json({ id: "user_created" }); + }) as unknown as typeof fetch; +}); + +afterEach(() => { + process.exitCode = 0; +}); + +const created = () => + requests.filter((r) => r.method === "POST" && new URL(r.url).pathname === "/v1/users"); + +const importOptions = { input: "export.json", source: "clerk", secretKey: "sk_test_x" }; + +describe("prompts for what was not passed", () => { + test("bare `clerk migrate import` asks for the file, then the source, then imports", async () => { + await run({ secretKey: "sk_test_x" }); + + expect(mockText).toHaveBeenCalledTimes(1); + expect(mockSelect).toHaveBeenCalledTimes(1); + expect(created()).toHaveLength(2); + }); + + test("asks only for what was not passed", async () => { + await run(importOptions); + + expect(mockText).not.toHaveBeenCalled(); + expect(mockSelect).not.toHaveBeenCalled(); + }); +}); + +describe("consent", () => { + test("asks before writing, naming how many users", async () => { + await run(importOptions); + + expect(confirmMessages).toEqual(["Import 2 users?"]); + }); + + test("names the users it would skip", async () => { + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify([...EXPORT, { id: "u3" }])); + await run({ ...importOptions, allowPartial: true }); + + expect(confirmMessages).toEqual(["Create 2 users and skip 1 user?"]); + }); + + test("prints the checks before the question", async () => { + await run(importOptions); + + expect(captured.err).toContain("Checks"); + }); + + test("declining writes nothing to Clerk, and records no run", async () => { + confirmAnswer = false; + + await expect(run(importOptions)).rejects.toThrow(UserAbortError); + + expect(created()).toHaveLength(0); + expect(fs.existsSync(runsDir())).toBe(false); + }); + + test("--yes does not ask", async () => { + await run({ ...importOptions, yes: true }); + + expect(confirmMessages).toEqual([]); + expect(created()).toHaveLength(2); + }); + + // `--json` means nobody reads a prompt, even at a terminal. + test("--json never asks, and without --yes refuses", async () => { + await expect(run({ ...importOptions, json: true })).rejects.toThrow(/Pass --yes to confirm/); + + expect(confirmMessages).toEqual([]); + expect(created()).toHaveLength(0); + }); + + test("an unrecognized password hasher is rejected before any request", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { + id: "u1", + primary_email_address: "a@x.dev", + password_digest: "d", + password_hasher: "rot13", + }, + ]), + ); + + await expect(run(importOptions)).rejects.toThrow(/1 user would be rejected/); + expect(created()).toHaveLength(0); + }); +}); + +describe("legal consent", () => { + beforeEach(() => { + instanceSettings = { + attributes: { email_address: { enabled: true } }, + sign_up: { legal_consent_enabled: true }, + // A way in besides a password, so users without one import. + enterprise_sso: { enabled: true }, + }; + }); + + test("asks, and a yes imports the users without it", async () => { + await run(importOptions); + + expect(confirmMessages[0]).toContain("no legal acceptance on record"); + expect(created()).toHaveLength(2); + }); + + // A rerun with --yes skips the prompt, so the command has to carry the answer. + test("a yes carries into the command the refusal suggests", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + ...EXPORT, + { + id: "u3", + primary_email_address: "c@x.dev", + password_digest: "d", + password_hasher: "rot13", + }, + ]), + ); + + const error = (await run(importOptions).catch((caught: unknown) => caught)) as { + examples?: { command: string }[]; + }; + + expect(confirmMessages[0]).toContain("no legal acceptance on record"); + expect(error.examples?.[0]?.command).toContain("--skip-legal-checks"); + }); + + // `-y` is "import without prompting": the checks reject these users instead. + test("--yes does not ask, and the checks reject them", async () => { + await expect(run({ ...importOptions, yes: true })).rejects.toThrow(/2 users would be rejected/); + + expect(confirmMessages).toEqual([]); + expect(created()).toHaveLength(0); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/run.subprocess.test.ts b/packages/cli-core/src/commands/migrate/run.subprocess.test.ts new file mode 100644 index 000000000..124c55b0c --- /dev/null +++ b/packages/cli-core/src/commands/migrate/run.subprocess.test.ts @@ -0,0 +1,77 @@ +/** + * A real Ctrl-C mid-import, in a child process. + * + * `CLI_SIGINT_HANDLER` exits once telemetry is flushed, before the import gets + * back to `run()`, so nothing `run()` prints after the import reaches the + * terminal. A test that latches the interrupt directly never runs the handler, + * so this one sends the child a real SIGINT and reads its stderr. + */ + +import { afterAll, beforeAll, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +let workDir: string; + +beforeAll(() => { + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-sigint-"))); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([{ id: "u1", primary_email_address: "a@x.dev" }]), + ); +}); + +afterAll(() => { + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +test("a real Ctrl-C mid-import leaves the run ID and folder on screen", async () => { + const source = ` + const { CLI_SIGINT_HANDLER } = await import(${JSON.stringify(`${import.meta.dir}/../../lib/signals.ts`)}); + process.on("SIGINT", CLI_SIGINT_HANDLER); + setTimeout(() => process.exit(99), 10_000); // guard against a hang + globalThis.fetch = async (input, init) => { + const url = new URL(input.toString()); + const method = init?.method ?? "GET"; + if (url.pathname === "/v1/instance") { + return Response.json({ object: "instance", id: "ins_1", environment_type: "development" }); + } + if (url.pathname === "/v1/users/count") return Response.json({ total_count: 0 }); + if (method === "GET" && url.pathname === "/v1/users") return Response.json([]); + if (method === "POST" && url.pathname === "/v1/users") { + process.kill(process.pid, "SIGINT"); + return new Promise(() => {}); + } + return new Response("unavailable", { status: 503 }); + }; + const { run } = await import(${JSON.stringify(`${import.meta.dir}/run.ts`)}); + await run({ source: "clerk", input: "export.json", yes: true, secretKey: "sk_test_x" }); + `; + + // An inherited runs-dir override would send the run somewhere this test + // does not look. + const { + CLERK_CLI_NO_SIGNAL_RERAISE: _suppressed, + CLERK_MIGRATE_DIR: _runsDir, + ...cleanEnv + } = process.env; + const proc = Bun.spawn(["bun", "-e", source], { + cwd: workDir, + stdout: "ignore", + stderr: "pipe", + env: { + ...cleanEnv, + CLERK_CONFIG_DIR: path.join(workDir, "config"), + CLERK_MODE: "agent", + // Never emit a real event from a test run. + CLERK_TELEMETRY_DISABLED: "1", + }, + }); + const stderr = await new Response(proc.stderr).text(); + await proc.exited; + + expect(proc.signalCode).toBe("SIGINT"); + const [runId] = fs.readdirSync(path.join(workDir, ".clerk", "migrate")); + expect(stderr).toContain(`Run ${runId}: ${path.join(workDir, ".clerk", "migrate", runId ?? "")}`); +}, 15_000); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts new file mode 100644 index 000000000..54ad5a9a8 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -0,0 +1,872 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { _setConfigDir } from "../../lib/config.ts"; +import { type CliError, EXIT_CODE } from "../../lib/errors.ts"; +import { credentialStoreStubs, useCaptureLog } from "../../test/lib/stubs.ts"; + +// Every test below names its own `--secret-key`, which short-circuits the +// signed-in check — except the one that asserts what happens without it. +mock.module("../../lib/credential-store.ts", () => credentialStoreStubs); +import { _resetInterruptState, abortInFlight, beginInterrupt } from "../../lib/signals.ts"; +import { latestUserLines, readRun, type RunRecord } from "./lib/run-store.ts"; +import { explainErrors, run, validateRunOptions } from "./run.ts"; + +/** A real-shaped bcrypt digest: the checks reject anything that is not. */ +const BCRYPT = "$2a$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy"; + +let workDir: string; +let configDir: string; +let originalCwd: string; + +/** Where runs land for a project rooted at `workDir`. */ +const runsDir = () => path.join(workDir, ".clerk", "migrate"); + +/** The runs in `dir`, newest first. */ +const runsIn = (dir: string): RunRecord[] => + (fs.existsSync(dir) ? fs.readdirSync(dir) : []) + .map((id) => readRun(dir, id)) + .filter((record): record is RunRecord => record !== undefined) + .sort((a, b) => b.id.localeCompare(a.id)); + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-run-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-run-config-")); + _setConfigDir(configDir); + process.chdir(workDir); + fs.writeFileSync(path.join(workDir, "users.json"), "[]"); + fs.writeFileSync(path.join(workDir, "users.txt"), ""); +}); + +afterAll(() => { + _setConfigDir(undefined); + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); +}); + +describe("validateRunOptions", () => { + test("accepts a source and an existing JSON file", () => { + expect(validateRunOptions({ source: "clerk", file: "users.json" })).toEqual({ + source: "clerk", + file: "users.json", + }); + }); + + test.each([ + ["no source", { file: "users.json" }, /Missing --source/], + ["no file", { source: "clerk" }, /Missing the file to import/], + ["a missing file", { source: "clerk", file: "nope.json" }, /File not found/], + ["an unsupported extension", { source: "clerk", file: "users.txt" }, /Unsupported file type/], + ])("rejects %s", (_label, options, message) => { + expect(() => validateRunOptions(options)).toThrow(message); + }); + + test("names the valid sources when one is missing", () => { + expect(() => validateRunOptions({ file: "users.json" })).toThrow(/clerk/); + }); +}); + +type Stub = { + /** What `/v1/environment` reports; `null` makes the settings unreadable. */ + settings?: { + attributes?: object; + social?: object; + sign_up?: object; + enterprise_sso?: object; + } | null; + /** Users already in the instance, as `GET /v1/users` returns them. */ + existing?: { + id: string; + external_id?: string; + username?: string; + email_addresses?: { email_address: string }[]; + }[]; + /** `GET /v1/users/count`. */ + count?: number; + /** Source IDs whose `POST /v1/users` fails with a 422. */ + failing?: Set; +}; + +describe("run", () => { + const captured = useCaptureLog(); + let originalFetch: typeof globalThis.fetch; + let requests: { method: string; url: string; body: unknown }[]; + + const export2 = [ + { + id: "u1", + primary_email_address: "a@x.dev", + password_digest: BCRYPT, + password_hasher: "bcrypt", + }, + { id: "u2", primary_email_address: "b@x.dev" }, + ]; + + /** A fake Clerk: the instance, its settings, its users, and the writes. */ + function stubClerk(stub: Stub = {}): void { + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(input.toString()); + const method = init?.method ?? "GET"; + const body = init?.body ? (JSON.parse(init.body as string) as Record) : null; + requests.push({ method, url: url.toString(), body }); + + if (url.pathname === "/v1/instance") { + return Response.json({ object: "instance", id: "ins_1", environment_type: "development" }); + } + if (url.pathname === "/v1/domains") { + if (!stub.settings) return new Response("nope", { status: 500 }); + return Response.json({ + data: [{ is_satellite: false, frontend_api_url: "https://fapi.example.com" }], + }); + } + if (url.pathname.includes("/v1/dev_browser")) return Response.json({ token: "jwt" }); + if (url.pathname.includes("/v1/environment")) { + return Response.json({ user_settings: stub.settings }); + } + if (url.pathname === "/v1/users/count") { + return Response.json({ object: "total_count", total_count: stub.count ?? 0 }); + } + if (method === "GET" && url.pathname === "/v1/users") { + // BAPI strips the `+` the lookup puts on each external_id. + const wanted = new Set( + [...url.searchParams.values()].map((value) => value.replace(/^\+(?!\d)/, "")), + ); + return Response.json( + (stub.existing ?? []).filter( + (user) => + wanted.has(user.external_id ?? "") || + wanted.has(user.username ?? "") || + (user.email_addresses ?? []).some((email) => wanted.has(email.email_address)), + ), + ); + } + if (method === "POST" && url.pathname === "/v1/users") { + const externalId = body?.external_id as string; + if (stub.failing?.has(externalId)) { + return Response.json( + { + errors: [{ code: "form_identifier_exists", message: "That email address is taken." }], + }, + { status: 422 }, + ); + } + return Response.json({ id: `user_${externalId}` }); + } + return Response.json({ id: "ok" }); + }) as typeof fetch; + } + + beforeAll(() => { + originalFetch = globalThis.fetch; + }); + + beforeEach(() => { + requests = []; + delete process.env.CLERK_MIGRATE_RATE_LIMIT; + fs.rmSync(runsDir(), { recursive: true, force: true }); + fs.rmSync(path.join(configDir, "config.json"), { force: true }); + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(export2)); + stubClerk(); + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + process.exitCode = 0; + }); + + const baseOptions = { + source: "clerk", + input: "export.json", + yes: true, + secretKey: "sk_test_x", + }; + + const created = () => + requests + .filter((r) => r.method === "POST" && r.url.endsWith("/v1/users")) + .map((r) => (r.body as { external_id: string }).external_id); + + const exitCodeOf = async (promise: Promise) => + ((await promise.catch((caught: unknown) => caught)) as CliError | undefined)?.exitCode; + + test("refuses before anything else when nobody is signed in", async () => { + const previous = process.env.CLERK_SECRET_KEY; + delete process.env.CLERK_SECRET_KEY; + try { + await expect(run({ source: "clerk", input: "export.json", yes: true })).rejects.toThrow( + /Not logged in/, + ); + expect(requests).toHaveLength(0); + } finally { + if (previous !== undefined) process.env.CLERK_SECRET_KEY = previous; + } + }); + + // --app needs an account to resolve its key, unlike --secret-key. + test("refuses an --app import when nobody is signed in", async () => { + const previous = process.env.CLERK_SECRET_KEY; + delete process.env.CLERK_SECRET_KEY; + try { + await expect( + run({ source: "clerk", input: "export.json", yes: true, app: "app_123" }), + ).rejects.toThrow(/Not logged in/); + expect(requests).toHaveLength(0); + } finally { + if (previous !== undefined) process.env.CLERK_SECRET_KEY = previous; + } + }); + + // resolveBapiSecretKey takes --app over the exported key, so the sign-in is + // still needed: without it the Platform API call fails instead. + test("refuses an --app import when nobody is signed in, even with CLERK_SECRET_KEY set", async () => { + const previous = process.env.CLERK_SECRET_KEY; + process.env.CLERK_SECRET_KEY = "sk_test_x"; + try { + await expect( + run({ source: "clerk", input: "export.json", yes: true, app: "app_123" }), + ).rejects.toThrow(/Not logged in/); + expect(requests).toHaveLength(0); + } finally { + if (previous === undefined) delete process.env.CLERK_SECRET_KEY; + else process.env.CLERK_SECRET_KEY = previous; + } + }); + + test("imports every user in the file end to end", async () => { + await run(baseOptions); + + expect(created()).toEqual(["u1", "u2"]); + expect(captured.err).toContain("Imported:"); + }); + + test("prints the target first", async () => { + await run(baseOptions); + expect(captured.err).toContain("Target: development instance ins_1"); + expect(captured.err.indexOf("Target:")).toBeLessThan(captured.err.indexOf("Checks")); + }); + + test("records the run in the project's run store", async () => { + await run(baseOptions); + + const [record, ...rest] = runsIn(runsDir()); + expect(rest).toHaveLength(0); + expect(record).toMatchObject({ + kind: "import", + status: "complete", + source: "clerk", + counts: { total: 2, created: 2 }, + target: { keySource: "--secret-key", instanceType: "dev", instanceId: "ins_1" }, + }); + expect(record?.file?.path).toBe(path.join(workDir, "export.json")); + expect(record?.file?.sha256).toMatch(/^[0-9a-f]{64}$/); + + const lines = [...latestUserLines(runsDir(), record!.id).values()]; + expect(lines.map((line) => [line.sourceId, line.status, line.clerkId])).toEqual([ + ["u1", "created", "user_u1"], + ["u2", "created", "user_u2"], + ]); + }); + + test("gitignores the project's .clerk folder before writing a run", async () => { + await run(baseOptions); + expect(fs.readFileSync(path.join(workDir, ".gitignore"), "utf-8")).toContain(".clerk/"); + }); + + test("leaves .gitignore alone when the import is refused for lack of consent", async () => { + fs.rmSync(path.join(workDir, ".gitignore"), { force: true }); + + await run({ ...baseOptions, yes: false }).catch(() => undefined); + + expect(fs.existsSync(path.join(workDir, ".gitignore"))).toBe(false); + }); + + test("--runs-dir puts the run somewhere else", async () => { + await run({ ...baseOptions, runsDir: "elsewhere" }); + expect(runsIn(path.join(workDir, "elsewhere"))).toHaveLength(1); + expect(runsIn(runsDir())).toHaveLength(0); + }); + + test("--require-password leaves out the users without one", async () => { + await run({ ...baseOptions, requirePassword: true }); + + expect(created()).toEqual(["u1"]); + expect(captured.err).toContain("leaving out 1 user without a password"); + const [record] = runsIn(runsDir()); + expect(latestUserLines(runsDir(), record!.id).get("u2")).toMatchObject({ + status: "skipped", + reason: "no password (--require-password)", + }); + expect(record?.status).toBe("partial"); + }); + + // They never reach the checks, so the preview has to count them itself. + test("--require-password counts the users it left out in the --json preview", async () => { + await run({ ...baseOptions, requirePassword: true, dryRun: true, json: true }); + + expect(JSON.parse(captured.out)).toMatchObject({ + dryRun: true, + withoutPassword: 1, + checks: { total: 1 }, + }); + }); + + test("rejects a user with an unrecognized hasher, naming it", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { + id: "u1", + primary_email_address: "a@x.dev", + password_digest: "d", + password_hasher: "rot13", + }, + ]), + ); + + const error = (await run(baseOptions).catch((caught: unknown) => caught)) as CliError; + expect(error.message).toContain("1 user would be rejected, so nothing was imported"); + expect(captured.err).toContain('Unknown password hasher "rot13"'); + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(created()).toHaveLength(0); + }); + + test("exits 1 when some users failed", async () => { + stubClerk({ failing: new Set(["u2"]) }); + + await run(baseOptions); + + expect(process.exitCode).toBe(1); + expect(runsIn(runsDir())[0]).toMatchObject({ status: "partial", counts: { failed: 1 } }); + }); + + // Tests run non-TTY, the same signal an agent gives. + describe("without a file or a source", () => { + test("names what to pass rather than prompting for the file", async () => { + await expect(run({ source: "clerk", yes: true, secretKey: "sk_test_x" })).rejects.toThrow( + /needs the file to import, and cannot prompt here/, + ); + expect(requests).toHaveLength(0); + }); + + test("asks for --source when the file does not name its own", async () => { + await expect( + run({ input: "export.json", yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/Missing --source/); + }); + }); + + describe("checks", () => { + test("a reject stops the import, with the command that imports the rest", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([...export2, { id: "u3" }]), + ); + + const error = (await run(baseOptions).catch((caught: unknown) => caught)) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("1 user would be rejected, so nothing was imported"); + // An agent reading only the error learns why. + expect(error.message).toMatch(/nothing was imported\. Reason: .+\. Fix them/); + expect(error.examples?.[0]?.command).toContain("--allow-partial --yes"); + expect(created()).toHaveLength(0); + expect(runsIn(runsDir())).toHaveLength(0); + }); + + test("with several reasons, names the most common and points --json at the rest", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + ...export2, + { id: "u3" }, + { id: "u4" }, + { + id: "u5", + primary_email_address: "e@x.dev", + password_digest: "d", + password_hasher: "rot13", + }, + ]), + ); + + const error = (await run({ ...baseOptions, json: true }).catch( + (caught: unknown) => caught, + )) as CliError; + + expect(error.message).toContain("3 users would be rejected, so nothing was imported"); + expect(error.message).toMatch(/Most common reason \(2\): .+; and 1 other reason\./); + expect(error.message).toContain("See `checks.rejects` for each user."); + }); + + // "will create 0 users" hides that the run records the rest as skipped. + test("--allow-partial without --yes names the users it would skip", async () => { + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify([{ id: "u3" }])); + + await expect(run({ ...baseOptions, allowPartial: true, yes: false })).rejects.toThrow( + "will create 0 users and skip 1 user. Pass --yes to confirm.", + ); + }); + + test("--allow-partial imports the rest and records each reject as skipped", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([...export2, { id: "u3" }]), + ); + + await run({ ...baseOptions, allowPartial: true }); + + expect(created()).toEqual(["u1", "u2"]); + const [record] = runsIn(runsDir()); + expect(record).toMatchObject({ status: "partial", counts: { created: 2, skipped: 1 } }); + expect(latestUserLines(runsDir(), record!.id).get("u3")).toMatchObject({ + status: "skipped", + reason: expect.stringContaining("invalid:"), + }); + // The operator accepted the skips; only a failed user exits 1. + expect(process.exitCode).toBe(0); + }); + + test("--dry-run writes nothing, and exits 2 when the import would be refused", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([...export2, { id: "u3" }]), + ); + + await run({ ...baseOptions, dryRun: true }); + + expect(created()).toHaveLength(0); + expect(runsIn(runsDir())).toHaveLength(0); + expect(captured.err).toContain("Dry run: nothing was written."); + expect(process.exitCode).toBe(2); + }); + + test("--dry-run exits 0 when nothing would be rejected", async () => { + await run({ ...baseOptions, dryRun: true }); + + expect(created()).toHaveLength(0); + expect(process.exitCode).toBe(0); + }); + + test("a user whose only email is unverified is rejected where email is required, with a fix", async () => { + stubClerk({ + settings: { + attributes: { email_address: { enabled: true, required: true } }, + // A way in besides a password, so users without one import. + enterprise_sso: { enabled: true }, + }, + }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", unverified_email_addresses: "b@x.dev" }, + ]), + ); + + await run({ ...baseOptions, dryRun: true }); + + expect(captured.err).toContain( + "only has an unverified email, and this instance requires an email", + ); + expect(captured.err).toContain( + `clerk config patch --app APP_ID --instance ins_1 --json '{"auth_email":{"required_for_sign_up":false}}'`, + ); + }); + + test("legal consent: refused without --skip-legal-checks, sent with skip_legal_checks with it", async () => { + stubClerk({ + settings: { + attributes: { email_address: { enabled: true } }, + sign_up: { legal_consent_enabled: true }, + // A way in besides a password, so users without one import. + enterprise_sso: { enabled: true }, + }, + }); + + const error = (await run(baseOptions).catch((caught: unknown) => caught)) as CliError; + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(captured.err).toContain("no legal acceptance on record"); + expect(created()).toEqual([]); + + await run({ ...baseOptions, skipLegalChecks: true }); + const bodies = requests + .filter((r) => r.method === "POST" && r.url.endsWith("/v1/users")) + .map((r) => r.body); + expect(bodies).toEqual([ + expect.objectContaining({ external_id: "u1", skip_legal_checks: true }), + expect.objectContaining({ external_id: "u2", skip_legal_checks: true }), + ]); + }); + + test("a user already in the instance is rejected", async () => { + stubClerk({ + existing: [{ id: "user_old", email_addresses: [{ email_address: "b@x.dev" }] }], + }); + + await run({ ...baseOptions, dryRun: true }); + + expect(captured.err).toContain("email is already used by a user in the instance"); + expect(Bun.stripANSI(captured.err)).toContain("u2"); + }); + + test("the dev quota rejects users past the headroom; --allow-partial imports up to it", async () => { + stubClerk({ count: 99 }); + + expect(await exitCodeOf(run(baseOptions))).toBe(EXIT_CODE.USAGE); + expect(created()).toHaveLength(0); + + await run({ ...baseOptions, allowPartial: true }); + expect(created()).toEqual(["u1"]); + const [record] = runsIn(runsDir()); + expect(latestUserLines(runsDir(), record!.id).get("u2")?.reason).toContain("100-user limit"); + }); + + test("supabase users whose only provider is disabled are rejected", async () => { + stubClerk({ + settings: { + attributes: { email_address: { enabled: true } }, + social: { oauth_google: { enabled: true } }, + // A way in besides a password, so users without one import. + enterprise_sso: { enabled: true }, + }, + }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { + id: "sb1", + email: "a@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["discord"]}', + }, + { + id: "sb2", + email: "b@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["email","discord"]}', + }, + ]), + ); + + await run({ ...baseOptions, source: "supabase", dryRun: true }); + + expect(captured.err).toContain("only signs in with Discord, which is not enabled in Clerk"); + expect(captured.err).toContain( + "1 user signed in with Discord, which is not enabled in Clerk", + ); + }); + + test("names the fields Clerk won't store", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify(export2.map((user) => ({ ...user, department: "Sales" }))), + ); + + await run({ ...baseOptions, dryRun: true }); + + expect(captured.err).toContain("Clerk won't store: department (2 users)"); + }); + }); + + describe("consent", () => { + test("without --yes where nobody can be asked: the preview, then exit 2 with the command", async () => { + const error = (await run({ ...baseOptions, yes: false }).catch( + (caught: unknown) => caught, + )) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain(". Pass --yes to confirm"); + expect(error.examples?.[0]?.command).toBe( + "clerk migrate import export.json --source clerk --secret-key --yes", + ); + expect(captured.err).toContain("Checks"); + expect(created()).toHaveLength(0); + }); + + test("--json --yes returns { target, run, checks, result }", async () => { + await run({ ...baseOptions, json: true }); + + expect(JSON.parse(captured.out)).toMatchObject({ + target: { instanceId: "ins_1" }, + run: { kind: "import", status: "complete" }, + checks: { total: 2, importable: 2, rejects: [] }, + result: { created: 2, failed: 0, skipped: 0 }, + }); + }); + + // An agent reads stderr as text. + test("--json leaves colour codes out of stderr", async () => { + await run({ ...baseOptions, json: true }); + expect(captured.err).not.toContain("\x1b["); + }); + + test("--json without --yes returns the preview with consent required, and exits 2", async () => { + expect(await exitCodeOf(run({ ...baseOptions, yes: false, json: true }))).toBe( + EXIT_CODE.USAGE, + ); + expect(JSON.parse(captured.out)).toMatchObject({ + consent: "required", + checks: { importable: 2 }, + }); + expect(created()).toHaveLength(0); + }); + }); + + // The two `--json` outcomes that stop before a run exists: each says why, + // with `run: null`. + describe("--json, stopped before any run", () => { + test("a refused import reports refused, and exits 2", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([...export2, { id: "u3" }]), + ); + expect(await exitCodeOf(run({ ...baseOptions, json: true }))).toBe(EXIT_CODE.USAGE); + expect(JSON.parse(captured.out)).toMatchObject({ run: null, refused: true }); + expect(created()).toHaveLength(0); + }); + + test("a file with nobody to import reports nothingToImport", async () => { + fs.writeFileSync(path.join(workDir, "export.json"), "[]"); + await run({ ...baseOptions, json: true }); + expect(JSON.parse(captured.out)).toMatchObject({ run: null, nothingToImport: true }); + expect(created()).toHaveLength(0); + }); + }); + + describe("stopped part-way", () => { + // `importUsers` returns normally on a Ctrl-C; the run must not read as done. + test("a Ctrl-C after the first create leaves the run unfinished, as interrupted", async () => { + process.env.CLERK_MIGRATE_CONCURRENCY_LIMIT = "1"; + const clerk = globalThis.fetch; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const response = await clerk(input, init); + if (init?.method === "POST" && new URL(input.toString()).pathname === "/v1/users") { + beginInterrupt(); + abortInFlight(); + } + return response; + }) as typeof fetch; + try { + // Thrown, so the gutter closes as paused rather than done. + await expect(run(baseOptions)).rejects.toThrow(); + } finally { + _resetInterruptState(); + delete process.env.CLERK_MIGRATE_CONCURRENCY_LIMIT; + } + + const [record] = runsIn(runsDir()); + // The run folder is the only record of who was created. + expect(captured.err).toContain(`Run ${record?.id}: `); + expect(record?.finishedAt).toBeUndefined(); + expect(fs.existsSync(path.join(runsDir(), record?.id ?? "", "lock"))).toBe(false); + expect(created()).toHaveLength(1); + }); + + test("the user quota leaves the rest not sent, and the run partial", async () => { + stubClerk(); + const clerk = globalThis.fetch; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => + init?.method === "POST" && new URL(input.toString()).pathname === "/v1/users" + ? Response.json( + { + errors: [ + { + code: "user_quota_exceeded", + message: "user quota exceeded", + long_message: "You have reached your limit of 100 users.", + }, + ], + }, + { status: 403 }, + ) + : clerk(input, init)) as typeof fetch; + process.env.CLERK_MIGRATE_CONCURRENCY_LIMIT = "1"; + try { + await run({ ...baseOptions, json: true }); + } finally { + delete process.env.CLERK_MIGRATE_CONCURRENCY_LIMIT; + } + + expect(JSON.parse(captured.out)).toMatchObject({ + run: { status: "partial" }, + result: { + created: 0, + failed: 1, + notSent: 1, + stopReason: "You have reached your limit of 100 users.", + }, + }); + }); + + // In the summary, after the progress bar: printed mid-run, the bar redrew + // over it. + test("the summary says why the import stopped", async () => { + stubClerk(); + const clerk = globalThis.fetch; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => + init?.method === "POST" && new URL(input.toString()).pathname === "/v1/users" + ? Response.json( + { + errors: [ + { + code: "user_quota_exceeded", + message: "user quota exceeded", + long_message: "You have reached your limit of 100 users.", + }, + ], + }, + { status: 403 }, + ) + : clerk(input, init)) as typeof fetch; + await run(baseOptions); + + const err = Bun.stripANSI(captured.err); + expect(err).toContain("Not sent: 1"); + expect(err).toContain( + "No more users were sent. Once the limit is raised, run the import again with `--allow-partial --yes` to send them.", + ); + }); + }); + + describe("a phone Clerk refuses", () => { + const COUNTRY = + "Phone numbers from this country (France) are currently not supported. For more information, please contact support."; + + beforeEach(() => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "u1", primary_email_address: "a@x.dev", primary_phone_number: "+33612345678" }, + ]), + ); + const clerk = globalThis.fetch; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const body = init?.body ? (JSON.parse(init.body as string) as Record) : {}; + const isCreate = + init?.method === "POST" && new URL(input.toString()).pathname === "/v1/users"; + if (isCreate && body.phone_number) { + requests.push({ method: "POST", url: input.toString(), body }); + return Response.json( + { + errors: [ + { code: "unsupported_country_code", message: "unsupported", long_message: COUNTRY }, + ], + }, + { status: 403 }, + ); + } + return clerk(input, init); + }) as typeof fetch; + }); + + // The user imports, so it is no failure, but the phone is gone. + test("is counted in the summary, with the SMS note", async () => { + await run(baseOptions); + const err = Bun.stripANSI(captured.err); + expect(err).toContain("Failed: 0"); + expect(err).toContain(`Imported without their phone:\n 1 user: ${COUNTRY}`); + expect(err).toContain("Development instances block SMS"); + }); + + test("is a warning in --json", async () => { + await run({ ...baseOptions, json: true }); + expect(JSON.parse(captured.out)).toMatchObject({ + result: { + created: 1, + failed: 0, + errors: [], + warnings: [{ warning: `imported without their phone: ${COUNTRY}`, count: 1 }], + }, + }); + }); + }); + + describe("re-running", () => { + // Slice 1 has no continuing: the second run sees the first run's users in + // the instance, and refuses until --allow-partial. + test("a second import of the same file rejects the users the first created", async () => { + await run(baseOptions); + stubClerk({ + existing: [ + { id: "user_u1", external_id: "u1" }, + { id: "user_u2", external_id: "u2" }, + ], + }); + + expect(await exitCodeOf(run(baseOptions))).toBe(EXIT_CODE.USAGE); + expect(captured.err).toContain("already in the instance, with this source ID"); + expect(created()).toEqual(["u1", "u2"]); + expect(runsIn(runsDir())).toHaveLength(1); + }); + }); + + describe("per-platform imports", () => { + test("supabase transforms, validates and imports its export", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { + id: "sb1", + email: "a@x.dev", + email_confirmed_at: "2024-06-29 20:25:06+00", + encrypted_password: BCRYPT, + }, + ]), + ); + + await run({ ...baseOptions, source: "supabase" }); + + expect(created()).toEqual(["sb1"]); + }); + + test("an unknown source fails before signing in", async () => { + const { secretKey: _key, ...signedOut } = baseOptions; + const error = (await run({ ...signedOut, source: "nope" }).catch( + (caught: unknown) => caught, + )) as CliError; + expect(error.message).toContain('Unknown source "nope"'); + }); + + test("an unknown source fails listing the valid keys", async () => { + const error = (await run({ ...baseOptions, source: "nope" }).catch( + (caught: unknown) => caught, + )) as CliError; + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain('Unknown source "nope". Valid sources: clerk, supabase.'); + expect(requests).toHaveLength(0); + }); + }); +}); + +describe("explainErrors", () => { + const COUNTRY = + "Phone numbers from this country (France) are currently not supported. For more information, please contact support."; + const QUOTA = + "You have reached your limit of 100 users. If you need more users, please use a Production instance."; + + test("names the development instance as the reason countries are blocked", () => { + const [note] = explainErrors([COUNTRY], "dev"); + expect(note).toContain("Development instances block SMS to most countries"); + expect(note).toContain("test-emails-and-phones"); + }); + + test("sends a production operator to the Dashboard instead of support", () => { + const [note] = explainErrors([COUNTRY], "prod"); + expect(note).toContain("customization/sms/settings"); + expect(note).not.toContain("Development instances"); + }); + + test("explains the user quota only where one applies", () => { + expect(explainErrors([QUOTA], "dev").join(" ")).toContain("development-instance quota"); + // Production has no such quota by default, and the API's message already + // names the plan upgrade when a plan sets one. + expect(explainErrors([QUOTA], "prod")).toEqual([]); + }); + + test("says nothing about errors it does not recognize", () => { + expect(explainErrors(["Something else went wrong."], "dev")).toEqual([]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts new file mode 100644 index 000000000..e089cd8eb --- /dev/null +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -0,0 +1,713 @@ +/** + * `clerk migrate import ` — the user import itself. + * + * Registered as the `import` subcommand. The exported handler keeps the name + * `run` because `import` is a reserved word. + * + * An import goes through the same steps every time: + * + * 1. Settle the file, the source and the target instance, and print the target. + * 2. Load the users: read the file, map it through the source, then normalize + * and validate it. + * 3. Run `checkImport()` against the real instance. `--dry-run` stops here. + * 4. Refuse on any reject unless `--allow-partial`, then ask for consent. + * Nothing is written without `--yes` or a yes at the prompt. + * 5. Import, one run line per user. + */ + +import { bold, dim, green, red, yellow } from "../../lib/color.ts"; +import { resolveProfile } from "../../lib/config.ts"; +import { hasAccountCredentials } from "../../lib/credential-store.ts"; +import { + AUTH_ERROR_REASON, + AuthError, + CliError, + ERROR_CODE, + throwUsageError, + throwUserAbort, +} from "../../lib/errors.ts"; +import { resolveKeylessTarget } from "../../lib/keyless-target.ts"; +import { quoteArg } from "../../lib/json-body.ts"; +import { log } from "../../lib/log.ts"; +import { NEXT_STEPS, printAgentNextSteps } from "../../lib/next-steps.ts"; +import { confirm } from "../../lib/prompts.ts"; +import { interruptedExitCode } from "../../lib/signals.ts"; +import { withGutter, withSpinner } from "../../lib/spinner.ts"; +import { isAgent, isHuman } from "../../mode.ts"; +import { importUsers } from "./import-users.ts"; +import { checkImport, type ImportChecks } from "./lib/checks.ts"; +import { fetchInstanceSettings, fetchUserCount } from "./lib/clerk-config.ts"; +import { DEV_USER_LIMIT, resolveLimits, type InstanceType } from "./lib/instance.ts"; +import { resolveRunsDir, sha256File, startRun, type Run, type RunRecord } from "./lib/run-store.ts"; +import { withProgress } from "./lib/progress.ts"; +import { createApiScheduler } from "./lib/scheduler.ts"; +import { readSupabaseRows } from "./lib/supabase-providers.ts"; +import { printTarget, resolveClerkTarget } from "./lib/target.ts"; +import { + fileExists, + getFileType, + loadUsersFromFile, + resolveImportFilePath, +} from "./lib/transform.ts"; +import { resolveSource, sourceKeys } from "./sources/registry.ts"; +import type { ImportSummary, User } from "./types.ts"; +import { promptForFile, promptForSource } from "./wizard.ts"; +import { login } from "../auth/login.ts"; +import { link } from "../link/index.ts"; + +export type MigrateRunOptions = { + /** The export file. */ + input?: string; + /** A built-in source key. */ + source?: string; + /** The file to read, once `input` is resolved. Not a flag. */ + file?: string; + requirePassword?: boolean; + /** + * Import users with no legal acceptance into an instance that requires it. + * Without it, a prompt asks; where nobody can be asked, they are rejected. + */ + skipLegalChecks?: boolean; + /** Check against the instance, report, and write nothing. */ + dryRun?: boolean; + /** Import the users that pass, and record the rest as skipped. */ + allowPartial?: boolean; + yes?: boolean; + json?: boolean; + secretKey?: string; + app?: string; + instance?: string; + /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ + runsDir?: string; +}; + +const plural = (count: number, word: string) => `${count} ${word}${count === 1 ? "" : "s"}`; + +/** Nobody can answer a prompt: an agent, a non-TTY run, or `--json`. */ +function canPrompt(options: MigrateRunOptions): boolean { + return !options.json && isHuman() && !isAgent(); +} + +/** + * Validates what a run needs before anything is read or sent. + * + * `--source` has already been resolved to a registered key by the time this + * runs. + * + * @returns The source key and file path, both guaranteed present. + */ +export function validateRunOptions(options: MigrateRunOptions): { + source: string; + file: string; +} { + if (!options.source) { + throwUsageError( + `Missing --source. Valid values: ${sourceKeys().join(", ")}.`, + undefined, + ERROR_CODE.USAGE_ERROR, + [ + { + command: "clerk migrate import users.json --source clerk --yes", + description: "Import a Clerk export", + }, + ], + ); + } + if (!options.file) { + throwUsageError( + "Missing the file to import (a JSON or CSV export).", + undefined, + ERROR_CODE.USAGE_ERROR, + [ + { + command: "clerk migrate import users.json --source clerk --yes", + description: "Import a Clerk export", + }, + ], + ); + } + if (!fileExists(options.file)) { + throw new CliError(`File not found: ${options.file}`, { code: ERROR_CODE.FILE_NOT_FOUND }); + } + if (!getFileType(options.file)) { + throwUsageError(`Unsupported file type for ${options.file}. Provide a .json or .csv file.`); + } + + return { source: options.source, file: options.file }; +} + +/** Where a production instance's operator changes the SMS country blocklist. */ +const SMS_SETTINGS_URL = "https://dashboard.clerk.com/~/customization/sms/settings"; + +/** Clerk's fictional email addresses and phone numbers, for development. */ +const TEST_NUMBERS_URL = "https://clerk.com/docs/guides/development/testing/test-emails-and-phones"; + +/** + * What the API's error messages leave out: whether the operator can do + * something about them, and where. + * + * Both of these read as account-level restrictions and are not. Blocked + * countries are a per-instance SMS blocklist that development instances are + * created with far more of, and the user limit is a development-instance quota + * that production does not have by default — so "contact support", which both + * messages point at, is the wrong first move for most readers. + * + * @returns One note per recognized error family, empty when none apply. + */ +export function explainErrors(errors: Iterable, instanceType: InstanceType): string[] { + const all = [...errors]; + const notes: string[] = []; + + if (all.some((error) => error.includes("Phone numbers from this country"))) { + notes.push( + instanceType === "dev" + ? `Development instances block SMS to most countries by default — this is not a limit on your account. ` + + `Use Clerk's test phone numbers while developing (${TEST_NUMBERS_URL}), and contact support only if ` + + `you need real numbers in a specific country before going to production.` + : `Unblock the countries you need under SMS settings in the Dashboard (${SMS_SETTINGS_URL}). ` + + `Plans without SMS support cannot remove them; contact support if the setting is refused.`, + ); + } + + // Production has no user limit unless a plan imposes one, and the API's own + // message already names the fix ("upgrade to a paid plan") in that case. + if ( + instanceType === "dev" && + all.some((error) => /You have reached your limit of \d+ users/.test(error)) + ) { + notes.push( + `The user limit is a development-instance quota (${DEV_USER_LIMIT} by default). Import into a production ` + + `instance to bring everyone across, or contact support to raise this instance's limit.`, + ); + } + + return notes; +} + +// --- Settling the file, source and target ---------------------------------- + +/** + * Makes sure there is somewhere to import *into* before anything else happens. + * + * Without this the first complaint comes from deep inside the secret-key chain, + * which resolves the linked profile before it ever asks for a token — so a + * signed-out operator in an unlinked directory is told to `clerk link`, a + * command that will only turn around and ask them to sign in. + * + * A human gets the same sign-in-then-link flow `clerk link` already runs. An + * agent cannot answer a browser login or an application picker, so it gets the + * error naming whichever half is missing. + */ +async function ensureImportTarget(options: MigrateRunOptions): Promise { + // A secret key names the destination instance on its own, with no account + // and no linked directory involved — mirroring resolveBapiSecretKey, which + // takes `--app` over an exported CLERK_SECRET_KEY. + if (options.secretKey || (!options.app && process.env.CLERK_SECRET_KEY)) return; + // `--app` names it too, but resolves its key through the Platform API, which + // needs an account: it goes through the sign-in below, though not the link. + // An unclaimed accountless application keeps its only secret key on disk. + if (!options.app && (await resolveKeylessTarget({ instance: options.instance }))) return; + + const interactive = canPrompt(options); + + if (!(await hasAccountCredentials())) { + if (!interactive) { + throw new AuthError({ + reason: AUTH_ERROR_REASON.NOT_LOGGED_IN, + message: + "Not logged in, so there is no Clerk instance to import into. Run `clerk auth login`, then `clerk link`.", + examples: [ + { command: "clerk auth login", description: "Sign in, then re-run the import" }, + { + command: + "clerk migrate import users.json --source clerk --yes --secret-key sk_test_...", + description: "Import without signing in", + }, + ], + }); + } + log.info("Not logged in. Signing in first..."); + await login({ showNextSteps: false }); + } + + // Left to the secret-key chain when non-interactive: its `not_linked` error + // is the one every other command raises, and there is nothing to add to it. + if (interactive && !options.app && !(await resolveProfile(process.cwd()))) { + log.info("This directory isn't linked to a Clerk application. Linking one first..."); + await link({ skipIfLinked: true }); + } +} + +/** + * Settles which file to read. A human who gave none is asked for a path. + */ +async function resolveInput(options: MigrateRunOptions): Promise { + if (options.input) return options.input; + if (!canPrompt(options)) { + throwUsageError( + "`clerk migrate import` needs the file to import, and cannot prompt here.", + undefined, + undefined, + [ + { + command: "clerk migrate import users.json --source clerk --yes", + description: "Import a file", + }, + ], + ); + } + return promptForFile(); +} + +/** + * Resolves `--source` to a registered key. + * + * @throws UsageError for an unknown key. + */ +async function applySource(options: MigrateRunOptions): Promise { + if (!options.source) return options; + const resolved = await resolveSource(options.source); + return { ...options, source: resolved.key }; +} + +// --- Reporting ------------------------------------------------------------- + +/** How many rejected source IDs to name per reason before summing the rest. */ +const REJECT_SAMPLE = 5; + +function printChecks(checks: ImportChecks): void { + log.blank(); + log.info(bold("Checks")); + log.info(` ${plural(checks.total, "user")} checked`); + + if (checks.rejects.length > 0) { + log.info(` ${red("✗")} ${red(`${plural(checks.rejects.length, "user")} rejected`)}`); + for (const { reason, count } of checks.rejectReasons) { + const ids = checks.rejects + .filter((reject) => reject.reason === reason) + .map((reject) => + reject.keptSourceId + ? `${reject.sourceId} (kept: ${reject.keptSourceId})` + : reject.sourceId, + ); + const sample = ids.slice(0, REJECT_SAMPLE).join(", "); + const more = ids.length > REJECT_SAMPLE ? `, and ${ids.length - REJECT_SAMPLE} more` : ""; + log.info(` ${count}: ${reason}`); + log.info(` ${dim(sample + more)}`); + } + } + + if (checks.warnings.length > 0) { + log.info(` ${yellow("⚠")} ${yellow("Imported, but not everything comes across")}`); + for (const warning of checks.warnings) log.info(` ${dim(warning)}`); + } + + log.info(` ${green("✓")} ${green(`${plural(checks.importable.length, "user")} to import`)}`); + + if (checks.settingsUnavailable) { + log.info( + ` ${yellow("!")} ${dim("Could not read this instance's settings, so required fields were not checked.")}`, + ); + } + + if (checks.fixes.length > 0) { + log.blank(); + log.info(bold("Or change the instance instead")); + for (const fix of checks.fixes) { + log.info(` ${fix.label}`); + log.info(dim(` ${fix.command ?? fix.url}`)); + } + } +} + +function checksJson(checks: ImportChecks) { + return { + total: checks.total, + importable: checks.importable.length, + rejects: checks.rejects, + rejectReasons: checks.rejectReasons, + warnings: checks.warnings, + fixes: checks.fixes, + ...(checks.quota ? { quota: checks.quota } : {}), + settingsUnavailable: checks.settingsUnavailable, + }; +} + +function formatSummary( + summary: ImportSummary, + skipped: number, + record: RunRecord, + runFolder: string, + instanceType: InstanceType, +): string[] { + const lines = [ + `${green("Imported:")} ${summary.successful}`, + `${red("Failed:")} ${summary.failed}`, + ]; + if (skipped > 0) lines.push(`${yellow("Skipped:")} ${skipped}`); + // The user quota stopped the run: these go out on a re-run. + if (summary.notSent > 0) lines.push(`${yellow("Not sent:")} ${summary.notSent}`); + // Printed here, after the progress bar, which would redraw over it mid-run. + if (summary.stopReason) { + lines.push( + "", + yellow( + "No more users were sent. Once the limit is raised, run the import again with `--allow-partial --yes` to send them.", + ), + ); + } + + if (summary.errorBreakdown.size > 0) { + lines.push("", bold("Error breakdown:")); + for (const [error, count] of summary.errorBreakdown) { + lines.push(` ${plural(count, "user")}: ${error}`); + } + } + // Imported, so not failures, but the phone is gone: say so, and why. + if (summary.droppedPhones.size > 0) { + lines.push("", bold("Imported without their phone:")); + for (const [reason, count] of summary.droppedPhones) { + lines.push(` ${plural(count, "user")}: ${reason}`); + } + } + const explained = [...summary.errorBreakdown.keys(), ...summary.droppedPhones.keys()]; + for (const note of explainErrors(explained, instanceType)) lines.push("", note); + lines.push("", dim(`Run ${record.id}: ${runFolder}`)); + return lines; +} + +// --- The import ------------------------------------------------------------ + +/** + * The exact command that would carry on from here, for the consent and + * refusal messages. Every value is shell-quoted; secrets are placeholders. + */ +function commandFor(options: MigrateRunOptions, extra: string[]) { + const input = options.input ?? options.file ?? ""; + const parts = ["clerk migrate import", quoteArg(input)]; + if (options.source) parts.push("--source", quoteArg(options.source)); + if (options.allowPartial) parts.push("--allow-partial"); + if (options.requirePassword) parts.push("--require-password"); + if (options.skipLegalChecks) parts.push("--skip-legal-checks"); + if (options.secretKey) parts.push("--secret-key", ""); + if (options.app) parts.push("--app", quoteArg(options.app)); + if (options.instance) parts.push("--instance", quoteArg(options.instance)); + if (options.runsDir) parts.push("--runs-dir", quoteArg(options.runsDir)); + if (options.json) parts.push("--json"); + return [...parts, ...extra].join(" "); +} + +/** + * Why the checks refused, in the error itself: an agent may read only that, + * not the report above it. + */ +function rejectSummary(checks: ImportChecks, json = false): string { + const [top, ...rest] = checks.rejectReasons.toSorted((a, b) => b.count - a.count); + if (!top) return ""; + const why = + rest.length === 0 + ? `Reason: ${top.reason}.` + : `Most common reason (${top.count}): ${top.reason}; and ${plural(rest.length, "other reason")}.`; + return ` ${why}${json ? " See `checks.rejects` for each user." : ""}`; +} + +/** Records the checks' rejects as skipped users, so the run says who they were. */ +function recordRejects(run: Run, checks: ImportChecks): void { + for (const { sourceId, reason, keptSourceId } of checks.rejects) { + run.append({ + sourceId, + status: "skipped", + reason: keptSourceId ? `${reason} (kept: ${keptSourceId})` : reason, + }); + } +} + +export async function run(rawOptions: MigrateRunOptions): Promise { + // A mistyped --source fails before any sign-in or link. + let options = await applySource(rawOptions); + await ensureImportTarget(options); + + const input = await resolveInput(options); + options = { ...options, file: input }; + + if (!options.source && canPrompt(options) && fileExists(input)) { + options = { ...options, source: await promptForSource() }; + } + + const { source, file } = validateRunOptions(options); + + await withGutter( + "Migrating users to Clerk", + async ({ setNextSteps }) => { + const { secretKey, target } = await resolveClerkTarget(options); + if (!options.json) printTarget(target); + const limits = resolveLimits(secretKey); + + const filePath = resolveImportFilePath(file); + const sha256 = sha256File(filePath); + const runsDir = await resolveRunsDir(options.runsDir); + const schedule = createApiScheduler(limits.concurrencyLimit, limits.rateLimit); + + const loaded = await withSpinner(`Loading users from ${file}...`, async () => + loadUsersFromFile(file, source), + ); + let users = loaded.users; + + // An instruction about this import, not a prediction: users without a + // password are left out of the job rather than recorded as rejects. + let withoutPassword: User[] = []; + if (options.requirePassword) { + withoutPassword = users.filter((user) => !user.password); + if (withoutPassword.length > 0 && !options.json) { + log.info( + `--require-password: leaving out ${plural(withoutPassword.length, "user")} without a password.`, + ); + } + // A Set, not `includes`: half a large Supabase export can lack a + // password, and a scan per user would stall before any check runs. + const leftOut = new Set(withoutPassword); + users = users.filter((user) => !leftOut.has(user)); + } + + let supabaseRows: Record[] | undefined; + if (source === "supabase") { + try { + supabaseRows = await readSupabaseRows(file); + } catch (error) { + log.debug(`migrate: could not read Supabase providers: ${String(error)}`); + } + } + + const [settings, existingUsers] = await withSpinner("Checking the instance...", async () => + Promise.all([ + fetchInstanceSettings(secretKey), + limits.instanceType === "dev" ? fetchUserCount(secretKey) : Promise.resolve(null), + ]), + ); + + // Creating a user without legal acceptance needs consent of its own: + // the flag, or a yes at a prompt. Otherwise the checks reject them. + let skipLegalChecks = options.skipLegalChecks ?? false; + const withoutLegal = settings?.sign_up?.legal_consent_enabled + ? users.filter((user) => !user.legalAcceptedAt && !user.skipLegalChecks).length + : 0; + // `-y` imports without prompting, so it does not stop here either: the + // checks reject these users, and --skip-legal-checks is the way through. + if ( + !skipLegalChecks && + withoutLegal > 0 && + !options.dryRun && + !options.yes && + canPrompt(options) + ) { + skipLegalChecks = await confirm({ + message: `${plural(withoutLegal, "user")} ${withoutLegal === 1 ? "has" : "have"} no legal acceptance on record, which this instance requires. Import them without it?`, + default: false, + }); + // The suggested commands carry the answer, so a rerun with --yes keeps it. + options = { ...options, skipLegalChecks }; + } + + const checks = await withSpinner("Checking users against the instance...", async (spinner) => + checkImport({ + users, + skipLegalChecks, + failures: loaded.failures, + unknownFields: loaded.unknownFields, + ...(supabaseRows ? { supabaseRows } : {}), + settings, + existingUsers, + instanceType: limits.instanceType, + target, + secretKey, + schedule, + spinner, + }), + ); + + const refused = checks.rejects.length > 0 && !options.allowPartial; + // The users --require-password left out never reach the checks, so the + // JSON says how many, or `checks.total` would not add up to the file. + const leftOut = withoutPassword.length > 0 ? { withoutPassword: withoutPassword.length } : {}; + const preview = (extra: Record) => + log.data( + JSON.stringify( + { target, run: null, checks: checksJson(checks), ...leftOut, ...extra }, + null, + 2, + ), + ); + + if (!options.json) printChecks(checks); + + if (options.dryRun) { + if (options.json) preview({ dryRun: true }); + else { + log.blank(); + log.info(dim("Dry run: nothing was written.")); + } + // The exit code says what the real run would do. + if (refused) process.exitCode = 2; + return; + } + + if (refused) { + if (options.json) preview({ refused: true }); + throwUsageError( + `${plural(checks.rejects.length, "user")} would be rejected, so nothing was imported.` + + `${rejectSummary(checks, options.json)} ` + + "Fix them, or pass --allow-partial to import the rest and record them as skipped.", + undefined, + undefined, + [ + { + command: commandFor(options, ["--allow-partial", "--yes"]), + description: "Import the users that pass", + }, + ], + ); + } + + if ( + checks.importable.length === 0 && + checks.rejects.length === 0 && + withoutPassword.length === 0 + ) { + if (options.json) preview({ nothingToImport: true }); + else log.warn("No users left to import."); + return; + } + + // Rule 1: nothing is written without consent — `--yes`, or a yes at a + // prompt. An agent, a non-TTY run and `--json` never prompt. With users + // skipped, the question names them: "Import 0 users?" hides the skips. + const toCreate = plural(checks.importable.length, "user"); + const skipping = checks.rejects.length + withoutPassword.length; + const skips = skipping > 0 ? ` and skip ${plural(skipping, "user")}` : ""; + if (!options.yes) { + if (!canPrompt(options)) { + if (options.json) preview({ consent: "required" }); + throwUsageError( + `\`clerk migrate import\` will create ${toCreate}${skips}. Pass --yes to confirm.`, + undefined, + undefined, + [ + { + command: commandFor(options, ["--yes"]), + description: "Run the import", + }, + ], + ); + } + log.blank(); + const proceed = await confirm({ + message: skips ? `Create ${toCreate}${skips}?` : `Import ${toCreate}?`, + default: false, + }); + if (!proceed) throwUserAbort(); + } + + // Gitignored only now, once there is consent to write a run. + await resolveRunsDir(options.runsDir, { write: true }); + const run = startRun(runsDir, { + kind: "import", + target, + source, + file: { path: filePath, sha256 }, + }); + // Printed now, not on the way out: on a Ctrl-C the signal handler exits + // before the import returns, and the folder is the only record of who + // was created. + log.info(`Run ${run.record.id}: ${run.dir}`); + recordRejects(run, checks); + for (const user of withoutPassword) { + run.append({ + sourceId: user.userId, + status: "skipped", + reason: "no password (--require-password)", + }); + } + + const summary: ImportSummary = + checks.importable.length > 0 + ? await withProgress( + { total: checks.importable.length, verb: "created" }, + async (progress) => + importUsers({ + users: checks.importable, + secretKey, + limits, + record: run.append, + skipPasswordRequirement: !options.requirePassword, + progress, + }), + ) + : { + totalProcessed: 0, + successful: 0, + failed: 0, + notSent: 0, + droppedPhones: new Map(), + validationFailed: 0, + errorBreakdown: new Map(), + }; + // A Ctrl-C returns the import early, and the users it never sent have + // no line to count. Left unfinished, the run reads as interrupted, and + // the throw closes the gutter as paused, when the signal handler has not + // already exited. + if (interruptedExitCode() !== null) { + run.release(); + throwUserAbort(); + } + const record = run.finish({ notSent: summary.notSent }); + if (summary.failed > 0) process.exitCode = 1; + + if (options.json) { + log.data( + JSON.stringify( + { + target, + run: record, + checks: checksJson(checks), + ...leftOut, + result: { + created: summary.successful, + failed: summary.failed, + notSent: summary.notSent, + ...(summary.stopReason ? { stopReason: summary.stopReason } : {}), + skipped: checks.rejects.length + withoutPassword.length, + errors: [...summary.errorBreakdown].map(([error, count]) => ({ error, count })), + warnings: [...summary.droppedPhones].map(([reason, count]) => ({ + warning: `imported without their phone: ${reason}`, + count, + })), + }, + }, + null, + 2, + ), + ); + return; + } + + log.blank(); + for (const line of formatSummary( + summary, + checks.rejects.length + withoutPassword.length, + record, + run.dir, + limits.instanceType, + )) { + log.info(line); + } + + const steps = + summary.failed > 0 + ? NEXT_STEPS.MIGRATE_DONE_WITH_ERRORS(run.dir) + : NEXT_STEPS.MIGRATE_DONE(run.dir); + setNextSteps(steps); + printAgentNextSteps(steps); + }, + { skip: Boolean(options.json) }, + ); +} diff --git a/packages/cli-core/src/commands/migrate/wizard.test.ts b/packages/cli-core/src/commands/migrate/wizard.test.ts new file mode 100644 index 000000000..387d42d79 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/wizard.test.ts @@ -0,0 +1,98 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { listageStubs, useCaptureLog } from "../../test/lib/stubs.ts"; + +useCaptureLog(); + +type Prompt = { message: string; default?: string; validate?: (v?: string) => string | undefined }; +type SelectPrompt = Prompt & { choices: { name: string; value: string }[] }; + +// Registered at file top, before the wizard (or anything it imports) loads. +// This file is the only consumer of the mocked prompt modules. +const mockSelect = mock(async (_config: SelectPrompt) => undefined as unknown); +const mockText = mock(async (_config: Prompt) => "" as unknown); + +mock.module("../../lib/listage.ts", () => ({ + ...listageStubs, + select: (config: SelectPrompt) => mockSelect(config), +})); + +mock.module("../../lib/prompts.ts", () => ({ + confirm: async () => true, + text: (config: Prompt) => mockText(config), + password: async () => "", + editor: async () => "{}", +})); + +const { promptForFile, promptForSource } = await import("./wizard.ts"); + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-wizard-"))); + process.chdir(workDir); + fs.writeFileSync(path.join(workDir, "users.json"), "[]"); + fs.writeFileSync(path.join(workDir, "other.csv"), ""); + fs.writeFileSync(path.join(workDir, "notes.txt"), ""); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + mockSelect.mockReset(); + mockText.mockReset(); +}); + +/** The config object passed to the Nth `text`/`select` prompt. */ +const textCall = (index: number): Prompt | undefined => mockText.mock.calls[index]?.[0]; +const selectCall = (index: number): SelectPrompt | undefined => mockSelect.mock.calls[index]?.[0]; + +describe("promptForSource", () => { + test("is built from the registry, so every platform appears", async () => { + mockSelect.mockResolvedValue("supabase"); + + expect(await promptForSource()).toBe("supabase"); + expect(selectCall(0)?.choices.map((choice) => choice.value)).toEqual(["clerk", "supabase"]); + }); + + test("labels each choice with the source's display name", async () => { + mockSelect.mockResolvedValue("clerk"); + await promptForSource(); + expect(selectCall(0)?.choices.map((choice) => choice.name)).toContain("Supabase"); + }); +}); + +describe("promptForFile", () => { + const validate = async () => { + mockText.mockResolvedValue("users.json"); + await promptForFile(); + return textCall(0)?.validate as (value?: string) => string | undefined; + }; + + test("returns the trimmed path", async () => { + mockText.mockResolvedValue(" users.json "); + expect(await promptForFile()).toBe("users.json"); + }); + + test("accepts an existing JSON or CSV file", async () => { + const check = await validate(); + expect(check("users.json")).toBeUndefined(); + expect(check("other.csv")).toBeUndefined(); + }); + + test.each([ + ["", /required/], + ["nope.json", /File not found/], + ["notes.txt", /\.json or \.csv/], + ])("rejects %p", async (value, message) => { + const check = await validate(); + expect(check(value)).toMatch(message); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/wizard.ts b/packages/cli-core/src/commands/migrate/wizard.ts new file mode 100644 index 000000000..bb0fbd70a --- /dev/null +++ b/packages/cli-core/src/commands/migrate/wizard.ts @@ -0,0 +1,47 @@ +/** + * The prompts behind an interactive `clerk migrate import`. + * + * Each one fills in exactly one thing the command was not given: the file, or + * the source. + * + * Nothing here runs for an agent, a non-TTY run or `--json`: `run` raises a + * usage error naming what to pass instead. + */ + +import { select } from "../../lib/listage.ts"; +import { text } from "../../lib/prompts.ts"; +import { fileExists, getFileType } from "./lib/transform.ts"; +import { sources } from "./sources/registry.ts"; + +/** Trims a description down to a single readable hint line. */ +function hint(description: string): string { + const firstSentence = description.split(". ")[0] ?? description; + return firstSentence.length > 96 ? `${firstSentence.slice(0, 93)}...` : firstSentence; +} + +/** Asks which platform the file came from. Built from the registry. */ +export async function promptForSource(): Promise { + return select({ + message: "Which platform did this file come from?", + choices: sources.map((entry) => ({ + name: entry.label, + value: entry.key, + description: hint(entry.description), + })), + }); +} + +/** Asks for the file to import. */ +export async function promptForFile(): Promise { + const answer = await text({ + message: "Path to the exported user file (JSON or CSV)", + validate: (value) => { + const file = value?.trim(); + if (!file) return "A file path is required"; + if (!fileExists(file)) return `File not found: ${file}`; + if (!getFileType(file)) return "Provide a .json or .csv file"; + return undefined; + }, + }); + return answer.trim(); +} diff --git a/packages/cli-core/src/lib/json-body.ts b/packages/cli-core/src/lib/json-body.ts index f5ff5a7d2..31b18cc5f 100644 --- a/packages/cli-core/src/lib/json-body.ts +++ b/packages/cli-core/src/lib/json-body.ts @@ -85,7 +85,7 @@ function preview(raw: string): string { * literal `"` inside doubled; everywhere else single quotes pass the value * through untouched. */ -function quoteArg(value: string): string { +export function quoteArg(value: string): string { if (/^[A-Za-z0-9_\-./:@]+$/.test(value)) return value; if (process.platform === "win32") return `"${value.replace(/"/g, '""')}"`; return `'${value.replace(/'/g, `'\\''`)}'`; diff --git a/packages/cli-core/src/lib/next-steps.ts b/packages/cli-core/src/lib/next-steps.ts index f0967af8f..e11e94a50 100644 --- a/packages/cli-core/src/lib/next-steps.ts +++ b/packages/cli-core/src/lib/next-steps.ts @@ -75,6 +75,10 @@ export const NEXT_STEPS = { "Run `clerk apps list` to see your other applications", "Run `clerk config pull` to inspect the live configuration of this instance", ], + MIGRATE_DONE: (runFolder: string) => [`See each user's outcome in ${runFolder}/users.ndjson`], + MIGRATE_DONE_WITH_ERRORS: (runFolder: string) => [ + `See every user that failed, and why, in ${runFolder}/users.ndjson`, + ], } as const; /** @@ -82,7 +86,30 @@ export const NEXT_STEPS = { * Only shown in human/interactive mode — agents get AGENT_PROMPT instead. */ export function printNextSteps(steps: readonly string[]): void { - if (!isHuman() || steps.length === 0) return; + if (!isHuman()) return; + renderNextSteps(steps); +} + +/** + * The same suggestions, on the paths a human never takes: agent mode and a + * non-TTY, where `printNextSteps` and `withGutter`'s outro both print nothing. + * + * A no-op for a human, who gets them from the outro — so a caller pairs this + * with `setNextSteps` rather than choosing between the two. + * + * Opt-in rather than folded into `printNextSteps`, because most steps are a + * nudge towards a command someone might like to run next. The migrate ones are + * not: they name the file holding the per-user record of what landed. An agent + * that imported 10,000 users and lost 300 of them needs it, and has no gutter + * to read it from. + */ +export function printAgentNextSteps(steps: readonly string[]): void { + if (isHuman()) return; + renderNextSteps(steps); +} + +function renderNextSteps(steps: readonly string[]): void { + if (steps.length === 0) return; for (const step of steps) { log.info(` ${cyan("\u2192")} ${step}`); } diff --git a/packages/cli-core/src/lib/spinner.ts b/packages/cli-core/src/lib/spinner.ts index 2222cb838..886248b33 100644 --- a/packages/cli-core/src/lib/spinner.ts +++ b/packages/cli-core/src/lib/spinner.ts @@ -107,7 +107,9 @@ export async function withGutter( let nextSteps: readonly string[] | undefined; const controls: GutterControls = { setNextSteps(steps) { - nextSteps = steps; + // Empty is ignored rather than stored: `outro([])` would render the + // "Next steps" header with no bullets under it. Matches printNextSteps. + if (steps.length > 0) nextSteps = steps; }, }; diff --git a/packages/cli-core/src/test/integration/completion.test.ts b/packages/cli-core/src/test/integration/completion.test.ts index e1e10d2e1..3c148aee4 100644 --- a/packages/cli-core/src/test/integration/completion.test.ts +++ b/packages/cli-core/src/test/integration/completion.test.ts @@ -220,6 +220,23 @@ describe("generateCompletions", () => { expect(names).toContain("DELETE"); }); + test("completes --source with the built-in migrate sources", () => { + const saved = process.env.CLERK_EXPERIMENTAL; + process.env.CLERK_EXPERIMENTAL = "migrate"; + try { + const names = generateCompletions(createProgram(), [ + "migrate", + "import", + "--source", + "", + ]).completions.map((c) => c.name); + expect(names).toEqual(["clerk", "supabase"]); + } finally { + if (saved === undefined) delete process.env.CLERK_EXPERIMENTAL; + else process.env.CLERK_EXPERIMENTAL = saved; + } + }); + test("returns empty for options with unknown values (file paths)", () => { const result = complete("config", "pull", "--output", ""); expect(result.completions).toEqual([]); diff --git a/test/e2e/migrate.test.ts b/test/e2e/migrate.test.ts new file mode 100644 index 000000000..df94ca644 --- /dev/null +++ b/test/e2e/migrate.test.ts @@ -0,0 +1,217 @@ +/** + * Live-BAPI tests for `clerk migrate import`, covering what only a real + * instance can answer: + * + * - A Supabase export round-trips: the dry run counts it without writing, an + * import without `--yes` writes nothing, the import creates every user and + * records the run, and each bcrypt hash verifies against the password it + * was made from. + * - A user whose only email is unverified, imported into an instance that + * requires an email, is refused by Clerk. The import's checks reject that + * user up front on the strength of this test, so it checks both halves. + * + * Requires `CLERK_PLATFORM_API_KEY` and `CLERK_CLI_TEST_APP_ID`. Locally, run + * via `bun run test:e2e:op` so 1Password resolves both in-memory. + */ + +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { randomBytes } from "node:crypto"; +import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const CLI_PATH = join(import.meta.dir, "../../packages/cli-core/src/cli.ts"); + +let APP_ID: string; +let workDir: string; +let emailRequired = false; +/** Every Clerk user an import created, deleted in `afterAll`. */ +const createdIds: string[] = []; + +async function cli(args: string[]) { + return Bun.$`bun ${CLI_PATH} ${args} --app ${APP_ID}` + .env({ + ...process.env, + CLERK_CONFIG_DIR: join(workDir, "config"), + CLERK_MIGRATE_DIR: join(workDir, "runs"), + CLERK_EXPERIMENTAL: "migrate", + CLERK_TELEMETRY_DISABLED: "1", + }) + .cwd(workDir) + .quiet() + .nothrow(); +} + +beforeAll(async () => { + const appId = process.env.CLERK_CLI_TEST_APP_ID; + if (!appId || !process.env.CLERK_PLATFORM_API_KEY) { + throw new Error( + "CLERK_CLI_TEST_APP_ID and CLERK_PLATFORM_API_KEY are required. " + + "Run via `bun run test:e2e:op` for local 1Password injection.", + ); + } + APP_ID = appId; + workDir = mkdtempSync(join(tmpdir(), "clerk-cli-e2e-migrate-")); + + const config = await cli(["config", "pull", "--keys", "auth_email"]); + const authEmail = ( + JSON.parse(config.stdout.toString()) as { + auth_email?: { required_for_sign_up?: boolean }; + } + ).auth_email; + emailRequired = Boolean(authEmail?.required_for_sign_up); +}); + +// Delete every imported user, so the test app does not fill up. +afterAll(async () => { + await Promise.all(createdIds.map(async (id) => cli(["api", `/users/${id}`, "-X", "DELETE"]))); + rmSync(workDir, { recursive: true, force: true }); +}, 60_000); + +/** + * Each user's latest line in run `runId`. A user has several (`creating`, + * then `created`); the last one wins. + */ +function latestLines(runId: string): Record[] { + const latest = new Map>(); + for (const line of readFileSync(join(workDir, "runs", runId, "users.ndjson"), "utf-8") + .trim() + .split("\n")) { + const parsed = JSON.parse(line) as Record; + latest.set(parsed.sourceId, parsed); + } + for (const line of latest.values()) { + if (line.status === "created" && typeof line.clerkId === "string") + createdIds.push(line.clerkId); + } + return [...latest.values()]; +} + +function writeExport(users: Record[]): string { + const file = join(workDir, `supabase-${randomBytes(4).toString("hex")}.json`); + writeFileSync(file, JSON.stringify(users)); + return file; +} + +test("a Supabase export dry-runs, imports, and its passwords verify", async () => { + const users = await Promise.all( + [0, 1].map(async () => { + const hex = randomBytes(6).toString("hex"); + const password = `Migrate${hex}!1`; + return { + password, + record: { + id: `sb_${hex}`, + email: `e2e-${hex}+clerk_test@clerkcookie.com`, + email_confirmed_at: "2024-06-29 20:25:06+00", + encrypted_password: await Bun.password.hash(password, { algorithm: "bcrypt", cost: 10 }), + }, + }; + }), + ); + const file = writeExport(users.map((user) => user.record)); + + const dryRun = await cli([ + "migrate", + "import", + file, + "--source", + "supabase", + "--dry-run", + "--json", + ]); + expect(dryRun.exitCode).toBe(0); + expect(JSON.parse(dryRun.stdout.toString())).toMatchObject({ + dryRun: true, + checks: { importable: 2 }, + }); + expect(readdirSync(workDir)).not.toContain("runs"); + + // Rule 1, against a real instance: without --yes nobody has consented, so + // nothing is written, here or in Clerk. + const unconsented = await cli(["migrate", "import", file, "--source", "supabase", "--json"]); + expect(unconsented.exitCode).toBe(2); + expect(JSON.parse(unconsented.stdout.toString())).toMatchObject({ + consent: "required", + checks: { importable: 2 }, + }); + expect(readdirSync(workDir)).not.toContain("runs"); + for (const { record } of users) { + const found = await cli(["api", `/users?external_id=${record.id}`]); + expect(JSON.parse(found.stdout.toString())).toEqual([]); + } + + const imported = await cli([ + "migrate", + "import", + file, + "--source", + "supabase", + "--yes", + "--json", + ]); + const result = JSON.parse(imported.stdout.toString()) as { + run: { id: string }; + result: { created: number }; + }; + // Read first: it registers the created users for cleanup, which a failed + // assertion would otherwise skip, leaving them in the shared test app. + const lines = latestLines(result.run.id); + expect(imported.exitCode).toBe(0); + expect(result.result.created).toBe(2); + expect( + JSON.parse(readFileSync(join(workDir, "runs", result.run.id, "run.json"), "utf-8")), + ).toMatchObject({ status: "complete", counts: { total: 2, created: 2 } }); + for (const { password, record } of users) { + const line = lines.find((candidate) => candidate.sourceId === record.id); + expect(line).toMatchObject({ status: "created" }); + const verify = await cli([ + "api", + `/users/${line?.clerkId as string}/verify_password`, + "-X", + "POST", + "-d", + JSON.stringify({ password }), + ]); + expect(JSON.parse(verify.stdout.toString())).toMatchObject({ verified: true }); + } +}, 60_000); + +test("a user whose only email is unverified is refused where email is required", async () => { + if (!emailRequired) { + // The test app decides this, not the test; say so rather than pass silently. + console.warn("Skipped: the test app's development instance does not require an email."); + return; + } + const hex = randomBytes(6).toString("hex"); + + // The premise: Clerk itself refuses a user created with no email, which is + // what an unverified-only user is at `POST /v1/users`. + const direct = await cli([ + "api", + "/users", + "-d", + JSON.stringify({ external_id: `direct_${hex}`, skip_password_requirement: true }), + ]); + expect(direct.exitCode).not.toBe(0); + + // And the import's checks reject them before asking Clerk. + const file = writeExport([{ id: `sb_${hex}`, email: `e2e-${hex}+clerk_test@clerkcookie.com` }]); + const imported = await cli([ + "migrate", + "import", + file, + "--source", + "supabase", + "--allow-partial", + "--yes", + "--json", + ]); + const { run } = JSON.parse(imported.stdout.toString()) as { run: { id: string } }; + expect(latestLines(run.id)).toEqual([ + expect.objectContaining({ + status: "skipped", + reason: "only has an unverified email, and this instance requires an email", + }), + ]); +}, 60_000);