feat(migrate): add runs and undo, and continue a stopped import

.changeset/migrate-runs-undo.md

@@ -0,0 +1,7 @@
+---
+"clerk": minor
+---
+
+Add `clerk migrate runs` and `clerk migrate undo`, and let a re-run of `clerk migrate import` continue where a stopped one left off. `clerk migrate` is still experimental and needs `CLERK_EXPERIMENTAL=migrate`.
+
+Every user an import creates now carries `clerkMigrateRun` (the run's ID) in its `private_metadata`, replacing any value the source had under that key. A re-run reads it to adopt a user whose create was cut off. Nothing removes it afterwards, and your backend can read it.

packages/cli-core/src/commands/migrate/import-users.test.ts

@@ -5,10 +5,10 @@ import { useCaptureLog } from "../../test/lib/stubs.ts";
 import {
   buildCreateUserBody,
   importUsers,
-  normalizeErrorMessage,
   readRetryAfter,
   splitIdentifiers,
 } from "./import-users.ts";
+import { normalizeErrorMessage } from "./lib/failures.ts";
 import type { ResolvedLimits } from "./lib/instance.ts";
 import type { UserLine } from "./lib/run-store.ts";
 import type { User } from "./types.ts";
@@ -210,6 +210,42 @@ describe("importUsers", () => {
       headers,
     });
 
+  // How a continue or an undo tells its own cut-off create from a user
+  // someone else made with the same external_id.
+  test("marks each create with its run, keeping the source's private metadata", async () => {
+    stub(() => ok("user_created"));
+
+    await importUsers({
+      users: [user({ userId: "u1", privateMetadata: { plan: "pro" } })],
+      secretKey: "sk_test_x",
+      limits: LIMITS,
+      record,
+      runId: "20260101-000000-abcd",
+    });
+
+    expect(requests.find((r) => r.url.endsWith("/v1/users"))?.body).toMatchObject({
+      private_metadata: { plan: "pro", clerkMigrateRun: "20260101-000000-abcd" },
+    });
+  });
+
+  // A Clerk export of users an earlier migration imported carries that run's
+  // marker; this run's must win, or a cut-off create could never be adopted.
+  test("replaces a marker the source's private metadata already carries", async () => {
+    stub(() => ok("user_created"));
+
+    await importUsers({
+      users: [user({ userId: "u1", privateMetadata: { clerkMigrateRun: "20250101-000000-0000" } })],
+      secretKey: "sk_test_x",
+      limits: LIMITS,
+      record,
+      runId: "20260101-000000-abcd",
+    });
+
+    expect(requests.find((r) => r.url.endsWith("/v1/users"))?.body).toMatchObject({
+      private_metadata: { clerkMigrateRun: "20260101-000000-abcd" },
+    });
+  });
+
   test("creates each user and reports them as successful", async () => {
     stub(() => ok("user_created"));
 
@@ -327,14 +363,18 @@ describe("importUsers", () => {
     });
 
     expect(summary).toMatchObject({ successful: 1, failed: 0 });
-    // On record once created, then with its attach pending, then with what
-    // the attach added. A refused attach is not retried, so nothing is pending.
-    expect(lines).toHaveLength(3);
-    expect(lines[0]).toEqual({ sourceId: "u1", clerkId: "user_created", status: "created" });
-    expect(lines[1]?.pending).toEqual([{ kind: "email", value: "b@x.dev", verified: true }]);
-    expect(lines[2]).toMatchObject({ status: "created", clerkId: "user_created" });
-    expect(lines[2]?.error).toContain("Failed to add additional email b@x.dev");
-    expect(lines[2]).not.toHaveProperty("pending");
+    // On record once created with its attach pending, in one line, then with
+    // what the attach added. A refused attach is not retried, so nothing is pending.
+    expect(lines).toHaveLength(2);
+    expect(lines[0]).toEqual({
+      sourceId: "u1",
+      clerkId: "user_created",
+      status: "created",
+      pending: [{ kind: "email", value: "b@x.dev", verified: true }],
+    });
+    expect(lines[1]).toMatchObject({ status: "created", clerkId: "user_created" });
+    expect(lines[1]?.error).toContain("Failed to add additional email b@x.dev");
+    expect(lines[1]).not.toHaveProperty("pending");
   });
 
   // A Ctrl-C mid-backoff should not wait out the pause, nor send the attach.
@@ -380,7 +420,7 @@ describe("importUsers", () => {
     expect(lines.at(-1)).not.toHaveProperty("pending");
   });
 
-  test("keeps an attach with no answer pending", async () => {
+  test("keeps an attach with no answer pending, for a continued run", async () => {
     stub((url) =>
       url.endsWith("/v1/email_addresses") ? clerkError(503, "unavailable") : ok("user_created"),
     );
@@ -421,6 +461,103 @@ describe("importUsers", () => {
     ]);
   });
 
+  test("attachOnly sends just the pending attaches, and clears them", async () => {
+    stub(() => ok("idn_1"));
+
+    await importUsers({
+      users: [],
+      attachOnly: [
+        {
+          sourceId: "u1",
+          clerkId: "user_1",
+          status: "created",
+          pending: [{ kind: "phone", value: "+15555550100", verified: false }],
+        },
+      ],
+      secretKey: "sk_test_x",
+      limits: LIMITS,
+      record,
+    });
+
+    expect(requests.map((r) => [new URL(r.url).pathname, r.body])).toEqual([
+      [
+        "/v1/phone_numbers",
+        { user_id: "user_1", phone_number: "+15555550100", primary: false, verified: false },
+      ],
+    ]);
+    expect(lines.at(-1)).toEqual({ sourceId: "u1", clerkId: "user_1", status: "created" });
+  });
+
+  // A continue with only attaches left would otherwise read 0/0 throughout.
+  test("counts an attach-only user towards progress once its attaches are sent", async () => {
+    stub(() => ok("idn_1"));
+    const reported: { done: number; ok: number }[] = [];
+
+    const summary = await importUsers({
+      users: [],
+      attachOnly: [
+        {
+          sourceId: "u1",
+          clerkId: "user_1",
+          status: "created",
+          pending: [{ kind: "phone", value: "+15555550100", verified: false }],
+        },
+      ],
+      secretKey: "sk_test_x",
+      limits: LIMITS,
+      record,
+      progress: ({ done, ok }) => reported.push({ done, ok }),
+    });
+
+    // Created by the run's earlier attempt: the bar and the summary agree.
+    expect(reported.at(-1)).toEqual({ done: 1, ok: 1 });
+    expect(summary.successful).toBe(1);
+  });
+
+  // The pending phone attaches now, but the email Clerk refused stays refused.
+  test("an attach-only user keeps the error its earlier line recorded", async () => {
+    stub(() => ok("idn_1"));
+
+    await importUsers({
+      users: [],
+      attachOnly: [
+        {
+          sourceId: "u1",
+          clerkId: "user_1",
+          status: "created",
+          error: "b@x.dev: That email address is taken.",
+          pending: [{ kind: "phone", value: "+15555550100", verified: false }],
+        },
+      ],
+      secretKey: "sk_test_x",
+      limits: LIMITS,
+      record,
+    });
+
+    expect(lines.at(-1)).toMatchObject({
+      sourceId: "u1",
+      status: "created",
+      error: "b@x.dev: That email address is taken.",
+    });
+    expect(lines.at(-1)?.pending).toBeUndefined();
+  });
+
+  test("an adopted user is not created again; only its extras attach", async () => {
+    stub(() => ok("idn_1"));
+
+    const summary = await importUsers({
+      users: [user({ email: ["a@x.dev", "b@x.dev"] })],
+      adopted: new Map([["u1", "user_found"]]),
+      secretKey: "sk_test_x",
+      limits: LIMITS,
+      record,
+    });
+
+    expect(requests.map((r) => new URL(r.url).pathname)).toEqual(["/v1/email_addresses"]);
+    expect(summary.successful).toBe(1);
+    expect(lines.at(-1)).toMatchObject({ clerkId: "user_found", status: "created" });
+  });
+
   // Shapes from clerk_go's apierror: the country error carries its own code
   // and no param_name; the E.164 error is a form error on phone_number.
   test.each([
@@ -604,7 +741,9 @@ describe("importUsers", () => {
     });
 
     expect(summary.failed).toBe(1);
-    expect(allLines.map((line) => line.status)).toEqual(["creating"]);
+    // A second `creating` line, so `runs <id>` can say why.
+    expect(allLines.map((line) => line.status)).toEqual(["creating", "creating"]);
+    expect(allLines[1]?.error).toContain("a re-run checks");
     expect([...summary.errorBreakdown.keys()][0]).toContain("a re-run checks");
   });
 
@@ -619,7 +758,15 @@ describe("importUsers", () => {
     });
 
     expect(summary).toMatchObject({ successful: 0, failed: 1 });
-    expect(allLines).toEqual([{ sourceId: "u1", status: "creating" }]);
+    expect(allLines).toEqual([
+      { sourceId: "u1", status: "creating" },
+      {
+        sourceId: "u1",
+        status: "creating",
+        error: expect.stringContaining("without a user ID"),
+        code: "unknown",
+      },
+    ]);
   });
 
   // Every create after the first refusal would be refused too, so none is