[pull] main from firecrawl:main

apps/api/native/Cargo.toml

@@ -26,6 +26,10 @@ strsim = "0.11"
 texting_robots = "0.2.2"
 url = "2.5.7"
 tokio = "1.48.0"
+# Transitive dep (pdf-inspector -> unicode-normalization -> tinyvec). Pinned because
+# tinyvec 1.13.0 fails to compile (https://github.com/Lokathor/tinyvec/issues/225)
+# and Cargo.lock is not committed, so fresh builds would otherwise pick it up.
+tinyvec = ">=1.12, <1.13"
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", default-features = false, features = ["registry"] }
 

apps/api/package.json

@@ -87,6 +87,7 @@
     "@clickhouse/client": "^1.8.1",
     "@dqbd/tiktoken": "^1.0.22",
     "@google-cloud/bigtable": "^7.2.0",
+    "@google-cloud/pubsub": "^6.0.1",
     "@google-cloud/storage": "^7.21.0",
     "@mendable/firecrawl-rs": "workspace:*",
     "@openrouter/ai-sdk-provider": "^2.9.1",

apps/api/src/__tests__/routes/deprecations-existing-routes.routes.test.ts

@@ -0,0 +1,97 @@
+import express from "express";
+import request from "supertest";
+import { deprecationMiddleware } from "../../lib/deprecations";
+
+// Every pre-existing deprecation entry and its replacement. Pinned so the
+// github entry cannot change anyone else's wire output. All 13 shipped in
+// #3469 on 2026-05-06, which is the date they now emit.
+const LEGACY_DEPRECATED_AT = "@1778025600";
+
+const EXISTING = {
+  v1_extract: "/v2/scrape",
+  v1_extract_status: "/v2/scrape",
+  v2_extract: "/v2/scrape",
+  v2_extract_status: "/v2/scrape",
+  v1_deep_research: "/v2/search",
+  v1_deep_research_status: "/v2/search",
+  v1_llmstxt: undefined,
+  v1_llmstxt_status: undefined,
+  v0_scrape: "/v2/scrape",
+  v0_crawl: "/v2/crawl",
+  v0_crawl_status: "/v2/crawl/:jobId",
+  v0_crawl_cancel: "/v2/crawl/:jobId",
+  v0_search: "/v2/search",
+} as const;
+
+type Key = keyof typeof EXISTING;
+
+// What the old middleware did, replayed on a copy, so we can compare bytes.
+function legacyAnnotate(body: any, message: string, replacement?: string) {
+  const copy = JSON.parse(JSON.stringify(body));
+  const existing = Array.isArray(copy.warnings) ? copy.warnings : [];
+  copy.warnings = [...existing, message];
+  if (replacement && copy.replacement === undefined) {
+    copy.replacement = replacement;
+  }
+  return JSON.stringify(copy);
+}
+
+function appFor(key: Key, body: unknown) {
+  const app = express();
+  app.get("/probe", deprecationMiddleware(key), (_req, res) => {
+    res.status(200).json(body);
+  });
+  return app;
+}
+
+const BODIES: Record<string, unknown> = {
+  plain: { success: true, id: "abc" },
+  "with existing warnings": { success: true, warnings: ["upstream note"] },
+  "with existing replacement": { success: true, replacement: "/keep/me" },
+  "warnings first": { warnings: [], success: true },
+};
+
+describe("pre-existing deprecated routes are unchanged", () => {
+  for (const key of Object.keys(EXISTING) as Key[]) {
+    describe(key, () => {
+      it("emits the shared 2026-05-06 Deprecation date and no Sunset", async () => {
+        const res = await request(appFor(key, BODIES.plain)).get("/probe");
+
+        expect(res.headers["deprecation"]).toBe(LEGACY_DEPRECATED_AT);
+        expect(res.headers["sunset"]).toBeUndefined();
+        expect(res.headers["warning"]).toMatch(/^299 - "/);
+        if (EXISTING[key]) {
+          expect(res.headers["link"]).toBe(
+            `<${EXISTING[key]}>; rel="successor-version"`,
+          );
+        } else {
+          expect(res.headers["link"]).toBeUndefined();
+        }
+      });
+
+      for (const [label, body] of Object.entries(BODIES)) {
+        it(`serialises a ${label} body byte-for-byte as before`, async () => {
+          const res = await request(appFor(key, body)).get("/probe");
+          const message = res.headers["warning"].replace(/^299 - "|"$/g, "");
+
+          expect(res.text).toBe(legacyAnnotate(body, message, EXISTING[key]));
+        });
+      }
+
+      it("no longer mutates the object the controller passed in", async () => {
+        const body: any = { success: true };
+        await request(appFor(key, body)).get("/probe");
+
+        expect(body.warnings).toBeUndefined();
+        expect(body.replacement).toBeUndefined();
+      });
+    });
+  }
+
+  it("arrays and strings pass straight through", async () => {
+    const arr = await request(appFor("v0_search", [1, 2])).get("/probe");
+    expect(arr.text).toBe("[1,2]");
+    const str = await request(appFor("v0_search", "hello")).get("/probe");
+    expect(str.text).toBe('"hello"');
+  });
+});