diff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.toml index 38d607661c..a120495bf3 100644 --- a/apps/api/native/Cargo.toml +++ b/apps/api/native/Cargo.toml @@ -26,6 +26,10 @@ strsim = "0.11" texting_robots = "0.2.2" url = "2.5.7" tokio = "1.48.0" +# Transitive dep (pdf-inspector -> unicode-normalization -> tinyvec). Pinned because +# tinyvec 1.13.0 fails to compile (https://github.com/Lokathor/tinyvec/issues/225) +# and Cargo.lock is not committed, so fresh builds would otherwise pick it up. +tinyvec = ">=1.12, <1.13" tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["registry"] } diff --git a/apps/api/package.json b/apps/api/package.json index b580e71c47..a0240f4da8 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -87,6 +87,7 @@ "@clickhouse/client": "^1.8.1", "@dqbd/tiktoken": "^1.0.22", "@google-cloud/bigtable": "^7.2.0", + "@google-cloud/pubsub": "^6.0.1", "@google-cloud/storage": "^7.21.0", "@mendable/firecrawl-rs": "workspace:*", "@openrouter/ai-sdk-provider": "^2.9.1", diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml index 4e4d23815f..4afe587169 100644 --- a/apps/api/pnpm-lock.yaml +++ b/apps/api/pnpm-lock.yaml @@ -93,6 +93,9 @@ importers: '@google-cloud/bigtable': specifier: ^7.2.0 version: 7.2.0(@opentelemetry/core@2.9.0(@opentelemetry/api@1.9.1)) + '@google-cloud/pubsub': + specifier: ^6.0.1 + version: 6.0.1 '@google-cloud/storage': specifier: ^7.21.0 version: 7.21.0 @@ -834,6 +837,10 @@ packages: resolution: {integrity: sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==} engines: {node: '>=14.0.0'} + '@google-cloud/paginator@7.0.1': + resolution: {integrity: sha512-k32cWlHAF8yTgg8rciLI8mPMI6UzuJdKp53YRxISRwMFxUl2FYplvs+Mr2UHxKn0W7rXsqZnUZy73AOJFDP8iA==} + engines: {node: '>=22'} + '@google-cloud/precise-date@4.0.0': resolution: {integrity: sha512-1TUx3KdaU3cN7nfCdNf+UVqA/PSX29Cjcox3fZZBtINlRrXVTmUkQnCKv2MbBUbCopbK4olAT1IHl76uZyCiVA==} engines: {node: '>=14.0.0'} @@ -858,6 +865,14 @@ packages: resolution: {integrity: sha512-lpN2AtoQ/iimp7jjm5zJFWbpW7OJc5qWmQdt59CI4ENeoIRIx+yf2ShSR2kanQfjFOshA74eSbimXXuRaSCUVg==} engines: {node: '>=22'} + '@google-cloud/pubsub-api@0.3.0': + resolution: {integrity: sha512-x1L+lrIHgaMwN5fAhnNVS7rh0BZn9eKmqOBXnOUM3jkilF4GE4ooETS9SYnCtPCVHwQtByDTh5b7KIMUW7XAHQ==} + engines: {node: '>=22'} + + '@google-cloud/pubsub@6.0.1': + resolution: {integrity: sha512-Ms/cE5wyxWoHHj6IUWZO67j7nDhAKa7eAUkUYBJjRie0Yr1V82NclaaIntdqAmmpOgXbIzdDlrIyWXLVMeGSDg==} + engines: {node: '>=22'} + '@google-cloud/storage@7.21.0': resolution: {integrity: sha512-l+IFTkd+6Y5LoAuXyYCKNAKtw/Ci+rAMqgdTB1jv4iZiLhw0rtq+0qjIRbBizXkNzEFmXiXUW0H7sZQQvk1ffA==} engines: {node: '>=14'} @@ -1543,6 +1558,10 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.3.0 <1.10.0' + '@opentelemetry/semantic-conventions@1.39.0': + resolution: {integrity: sha512-R5R9tb2AXs2IRLNKLBJDynhkfmx7mX0vi8NkhZb3gUkPWHn6HXk5J8iQ/dql0U3ApfWym4kXXmBDRGO+oeOfjg==} + engines: {node: '>=14'} + '@opentelemetry/semantic-conventions@1.43.0': resolution: {integrity: sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==} engines: {node: '>=14'} @@ -3200,6 +3219,10 @@ packages: resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==} hasBin: true + heap-js@2.7.1: + resolution: {integrity: sha512-EQfezRg0NCZGNlhlDR3Evrw1FVL2G3LhU7EgPoxufQKruNBSYA8MiRPHeWbU+36o+Fhel0wMwM+sLEiBAlNLJA==} + engines: {node: '>=10.0.0'} + html-encoding-sniffer@6.0.0: resolution: {integrity: sha512-CV9TW3Y3f8/wT0BRFc1/KAVQ3TUHiXmaAb6VW9vtiMFf7SLoMd1PdAc4W3KFOFETBJUb90KatHqlsZMWV+R9Gg==} engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0} @@ -3327,6 +3350,9 @@ packages: is-promise@4.0.0: resolution: {integrity: sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==} + is-stream-ended@0.1.4: + resolution: {integrity: sha512-xj0XPvmr7bQFTvirqnFr50o0hQIh6ZItDqloxt5aJrR4NQsYeSsyFQERYGCAzfindAcnKjINnwEEgLx4IqVzQw==} + is-stream@2.0.1: resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} engines: {node: '>=8'} @@ -3837,6 +3863,10 @@ packages: oxlint-tsgolint: optional: true + p-defer@3.0.0: + resolution: {integrity: sha512-ugZxsxmtTln604yeYd29EGrNhazN2lywetzpKhfmQjW/VJmhpDmWbiX+h0zL8V91R0UXkhb3KtPmyq9PZw3aYw==} + engines: {node: '>=8'} + p-finally@1.0.0: resolution: {integrity: sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==} engines: {node: '>=4'} @@ -5253,6 +5283,10 @@ snapshots: arrify: 2.0.1 extend: 3.0.2 + '@google-cloud/paginator@7.0.1': + dependencies: + extend: 3.0.2 + '@google-cloud/precise-date@4.0.0': {} '@google-cloud/precise-date@6.0.1': {} @@ -5265,6 +5299,35 @@ snapshots: '@google-cloud/promisify@6.0.1': {} + '@google-cloud/pubsub-api@0.3.0': + dependencies: + google-gax: 6.1.0 + transitivePeerDependencies: + - supports-color + + '@google-cloud/pubsub@6.0.1': + dependencies: + '@google-cloud/paginator': 7.0.1 + '@google-cloud/precise-date': 6.0.1 + '@google-cloud/projectify': 6.0.1 + '@google-cloud/promisify': 6.0.1 + '@google-cloud/pubsub-api': 0.3.0 + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.9.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.39.0 + arrify: 2.0.1 + extend: 3.0.2 + google-auth-library: 11.0.2 + google-gax: 6.1.0 + google-logging-utils: 2.0.1 + heap-js: 2.7.1 + is-stream-ended: 0.1.4 + lodash.snakecase: 4.1.1 + long: 5.3.2 + p-defer: 3.0.0 + transitivePeerDependencies: + - supports-color + '@google-cloud/storage@7.21.0': dependencies: '@google-cloud/paginator': 5.0.2 @@ -5867,6 +5930,8 @@ snapshots: '@opentelemetry/resources': 2.9.0(@opentelemetry/api@1.9.1) '@opentelemetry/semantic-conventions': 1.43.0 + '@opentelemetry/semantic-conventions@1.39.0': {} + '@opentelemetry/semantic-conventions@1.43.0': {} '@oxc-project/types@0.133.0': {} @@ -7361,6 +7426,8 @@ snapshots: he@1.2.0: {} + heap-js@2.7.1: {} + html-encoding-sniffer@6.0.0(@noble/hashes@1.8.0): dependencies: '@exodus/bytes': 1.15.1(@noble/hashes@1.8.0) @@ -7510,6 +7577,8 @@ snapshots: is-promise@4.0.0: {} + is-stream-ended@0.1.4: {} + is-stream@2.0.1: {} is-utf8@0.2.1: {} @@ -7977,6 +8046,8 @@ snapshots: '@oxlint/win32-arm64': 1.14.0 '@oxlint/win32-x64': 1.14.0 + p-defer@3.0.0: {} + p-finally@1.0.0: {} p-limit@3.1.0: diff --git a/apps/api/src/__tests__/routes/deprecations-existing-routes.routes.test.ts b/apps/api/src/__tests__/routes/deprecations-existing-routes.routes.test.ts new file mode 100644 index 0000000000..60847f4a99 --- /dev/null +++ b/apps/api/src/__tests__/routes/deprecations-existing-routes.routes.test.ts @@ -0,0 +1,97 @@ +import express from "express"; +import request from "supertest"; +import { deprecationMiddleware } from "../../lib/deprecations"; + +// Every pre-existing deprecation entry and its replacement. Pinned so the +// github entry cannot change anyone else's wire output. All 13 shipped in +// #3469 on 2026-05-06, which is the date they now emit. +const LEGACY_DEPRECATED_AT = "@1778025600"; + +const EXISTING = { + v1_extract: "/v2/scrape", + v1_extract_status: "/v2/scrape", + v2_extract: "/v2/scrape", + v2_extract_status: "/v2/scrape", + v1_deep_research: "/v2/search", + v1_deep_research_status: "/v2/search", + v1_llmstxt: undefined, + v1_llmstxt_status: undefined, + v0_scrape: "/v2/scrape", + v0_crawl: "/v2/crawl", + v0_crawl_status: "/v2/crawl/:jobId", + v0_crawl_cancel: "/v2/crawl/:jobId", + v0_search: "/v2/search", +} as const; + +type Key = keyof typeof EXISTING; + +// What the old middleware did, replayed on a copy, so we can compare bytes. +function legacyAnnotate(body: any, message: string, replacement?: string) { + const copy = JSON.parse(JSON.stringify(body)); + const existing = Array.isArray(copy.warnings) ? copy.warnings : []; + copy.warnings = [...existing, message]; + if (replacement && copy.replacement === undefined) { + copy.replacement = replacement; + } + return JSON.stringify(copy); +} + +function appFor(key: Key, body: unknown) { + const app = express(); + app.get("/probe", deprecationMiddleware(key), (_req, res) => { + res.status(200).json(body); + }); + return app; +} + +const BODIES: Record = { + plain: { success: true, id: "abc" }, + "with existing warnings": { success: true, warnings: ["upstream note"] }, + "with existing replacement": { success: true, replacement: "/keep/me" }, + "warnings first": { warnings: [], success: true }, +}; + +describe("pre-existing deprecated routes are unchanged", () => { + for (const key of Object.keys(EXISTING) as Key[]) { + describe(key, () => { + it("emits the shared 2026-05-06 Deprecation date and no Sunset", async () => { + const res = await request(appFor(key, BODIES.plain)).get("/probe"); + + expect(res.headers["deprecation"]).toBe(LEGACY_DEPRECATED_AT); + expect(res.headers["sunset"]).toBeUndefined(); + expect(res.headers["warning"]).toMatch(/^299 - "/); + if (EXISTING[key]) { + expect(res.headers["link"]).toBe( + `<${EXISTING[key]}>; rel="successor-version"`, + ); + } else { + expect(res.headers["link"]).toBeUndefined(); + } + }); + + for (const [label, body] of Object.entries(BODIES)) { + it(`serialises a ${label} body byte-for-byte as before`, async () => { + const res = await request(appFor(key, body)).get("/probe"); + const message = res.headers["warning"].replace(/^299 - "|"$/g, ""); + + expect(res.text).toBe(legacyAnnotate(body, message, EXISTING[key])); + }); + } + + it("no longer mutates the object the controller passed in", async () => { + const body: any = { success: true }; + await request(appFor(key, body)).get("/probe"); + + expect(body.warnings).toBeUndefined(); + expect(body.replacement).toBeUndefined(); + }); + }); + } + + it("arrays and strings pass straight through", async () => { + const arr = await request(appFor("v0_search", [1, 2])).get("/probe"); + expect(arr.text).toBe("[1,2]"); + const str = await request(appFor("v0_search", "hello")).get("/probe"); + expect(str.text).toBe('"hello"'); + }); +}); diff --git a/apps/api/src/__tests__/routes/research-github-deprecation.routes.test.ts b/apps/api/src/__tests__/routes/research-github-deprecation.routes.test.ts new file mode 100644 index 0000000000..cd8cf4ab9b --- /dev/null +++ b/apps/api/src/__tests__/routes/research-github-deprecation.routes.test.ts @@ -0,0 +1,185 @@ +import express from "express"; +import request from "supertest"; +import { createResearchRouter } from "../../controllers/v2/research-proxy"; + +// Mounts the real research router against a stubbed upstream, so the +// deprecation contract is exercised through the actual controller rather than +// a fake. The one thing only this test can reach is the logged payload: on the +// canonical mount the controller responds with the same object it later writes +// to Postgres, so a mutating res.json would leak the notice into the +// research_github_searches row. +const upstreamCalls: string[] = []; +const loggedRows: any[] = []; + +let upstreamBody: any = { + success: true, + results: [ + { + resultType: "github_history", + repo: "milvus-io/milvus", + url: "https://github.com/milvus-io/milvus/issues/1", + pageType: "issue", + number: 1, + snippet: "hybrid search", + contentMd: "# hybrid search", + segmentCount: 2, + scores: { rrf: 0.5 }, + }, + ], +}; +let upstreamStatus = 200; + +vi.mock("../../lib/research-upstream", () => ({ + fetchResearchUpstream: async (options: { path: string }) => { + upstreamCalls.push(options.path); + return { + status: upstreamStatus, + ok: upstreamStatus >= 200 && upstreamStatus < 300, + headers: new Headers(), + text: async () => JSON.stringify(upstreamBody), + }; + }, +})); + +vi.mock("../../services/logging/log_job", () => ({ + logRequest: async () => {}, + logResearchEndpoint: async (row: any) => { + loggedRows.push(row); + }, +})); + +vi.mock("../../lib/keyless", () => ({ + chargeKeylessCredits: async () => {}, +})); + +vi.mock("../../services/billing/credit_billing", () => ({ + billTeam: async () => {}, +})); + +function appWith(legacy: boolean) { + const app = express(); + app.use((req: any, _res, next) => { + req.auth = { team_id: "team-test", plan: "standard" }; + req.acuc = { api_key_id: null }; + next(); + }); + app.use( + legacy ? "/v2/research" : "/v2/search/research", + createResearchRouter(legacy ? { legacy: true } : {}), + ); + return app; +} + +const canonical = appWith(false); +const legacy = appWith(true); + +beforeEach(() => { + upstreamCalls.length = 0; + loggedRows.length = 0; + upstreamStatus = 200; +}); + +describe("github search deprecation, canonical mount", () => { + it("sets every deprecation header", async () => { + const res = await request(canonical).get( + "/v2/search/research/github?query=milvus+hybrid+search&k=3", + ); + + expect(res.statusCode).toBe(200); + expect(res.headers["deprecation"]).toBe("@1788393600"); + expect(res.headers["sunset"]).toBe("Tue, 03 Nov 2026 23:59:59 GMT"); + expect(res.headers["link"]).toContain( + '; rel="deprecation"', + ); + expect(res.headers["link"]).toContain( + '; rel="successor-version"', + ); + expect(res.headers["warning"]).toMatch(/^299 - "/); + expect(res.headers["warning"]).toContain("2026-11-03"); + }); + + it("adds warnings and replacement to the body without losing the results", async () => { + const res = await request(canonical).get( + "/v2/search/research/github?query=x", + ); + + expect(res.body.success).toBe(true); + expect(res.body.results).toHaveLength(1); + expect(res.body.results[0].repo).toBe("milvus-io/milvus"); + expect(res.body.replacement).toBe("/v2/search/developer"); + expect(res.body.warnings).toHaveLength(1); + expect(res.body.warnings[0]).toContain("/v2/search/developer"); + expect(res.body.warnings[0]).toContain("2026-11-03"); + }); + + it("keeps the deprecation notice out of the logged row", async () => { + await request(canonical).get("/v2/search/research/github?query=x"); + + expect(loggedRows).toHaveLength(1); + const logged = loggedRows[0].response; + expect(logged.success).toBe(true); + expect(logged.results).toHaveLength(1); + expect(logged.warnings).toBeUndefined(); + expect(logged.replacement).toBeUndefined(); + expect(JSON.stringify(logged)).not.toContain("deprecated"); + }); + + it("still proxies to the unchanged upstream path", async () => { + await request(canonical).get("/v2/search/research/github?query=x"); + + expect(upstreamCalls).toEqual(["/v2/research/github"]); + }); + + it("warns on a rejected request too", async () => { + const res = await request(canonical).get("/v2/search/research/github"); + + expect(res.statusCode).toBe(400); + expect(res.body.success).toBe(false); + expect(res.headers["deprecation"]).toBe("@1788393600"); + expect(res.body.warnings).toHaveLength(1); + }); + + it("leaves the paper routes on the same router alone", async () => { + const res = await request(canonical).get( + "/v2/search/research/papers?query=rag", + ); + + expect(res.statusCode).toBe(200); + expect(res.headers["deprecation"]).toBeUndefined(); + expect(res.headers["sunset"]).toBeUndefined(); + expect(res.headers["link"]).toBeUndefined(); + expect(res.headers["warning"]).toBeUndefined(); + expect(res.body.warnings).toBeUndefined(); + expect(res.body.replacement).toBeUndefined(); + expect(loggedRows[0].response.warnings).toBeUndefined(); + }); +}); + +describe("github search deprecation, legacy mount", () => { + it("keeps the snake_case aliases alongside the notice", async () => { + const res = await request(legacy).get("/v2/research/github?query=x"); + + expect(res.statusCode).toBe(200); + expect(res.headers["deprecation"]).toBe("@1788393600"); + expect(res.body.results[0].result_type).toBe("github_history"); + expect(res.body.results[0].page_type).toBe("issue"); + expect(res.body.results[0].content_md).toBe("# hybrid search"); + expect(res.body.results[0].segment_count).toBe(2); + expect(res.body.warnings).toHaveLength(1); + expect(res.body.replacement).toBe("/v2/search/developer"); + }); + + it("does not grow a snake_case twin of the injected keys", async () => { + const res = await request(legacy).get("/v2/research/github?query=x"); + + expect(res.body).not.toHaveProperty("warnings_"); + expect(res.body).not.toHaveProperty("replacement_"); + }); + + it("keeps the notice out of the logged row here as well", async () => { + await request(legacy).get("/v2/research/github?query=x"); + + expect(loggedRows[0].response.warnings).toBeUndefined(); + expect(loggedRows[0].response.replacement).toBeUndefined(); + }); +}); diff --git a/apps/api/src/__tests__/routes/research-keyless.routes.test.ts b/apps/api/src/__tests__/routes/research-keyless.routes.test.ts index d5276f91c4..26b0097ec2 100644 --- a/apps/api/src/__tests__/routes/research-keyless.routes.test.ts +++ b/apps/api/src/__tests__/routes/research-keyless.routes.test.ts @@ -112,6 +112,20 @@ describe("RESEARCH_KEYLESS_DISABLED parsing", () => { expect(config.RESEARCH_KEYLESS_DISABLED).toEqual(["inspect", "similar"]); }); + it("accepts the deprecated github operation only when it is named", async () => { + const { config } = await loadWithFlag("github"); + + expect(config.RESEARCH_KEYLESS_DISABLED).toEqual(["github"]); + }); + + it("keeps github out of the true/all shorthand and out of the default", async () => { + for (const flag of [undefined, "true", "all", "on", "1", "yes"]) { + const { config } = await loadWithFlag(flag); + + expect(config.RESEARCH_KEYLESS_DISABLED).not.toContain("github"); + } + }); + it("refuses an unknown operation at boot instead of silently ignoring it", async () => { await expect(loadWithFlag("inspect,papers")).rejects.toThrow(); }); @@ -142,6 +156,33 @@ describe("Research Index paper operation gate", () => { ).toBe(false); }); + it("blocks the deprecated github operation once the flag names it", async () => { + const { isResearchKeylessDisabled } = await loadWithFlag("github"); + + expect( + isResearchKeylessDisabled(fakeRequest("/github", { query: "x" })), + ).toBe(true); + expect( + isResearchKeylessDisabled( + fakeRequest("/v2/search/research/github", { query: "x" }), + ), + ).toBe(true); + expect( + isResearchKeylessDisabled(fakeRequest("/papers", { query: "rag" })), + ).toBe(false); + }); + + it("does not mistake a paper path for the github route", async () => { + const { isResearchKeylessDisabled } = await loadWithFlag("github"); + + expect(isResearchKeylessDisabled(fakeRequest("/papers/github"))).toBe( + false, + ); + expect( + isResearchKeylessDisabled(fakeRequest("/papers/github/similar")), + ).toBe(false); + }); + it("blocks only the named operation when the scope is narrowed", async () => { const { isResearchKeylessDisabled } = await loadWithFlag("inspect"); diff --git a/apps/api/src/__tests__/snips/v1/deprecation.test.ts b/apps/api/src/__tests__/snips/v1/deprecation.test.ts index ed2dd69eca..0e153706a1 100644 --- a/apps/api/src/__tests__/snips/v1/deprecation.test.ts +++ b/apps/api/src/__tests__/snips/v1/deprecation.test.ts @@ -23,7 +23,7 @@ describe("Deprecation warnings on legacy endpoints", () => { expect(res.statusCode).toBe(200); expect(res.body.success).toBe(true); - expect(res.headers["deprecation"]).toBe("true"); + expect(res.headers["deprecation"]).toBe("@1778025600"); expect(res.headers["warning"]).toMatch(/^299 - "/); expect(res.headers["warning"]).toMatch(/llmstxt/i); expect(Array.isArray(res.body.warnings)).toBe(true); @@ -43,7 +43,7 @@ describe("Deprecation warnings on legacy endpoints", () => { .set("Authorization", `Bearer ${identity.apiKey}`); expect(res.statusCode).toBe(404); - expect(res.headers["deprecation"]).toBe("true"); + expect(res.headers["deprecation"]).toBe("@1778025600"); expect(res.headers["warning"]).toMatch(/deprecated/i); expect(Array.isArray(res.body.warnings)).toBe(true); expect(res.body.warnings.some((w: string) => /deprecated/i.test(w))).toBe( @@ -64,7 +64,7 @@ describe("Deprecation warnings on legacy endpoints", () => { }); expect(res.statusCode).toBe(200); - expect(res.headers["deprecation"]).toBe("true"); + expect(res.headers["deprecation"]).toBe("@1778025600"); expect(res.headers["warning"]).toMatch(/deep-research/i); expect(res.headers["link"]).toContain( '; rel="successor-version"', diff --git a/apps/api/src/__tests__/snips/v2/research.test.ts b/apps/api/src/__tests__/snips/v2/research.test.ts index 6815f5caba..64a433b096 100644 --- a/apps/api/src/__tests__/snips/v2/research.test.ts +++ b/apps/api/src/__tests__/snips/v2/research.test.ts @@ -101,9 +101,10 @@ async function snapshotKeylessCounts(): Promise> { // our snapshot window resets the counters. Ambiguity retries the whole probe // (the requests under test are idempotent zero-credit GETs) instead of // asserting on a poisoned window. -async function issueAndResolveKeylessIp< - T extends { statusCode: number }, ->(forwardedIp: string, issue: () => Promise): Promise<{ res: T; ip: string }> { +async function issueAndResolveKeylessIp( + forwardedIp: string, + issue: () => Promise, +): Promise<{ res: T; ip: string }> { if (KEYLESS_PROXY_SECRET) { return { res: await issue(), ip: forwardedIp }; } @@ -116,7 +117,9 @@ async function issueAndResolveKeylessIp< .filter(([ip, count]) => count > (before.get(ip) ?? 0)) .map(([ip]) => ip); if (bumped.length === 1) return { res, ip: bumped[0] }; - attempts.push(bumped.length === 0 ? "none" : `multiple[${bumped.join(", ")}]`); + attempts.push( + bumped.length === 0 ? "none" : `multiple[${bumped.join(", ")}]`, + ); } // Distinguish the two failure modes: "none" means the request was never // counted against any candidate key (not treated as keyless, or the server @@ -536,3 +539,94 @@ describeIf(HAS_RESEARCH)("Research API", () => { }, 180000); }); }); + +const GITHUB_DEPRECATED_AT = "@1788393600"; +const GITHUB_SUNSET = "Tue, 03 Nov 2026 23:59:59 GMT"; + +describeIf(HAS_RESEARCH)("Research API github search deprecation", () => { + it("carries the deprecation headers and body warning on the canonical mount", async () => { + const identity = await idmux({ + name: "research/github deprecation canonical", + credits: 100, + }); + + const res = await researchRaw( + "/v2/search/research/github", + { query: "milvus hybrid search", k: 3 }, + identity, + ); + + expect(res.statusCode).toBe(200); + expect(res.body.success).toBe(true); + + expect(res.headers["deprecation"]).toBe(GITHUB_DEPRECATED_AT); + expect(res.headers["sunset"]).toBe(GITHUB_SUNSET); + expect(res.headers["link"]).toContain('rel="deprecation"'); + expect(res.headers["link"]).toContain( + '; rel="successor-version"', + ); + expect(res.headers["warning"]).toMatch(/^299 - "/); + + expect(Array.isArray(res.body.warnings)).toBe(true); + expect( + res.body.warnings.some((w: string) => /\/v2\/search\/developer/.test(w)), + ).toBe(true); + expect(res.body.replacement).toBe("/v2/search/developer"); + }, 120000); + + it("leaves the paper routes on the same router undeprecated", async () => { + const identity = await idmux({ + name: "research/github deprecation scope", + credits: 100, + }); + + const res = await researchRaw( + "/v2/search/research/papers", + { query: "retrieval augmented generation", k: 1 }, + identity, + ); + + expect(res.statusCode).toBe(200); + expect(res.headers["deprecation"]).toBeUndefined(); + expect(res.headers["sunset"]).toBeUndefined(); + expect(res.body.warnings).toBeUndefined(); + expect(res.body.replacement).toBeUndefined(); + }, 120000); + + it("keeps the legacy snake_case aliases alongside the new warning", async () => { + const identity = await idmux({ + name: "research/github deprecation legacy", + credits: 100, + }); + + const res = await researchRaw( + "/v2/research/github", + { query: "milvus hybrid search", k: 3 }, + identity, + ); + + expect(res.statusCode).toBe(200); + expect(res.headers["deprecation"]).toBe(GITHUB_DEPRECATED_AT); + expect(Array.isArray(res.body.warnings)).toBe(true); + expect(res.body.results.length).toBeGreaterThan(0); + // resultType is optional, so only assert the alias when the camel key is set. + const first = res.body.results[0]; + if (first.resultType !== undefined) { + expect(first.result_type).toBe(first.resultType); + } + }, 120000); + + it("still warns when the request is rejected", async () => { + const identity = await idmux({ + name: "research/github deprecation 400", + credits: 100, + }); + + const res = await researchRaw("/v2/search/research/github", {}, identity); + + expect(res.statusCode).toBe(400); + expect(res.body.success).toBe(false); + expect(res.headers["deprecation"]).toBe(GITHUB_DEPRECATED_AT); + expect(Array.isArray(res.body.warnings)).toBe(true); + }, 60000); +}); diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts index 72cb20029f..568c8380df 100644 --- a/apps/api/src/config.ts +++ b/apps/api/src/config.ts @@ -23,7 +23,16 @@ const RESEARCH_PAPER_OPERATIONS = [ "similar", ] as const; -export type ResearchPaperOperation = (typeof RESEARCH_PAPER_OPERATIONS)[number]; +// "github" is out of the default and the true/all shorthand, so keyless +// behaviour is unchanged until someone names it. An explicit list replaces the +// default, so closing it means RESEARCH_KEYLESS_DISABLED=search,inspect,read,similar,github +const RESEARCH_KEYLESS_OPERATIONS = [ + ...RESEARCH_PAPER_OPERATIONS, + "github", +] as const; + +export type ResearchKeylessOperation = + (typeof RESEARCH_KEYLESS_OPERATIONS)[number]; const researchKeylessDisabled = z.preprocess( value => { @@ -40,7 +49,7 @@ const researchKeylessDisabled = z.preprocess( .filter(Boolean); }, z - .array(z.enum(RESEARCH_PAPER_OPERATIONS)) + .array(z.enum(RESEARCH_KEYLESS_OPERATIONS)) .default([...RESEARCH_PAPER_OPERATIONS]), ); @@ -239,6 +248,9 @@ const configSchema = z.object({ PARSE_UPLOAD_REF_SECRET: emptyStringAsUndefined(z.string().trim().min(1)), PARSE_UPLOAD_PUBLIC_BASE_URL: z.string().url().optional(), + // Google Cloud Pub/Sub + PUBSUB_CREDENTIALS: z.string().optional(), + // Cloud Bigtable (change tracking bookkeeping store). The client // auto-detects BIGTABLE_EMULATOR_HOST, so local dev only needs the // emulator plus these vars. BIGTABLE_CREDENTIALS mirrors diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.ts index 764925e97b..3a0eef5896 100644 --- a/apps/api/src/controllers/v2/research-proxy.ts +++ b/apps/api/src/controllers/v2/research-proxy.ts @@ -17,6 +17,7 @@ import type { } from "../../services/logging/log_job"; import type { RequestWithAuth } from "../v1/types"; import { wrap } from "../../routes/shared"; +import { deprecationMiddleware } from "../../lib/deprecations"; import { integrationSchema } from "../../utils/integration"; import { requestOrigin } from "../../lib/request-origin"; @@ -521,8 +522,10 @@ export function createResearchRouter(options: { legacy?: boolean } = {}) { }), ); + // On the route, not the mounts, so the paper routes stay undeprecated. router.get( "/github", + deprecationMiddleware("v2_research_github_search"), wrap( createResearchController( githubSearchSchema, diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index cdc08bea3b..4c8af690c0 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -44,6 +44,7 @@ import responseTime from "response-time"; import { shutdownWebhookQueue } from "./services/webhook"; import { shutdownIndexerQueue } from "./services/indexing/indexer-queue"; import { isKeylessConfigured } from "./lib/keyless"; +import { shutdownPubSubLogging } from "./services/logging/log_job"; const { createBullBoard } = require("@bull-board/api"); const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter"); @@ -183,7 +184,8 @@ async function startServer(port = DEFAULT_PORT) { logger.info("Server closed."); nuqShutdown().finally(() => { shutdownWebhookQueue().finally(() => { - shutdownIndexerQueue().finally(() => { + shutdownIndexerQueue().finally(async () => { + await shutdownPubSubLogging(); logger.info("NUQ shutdown complete"); process.exit(0); }); diff --git a/apps/api/src/lib/deprecations.ts b/apps/api/src/lib/deprecations.ts index 02ede0a01e..752599543a 100644 --- a/apps/api/src/lib/deprecations.ts +++ b/apps/api/src/lib/deprecations.ts @@ -3,66 +3,90 @@ import { NextFunction, Request, Response } from "express"; interface Deprecation { message: string; replacement?: string; + // RFC 9745 requires a Date here, e.g. "@1788393600". + deprecatedAt: string; sunset?: string; docs?: string; } +// Every legacy entry shipped together in #3469, so they share one date. const DEPRECATIONS = { v1_extract: { message: "/v1/extract is deprecated. Use /v2/scrape with formats including a 'json' format object.", replacement: "/v2/scrape", + deprecatedAt: "@1778025600", }, v1_extract_status: { message: "/v1/extract/:jobId is deprecated. Use /v2/scrape with formats including a 'json' format object.", replacement: "/v2/scrape", + deprecatedAt: "@1778025600", }, v2_extract: { message: "/v2/extract is deprecated. Use /v2/scrape with formats including a 'json' format object.", replacement: "/v2/scrape", + deprecatedAt: "@1778025600", }, v2_extract_status: { message: "/v2/extract/:jobId is deprecated. Use /v2/scrape with formats including a 'json' format object.", replacement: "/v2/scrape", + deprecatedAt: "@1778025600", + }, + v2_research_github_search: { + message: + "The research index GitHub search (GET /v2/search/research/github, legacy GET /v2/research/github) is deprecated and stops responding after 2026-11-03. Use GET or POST /v2/search/developer instead: it searches GitHub issues, pull requests and READMEs plus curated documentation sources, returns matched passages, and adds filters for repo, language, license and stars. Response changes: 'snippet' becomes 'passages', results gain an 'id', and there is no score breakdown and no web fallback result type.", + replacement: "/v2/search/developer", + docs: "https://docs.firecrawl.dev/features/developer", + deprecatedAt: "@1788393600", + sunset: "Tue, 03 Nov 2026 23:59:59 GMT", }, v1_deep_research: { message: "/v1/deep-research is deprecated. Use /v2/search instead.", replacement: "/v2/search", + deprecatedAt: "@1778025600", }, v1_deep_research_status: { message: "/v1/deep-research/:jobId is deprecated. Use /v2/search instead.", replacement: "/v2/search", + deprecatedAt: "@1778025600", }, v1_llmstxt: { message: "/v1/llmstxt is deprecated and will not be replaced.", + deprecatedAt: "@1778025600", }, v1_llmstxt_status: { message: "/v1/llmstxt/:jobId is deprecated and will not be replaced.", + deprecatedAt: "@1778025600", }, v0_scrape: { message: "/v0/scrape is deprecated. Use /v2/scrape instead.", replacement: "/v2/scrape", + deprecatedAt: "@1778025600", }, v0_crawl: { message: "/v0/crawl is deprecated. Use /v2/crawl instead.", replacement: "/v2/crawl", + deprecatedAt: "@1778025600", }, v0_crawl_status: { message: "/v0/crawl/status/:jobId is deprecated. Use /v2/crawl/:jobId instead.", replacement: "/v2/crawl/:jobId", + deprecatedAt: "@1778025600", }, v0_crawl_cancel: { message: "/v0/crawl/cancel/:jobId is deprecated. Use DELETE /v2/crawl/:jobId instead.", replacement: "/v2/crawl/:jobId", + deprecatedAt: "@1778025600", }, v0_search: { message: "/v0/search is deprecated. Use /v2/search instead.", replacement: "/v2/search", + deprecatedAt: "@1778025600", }, } as const satisfies Record; @@ -77,7 +101,7 @@ export function deprecationMiddleware(key: DeprecationKey) { const dep: Deprecation = DEPRECATIONS[key]; return (req: Request, res: Response, next: NextFunction) => { // RFC 9745 Deprecation header. - res.setHeader("Deprecation", "true"); + res.setHeader("Deprecation", dep.deprecatedAt); // RFC 8594 Sunset header. if (dep.sunset) res.setHeader("Sunset", dep.sunset); @@ -96,11 +120,16 @@ export function deprecationMiddleware(key: DeprecationKey) { const originalJson = res.json.bind(res); res.json = (body: any) => { if (body && typeof body === "object" && !Array.isArray(body)) { + // Copy: the research controllers log the same object after responding. const existing = Array.isArray(body.warnings) ? body.warnings : []; - body.warnings = [...existing, dep.message]; - if (dep.replacement && body.replacement === undefined) { - body.replacement = dep.replacement; + const annotated: any = { + ...body, + warnings: [...existing, dep.message], + }; + if (dep.replacement && annotated.replacement === undefined) { + annotated.replacement = dep.replacement; } + return originalJson(annotated); } return originalJson(body); }; diff --git a/apps/api/src/lib/research-keyless.ts b/apps/api/src/lib/research-keyless.ts index 45ca7f6ea2..43a2ebd514 100644 --- a/apps/api/src/lib/research-keyless.ts +++ b/apps/api/src/lib/research-keyless.ts @@ -1,10 +1,15 @@ import type { Request } from "express"; -import { config, type ResearchPaperOperation } from "../config"; +import { config, type ResearchKeylessOperation } from "../config"; -function researchPaperOperation(req: Request): ResearchPaperOperation | null { +function researchKeylessOperation( + req: Request, +): ResearchKeylessOperation | null { const segments = req.path.toLowerCase().split("/").filter(Boolean); const papersIndex = segments.indexOf("papers"); - if (papersIndex === -1) return null; + // Paper paths win, so a paper id of "github" is not the github route. + if (papersIndex === -1) { + return segments.includes("github") ? "github" : null; + } const rest = segments.slice(papersIndex + 1); if (rest.length === 0) return "search"; @@ -19,6 +24,6 @@ export function isResearchKeylessDisabled(req: Request): boolean { const disabledOperations = config.RESEARCH_KEYLESS_DISABLED; if (disabledOperations.length === 0) return false; - const operation = researchPaperOperation(req); + const operation = researchKeylessOperation(req); return operation !== null && disabledOperations.includes(operation); } diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.ts index 037eb19a06..15e07efb41 100644 --- a/apps/api/src/services/extract-worker.ts +++ b/apps/api/src/services/extract-worker.ts @@ -20,7 +20,7 @@ import { shutdownExtractQueue, ExtractJobData, } from "./extract-queue"; -import { logExtract } from "./logging/log_job"; +import { logExtract, shutdownPubSubLogging } from "./logging/log_job"; import { jobDurationSeconds } from "../lib/job-metrics"; import { register } from "prom-client"; @@ -215,6 +215,7 @@ app.listen(workerPort, (error?: Error) => { async function shutdown() { _logger.info("Shutting down extract worker..."); await shutdownExtractQueue(); + await shutdownPubSubLogging(); _logger.info("Extract worker shut down"); process.exit(0); } diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts index 5e74be8be4..2de767688f 100644 --- a/apps/api/src/services/indexing/index-worker.ts +++ b/apps/api/src/services/indexing/index-worker.ts @@ -41,7 +41,7 @@ import { withSpan, setSpanAttributes } from "../../lib/otel-tracer"; import { crawlGroup, resolveNewGroupBackend } from "../worker/nuq-router"; import { getACUCTeam } from "../../controllers/auth"; import { processEngpickerJob } from "../../lib/engpicker"; -import { logRequest } from "../logging/log_job"; +import { logRequest, shutdownPubSubLogging } from "../logging/log_job"; import { startSiemLoggingConsumer } from "../siem-logging/worker"; import { closeSiemLoggingTransport } from "../../lib/siem-logging/transport"; @@ -686,6 +686,7 @@ const workerFun = async ( await new Promise(resolve => setTimeout(resolve, 500)); } logger.info("All jobs finished. Worker exiting!"); + await shutdownPubSubLogging(); process.exit(0); }; diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.ts index cb70ecdfda..d553ccd2ae 100644 --- a/apps/api/src/services/logging/log_job.test.ts +++ b/apps/api/src/services/logging/log_job.test.ts @@ -3,7 +3,17 @@ import { vi } from "vitest"; // vi.mock is hoisted; anything its factories reference must be created in // vi.hoisted() (also hoisted). Under Jest these worked because importing `jest` // from @jest/globals disables jest.mock hoisting. -const { captureException, logger, values, insert } = vi.hoisted(() => { +const { + captureException, + logger, + values, + insert, + topic, + publishes, + publishMessage, + flush, + close, +} = vi.hoisted(() => { const logger: any = { info: vi.fn(), warn: vi.fn(), @@ -13,9 +23,39 @@ const { captureException, logger, values, insert } = vi.hoisted(() => { }; const values = vi.fn<(data: any) => Promise>(); const insert = vi.fn(() => ({ values })); - return { captureException: vi.fn(), logger, values, insert }; + const publishMessage = vi.fn(async (_message: any) => "message-id"); + const flush = vi.fn(async () => {}); + const close = vi.fn(async () => {}); + const publishes: { name: string; options: any }[] = []; + const topic = vi.fn((name: string, options: any) => { + return { + publishMessage: (message: any) => { + publishes.push({ name, options }); + return publishMessage(message); + }, + flush, + }; + }); + return { + captureException: vi.fn(), + logger, + values, + insert, + topic, + publishes, + publishMessage, + flush, + close, + }; }); +vi.mock("@google-cloud/pubsub", () => ({ + PubSub: class { + topic = topic; + close = close; + }, +})); + vi.mock("@sentry/node", () => ({ captureException, })); @@ -23,6 +63,9 @@ vi.mock("@sentry/node", () => ({ vi.mock("../../config", () => ({ config: { GCS_BUCKET_NAME: undefined, + PUBSUB_CREDENTIALS: Buffer.from( + JSON.stringify({ project_id: "firecrawl" }), + ).toString("base64"), USE_DB_AUTHENTICATION: true, }, })); @@ -52,7 +95,12 @@ vi.mock("../posthog", () => ({ trackFirstSurfaceUse: vi.fn(), })); -import { logRequest, logSearch, type LoggedSearch } from "./log_job"; +import { + logRequest, + logSearch, + shutdownPubSubLogging, + type LoggedSearch, +} from "./log_job"; import * as schema from "../../db/schema"; function makeSearch(overrides: Partial = {}): LoggedSearch { @@ -79,6 +127,8 @@ describe("logSearch", () => { beforeEach(() => { vi.clearAllMocks(); values.mockResolvedValue(undefined); + publishMessage.mockResolvedValue("message-id"); + publishes.length = 0; }); it("removes null bytes from search query log fields", async () => { @@ -127,6 +177,8 @@ describe("logRequest", () => { beforeEach(() => { vi.clearAllMocks(); values.mockResolvedValue(undefined); + publishMessage.mockResolvedValue("message-id"); + publishes.length = 0; }); function makeRequest(externalRequestId: string | null) { @@ -147,7 +199,51 @@ describe("logRequest", () => { await logRequest(makeRequest("op_integration_42")); expect(insert).toHaveBeenCalledWith(schema.requests); - expect(values.mock.calls[0][0].external_request_id).toBe("op_integration_42"); + expect(values.mock.calls[0][0].external_request_id).toBe( + "op_integration_42", + ); + }); + + it("writes the request to the database and its Pub/Sub topic", async () => { + await logRequest(makeRequest("op_integration_42")); + + expect(insert).toHaveBeenCalledWith(schema.requests); + expect(publishes).toContainEqual({ + name: "requests", + options: { gaxOpts: { timeout: 60_000 } }, + }); + + const published = JSON.parse( + publishMessage.mock.calls[0][0].data.toString("utf8"), + ); + expect(published.id).toBe("019e6f45-7778-727d-adf0-0abe9d5062b6"); + expect(published.external_request_id).toBe("op_integration_42"); + expect(new Date(published.created_at).toISOString()).toBe( + published.created_at, + ); + expect(values.mock.calls[0][0].created_at.toISOString()).toBe( + published.created_at, + ); + }); + + it("keeps the database write when Pub/Sub fails", async () => { + publishMessage.mockRejectedValueOnce(new Error("Pub/Sub unavailable")); + + await expect(logRequest(makeRequest(null))).resolves.toBeUndefined(); + + expect(values).toHaveBeenCalled(); + expect(logger.error).toHaveBeenCalledWith( + "Failed to publish log to Pub/Sub", + expect.objectContaining({ error: expect.any(Error) }), + ); + expect(captureException).toHaveBeenCalled(); + }); + + it("does not hold the caller on a slow publish", async () => { + publishMessage.mockReturnValueOnce(new Promise(() => {})); + + await expect(logRequest(makeRequest(null))).resolves.toBeUndefined(); + expect(values).toHaveBeenCalled(); }); it("stores null, not a truncation, when the id exceeds the byte cap", async () => { @@ -173,4 +269,13 @@ describe("logRequest", () => { await logRequest(makeRequest("é".repeat(1024))); expect(values.mock.calls[1][0].external_request_id).toBe("é".repeat(1024)); }); + + it("flushes Pub/Sub messages during shutdown", async () => { + await logRequest(makeRequest(null)); + + await Promise.all([shutdownPubSubLogging(), shutdownPubSubLogging()]); + + expect(flush).toHaveBeenCalled(); + expect(close).toHaveBeenCalledOnce(); + }); }); diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts index f81f9011af..8d3169e2fb 100644 --- a/apps/api/src/services/logging/log_job.ts +++ b/apps/api/src/services/logging/log_job.ts @@ -23,6 +23,7 @@ import type { CostTracking } from "../../lib/cost-tracking"; import type { Logger } from "winston"; import { saveExtractResult } from "../../lib/extract/extract-redis"; import { trackFirstSurfaceUse } from "../posthog"; +import { PubSub, type Topic } from "@google-cloud/pubsub"; configDotenv(); const previewTeamId = "3adefd26-77ec-5968-8dcf-c94b5630d1de"; @@ -58,6 +59,101 @@ const tableMap: Record = { deep_researches: schema.deep_researches, }; +let pubSubClient: PubSub | null | undefined; +const pubSubTopics = new Map(); +let pubSubShutdown: Promise | undefined; +const PUBSUB_PUBLISH_TIMEOUT_MS = 60_000; + +function getPubSubClient(logger: Logger): PubSub | null { + if (pubSubClient !== undefined) return pubSubClient; + if (!config.PUBSUB_CREDENTIALS) return (pubSubClient = null); + + try { + const credentials = JSON.parse( + Buffer.from(config.PUBSUB_CREDENTIALS, "base64").toString("utf8"), + ); + return (pubSubClient = new PubSub({ + projectId: credentials.project_id, + credentials, + })); + } catch (error) { + pubSubClient = null; + logger.error("Failed to initialize Pub/Sub log publisher", { error }); + Sentry.captureException(error, { + tags: { operation: "initializePubSubLogPublisher" }, + }); + return null; + } +} + +// One Topic per table so publishes share a batch. +function getTopic(client: PubSub, table: string): Topic { + let topic = pubSubTopics.get(table); + if (!topic) { + topic = client.topic(table, { + gaxOpts: { timeout: PUBSUB_PUBLISH_TIMEOUT_MS }, + }); + pubSubTopics.set(table, topic); + } + return topic; +} + +async function publishLog(table: string, data: any, logger: Logger) { + const client = getPubSubClient(logger); + if (!client) return; + + try { + await getTopic(client, table).publishMessage({ + data: Buffer.from(JSON.stringify(data)), + }); + } catch (error) { + logger.error("Failed to publish log to Pub/Sub", { error }); + Sentry.captureException(error, { + tags: { table, operation: "publishPubSubLog" }, + }); + } +} + +export function shutdownPubSubLogging(): Promise { + if (pubSubShutdown) return pubSubShutdown; + + pubSubShutdown = shutdownPubSubLoggingOnce(); + return pubSubShutdown; +} + +async function shutdownPubSubLoggingOnce(): Promise { + const client = pubSubClient; + if (!client) return; + + const logger = _logger.child({ + module: "log_job", + method: "shutdownPubSubLogging", + }); + const results = await Promise.allSettled( + [...pubSubTopics.values()].map(topic => topic.flush()), + ); + const errors = results.flatMap(result => + result.status === "rejected" ? [result.reason] : [], + ); + + if (errors.length > 0) { + logger.error("Failed to flush Pub/Sub log publisher", { errors }); + Sentry.captureException(errors[0], { + tags: { operation: "flushPubSubLogPublisher" }, + extra: { failures: errors.length }, + }); + } + + try { + await client.close(); + } catch (error) { + logger.error("Failed to close Pub/Sub log publisher", { error }); + Sentry.captureException(error, { + tags: { operation: "closePubSubLogPublisher" }, + }); + } +} + async function robustInsert( table: string, data: any, @@ -79,6 +175,8 @@ async function robustInsert( } const target = tableMap[table]; + data = { ...data, created_at: data.created_at ?? new Date() }; + void publishLog(table, data, logger); const attempts: { error: any; timeMs: number; backoffMs: number }[] = []; diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts index feeb46d93b..ffe3085c0b 100644 --- a/apps/api/src/services/queue-worker.ts +++ b/apps/api/src/services/queue-worker.ts @@ -34,6 +34,7 @@ import { consumeMonitorCheckJobs, consumeMonitorSearchCheckJobs, } from "./monitoring/queue"; +import { shutdownPubSubLogging } from "./logging/log_job"; configDotenv(); @@ -512,5 +513,6 @@ app.listen(workerPort, (error?: Error) => { } _logger.info("All jobs finished. Shutting down..."); + await shutdownPubSubLogging(); process.exit(0); })(); diff --git a/apps/api/src/services/worker/nuq-worker-runner.ts b/apps/api/src/services/worker/nuq-worker-runner.ts index 63dfd6a91c..bbffc593f7 100644 --- a/apps/api/src/services/worker/nuq-worker-runner.ts +++ b/apps/api/src/services/worker/nuq-worker-runner.ts @@ -7,6 +7,7 @@ import { register } from "prom-client"; import Express from "express"; import { initializeBlocklist } from "../../scraper/WebScraper/utils/blocklist"; import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-forcing"; +import { shutdownPubSubLogging } from "../logging/log_job"; export type WorkerQueue = { getJobToProcess(logger?: any): Promise | null>; @@ -204,6 +205,7 @@ export async function runNuqWorker(options: { server.close(async () => { await options.beforeShutdown?.(); await options.shutdown?.(); + await shutdownPubSubLogging(); _logger.info("NuQ worker shut down", { module: options.serviceName }); process.exit(0); }); diff --git a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs index 7bbaf6f5d2..4e478c531a 100644 --- a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs +++ b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs @@ -551,6 +551,7 @@ public async Task RelatedPapersAsync( /// /// Searches GitHub research content. /// + [Obsolete("Stops responding after 2026-11-03. Use the developer index at GET or POST /v2/search/developer; this SDK does not wrap it yet, so call it directly. It does not carry over the score breakdown or the web fallback results.")] public async Task SearchGitHubAsync( string query, SearchGitHubOptions? options = null, diff --git a/apps/elixir-sdk/generate.exs b/apps/elixir-sdk/generate.exs index eb21afc265..959b825b92 100644 --- a/apps/elixir-sdk/generate.exs +++ b/apps/elixir-sdk/generate.exs @@ -23,6 +23,10 @@ defmodule Firecrawl.Generator do "getHistoricalTokenUsage" ]) + # The method key becomes the Req function name in generated code, a position + # no escaping can protect, so anything else stops generation outright. + @http_methods ~w(get post put patch delete head options) + def run do IO.puts("Fetching OpenAPI spec...") {:ok, spec} = fetch_spec() @@ -51,6 +55,14 @@ defmodule Firecrawl.Generator do defp fetch_spec do Application.ensure_all_started(:req) + # Set FIRECRAWL_OPENAPI_SPEC to a local file to generate without the network. + case System.get_env("FIRECRAWL_OPENAPI_SPEC") do + nil -> fetch_remote_spec() + path -> {:ok, path |> File.read!() |> Jason.decode!()} + end + end + + defp fetch_remote_spec do case Req.get(@openapi_url) do {:ok, %Req.Response{status: 200, body: body}} when is_map(body) -> {:ok, body} @@ -88,6 +100,10 @@ defmodule Firecrawl.Generator do methods |> Enum.reject(fn {key, _} -> key == "parameters" end) |> Enum.map(fn {method, operation} -> + unless method in @http_methods do + raise ArgumentError, "refusing unknown HTTP method #{inspect(method)} at #{inspect(path)}" + end + {method, path, operation, path_level_params} end) end) @@ -107,7 +123,7 @@ defmodule Firecrawl.Generator do defmodule Firecrawl do @moduledoc \"\"\" - Auto-generated Firecrawl API #{api_version} client. + Auto-generated Firecrawl API #{escape_source_text(api_version)} client. Generated from the OpenAPI spec at: #{@openapi_url} @@ -144,7 +160,7 @@ defmodule Firecrawl.Generator do @type response :: {:ok, Req.Response.t()} | {:error, Exception.t() | Firecrawl.Error.t()} - @base_url "#{base_url}" + @base_url #{inspect(base_url)} # Sourced from mix.exs at compile time so the origin header cannot drift # from the published package version. @version Mix.Project.config()[:version] @@ -349,6 +365,7 @@ defmodule Firecrawl.Generator do ) # Build typespecs + deprecated_code = build_deprecated(operation) spec_code = build_typespec(func_name, path_params, has_body || has_query_schema, false) bang_spec_code = build_typespec(func_name, path_params, has_body || has_query_schema, true) @@ -358,11 +375,13 @@ defmodule Firecrawl.Generator do query_schema_code, query_key_mapping_code, doc, + deprecated_code, spec_code, " #{sig}", body, "", doc_bang(func_name), + deprecated_code, bang_spec_code, " #{bang_sig}", bang_body, @@ -434,6 +453,7 @@ defmodule Firecrawl.Generator do {sig, body} = build_multipart_function_body(func_name, method, path, meta, has_options?, false) {bang_sig, bang_body} = build_multipart_function_body(func_name, method, path, meta, has_options?, true) + deprecated_code = build_deprecated(operation) spec_code = build_multipart_typespec(func_name, has_options?, false) bang_spec_code = build_multipart_typespec(func_name, has_options?, true) @@ -441,11 +461,13 @@ defmodule Firecrawl.Generator do body_schema_code, body_key_mapping_code, doc, + deprecated_code, spec_code, " #{sig}", body, "", doc_bang(func_name), + deprecated_code, bang_spec_code, " #{bang_sig}", bang_body, @@ -472,13 +494,13 @@ defmodule Firecrawl.Generator do options_part_text = cond do has_options? and not is_nil(options_field) -> - "{\"#{options_field}\", Jason.encode!(to_body(params, @#{func_name}_key_mapping))}, " + "{#{inspect(options_field)}, Jason.encode!(to_body(params, @#{func_name}_key_mapping))}, " true -> "" end - file_part_text = "{\"#{file_field}\", file_part}" + file_part_text = "{#{inspect(file_field)}, file_part}" indent = if not bang? and has_options?, do: " ", else: " " @@ -503,7 +525,7 @@ defmodule Firecrawl.Generator do "", "#{indent}multipart = [#{options_part_text}#{file_part_text}]", "", - "#{indent}Req.#{req_fn}(client(opts), url: \"#{path}\", form_multipart: multipart)" + "#{indent}Req.#{req_fn}(client(opts), url: \"#{escape_string_literal(path)}\", form_multipart: multipart)" ] core = Enum.join(core_lines, "\n") @@ -542,14 +564,14 @@ defmodule Firecrawl.Generator do parts = [ " @doc \"\"\"", - " #{summary}", + " #{escape_source_text(summary)}", "", - " #{bt}#{http_method} #{path}#{bt}", + " #{bt}#{escape_source_text(http_method)} #{escape_source_text(path)}#{bt}", "", " Sends a #{bt}multipart/form-data#{bt} request." ] - parts = if tag != "", do: parts ++ ["", " Tag: #{tag}"], else: parts + parts = if tag != "", do: parts ++ ["", " Tag: #{escape_source_text(tag)}"], else: parts parts = parts ++ @@ -572,7 +594,7 @@ defmodule Firecrawl.Generator do " ## Parameters", "", " Validated by #{bt}NimbleOptions#{bt}. Pass options as a keyword list with snake_case keys.", - " These are JSON-encoded and sent as the #{bt}#{meta.options_field}#{bt} multipart field.", + " These are JSON-encoded and sent as the #{bt}#{escape_source_text(meta.options_field)}#{bt} multipart field.", " See #{bt}@#{func_name}_schema#{bt} for the full schema." ] else @@ -685,7 +707,7 @@ defmodule Firecrawl.Generator do properties |> Enum.map(fn {name, _} -> snake = to_snake_case(name) - "#{snake}: \"#{name}\"" + "#{snake}: #{inspect(name)}" end) |> Enum.join(", ") @@ -724,7 +746,7 @@ defmodule Firecrawl.Generator do |> Enum.map(fn param -> name = Map.get(param, "name") snake = to_snake_case(name) - "#{snake}: \"#{name}\"" + "#{snake}: #{inspect(name)}" end) |> Enum.join(", ") @@ -797,6 +819,40 @@ defmodule Firecrawl.Generator do # Doc Generation # --------------------------------------------------------------------------- + # OpenAPI marks a retiring operation with `deprecated: true`. Elixir's + # @deprecated turns that into a compiler warning at the caller. + def build_deprecated(operation) do + if Map.get(operation, "deprecated", false) do + note = + Map.get(operation, "x-deprecation-note") || + "Deprecated in the Firecrawl API. See the function docs for the replacement." + + " @deprecated #{inspect(to_string(note))}" + end + end + + # Spec text is fetched from the network and lands inside generated heredocs, + # where Elixir would run #{} as code at compile time. Neutralise that, the + # heredoc terminator, and stray backslashes. + def escape_source_text(text) do + text + |> to_string() + |> String.replace("\\", "\\\\") + |> String.replace(~S(#{), ~S(\#{)) + |> String.replace(~S("""), ~S(\"\"\")) + end + + # Same job for text that lands inside a generated "..." literal, where a + # quote or #{} would end or execute it. Path templates keep their {param} + # holes untouched so build_elixir_path can turn them into interpolations. + def escape_string_literal(text) do + text + |> to_string() + |> String.replace("\\", "\\\\") + |> String.replace("\"", "\\\"") + |> String.replace(~S(#{), ~S(\#{)) + end + defp build_doc( summary, http_method, @@ -812,18 +868,18 @@ defmodule Firecrawl.Generator do parts = [ " @doc \"\"\"", - " #{summary}", + " #{escape_source_text(summary)}", "", - " #{bt}#{http_method} #{path}#{bt}" + " #{bt}#{escape_source_text(http_method)} #{escape_source_text(path)}#{bt}" ] - parts = if tag != "", do: parts ++ ["", " Tag: #{tag}"], else: parts + parts = if tag != "", do: parts ++ ["", " Tag: #{escape_source_text(tag)}"], else: parts parts = if path_params != [] do param_docs = Enum.map(path_params, fn p -> - " * #{bt}#{to_snake_case(p)}#{bt} - Path parameter #{bt}#{p}#{bt}" + " * #{bt}#{to_snake_case(p)}#{bt} - Path parameter #{bt}#{escape_source_text(p)}#{bt}" end) parts ++ ["", " ## Path Parameters", ""] ++ param_docs @@ -849,7 +905,7 @@ defmodule Firecrawl.Generator do if has_query_schema do param_docs = Enum.map(query_param_names, fn p -> - " * #{bt}#{to_snake_case(p)}#{bt} — query parameter #{bt}#{p}#{bt}" + " * #{bt}#{to_snake_case(p)}#{bt} — query parameter #{bt}#{escape_source_text(p)}#{bt}" end) parts ++ ["", " ## Query Parameters", ""] ++ param_docs @@ -1018,10 +1074,10 @@ defmodule Firecrawl.Generator do end end - defp build_elixir_path(path, []), do: path + defp build_elixir_path(path, []), do: escape_string_literal(path) defp build_elixir_path(path, path_params) do - Enum.reduce(path_params, path, fn param, acc -> + Enum.reduce(path_params, escape_string_literal(path), fn param, acc -> snake = to_snake_case(param) String.replace(acc, "{#{param}}", "\#{#{snake}}") end) @@ -1036,7 +1092,7 @@ defmodule Firecrawl.Generator do if Regex.match?(~r/^[a-zA-Z_][a-zA-Z0-9_]*$/, value) do ":#{value}" else - ":\"#{value}\"" + ":" <> inspect(to_string(value)) end end @@ -1111,4 +1167,4 @@ defmodule Firecrawl.Generator do end end -Firecrawl.Generator.run() +unless Code.ensure_loaded?(Mix) and Mix.env() == :test, do: Firecrawl.Generator.run() diff --git a/apps/elixir-sdk/test/generator_test.exs b/apps/elixir-sdk/test/generator_test.exs new file mode 100644 index 0000000000..e6018d8bd3 --- /dev/null +++ b/apps/elixir-sdk/test/generator_test.exs @@ -0,0 +1,162 @@ +Code.require_file("../generate.exs", __DIR__) + +defmodule Firecrawl.GeneratorTest do + use ExUnit.Case, async: false + import ExUnit.CaptureIO + + # The generator pulls the OpenAPI spec over the network and writes its text + # into Elixir source. Anything that reaches a string literal or heredoc must + # arrive as data, never as code that runs while the SDK compiles. + @marker Path.join(System.tmp_dir!(), "firecrawl_generator_test_pwned") + + # No double quotes anywhere, so an escaper that only handles quotes leaves + # the interpolation intact. + @hostile "see \#{File.write!(~c'#{@marker}', ~c'x')} for details" + + setup do + File.rm(@marker) + on_exit(fn -> File.rm(@marker) end) + :ok + end + + defp compile_quietly(source) do + capture_io(:stderr, fn -> Code.compile_string(source) end) + end + + describe "build_deprecated/1" do + test "emits nothing when the operation is not deprecated" do + assert Firecrawl.Generator.build_deprecated(%{}) == nil + assert Firecrawl.Generator.build_deprecated(%{"deprecated" => false}) == nil + end + + test "falls back to a generic note" do + line = Firecrawl.Generator.build_deprecated(%{"deprecated" => true}) + assert line =~ ~r/^ @deprecated "Deprecated in the Firecrawl API/ + end + + test "a hostile note is carried as data and cannot run at compile time" do + line = + Firecrawl.Generator.build_deprecated(%{ + "deprecated" => true, + "x-deprecation-note" => @hostile + }) + + warnings = + compile_quietly(""" + defmodule Firecrawl.GeneratorTest.Hostile do + #{line} + def f, do: :ok + end + + defmodule Firecrawl.GeneratorTest.HostileCaller do + def g, do: Firecrawl.GeneratorTest.Hostile.f() + end + """) + + refute File.exists?(@marker), "the note was evaluated while compiling" + # The caller's deprecation warning must quote the note verbatim, which + # proves it survived as a plain string rather than being interpreted. + assert warnings =~ "is deprecated. " <> @hostile + end + + test "quotes, backslashes and newlines cannot break out of the literal" do + note = ~S(a "quoted" note with a \ backslash) <> "\nand a newline" + + line = + Firecrawl.Generator.build_deprecated(%{ + "deprecated" => true, + "x-deprecation-note" => note + }) + + warnings = + compile_quietly(""" + defmodule Firecrawl.GeneratorTest.Escapes do + #{line} + def f, do: :ok + end + + defmodule Firecrawl.GeneratorTest.EscapesCaller do + def g, do: Firecrawl.GeneratorTest.Escapes.f() + end + """) + + assert warnings =~ "is deprecated. " <> note + end + end + + test "the hostile payload really is a live interpolation, not a reference to @marker" do + assert String.contains?(@hostile, @marker) + refute String.contains?(@hostile, "@marker") + + # Unescaped, it must fire, or none of the tests below prove anything. + compile_quietly(""" + defmodule Firecrawl.GeneratorTest.Unescaped do + @doc "#{@hostile}" + def f, do: :ok + end + """) + + assert File.exists?(@marker) + end + + describe "escape_string_literal/1" do + test "leaves an ordinary path untouched" do + assert Firecrawl.Generator.escape_string_literal("/search/research/papers/{id}") == + "/search/research/papers/{id}" + end + + test "a hostile path inside a url literal cannot run or break out" do + path = Firecrawl.Generator.escape_string_literal(~S(/x" <> ) <> @hostile <> ~S( <> "/y)) + + [{mod, _}] = + Code.compile_string(""" + defmodule Firecrawl.GeneratorTest.HostilePath do + def url, do: "#{path}" + end + """) + + refute File.exists?(@marker), "the path was evaluated while compiling" + assert mod.url() =~ "firecrawl_generator_test_pwned" + end + end + + describe "escape_source_text/1" do + test "leaves ordinary spec text untouched" do + assert Firecrawl.Generator.escape_source_text("Scrape a single URL") == + "Scrape a single URL" + end + + test "a hostile summary inside a @doc heredoc cannot run at compile time" do + summary = Firecrawl.Generator.escape_source_text(@hostile) + + compile_quietly(""" + defmodule Firecrawl.GeneratorTest.HostileDoc do + @doc \"\"\" + #{summary} + \"\"\" + def f, do: :ok + end + """) + + refute File.exists?(@marker), "the summary was evaluated while compiling" + end + + test "a heredoc terminator in the summary does not end the docstring early" do + summary = + Firecrawl.Generator.escape_source_text(~S(ends the doc """ def injected, do: :owned)) + + [{mod, _}] = + Code.compile_string(""" + defmodule Firecrawl.GeneratorTest.Terminator do + @doc \"\"\" + #{summary} + \"\"\" + def f, do: :ok + end + """) + + refute function_exported?(mod, :injected, 0) + assert function_exported?(mod, :f, 0) + end + end +end diff --git a/apps/go-sdk/firecrawl.go b/apps/go-sdk/firecrawl.go index b6dda544ba..1f85d2ced5 100644 --- a/apps/go-sdk/firecrawl.go +++ b/apps/go-sdk/firecrawl.go @@ -672,6 +672,11 @@ func (c *Client) RelatedPapers(ctx context.Context, paperID, intent string, opts } // SearchGitHub searches GitHub research content. +// +// Deprecated: stops responding after 2026-11-03. Use the developer index at +// GET or POST /v2/search/developer. This SDK does not wrap it yet, so call it +// directly. It does not carry over the score breakdown or the web fallback +// results. func (c *Client) SearchGitHub(ctx context.Context, query string, opts *SearchGitHubOptions) (*GitHubSearchResponse, error) { if query == "" { return nil, &FirecrawlError{Message: "query is required"} diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java index a04498dc69..c6506a0c21 100644 --- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java +++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java @@ -602,10 +602,24 @@ public ResearchModels.SimilarPapersResponse relatedPapers(String paperId, String return http.get("/v2/search/research/papers/" + urlEncode(paperId) + "/similar" + researchQuery(params), ResearchModels.SimilarPapersResponse.class); } + /** + * @deprecated Stops responding after 2026-11-03. Use the developer index at + * GET or POST /v2/search/developer, which this SDK does not wrap + * yet, so call it directly. It does not carry over the score + * breakdown or the web fallback results. + */ + @Deprecated public ResearchModels.GitHubSearchResponse searchGitHub(String query) { return searchGitHub(query, null); } + /** + * @deprecated Stops responding after 2026-11-03. Use the developer index at + * GET or POST /v2/search/developer, which this SDK does not wrap + * yet, so call it directly. It does not carry over the score + * breakdown or the web fallback results. + */ + @Deprecated public ResearchModels.GitHubSearchResponse searchGitHub(String query, ResearchModels.SearchGitHubOptions options) { Objects.requireNonNull(query, "Query is required"); Map params = new LinkedHashMap<>(); @@ -1056,6 +1070,13 @@ public CompletableFuture relatedPapersAsyn return CompletableFuture.supplyAsync(() -> relatedPapers(paperId, intent, options), asyncExecutor); } + /** + * @deprecated Stops responding after 2026-11-03. Use the developer index at + * GET or POST /v2/search/developer, which this SDK does not wrap + * yet, so call it directly. It does not carry over the score + * breakdown or the web fallback results. + */ + @Deprecated public CompletableFuture searchGitHubAsync(String query, ResearchModels.SearchGitHubOptions options) { return CompletableFuture.supplyAsync(() -> searchGitHub(query, options), asyncExecutor); } diff --git a/apps/js-sdk/firecrawl/README.md b/apps/js-sdk/firecrawl/README.md index a0b034a04c..54ccc7f43a 100644 --- a/apps/js-sdk/firecrawl/README.md +++ b/apps/js-sdk/firecrawl/README.md @@ -261,8 +261,13 @@ const related = await app.research.similarPapers('pmid:', { }); ``` -A companion `app.research.searchGithub` searches indexed GitHub issue/PR history -and repository readmes. +> **`app.research.searchGithub` is deprecated.** The research index GitHub +> endpoint stops responding after 2026-11-03. Use `app.developerSearch` +> instead: it searches GitHub issues, pull requests and readmes plus curated +> documentation sources, returns matched passages, and adds filters for repo, +> language, license and stars. It does not carry over the `scores` breakdown +> or the `resultType: "web"` fallback results. See +> [the developer index docs](https://docs.firecrawl.dev/features/developer). ### Scrape-bound interactive browsing (v2) diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.ts index 8c3091286a..e6a85b64bf 100644 --- a/apps/js-sdk/firecrawl/src/v2/client.ts +++ b/apps/js-sdk/firecrawl/src/v2/client.ts @@ -285,6 +285,9 @@ export class FirecrawlClient { * Access the v2 research endpoints — Firecrawl's **research paper index** * (~43M paper abstracts) plus GitHub history/readmes. * + * `research.searchGithub()` is deprecated and stops responding after + * 2026-11-03. Use `developerSearch()` instead. + * * The paper corpus is roughly 90% biomedical and life sciences — PubMed, * bioRxiv and medRxiv — with arXiv covering physics, mathematics and * computer science. diff --git a/apps/js-sdk/firecrawl/src/v2/methods/research.ts b/apps/js-sdk/firecrawl/src/v2/methods/research.ts index a05d322494..5292b29977 100644 --- a/apps/js-sdk/firecrawl/src/v2/methods/research.ts +++ b/apps/js-sdk/firecrawl/src/v2/methods/research.ts @@ -159,9 +159,9 @@ export class ResearchClient { appendParam(params, "query", options.query); appendParam(params, "k", options.k); try { - const res = await this.http.get( - withQuery(`${BASE}/papers/${encodeURIComponent(id)}`, params), - ); + const res = await this.http.get< + PaperMetadataResponse | ReadPaperResponse + >(withQuery(`${BASE}/papers/${encodeURIComponent(id)}`, params)); if (res.status !== 200) throwForBadResponse(res, "get paper"); return res.data; } catch (err) { @@ -191,10 +191,7 @@ export class ResearchClient { appendParam(params, "anchor", options.anchor); try { const res = await this.http.get( - withQuery( - `${BASE}/papers/${encodeURIComponent(id)}/similar`, - params, - ), + withQuery(`${BASE}/papers/${encodeURIComponent(id)}/similar`, params), ); if (res.status !== 200) throwForBadResponse(res, "find similar papers"); return res.data; @@ -204,7 +201,16 @@ export class ResearchClient { } /** - * Search GitHub issue/PR history and repository readmes. + * Search the research index GitHub slice: issue/PR history and repository + * readmes. + * + * @deprecated Use `developerSearch()` on the client instead. This + * endpoint stops responding after 2026-11-03. The developer index searches + * GitHub issues, pull requests and readmes plus curated documentation + * sources, returns matched passages, and adds filters for repo, language, + * license and stars. It does not carry over the `scores` breakdown or the + * `resultType: "web"` fallback results. See + * https://docs.firecrawl.dev/features/developer. * @param query Search query. * @param options Optional `k`. */ diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts index b6b750558d..a7d16d4901 100644 --- a/apps/js-sdk/firecrawl/src/v2/types.ts +++ b/apps/js-sdk/firecrawl/src/v2/types.ts @@ -1917,7 +1917,14 @@ export interface SimilarPapersResponse { note?: string | null; } -/** Component scores; each field is present only when that signal contributed. */ +/** + * Component scores; each field is present only when that signal contributed. + * + * @deprecated Use the developer index instead. The research index GitHub + * search stops responding after 2026-11-03. The developer index does not + * expose a score breakdown. See + * https://docs.firecrawl.dev/features/developer. + */ export interface GitHubScoreBreakdown { rrf?: number; semantic?: number; @@ -1926,6 +1933,11 @@ export interface GitHubScoreBreakdown { rerank?: number; } +/** + * @deprecated Use the developer index instead. The research index GitHub + * search stops responding after 2026-11-03. See + * https://docs.firecrawl.dev/features/developer. + */ export interface GitHubSearchItem { resultType?: "github_history" | "repo_readme" | "web"; /** `owner/name`; empty for web results whose URL is not a repo page. */ @@ -1948,9 +1960,18 @@ export interface GitHubSearchItem { scores: GitHubScoreBreakdown; } +/** + * @deprecated Use the developer index instead. The research index GitHub + * search stops responding after 2026-11-03. See + * https://docs.firecrawl.dev/features/developer. + */ export interface GitHubSearchResponse { success: boolean; results: GitHubSearchItem[]; + /** Deprecation notice while the sunset window is live. */ + warnings?: string[]; + /** Replacement endpoint path, while the sunset window is live. */ + replacement?: string; } /** Options for `research.searchPapers`. */ @@ -1989,7 +2010,13 @@ export interface SimilarPapersOptions { anchor?: string[]; } -/** Options for `research.searchGithub`. */ +/** + * Options for `research.searchGithub`. + * + * @deprecated Use the developer index instead. The research index GitHub + * search stops responding after 2026-11-03. See + * https://docs.firecrawl.dev/features/developer. + */ export interface SearchGithubOptions { /** Number of results to return (1–100, default 20). */ k?: number; diff --git a/apps/php-sdk/src/Client/FirecrawlClient.php b/apps/php-sdk/src/Client/FirecrawlClient.php index 36a82a9b74..836f5dbcd8 100644 --- a/apps/php-sdk/src/Client/FirecrawlClient.php +++ b/apps/php-sdk/src/Client/FirecrawlClient.php @@ -181,6 +181,10 @@ public function relatedPapers(string $paperId, string $intent, array $options = /** * Search GitHub research content. * + * @deprecated Stops responding after 2026-11-03. Use the developer index at + * GET or POST /v2/search/developer, which this SDK does not wrap yet, so + * call it directly. It does not carry over the score breakdown or the web + * fallback results. * @param array $options * @return array */ diff --git a/apps/python-sdk/README.md b/apps/python-sdk/README.md index 91d9d4e4e6..d3a2b40eba 100644 --- a/apps/python-sdk/README.md +++ b/apps/python-sdk/README.md @@ -273,8 +273,13 @@ related = firecrawl.related_papers( ) ``` -A companion `search_github` searches indexed GitHub issue/PR history and repo -readmes. +> **`search_github` is deprecated.** The research index GitHub endpoint stops +> responding after 2026-11-03. Use `developer_search` instead: it searches +> GitHub issues, pull requests and readmes plus curated documentation sources, +> returns matched passages, and adds filters for repo, language, license and +> stars. It does not carry over the `scores` breakdown or the +> `resultType: "web"` fallback results. See +> [the developer index docs](https://docs.firecrawl.dev/features/developer). > **Response keys are camelCase.** Unlike the rest of the SDK, the research > methods return the raw JSON body as a `dict` — they are not parsed into typed diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/research.py b/apps/python-sdk/firecrawl/v2/methods/aio/research.py index e8725987ab..33980e7bed 100644 --- a/apps/python-sdk/firecrawl/v2/methods/aio/research.py +++ b/apps/python-sdk/firecrawl/v2/methods/aio/research.py @@ -29,11 +29,13 @@ from typing import Any, Dict, List, Optional from urllib.parse import quote +import warnings from ...utils import handle_response_error from ...utils.http_client_async import AsyncHttpClient from ...utils.get_version import get_version from ..research_docs import ( + GITHUB_SEARCH_DEPRECATION_MSG, AIO_INSPECT_PAPER_DOC, AIO_READ_PAPER_DOC, AIO_RELATED_PAPERS_DOC, @@ -152,6 +154,9 @@ async def search_github( *, k: Optional[int] = None, ) -> Dict[str, Any]: + # FutureWarning, not DeprecationWarning: the default filters hide the latter + # outside __main__, and every entry point here is several SDK frames deep. + warnings.warn(GITHUB_SEARCH_DEPRECATION_MSG, FutureWarning, stacklevel=2) return await _get( client, BASE + "/github" + _query({"query": query, "k": k, "origin": ORIGIN}), diff --git a/apps/python-sdk/firecrawl/v2/methods/research.py b/apps/python-sdk/firecrawl/v2/methods/research.py index 79d4d92cca..6bf51b8b43 100644 --- a/apps/python-sdk/firecrawl/v2/methods/research.py +++ b/apps/python-sdk/firecrawl/v2/methods/research.py @@ -29,10 +29,12 @@ from typing import Any, Dict, List, Optional from urllib.parse import quote +import warnings from ..utils import HttpClient, handle_response_error from ..utils.get_version import get_version from .research_docs import ( + GITHUB_SEARCH_DEPRECATION_MSG, INSPECT_PAPER_DOC, READ_PAPER_DOC, RELATED_PAPERS_DOC, @@ -151,6 +153,9 @@ def search_github( *, k: Optional[int] = None, ) -> Dict[str, Any]: + # FutureWarning, not DeprecationWarning: the default filters hide the latter + # outside __main__, and every entry point here is several SDK frames deep. + warnings.warn(GITHUB_SEARCH_DEPRECATION_MSG, FutureWarning, stacklevel=2) return _get( client, BASE + "/github" + _query({"query": query, "k": k, "origin": ORIGIN}), diff --git a/apps/python-sdk/firecrawl/v2/methods/research_docs.py b/apps/python-sdk/firecrawl/v2/methods/research_docs.py index 57a4d9c438..5d0d1c5fd8 100644 --- a/apps/python-sdk/firecrawl/v2/methods/research_docs.py +++ b/apps/python-sdk/firecrawl/v2/methods/research_docs.py @@ -55,6 +55,18 @@ def decorate(fn: F) -> F: # Implementation functions (``research.py`` / ``aio/research.py``) # --------------------------------------------------------------------------- +GITHUB_SEARCH_DEPRECATION_MSG = ( + "search_github() is deprecated and the research index GitHub endpoint " + "stops responding after 2026-11-03. Use developer_search() instead: it " + "searches GitHub issues, pull requests and READMEs plus curated " + "documentation sources, returns matched passages, and adds filters for " + "repo, language, license and stars. Response changes: 'snippet' becomes " + "'passages', results gain an 'id', and there is no score breakdown and no " + "web fallback result type. See " + "https://docs.firecrawl.dev/features/developer." +) + + _SEARCH_PAPERS_TEMPLATE = """ Search the research paper index by abstract relevance. @@ -169,12 +181,21 @@ def decorate(fn: F) -> F: """ _SEARCH_GITHUB_TEMPLATE = """ -Search the developer index: GitHub issue/PR history and repository readmes. +Search the **research index** GitHub slice: GitHub issue/PR history and +repository readmes. + +.. deprecated:: + Use ``developer_search()`` instead. This endpoint stops responding after + 2026-11-03. The developer index searches GitHub issues, pull requests and + readmes plus curated documentation sources, returns matched passages, and + adds filters for repo, language, license and stars. It does **not** carry + over the ``scores`` breakdown or the ``resultType: "web"`` fallback + results. See https://docs.firecrawl.dev/features/developer. This is the code-and-discussion companion to ``search_papers`` and is served -by the same ``/v2/search/research`` surface. It searches indexed GitHub -history and readmes — it does **not** search the paper corpus, and it is not -the same as ``search(categories=["github"])`` (which is a ``site:github.com`` +by the same ``/v2/search/research`` surface. It is a separate index from the +developer index, it does **not** search the paper corpus, and it is not the +same as ``search(categories=["github"])`` (which is a ``site:github.com`` filter on ordinary web search). Args: @@ -185,7 +206,9 @@ def decorate(fn: F) -> F: Returns: Raw API ``dict`` with ``success`` and ``results``. Keys are camelCase and are **not** normalized to snake_case — expect ``resultType``, ``repo``, - ``url``, ``pageType``, ``number`` and a ``scoreBreakdown`` object. + ``url``, ``pageType``, ``number`` and a ``scores`` object. The response + also carries a ``warnings`` list and a ``replacement`` field while the + deprecation is live. """ SEARCH_PAPERS_DOC = _SEARCH_PAPERS_TEMPLATE.replace(_CLIENT_ARG, _SYNC_CLIENT_ARG) @@ -287,12 +310,22 @@ def decorate(fn: F) -> F: """ _CLIENT_SEARCH_GITHUB_TEMPLATE = """ -Search the developer index: GitHub issue/PR history and repo readmes. +Search the **research index** GitHub slice: issue/PR history and repo +readmes. + +.. deprecated:: + Use ``developer_search()`` instead. This endpoint stops responding + after 2026-11-03. The developer index searches GitHub issues, pull + requests and readmes plus curated documentation sources, and returns + matched passages. It does **not** carry over the ``scores`` breakdown + or the ``resultType: "web"`` fallback results. See + https://docs.firecrawl.dev/features/developer. The code-and-discussion companion to ``search_papers``, served by the -same ``/v2/search/research`` surface. It does not search the paper -corpus, and it is not ``search(categories=["github"])`` (which is just a -``site:github.com`` filter on ordinary web search). +same ``/v2/search/research`` surface. A separate index from the developer +index. It does not search the paper corpus, and it is not +``search(categories=["github"])`` (which is just a ``site:github.com`` +filter on ordinary web search). Args: query: Natural-language query, e.g. ``"pysam VCF parsing memory leak"``. diff --git a/apps/ruby-sdk/lib/firecrawl/client.rb b/apps/ruby-sdk/lib/firecrawl/client.rb index 9182ae0136..25b9b334d4 100644 --- a/apps/ruby-sdk/lib/firecrawl/client.rb +++ b/apps/ruby-sdk/lib/firecrawl/client.rb @@ -129,6 +129,10 @@ def related_papers(paper_id, intent, options = {}) # Search GitHub research content. # + # @deprecated Stops responding after 2026-11-03. Use the developer index at + # GET or POST /v2/search/developer, which this SDK does not wrap yet, so + # call it directly. It does not carry over the score breakdown or the + # web fallback results. # @param query_text [String] GitHub query # @param options [Hash] optional query parameters # @return [Hash] diff --git a/apps/rust-sdk/src/research.rs b/apps/rust-sdk/src/research.rs index 232c523ebe..a735edc653 100644 --- a/apps/rust-sdk/src/research.rs +++ b/apps/rust-sdk/src/research.rs @@ -289,6 +289,11 @@ impl Client { self.handle_response(response, "related papers").await } + /// Search the research index GitHub slice. + /// + /// Stops responding after 2026-11-03. Call `/v2/search/developer` directly; + /// this SDK does not wrap it yet. + #[deprecated(note = "Use the developer index at /v2/search/developer; sunset 2026-11-03")] pub async fn search_github( &self, query_text: impl AsRef,