diff --git a/README.md b/README.md index 7536ef7c69..f96bb10d40 100644 --- a/README.md +++ b/README.md @@ -466,9 +466,21 @@ Paper ids accept `pmid:`, `pmcid:`, `doi:`, and `arxiv:` forms, plus canonical ` ### `feedback` - Send endpoint job feedback -Send concise feedback for a completed v2 `search`, `scrape`, `parse`, or `map` -job. For search-result quality, `search-feedback` is still the most guided -command; `feedback` is the generic endpoint/job surface. +Send evidence through `/v2/feedback`. Feedback on keyless `search`, `scrape`, and +`parse` jobs is optional. Consider submitting feedback through `firecrawl feedback`, especially if this result is wrong, incomplete, blocked, or an error. Include specific evidence to help improve Firecrawl. Keyless feedback submissions require `--rating`, `--task`, `--assessment`, and 1-20 observations +provided through `--observations` or `--observations-file`. Task, assessment, and each observation detail require 10-2000 characters after trimming whitespace. Keyless Parse also requires `--doc-class born_digital|scanned|mixed|unknown` once per submission. Use the returned job +reference and evidence already available; no user interview or additional +investigation is required. Run `firecrawl feedback --help` for category fields. + +Each keyless job accepts one submission; retrying returns the original feedback +ID. Submit from the same caller IP before the invitation's `expiresAt` deadline, +which provides a 24-hour feedback window for the job. +Submitting feedback does not consume or restore operation allowance. Invitations +and references appear in metadata or stderr, preserving ordinary stdout. + +Authenticated callers retain the existing fields. `search-feedback` remains an +authenticated Search command and cannot submit feedback for keyless jobs. The +following example uses the authenticated endpoint feedback contract: ```bash firecrawl feedback scrape 0193f6c5-1234-7890-abcd-1234567890ab \ @@ -483,25 +495,33 @@ firecrawl feedback scrape 0193f6c5-1234-7890-abcd-1234567890ab \ Keep notes and metadata small. Do not send raw scrape or parse outputs as feedback. -Set `FIRECRAWL_NO_ENDPOINT_FEEDBACK=1` to make `firecrawl feedback` skip -endpoint feedback calls silently. +Search observations identify delivered result positions or missing information. Scrape and Parse observations describe the requested output formats. Failed jobs use a `failure` observation based on the returned error. + +Run `firecrawl feedback --help` for endpoint-specific categories, fields and reason codes. Stored keyless feedback must fit within 8 KiB, including server defaults and verification flags. Submit from the same caller IP; attempts are rate limited. See the [API feedback contract](https://docs.firecrawl.dev/api-reference/endpoint/feedback) for examples, format constraints, and Parse retention behavior. + +Set `FIRECRAWL_NO_ENDPOINT_FEEDBACK=1` or `FIRECRAWL_DISABLE_ENDPOINT_FEEDBACK=1` to skip authenticated endpoint feedback calls. These flags do not suppress keyless invitations or submissions. The API includes a pointer on every eligible keyless job response. Feedback is optional and helps improve Firecrawl when a result is wrong, incomplete, blocked, or an error; keyless access does not depend on it. #### Feedback Options -| Option | Description | -| -------------------------------- | -------------------------------------------- | -| `--rating ` | Required: `good`, `partial`, or `bad` | -| `--issues ` | Comma-separated issue codes or JSON array | -| `--tags ` | Comma-separated tags or JSON array | -| `--note ` | Short human-readable feedback | -| `--valuable-sources ` | JSON array of `{url, reason}` entries | -| `--missing-content ` | JSON array of `{topic, description}` entries | -| `--query-suggestions ` | Search/query improvement notes | -| `--url ` | Relevant URL for scrape or parse feedback | -| `--page-numbers ` | Comma-separated page numbers or JSON array | -| `--metadata ` | Small JSON object with extra context | -| `--metadata-file ` | Path to small metadata JSON object | -| `--silent` | Suppress output for background agent calls | +| Option | Description | +| -------------------------------- | ---------------------------------------------------- | +| `--rating ` | Required: `good`, `partial`, or `bad` | +| `--task ` | Task intent, required for keyless feedback | +| `--doc-class ` | Document class, required for keyless Parse | +| `--assessment ` | Assessment, required for keyless feedback | +| `--observations ` | JSON array of category-specific keyless observations | +| `--observations-file ` | File containing the observations JSON array | +| `--issues ` | Comma-separated issue codes or JSON array | +| `--tags ` | Comma-separated tags or JSON array | +| `--note ` | Short human-readable feedback | +| `--valuable-sources ` | JSON array of `{url, reason}` entries | +| `--missing-content ` | JSON array of `{topic, description}` entries | +| `--query-suggestions ` | Search/query improvement notes | +| `--url ` | Relevant URL for scrape or parse feedback | +| `--page-numbers ` | Comma-separated page numbers or JSON array | +| `--metadata ` | Small JSON object with extra context | +| `--metadata-file ` | Path to small metadata JSON object | +| `--silent` | Suppress output for background agent calls | --- diff --git a/skills/firecrawl-search/SKILL.md b/skills/firecrawl-search/SKILL.md index 578badaa97..9b26004d0f 100644 --- a/skills/firecrawl-search/SKILL.md +++ b/skills/firecrawl-search/SKILL.md @@ -86,9 +86,30 @@ Keep large search responses in `--json -o` output and select the relevant result - Use `jq` to extract URLs or titles: `jq -r '.data.web[].url' .firecrawl/search.json` - Naming convention: `.firecrawl/search-{query}.json` or `.firecrawl/search-{query}-scraped.json` -## Send feedback after every search (refunds 1 credit) +## Feedback -Search costs 2 credits. After you've actually used the results (or decided they were useless), send structured feedback with `firecrawl search-feedback `. The first feedback per search refunds 1 credit and helps us improve search quality. **Do this once per search**, in the background, after you finish processing the results — it does not block your main task. +Choose the feedback contract that matches the search job's authentication. A returned `metadata.feedback` invitation identifies the keyless submission endpoint and job ID. Do not send the keyless fields to the authenticated Search feedback route, or add credentials to submit feedback for a keyless job. + +Keyless feedback is available for successful and failed jobs when the response includes a job reference. For an explicitly failed job, use observation `kind: "failure"` and `reason: "timeout"`, `"transport_error"`, `"proxy_error"`, or `"other"`; report only the error already returned. Keep the submission under 8 KiB including server defaults. Run `firecrawl feedback --help` for reason definitions and the complete contract. + +### Keyless Search + +Use `firecrawl feedback search ` with `--rating`, `--task`, `--assessment`, and `--observations-file`. The task describes what the search needed to answer; the assessment describes how well it answered that task. Supply 1-20 observations: + +- Useful or irrelevant results: `kind` and one-based `position` within the delivered group. `source` (`web`, `images`, or `news`) is required for multi-source jobs and for images-only or news-only jobs. Only web-only jobs can omit it, defaulting to `web`. Irrelevant results also require `reason`; see command help for allowed values. +- Missing information: `kind: "missing"` and `vertical`; `topic` is optional. `vertical` is optional on useful and irrelevant results. See command help for allowed verticals. +- Missing and irrelevant observations may include `knownSources`: up to 20 HTTP(S) URLs, only when already known. These identify absent content or the source that should have ranked instead. Unmentioned results are unassessed; a full ranking is not required. +- Every observation requires `detail` and `basis`: `output`, `source_comparison`, or `expectation`. A source comparison also requires `comparison: {reference, detail}`, with the correct content in `comparison.detail`. Use expectations for gaps that have not been verified against a source. + +Task, assessment, and observation detail each require 10-2000 characters. Use only evidence already available. Do not ask the user to complete an interview, investigate further, guess missing content, or diagnose causes merely to submit feedback. An empty result set can support a missing-information observation if the response includes an eligible job reference. + +Each job accepts one submission, and retrying returns the original feedback ID. Submit from the same caller IP before the invitation's `expiresAt` deadline, which provides a 24-hour feedback window for the job. Feedback does not consume or restore operation allowance. Consider submitting feedback, especially if a keyless result is wrong, incomplete, blocked, or an error. Include specific evidence to help improve Firecrawl; do not retry a rate-limit rejection in a loop. Client feedback flags do not suppress keyless invitations or submissions. + +Run `firecrawl feedback --help` for the full evidence contract. Old instructions using `firecrawl search-feedback` require authentication and do not work for keyless jobs. + +### Authenticated Search + +Authenticated callers can continue using `firecrawl search-feedback ` with `--valuable-sources`, `--missing-content`, and `--query-suggestions`. Its existing validation, feedback window, and refund policy are unchanged. If submitting, do so after processing the results, at most once per search, using only evidence already available. **Opt out:** if `FIRECRAWL_NO_SEARCH_FEEDBACK=1` (or `FIRECRAWL_DISABLE_SEARCH_FEEDBACK=1`) is set, the CLI silently skips the call and never sends anything. Respect that — do not try to work around it. (Team admins can also disable this server-side; the API will return `feedbackErrorCode: "TEAM_OPTED_OUT"` and the CLI will exit 0 silently.) diff --git a/skills/firecrawl/SKILL.md b/skills/firecrawl/SKILL.md index 67c83d0e4b..9512eaae19 100644 --- a/skills/firecrawl/SKILL.md +++ b/skills/firecrawl/SKILL.md @@ -115,11 +115,15 @@ Single format outputs raw content. Multiple formats (e.g., `--format markdown,li ## Feedback -After using search results, send `firecrawl search-feedback` (the first feedback per search refunds 1 credit). The full pattern, guard, and rules live in [firecrawl-search](../firecrawl-search/SKILL.md). +Consider submitting feedback, especially if a keyless Search, Scrape, or Parse result is wrong, incomplete, blocked, or an error. Include specific evidence to help improve Firecrawl. Submit it with `firecrawl feedback ` with `--rating`, `--task`, `--assessment`, and `--observations-file`. Use the job reference and invitation returned in metadata or stderr. Run `firecrawl feedback --help` for category-specific observations. For Alexandria feedback about a provider result or coverage gap, see [firecrawl-alexandria](../firecrawl-alexandria/SKILL.md). -For non-search endpoint jobs, use `firecrawl feedback ` to send concise job-level feedback through `/v2/feedback`. Supported endpoints are `search`, `scrape`, `parse`, and `map`. +Use only evidence already available, without interviewing the user or doing extra investigation. Each keyless job accepts one submission. Submit from the same caller IP before the invitation's `expiresAt` deadline, which provides a 24-hour feedback window for the job. Feedback does not consume or restore operation allowance. Do not send legacy issue/note fields as a substitute for keyless observations. + +Authenticated Search callers can continue using `firecrawl search-feedback` with its existing fields and policy. The two authentication modes use different request contracts; do not add credentials to submit feedback for a keyless job. Details live in [firecrawl-search](../firecrawl-search/SKILL.md). + +Authenticated callers can use `firecrawl feedback ` with the existing issue/note fields for `search`, `scrape`, `parse`, and `map`. The following example is for authenticated feedback: ```bash firecrawl feedback scrape "$SCRAPE_ID" \ @@ -134,7 +138,9 @@ firecrawl feedback scrape "$SCRAPE_ID" \ Keep generic feedback small: issue codes, tags, short notes, URLs, page numbers, and small metadata objects — never raw scrape/parse outputs or full page contents. -**Opt out:** `export FIRECRAWL_NO_ENDPOINT_FEEDBACK=1` makes the CLI skip every endpoint feedback call silently. Respect that flag — do not try to work around it. +Keyless feedback is available for successful and failed jobs when the response includes a job reference. For an explicitly failed job, use observation `kind: "failure"` and `reason: "timeout"`, `"transport_error"`, `"proxy_error"`, or `"other"`; report only the error already returned. Keep the submission under 8 KiB including server defaults. Run `firecrawl feedback --help` for reason definitions and the complete contract. + +**Authenticated feedback preference:** `FIRECRAWL_NO_ENDPOINT_FEEDBACK=1` or `FIRECRAWL_DISABLE_ENDPOINT_FEEDBACK=1` skips authenticated endpoint feedback calls. Respect these flags for authenticated jobs. Keyless jobs retain server-issued invitations and optional submissions regardless of these flags. ## Parallelization diff --git a/src/__tests__/cli-aliases.test.ts b/src/__tests__/cli-aliases.test.ts index d39949cea2..7ccf89588d 100644 --- a/src/__tests__/cli-aliases.test.ts +++ b/src/__tests__/cli-aliases.test.ts @@ -21,6 +21,8 @@ describe('CLI compatibility aliases', { timeout: 30000 }, () => { scrape.handleScrapeCommand = print; scrape.handleAllScrapeCommand = (_url, options) => print(options); require('./dist/commands/parse').handleParseCommand = print; + require('./dist/commands/search').handleSearchCommand = print; + require('./dist/commands/feedback').handleEndpointFeedbackCommand = print; require('./dist/commands/crawl').handleCrawlCommand = print; require('./dist/commands/agent').handleAgentCommand = print; process.argv = [process.execPath, ${JSON.stringify(cliPath)}, ...${JSON.stringify(args)}]; @@ -33,11 +35,49 @@ describe('CLI compatibility aliases', { timeout: 30000 }, () => { env: { ...process.env, FIRECRAWL_API_KEY: '', + FIRECRAWL_API_URL: '', FIRECRAWL_NO_UPDATE_CHECK: '1', }, }); } + testWithBuiltCli.each([ + { flags: [], sources: ['web'] }, + { flags: ['--api-key', 'fc-test-key'], sources: ['web', 'alexandria'] }, + { flags: ['--sources', 'news'], sources: ['news'] }, + ])('selects usable Search sources with $flags', ({ flags, sources }) => { + const result = run(['search', 'retry reference', ...flags]); + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout).sources).toEqual(sources); + expect(result.stdout).not.toContain('AUTH_CHECK'); + }); + + testWithBuiltCli.each([ + 'search', + 'scrape', + 'parse', + 'map', + 'Search', + 'Scrape', + 'Parse', + 'Map', + ])( + 'retains the authentication gate only for Map feedback: %s', + (endpoint) => { + const result = run([ + 'feedback', + endpoint, + '00000000-0000-4000-8000-000000000001', + '--rating', + 'good', + ]); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout.includes('AUTH_CHECK')).toBe( + endpoint.toLowerCase() === 'map' + ); + } + ); + testWithBuiltCli( 'sql preserves experimental aliases and execution options', () => { diff --git a/src/__tests__/cli-argv.test.ts b/src/__tests__/cli-argv.test.ts index 139c926bd1..4c40554166 100644 --- a/src/__tests__/cli-argv.test.ts +++ b/src/__tests__/cli-argv.test.ts @@ -7,6 +7,32 @@ describe('CLI argv parsing', () => { const cliPath = resolve(process.cwd(), 'dist/index.js'); const testWithBuiltCli = existsSync(cliPath) ? it : it.skip; + testWithBuiltCli( + 'describes substantive keyless evidence in feedback help', + () => { + const result = spawnSync( + process.execPath, + [cliPath, 'feedback', '--help'], + { + cwd: process.cwd(), + encoding: 'utf8', + } + ); + expect(result.status).toBe(0); + for (const field of [ + '--task', + '--assessment', + '--observations-file', + 'one-based position', + 'source_comparison', + 'one submission', + ]) { + expect(result.stdout).toContain(field); + } + expect(result.stdout).not.toContain('UTC day'); + } + ); + testWithBuiltCli('rejects invalid PDF page caps before scraping', () => { for (const value of ['0', '10001', '2.5', '3pages']) { const result = spawnSync( diff --git a/src/__tests__/commands/feedback.test.ts b/src/__tests__/commands/feedback.test.ts index 6cd746e5e0..75a8873b89 100644 --- a/src/__tests__/commands/feedback.test.ts +++ b/src/__tests__/commands/feedback.test.ts @@ -1,23 +1,19 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { executeEndpointFeedback, + parseEndpointFeedbackCliOptions, handleEndpointFeedbackCommand, parseEndpointFeedbackEndpoint, parseFeedbackListArg, parsePageNumbersArg, } from '../../commands/feedback'; import { parseAlexandriaFeedbackArray } from '../../commands/alexandria-feedback'; -import { getClient } from '../../utils/client'; import { initializeConfig } from '../../utils/config'; import { setupTest, teardownTest } from '../utils/mock-client'; -vi.mock('../../utils/client', async () => { - const actual = await vi.importActual('../../utils/client'); - return { - ...actual, - getClient: vi.fn(), - }; -}); +vi.mock('../../utils/credentials', () => ({ + loadCredentials: vi.fn(() => null), +})); describe('executeEndpointFeedback', () => { let mockFetch: ReturnType; @@ -36,10 +32,127 @@ describe('executeEndpointFeedback', () => { afterEach(() => { teardownTest(); vi.clearAllMocks(); + vi.unstubAllEnvs(); delete process.env.FIRECRAWL_NO_ENDPOINT_FEEDBACK; delete process.env.FIRECRAWL_DISABLE_ENDPOINT_FEEDBACK; }); + it.each([undefined, 'https://api.firecrawl.dev'])( + 'submits keyless evidence despite authenticated opt-out with API URL %s', + async (apiUrl) => { + vi.stubEnv('FIRECRAWL_API_KEY', ''); + vi.stubEnv('FIRECRAWL_NO_ENDPOINT_FEEDBACK', '1'); + vi.stubEnv('FIRECRAWL_DISABLE_ENDPOINT_FEEDBACK', '1'); + initializeConfig({ + apiKey: undefined, + apiUrl: 'https://api.firecrawl.dev', + }); + mockFetch.mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ + success: true, + feedbackId: 'feedback-1', + creditsRefunded: 0, + }), + }); + const observations = [ + { + kind: 'incorrect', + reason: 'missing_fields', + format: 'json', + basis: 'output', + detail: 'The table contains the expected column headings.', + page: 2, + }, + ]; + const result = await executeEndpointFeedback({ + apiUrl, + endpoint: 'parse', + docClass: 'born_digital', + jobId: '00000000-0000-4000-8000-000000000001', + rating: 'good', + task: 'Read the table headings', + assessment: 'The output preserved all table headings.', + observations, + }); + expect(result.success).toBe(true); + const [, init] = mockFetch.mock.calls[0]; + expect(init.headers.Authorization).toBeUndefined(); + expect(init.headers['X-Origin']).toBe('cli'); + expect(JSON.parse(init.body)).toMatchObject({ + endpoint: 'parse', + docClass: 'born_digital', + observations, + origin: 'cli', + integration: 'cli', + }); + } + ); + + it.each(['task', 'assessment', 'observations', 'docClass'] as const)( + 'rejects missing keyless %s before making a request', + async (field) => { + vi.stubEnv('FIRECRAWL_API_KEY', ''); + initializeConfig({ apiUrl: 'https://api.firecrawl.dev' }); + const options = { + endpoint: 'parse' as const, + jobId: '00000000-0000-4000-8000-000000000001', + rating: 'good' as const, + task: 'Read the retry reference', + assessment: 'The output preserves the retry interval.', + observations: [ + { + kind: 'correct', + basis: 'output', + detail: 'The retry interval is present.', + }, + ], + docClass: 'unknown' as const, + }; + const incomplete = { ...options, [field]: undefined }; + const result = await executeEndpointFeedback(incomplete); + expect(result.success).toBe(false); + expect(result.error).toMatch(/Keyless feedback requires/); + expect(mockFetch).not.toHaveBeenCalled(); + } + ); + + it('preserves replacement sources on keyless irrelevant Search observations', async () => { + vi.stubEnv('FIRECRAWL_API_KEY', ''); + initializeConfig({ + apiKey: undefined, + apiUrl: 'https://api.firecrawl.dev', + }); + mockFetch.mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ success: true, feedbackId: 'feedback-1' }), + }); + const observations = [ + { + kind: 'irrelevant', + reason: 'aggregator_over_official', + position: 1, + knownSources: ['https://example.com/official'], + basis: 'output', + detail: 'The official reference should rank before this aggregator.', + }, + ]; + await executeEndpointFeedback({ + endpoint: 'search', + jobId: '00000000-0000-4000-8000-000000000001', + rating: 'partial', + task: 'Find the official retry reference', + assessment: 'An aggregator ranked above the official reference.', + observations, + }); + const [, init] = mockFetch.mock.calls[0]; + expect(init.headers.Authorization).toBeUndefined(); + expect(init.headers['X-Origin']).toBe('cli'); + expect(JSON.parse(init.body).observations).toEqual(observations); + }); + it('posts Alexandria session feedback without job fields or legacy metadata', async () => { mockFetch.mockResolvedValue({ ok: true, @@ -114,10 +227,6 @@ describe('executeEndpointFeedback', () => { apiUrl: 'http://localhost:3002', }); - expect(getClient).toHaveBeenCalledWith({ - apiKey: undefined, - apiUrl: 'http://localhost:3002', - }); expect(result).toEqual({ success: true, feedbackId: '0193f6c5-1234-7890-abcd-1234567890ab', @@ -155,6 +264,75 @@ describe('executeEndpointFeedback', () => { }); }); + it.each([ + [400, false], + [429, false], + [400, true], + [429, true], + ] as const)( + 'shows API validation details and retry timing only for keyless feedback: HTTP %i, authenticated %s', + async (status, authenticated) => { + vi.stubEnv('FIRECRAWL_API_KEY', ''); + initializeConfig({ + apiKey: authenticated ? 'test-api-key' : undefined, + apiUrl: 'https://api.firecrawl.dev', + }); + const details = [ + { + path: ['observations', 0, 'basis'], + message: 'Invalid evidence basis', + }, + ]; + mockFetch.mockResolvedValue({ + ok: false, + status, + json: async () => ({ + success: false, + error: 'Feedback rejected', + ...(status === 400 + ? { feedbackErrorCode: 'INVALID_BODY', details } + : { retry_after_seconds: 2 }), + }), + }); + const exit = vi.spyOn(process, 'exit').mockImplementation(() => { + throw new Error('process.exit:1'); + }); + const stderr = vi.spyOn(console, 'error').mockImplementation(() => {}); + const stdout = vi.spyOn(process.stdout, 'write').mockReturnValue(true); + try { + await expect( + handleEndpointFeedbackCommand({ + endpoint: 'scrape', + jobId: '00000000-0000-4000-8000-000000000001', + rating: 'bad', + task: 'Retrieve the requested web page.', + assessment: 'The page request explicitly returned an error.', + observations: [ + { + kind: 'failure', + reason: 'other', + basis: 'invalid', + detail: 'The operation explicitly returned an error.', + }, + ], + }) + ).rejects.toThrow('process.exit:1'); + const output = stderr.mock.calls.flat().join(' '); + expect(output.includes('Invalid evidence basis')).toBe( + !authenticated && status === 400 + ); + expect(output.includes('Retry after: 2 seconds.')).toBe( + !authenticated && status === 429 + ); + expect(stdout).not.toHaveBeenCalled(); + } finally { + exit.mockRestore(); + stderr.mockRestore(); + stdout.mockRestore(); + } + } + ); + it('treats team opt-out as a disabled success', async () => { mockFetch.mockResolvedValue({ ok: false, @@ -199,7 +377,6 @@ describe('executeEndpointFeedback', () => { creditsRefunded: 0, }); - expect(getClient).not.toHaveBeenCalled(); expect(mockFetch).not.toHaveBeenCalled(); }); @@ -228,7 +405,6 @@ describe('executeEndpointFeedback', () => { expect(stderrSpy).not.toHaveBeenCalled(); expect(stdoutSpy).not.toHaveBeenCalled(); - expect(getClient).not.toHaveBeenCalled(); expect(mockFetch).not.toHaveBeenCalled(); } finally { exitSpy.mockRestore(); @@ -259,6 +435,26 @@ describe('feedback parsing', () => { }); }); +describe('keyless document class option', () => { + it.each(['born_digital', 'scanned', 'mixed', 'unknown'] as const)( + 'preserves %s for submission', + (docClass) => { + expect( + parseEndpointFeedbackCliOptions({ rating: 'partial', docClass }) + .docClass + ).toBe(docClass); + } + ); + it('rejects an unsupported class without changing authenticated defaults', () => { + expect(() => + parseEndpointFeedbackCliOptions({ rating: 'partial', docClass: 'pdf' }) + ).toThrow('--doc-class'); + expect( + parseEndpointFeedbackCliOptions({ rating: 'good' }).docClass + ).toBeUndefined(); + }); +}); + describe('parseAlexandriaFeedbackArray capability issues', () => { const base = { name: 'attachments', diff --git a/src/__tests__/commands/parse.test.ts b/src/__tests__/commands/parse.test.ts index 119a51124c..5de7d28070 100644 --- a/src/__tests__/commands/parse.test.ts +++ b/src/__tests__/commands/parse.test.ts @@ -20,6 +20,7 @@ describe('executeParse', () => { let mockFetch: ReturnType; beforeEach(() => { + vi.stubEnv('FIRECRAWL_API_KEY', ''); setupTest(); tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'firecrawl-parse-test-')); filePath = path.join(tmpDir, 'page.html'); @@ -38,6 +39,7 @@ describe('executeParse', () => { }); afterEach(() => { + vi.unstubAllEnvs(); vi.unstubAllGlobals(); fs.rmSync(tmpDir, { recursive: true, force: true }); teardownTest(); diff --git a/src/__tests__/commands/scrape.test.ts b/src/__tests__/commands/scrape.test.ts index b9c9606882..6a1642eb6e 100644 --- a/src/__tests__/commands/scrape.test.ts +++ b/src/__tests__/commands/scrape.test.ts @@ -44,6 +44,32 @@ describe('executeScrape', () => { vi.clearAllMocks(); }); + it('preserves keyless content and writes the API feedback invitation only to stderr', async () => { + const metadata = { jobId: 'job-1', feedback: { jobId: 'job-1' } }; + vi.mocked(isKeylessMode).mockReturnValue(true); + vi.mocked(keylessRequest).mockResolvedValue({ + success: true, + data: { markdown: 'Observed content', metadata }, + }); + const stdout = vi.spyOn(process.stdout, 'write').mockReturnValue(true); + const stderr = vi.spyOn(process.stderr, 'write').mockReturnValue(true); + try { + const result = await executeScrape({ url: 'https://example.com/' }); + expect(result).toMatchObject({ + success: true, + data: { markdown: 'Observed content', metadata }, + }); + expect(stdout).not.toHaveBeenCalled(); + expect(stderr.mock.calls.flat().join('')).toContain( + 'firecrawl feedback scrape job-1 --help' + ); + } finally { + vi.mocked(isKeylessMode).mockReturnValue(false); + stdout.mockRestore(); + stderr.mockRestore(); + } + }); + describe('API call generation', () => { it('should call scrape with correct URL and default markdown format', async () => { const mockResponse = { markdown: '# Test Content' }; diff --git a/src/__tests__/commands/search.test.ts b/src/__tests__/commands/search.test.ts index 246e102017..0daa8d3c59 100644 --- a/src/__tests__/commands/search.test.ts +++ b/src/__tests__/commands/search.test.ts @@ -9,6 +9,10 @@ import { initializeConfig } from '../../utils/config'; import { writeOutput } from '../../utils/output'; import { setupTest, teardownTest } from '../utils/mock-client'; +vi.mock('../../utils/credentials', () => ({ + loadCredentials: vi.fn(() => null), +})); + vi.mock('../../utils/output', () => ({ writeOutput: vi.fn() })); // Mock the Firecrawl client module @@ -822,6 +826,23 @@ describe('executeSearch', () => { }); }); + it.each([{ json: true }, { pretty: true }])( + 'preserves empty Search JSON and its feedback reference with %j', + async (format) => { + const metadata = { + jobId: '00000000-0000-4000-8000-000000000001', + feedback: { jobId: '00000000-0000-4000-8000-000000000001' }, + }; + mockHttpPost.mockResolvedValue( + mockSearchResponse({ web: [] }, { metadata }) + ); + await handleSearchCommand({ query: 'empty reference', ...format }); + expect( + JSON.parse(vi.mocked(writeOutput).mock.calls.at(-1)?.[0] as string) + ).toEqual({ success: true, data: { web: [] }, metadata }); + } + ); + describe('Developer results in the readable output', () => { // Read the text that `handleSearchCommand` sent to the writer. const writtenOutput = () => @@ -897,3 +918,53 @@ describe('executeSearch', () => { }); }); }); + +describe('keyless Search failure feedback', () => { + it.each([200, 500])( + 'prints the returned job and invitation when Search fails with HTTP %i', + async (status) => { + setupTest(); + vi.stubEnv('FIRECRAWL_API_KEY', ''); + initializeConfig({ + apiKey: undefined, + apiUrl: 'https://api.firecrawl.dev', + }); + const stderr = vi + .spyOn(process.stderr, 'write') + .mockImplementation(() => true); + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue({ + ok: status === 200, + status, + json: async () => ({ + success: false, + error: 'Search transport failed', + metadata: { + jobId: 'failed-search-job', + feedback: { + jobId: 'failed-search-job', + message: 'Optional feedback is available.', + }, + }, + }), + }) + ); + try { + const result = await executeSearch({ query: 'retry reference' }); + expect(result.success).toBe(false); + expect(result.error).toBe('Search transport failed'); + const printed = stderr.mock.calls.map((call) => call[0]).join(''); + expect(printed).toContain('Feedback job (search): failed-search-job'); + expect(printed).toContain( + 'firecrawl feedback search failed-search-job' + ); + } finally { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + stderr.mockRestore(); + teardownTest(); + } + } + ); +}); diff --git a/src/__tests__/utils/feedback-invitation.test.ts b/src/__tests__/utils/feedback-invitation.test.ts new file mode 100644 index 0000000000..f673df0cd5 --- /dev/null +++ b/src/__tests__/utils/feedback-invitation.test.ts @@ -0,0 +1,55 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { reportFeedbackInvitation } from '../../utils/feedback-invitation'; + +describe('feedback invitation output', () => { + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + }); + it('keeps content stdout unchanged and writes a short optional reminder to stderr', () => { + const stdout = vi.spyOn(process.stdout, 'write').mockReturnValue(true); + const stderr = vi.spyOn(process.stderr, 'write').mockReturnValue(true); + reportFeedbackInvitation( + { + jobId: 'job-1', + feedback: { jobId: 'job-1', message: 'Server feedback guidance.' }, + }, + 'parse' + ); + expect(stdout).not.toHaveBeenCalled(); + const printed = stderr.mock.calls.flat().join(''); + expect(printed).toBe( + 'Feedback job (parse): job-1\n' + + 'Consider submitting feedback, especially if this result is wrong, incomplete, blocked, or an error. Include specific evidence to help improve Firecrawl: firecrawl feedback parse job-1 --help\n' + ); + expect(printed).not.toContain('Server feedback guidance.'); + expect(printed).not.toContain('--observations-file'); + }); + it('retains keyless invitations despite authenticated feedback preferences', () => { + vi.stubEnv('FIRECRAWL_NO_ENDPOINT_FEEDBACK', 'true'); + const stderr = vi.spyOn(process.stderr, 'write').mockReturnValue(true); + reportFeedbackInvitation( + { + jobId: 'job-1', + feedback: { jobId: 'job-1', message: 'Optional feedback.' }, + }, + 'search' + ); + expect(stderr.mock.calls.flat().join('')).toContain( + 'firecrawl feedback search job-1' + ); + }); + it('prints only the job reference when the API did not issue an invitation', () => { + const stderr = vi.spyOn(process.stderr, 'write').mockReturnValue(true); + reportFeedbackInvitation({ jobId: 'job-1' }, 'parse'); + expect(stderr).toHaveBeenCalledExactlyOnceWith( + 'Feedback job (parse): job-1\n' + ); + }); + + it('does not invent invitations when metadata is absent', () => { + const stderr = vi.spyOn(process.stderr, 'write').mockReturnValue(true); + reportFeedbackInvitation(undefined, 'scrape'); + expect(stderr).not.toHaveBeenCalled(); + }); +}); diff --git a/src/commands/feedback.ts b/src/commands/feedback.ts index e137268bf4..0a0266a22b 100644 --- a/src/commands/feedback.ts +++ b/src/commands/feedback.ts @@ -1,7 +1,7 @@ +import { KEYLESS_CLI_HEADERS, isKeylessMode } from '../utils/client'; import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'; import { dirname } from 'path'; -import { getConfig, isCustomApiUrl, validateConfig } from '../utils/config'; -import { getClient } from '../utils/client'; +import { getConfig } from '../utils/config'; import { parseMissingContentArg, parseValuableSourcesArg, @@ -21,6 +21,10 @@ export interface EndpointFeedbackOptions { providerFeedback?: Record[]; capabilityFeedback?: Record[]; rating: SearchFeedbackRating; + task?: string; + assessment?: string; + docClass?: 'born_digital' | 'scanned' | 'mixed' | 'unknown'; + observations?: Record[]; issues?: string[]; tags?: string[]; note?: string; @@ -39,6 +43,7 @@ export interface EndpointFeedbackOptions { } export type EndpointFeedbackErrorCode = + | 'FEEDBACK_UNAVAILABLE' | 'JOB_NOT_FOUND' | 'SEARCH_NOT_FOUND' | 'FEEDBACK_WINDOW_EXPIRED' @@ -61,6 +66,8 @@ export interface EndpointFeedbackResult { error?: string; errorCode?: EndpointFeedbackErrorCode; status?: number; + details?: unknown; + retry_after_seconds?: number; disabled?: boolean; disabledSource?: 'env' | 'team'; } @@ -70,7 +77,14 @@ export const ENDPOINT_FEEDBACK_OPT_OUT_ENV_VARS = [ 'FIRECRAWL_DISABLE_ENDPOINT_FEEDBACK', ] as const; -const TRUTHY = new Set(['1', 'true', 'yes', 'on']); +export function isEndpointFeedbackDisabledLocally( + env: NodeJS.ProcessEnv = process.env +): boolean { + return ENDPOINT_FEEDBACK_OPT_OUT_ENV_VARS.some((key) => + /^(1|true|yes|on)$/i.test(env[key]?.trim() ?? '') + ); +} + const DEFAULT_API_URL = 'https://api.firecrawl.dev'; export const ENDPOINT_FEEDBACK_ENDPOINTS: EndpointFeedbackEndpoint[] = [ @@ -193,16 +207,32 @@ export function parseEndpointFeedbackRating( return rating as SearchFeedbackRating; } -export function isEndpointFeedbackDisabledLocally( - env: NodeJS.ProcessEnv = process.env -): boolean { - for (const key of ENDPOINT_FEEDBACK_OPT_OUT_ENV_VARS) { - const value = env[key]; - if (typeof value === 'string' && TRUTHY.has(value.trim().toLowerCase())) { - return true; - } +export function parseObservations( + raw?: string, + filePath?: string +): Record[] | undefined { + if (raw === undefined && filePath === undefined) return undefined; + if (raw !== undefined && filePath !== undefined) + throw new Error('Provide either --observations or --observations-file.'); + let value: unknown; + try { + value = JSON.parse(raw ?? readFileSync(filePath!, 'utf8')); + } catch (error) { + throw new Error( + `${filePath ? '--observations-file' : '--observations'}: ${error instanceof Error ? error.message : String(error)}` + ); } - return false; + if ( + !Array.isArray(value) || + value.length < 1 || + value.length > 20 || + value.some( + (item) => !item || typeof item !== 'object' || Array.isArray(item) + ) + ) { + throw new Error('Observations must be a JSON array of 1-20 objects.'); + } + return value; } export function parseEndpointFeedbackCliOptions(options: { @@ -214,8 +244,23 @@ export function parseEndpointFeedbackCliOptions(options: { valuableSources?: string; missingContent?: string | string[]; rating?: string; + observations?: string; + observationsFile?: string; + docClass?: string; }) { + if ( + options.docClass !== undefined && + !['born_digital', 'scanned', 'mixed', 'unknown'].includes(options.docClass) + ) + throw new Error( + '--doc-class must be one of: born_digital, scanned, mixed, unknown' + ); return { + docClass: options.docClass as EndpointFeedbackOptions['docClass'], + observations: parseObservations( + options.observations, + options.observationsFile + ), rating: parseEndpointFeedbackRating(String(options.rating || '')), issues: parseFeedbackListArg(options.issues, '--issues'), tags: parseFeedbackListArg(options.tags, '--tags'), @@ -229,28 +274,44 @@ export function parseEndpointFeedbackCliOptions(options: { export async function executeEndpointFeedback( options: EndpointFeedbackOptions ): Promise { - if (isEndpointFeedbackDisabledLocally()) { - return { - success: true, - disabled: true, - disabledSource: 'env', - creditsRefunded: 0, - }; - } - try { - if (options.apiKey || options.apiUrl) { - getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl }); - } - const config = getConfig(); const apiKey = options.apiKey || config.apiKey; + if (apiKey && isEndpointFeedbackDisabledLocally()) { + return { + success: true, + disabled: true, + disabledSource: 'env', + creditsRefunded: 0, + }; + } const apiUrl = (options.apiUrl || config.apiUrl || DEFAULT_API_URL).replace( /\/$/, '' ); - if (!isCustomApiUrl(apiUrl)) { - validateConfig(apiKey); + + if (isKeylessMode(apiKey, apiUrl)) { + if (!['search', 'scrape', 'parse'].includes(options.endpoint)) { + throw new Error( + 'Keyless feedback supports Search, Scrape, and Parse. Other endpoints require authentication.' + ); + } + const required = { + '--task': options.task, + '--assessment': options.assessment, + '--observations or --observations-file': options.observations, + ...(options.endpoint === 'parse' + ? { '--doc-class': options.docClass } + : {}), + }; + const missing = Object.entries(required) + .filter( + ([, value]) => + value === undefined || (typeof value === 'string' && !value.trim()) + ) + .map(([name]) => name); + if (missing.length) + throw new Error(`Keyless feedback requires ${missing.join(', ')}.`); } const body: Record = { @@ -277,6 +338,10 @@ export async function executeEndpointFeedback( ['issues', normalizeList(options.issues)], ['tags', normalizeList(options.tags)], ['note', options.note], + ['task', options.task], + ['assessment', options.assessment], + ['docClass', options.docClass], + ['observations', options.observations], ['valuableSources', options.valuableSources], ['missingContent', options.missingContent], ['querySuggestions', options.querySuggestions], @@ -295,7 +360,9 @@ export async function executeEndpointFeedback( const response = await fetch(`${apiUrl}/v2/feedback`, { method: 'POST', headers: { - ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}), + ...(apiKey + ? { Authorization: `Bearer ${apiKey}` } + : KEYLESS_CLI_HEADERS), 'Content-Type': 'application/json', }, body: JSON.stringify(body), @@ -330,6 +397,12 @@ export async function executeEndpointFeedback( error: errorMessage, errorCode, status: response.status, + ...(!apiKey + ? { + details: data.details, + retry_after_seconds: data.retry_after_seconds, + } + : {}), }; } @@ -411,6 +484,12 @@ export async function handleEndpointFeedbackCommand( if (result.errorCode) { console.error(`Code: ${result.errorCode}`); } + if (result.details !== undefined) { + console.error('Details:', JSON.stringify(result.details)); + } + if (typeof result.retry_after_seconds === 'number') { + console.error(`Retry after: ${result.retry_after_seconds} seconds.`); + } process.exit(1); } diff --git a/src/commands/parse.ts b/src/commands/parse.ts index 7e7e7ab8ba..4cfc6e6e8d 100644 --- a/src/commands/parse.ts +++ b/src/commands/parse.ts @@ -1,3 +1,4 @@ +import { reportFeedbackInvitation } from '../utils/feedback-invitation'; /** * Parse command implementation * @@ -200,6 +201,11 @@ export async function executeParse( const payload = (await response.json().catch(() => ({}))) as any; + if (keyless) + reportFeedbackInvitation( + payload?.data?.metadata ?? payload?.metadata, + 'parse' + ); if (!response.ok || payload?.success === false) { const message = payload?.error || diff --git a/src/commands/scrape.ts b/src/commands/scrape.ts index 5407275df6..43d6591604 100644 --- a/src/commands/scrape.ts +++ b/src/commands/scrape.ts @@ -1,3 +1,4 @@ +import { reportFeedbackInvitation } from '../utils/feedback-invitation'; /** * Scrape command implementation */ @@ -162,6 +163,7 @@ export async function executeScrape( ...scrapeParams, }); result = json?.data ?? json; + reportFeedbackInvitation(result?.metadata, 'scrape'); } else { const app = getClient({ apiKey: options.apiKey, diff --git a/src/commands/search.ts b/src/commands/search.ts index d13436ce49..f4ca8f9159 100644 --- a/src/commands/search.ts +++ b/src/commands/search.ts @@ -1,3 +1,4 @@ +import { reportFeedbackInvitation } from '../utils/feedback-invitation'; /** * Search command implementation */ @@ -120,6 +121,7 @@ export async function executeSearch( string, any >; + reportFeedbackInvitation(envelope.metadata, 'search'); } else { const app = getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl }); const httpResponse = await (app as any).http.post( @@ -146,6 +148,7 @@ export async function executeSearch( warning: envelope.warning, id: envelope.id, creditsUsed: envelope.creditsUsed, + metadata: envelope.metadata, }; } catch (error) { return { @@ -355,7 +358,7 @@ export async function handleSearchCommand( (result.data.news && result.data.news.length > 0) || (result.data.developer && result.data.developer.length > 0); - if (!hasResults && !(result.data.tools && (options.json || options.pretty))) { + if (!hasResults && !options.json && !options.pretty) { console.log('No results found.'); return; } @@ -373,6 +376,7 @@ export async function handleSearchCommand( if (result.warning) { jsonOutput.warning = result.warning; } + if (result.metadata) jsonOutput.metadata = result.metadata; if (result.id) { jsonOutput.id = result.id; } diff --git a/src/index.ts b/src/index.ts index aef4eb4bd0..575bff9a28 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,6 @@ #!/usr/bin/env node +import { KEYLESS_FEEDBACK_HELP } from './utils/feedback-invitation'; +import { isKeylessMode } from './utils/client'; /** * Firecrawl CLI @@ -341,7 +343,12 @@ program const commandName = actionCommand.name(); if (commandName === 'scrape') resolveScrapeTarget(actionCommand.args, commandOptions); - if (AUTH_REQUIRED_COMMANDS.includes(commandName)) { + const keylessFeedback = + commandName === 'feedback' && + ['search', 'scrape', 'parse'].includes( + actionCommand.args[0]?.toLowerCase() + ); + if (AUTH_REQUIRED_COMMANDS.includes(commandName) && !keylessFeedback) { // Skip auth for custom API URLs (e.g., local development) // Check both global and command-level options const { isCustomApiUrl } = await import('./utils/config'); @@ -437,6 +444,7 @@ function createScrapeCommand(): Command { .option('--actions-file ', 'Path to JSON actions file') .option('--proxy ', 'Proxy mode for scraping (e.g., auto, basic)') + .addHelpText('after', KEYLESS_FEEDBACK_HELP) .action(async (positionalArgs, options) => { const target = resolveScrapeTarget(positionalArgs ?? [], options); if (target.kind === 'alexandria') { @@ -879,6 +887,7 @@ Supported file types: .html, .htm, .pdf, .docx, .doc, .odt, .rtf, .xlsx, .xls Max upload size: 50 MB ` ) + .addHelpText('after', KEYLESS_FEEDBACK_HELP) .action(async (file: string, options) => { let format: string | undefined; if (options.html) { @@ -937,7 +946,7 @@ function createSearchCommand(): Command { ) .option( '--sources ', - 'Comma-separated sources: web, images, news, alexandria (default: web,alexandria; --sources web opts out of tools)' + 'Comma-separated sources: web, images, news, alexandria (keyless default: web; authenticated default: web,alexandria; --sources web opts out of tools)' ) .option( '--categories ', @@ -995,6 +1004,7 @@ function createSearchCommand(): Command { // false // ) .option('--json', 'Output as compact JSON', false) + .addHelpText('after', KEYLESS_FEEDBACK_HELP) .action(async (query, toolQuery, options) => { const alexandriaOnly = toolQuery !== undefined; if (alexandriaOnly && query !== 'alexandria') { @@ -1018,7 +1028,9 @@ function createSearchCommand(): Command { // Parse sources let sources: SearchSource[] = alexandriaOnly ? ['alexandria'] - : ['web', 'alexandria']; + : isKeylessMode(options.apiKey, options.apiUrl) + ? ['web'] + : ['web', 'alexandria']; if (options.sources) { sources = options.sources .split(',') @@ -1488,7 +1500,9 @@ function createSearchFeedbackCommand(): Command { */ function createFeedbackCommand(): Command { const cmd = new Command('feedback') - .description('Send feedback on a Firecrawl endpoint job.') + .description( + 'Submit evidence about a job. Consider submitting keyless Search, Scrape, or Parse feedback, especially if this result is wrong, incomplete, blocked, or an error. Include specific evidence to help improve Firecrawl.' + ) .argument('', 'Endpoint: search | scrape | parse | map') .argument('', 'The job id returned by the endpoint') .requiredOption('--rating ', 'Overall rating: good | bad | partial') @@ -1501,6 +1515,26 @@ function createFeedbackCommand(): Command { 'Comma-separated tags OR JSON array of tags' ) .option('--note ', 'Short note describing the feedback') + .option( + '--task ', + 'Task the output needed to support, required for keyless feedback' + ) + .option( + '--assessment ', + 'Meaningful assessment, required for keyless feedback' + ) + .option( + '--doc-class ', + 'Document class, required once for keyless Parse: born_digital | scanned | mixed | unknown' + ) + .option( + '--observations ', + 'JSON array of category-specific observations with kind, detail, and basis (output, source_comparison, or expectation)' + ) + .option( + '--observations-file ', + 'Read observations JSON from a file; use only evidence already available' + ) .option( '--valuable-sources ', 'Comma-separated URLs OR JSON array of {url, reason} entries' @@ -1535,6 +1569,51 @@ function createFeedbackCommand(): Command { 'Suppress output; useful when called in the background by another agent', false ) + .addHelpText( + 'after', + '\nKeyless evidence: task, assessment, and each observation detail must contain 10-2000 characters. Submit 1-20 observations.\n' + + 'Search: useful and irrelevant require a one-based position within the delivered group. source names the response group the position refers to: web, images, or news. It is required for multi-source jobs. Omission defaults to web, so images-only and news-only jobs must explicitly name their source. The position must exist in that requested group. irrelevant requires reason: aggregator_over_official, off_topic, stale, wrong_content_type, snippet_misleading, or blocked_or_paywalled. vertical is required on missing and optional on useful/irrelevant: web_general, social, business, research, developer, news, government, finance, or other. missing may include topic (up to 200 characters). missing and irrelevant may include knownSources (up to 20 HTTP(S) URLs): where absent content lives or the source that should have ranked instead. Unmentioned results are unassessed; a full ranking is not required. Do not submit engine attribution.\n' + + 'Scrape: kind correct, wrong_success, incomplete, or incorrect. wrong_success requires reason: blocked_shell, login_required, paywall, empty, wrong_page, stale, or wrong_locale. incomplete requires reason: partial_content, dynamic_content, pagination, main_content_stripped, or format_lost. incorrect requires reason: wrong, hallucinated, or missing_fields. correct has no reason. Optional location is up to 200 characters. No retryOutcome. hallucinated applies only to json, deterministicJson, summary, question, highlights, and changeTracking in json mode; missing_fields applies only to json and deterministicJson. For incomplete and incorrect, prefer source_comparison when the source is already available.\n' + + 'Parse: --doc-class is required once per submission: born_digital, scanned, mixed, or unknown. Observation kind: correct, text_ocr, table, formula, chart_figure, reading_order, headers_footers, headings_formatting, completeness, images_dropped, or incorrect. text_ocr requires reason: misread_chars, garbled, or missing_text. table requires reason: structure, cells_glued, or digits. completeness requires reason: pages_missing, truncated_at_max_pages, or sections_dropped. incorrect requires reason: wrong, hallucinated, or missing_fields. Other kinds have no reason subtype. Optional page is a one-based positive integer. incorrect applies to json and summary outputs. For text_ocr and table, include the correct text or cell values in comparison.detail when already known. Parse feedback does not automatically retain the document, extracted output, page images, or layout blocks; submitted observations and corrections are retained.\n' + + 'Scrape and Parse observations other than failure: format must be a format type the job requested. It is required for output and source_comparison observations when multiple formats were requested; optional for expectation observations and single-format jobs. All observations retain detail and basis; source_comparison requires comparison: {reference, detail}. comparison.detail contains the correct content from the inspected source.\n' + + 'Failed Search, Scrape, or Parse jobs: use kind failure with reason timeout, transport_error, proxy_error, or other. Accepted only for a failed job. Include detail and basis; do not supply position, source, format, location, or page. Parse still requires docClass (unknown is allowed).\n' + + 'If the saved Search response is unavailable, otherwise valid observations are accepted and stored with metadata.unverified: true because their positions could not be checked. Job ownership and requested sources are still checked. Available results must contain every referenced position.\n' + + 'Reason definitions:\n' + + '- aggregator_over_official: An intermediary was returned where the task needed an available official or primary source.\n' + + '- off_topic: The result addresses a different topic from the task.\n' + + '- stale: The content is outdated for the time or version the task requires.\n' + + '- wrong_content_type: The destination has the wrong content type for the task, such as a discussion instead of a reference.\n' + + '- snippet_misleading: The returned description misrepresents source content already inspected.\n' + + '- blocked_or_paywalled: Access to the destination was observed to be blocked or require a subscription; do not infer this from its URL or snippet.\n' + + '- blocked_shell: The successful response contains a bot challenge or access-blocking shell instead of the requested content.\n' + + '- login_required: The successful response contains a login requirement instead of the requested content.\n' + + '- paywall: The successful response contains a subscription barrier instead of the requested content.\n' + + '- empty: The successful response contains no meaningful requested content.\n' + + '- wrong_page: The successful response contains a different page or resource.\n' + + '- wrong_locale: The response uses the wrong language or region for the task.\n' + + '- partial_content: Only part of the expected content was returned, without a more specific known cause.\n' + + '- dynamic_content: Content loaded by client-side rendering or interaction is missing.\n' + + '- pagination: Expected content on additional pages is missing.\n' + + '- main_content_stripped: Content filtering removed requested primary content.\n' + + '- format_lost: Text is present, but meaningful structure such as headings, lists, or code formatting was lost.\n' + + '- wrong: Returned facts or values conflict with the inspected source.\n' + + '- hallucinated: The output asserts content unsupported by the inspected source.\n' + + '- missing_fields: Requested fields are absent from the structured output.\n' + + '- misread_chars: Characters were recognized incorrectly.\n' + + '- garbled: Extracted text is corrupted or unreadable.\n' + + '- missing_text: Visible source text was omitted.\n' + + '- structure: Table rows, columns, or header relationships were reconstructed incorrectly.\n' + + '- cells_glued: Distinct table cells were merged.\n' + + '- digits: Numeric table values were recognized incorrectly.\n' + + '- pages_missing: Source pages are absent from the output.\n' + + '- truncated_at_max_pages: Extraction ended at the configured page limit; this does not by itself imply a parser error.\n' + + '- sections_dropped: Sections within processed pages were omitted.\n' + + '- timeout: The operation explicitly reported a timeout.\n' + + '- transport_error: The operation explicitly reported a network, connection, or TLS failure.\n' + + '- proxy_error: The operation explicitly reported a proxy failure.\n' + + '- other: Another operation failure was reported; describe the returned error without guessing its cause.\n' + + 'Use only evidence already available. The stored keyless submission must fit within 8 KiB (8192 UTF-8 bytes), including server defaults and verification flags. Each job accepts one submission, and retrying returns the original feedback ID. Submission attempts are rate limited. Submit within 24 hours from the same caller IP. Contract and example: https://docs.firecrawl.dev/api-reference/endpoint/feedback.' + ) .action(async (endpointArg: string, jobId: string, options: any) => { let endpoint; try { @@ -1559,6 +1638,10 @@ function createFeedbackCommand(): Command { issues: parsed.issues, tags: parsed.tags, note: options.note, + task: options.task, + assessment: options.assessment, + docClass: parsed.docClass, + observations: parsed.observations, valuableSources: parsed.valuableSources, missingContent: parsed.missingContent, querySuggestions: options.querySuggestions, diff --git a/src/types/search.ts b/src/types/search.ts index ebf52aa5d8..b53d2ed5a4 100644 --- a/src/types/search.ts +++ b/src/types/search.ts @@ -128,6 +128,7 @@ export interface SearchResultData { } export interface SearchResult { + metadata?: Record; success: boolean; data?: SearchResultData; warning?: string; diff --git a/src/utils/client.ts b/src/utils/client.ts index d61d16e89a..0d85518d64 100644 --- a/src/utils/client.ts +++ b/src/utils/client.ts @@ -1,3 +1,4 @@ +import { reportFeedbackInvitation } from './feedback-invitation'; /** * Firecrawl client utility * Provides a singleton client instance initialized with global configuration @@ -51,7 +52,8 @@ export async function keylessRequest( body: JSON.stringify(body), }); const json: any = await response.json().catch(() => ({})); - if (!response.ok) { + if (!response.ok || json?.success === false) { + reportFeedbackInvitation(json?.metadata, path.split('/').pop()!); throw new Error( json?.error || `Firecrawl request failed (HTTP ${response.status})` ); diff --git a/src/utils/feedback-invitation.ts b/src/utils/feedback-invitation.ts new file mode 100644 index 0000000000..4963ba769b --- /dev/null +++ b/src/utils/feedback-invitation.ts @@ -0,0 +1,15 @@ +export const KEYLESS_FEEDBACK_HELP = + '\nConsider submitting feedback through firecrawl feedback, especially if a keyless result is wrong, incomplete, blocked, or an error. Include specific evidence to help improve Firecrawl. Run firecrawl feedback --help for submission fields. Invitations and job references appear in metadata or stderr. Feedback does not consume operation quota.'; + +export function reportFeedbackInvitation( + metadata: any, + endpoint: string +): void { + if (typeof metadata?.jobId !== 'string') return; + process.stderr.write(`Feedback job (${endpoint}): ${metadata.jobId}\n`); + if (metadata.feedback) { + process.stderr.write( + `Consider submitting feedback, especially if this result is wrong, incomplete, blocked, or an error. Include specific evidence to help improve Firecrawl: firecrawl feedback ${endpoint} ${metadata.jobId} --help\n` + ); + } +}