-
Notifications
You must be signed in to change notification settings - Fork 112
Add --objective to search-feedback #298
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,101 @@ | ||
| import { execFile } from 'node:child_process'; | ||
| import { createServer, type Server } from 'node:http'; | ||
| import { mkdtempSync, rmSync } from 'node:fs'; | ||
| import { tmpdir } from 'node:os'; | ||
| import { join } from 'node:path'; | ||
| import { promisify } from 'node:util'; | ||
| import { afterAll, beforeAll, beforeEach, expect, it } from 'vitest'; | ||
|
|
||
| const exec = promisify(execFile); | ||
| const requests: { url?: string; body: any }[] = []; | ||
| let server: Server; | ||
| let baseUrl: string; | ||
| const home = mkdtempSync(join(tmpdir(), 'search-feedback-cli-')); | ||
| const searchId = '00000000-0000-4000-8000-000000000001'; | ||
|
|
||
| beforeAll(async () => { | ||
| server = createServer(async (req, res) => { | ||
| let raw = ''; | ||
| for await (const chunk of req) raw += chunk; | ||
| requests.push({ url: req.url, body: raw ? JSON.parse(raw) : undefined }); | ||
| res.writeHead(200, { 'content-type': 'application/json' }); | ||
| res.end( | ||
| JSON.stringify({ success: true, feedbackId: 'f-1', creditsRefunded: 1 }) | ||
| ); | ||
| }); | ||
| await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve)); | ||
| baseUrl = `http://127.0.0.1:${(server.address() as { port: number }).port}`; | ||
| }); | ||
| afterAll(async () => { | ||
| await new Promise<void>((resolve, reject) => | ||
| server.close((error) => (error ? reject(error) : resolve())) | ||
| ); | ||
| rmSync(home, { recursive: true, force: true }); | ||
| }); | ||
| beforeEach(() => { | ||
| requests.length = 0; | ||
| }); | ||
|
|
||
| async function cli(args: string[]) { | ||
| try { | ||
| return { | ||
| code: 0, | ||
| ...(await exec(process.execPath, ['dist/index.js', ...args], { | ||
| timeout: 10000, | ||
| env: { | ||
| ...process.env, | ||
| HOME: home, | ||
| USERPROFILE: home, | ||
| FIRECRAWL_API_KEY: 'fc-test', | ||
| FIRECRAWL_API_URL: baseUrl, | ||
| FIRECRAWL_NO_UPDATE_CHECK: '1', | ||
| }, | ||
| })), | ||
| }; | ||
| } catch (error) { | ||
| return error as { code: number; stdout: string; stderr: string }; | ||
| } | ||
| } | ||
|
|
||
| const feedbackArgs = [ | ||
| 'search-feedback', | ||
| searchId, | ||
| '--rating', | ||
| 'bad', | ||
| '--missing-content', | ||
| 'Contract attachments', | ||
| '--objective', | ||
| ' Shortlist federal IT contracts to bid on this quarter ', | ||
| '--json', | ||
| ]; | ||
|
|
||
| it('sends the trimmed objective with search feedback', async () => { | ||
| const result = await cli(feedbackArgs); | ||
|
|
||
| expect(result.code).toBe(0); | ||
| expect(requests).toHaveLength(1); | ||
| expect(requests[0].url).toBe(`/v2/search/${searchId}/feedback`); | ||
| expect(requests[0].body).toMatchObject({ | ||
| rating: 'bad', | ||
| objective: 'Shortlist federal IT contracts to bid on this quarter', | ||
| }); | ||
| }); | ||
|
|
||
| it('still sends feedback without an objective', async () => { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: This test asserts that omitting Prompt for AI agents |
||
| const index = feedbackArgs.indexOf('--objective'); | ||
| const withoutObjective = feedbackArgs.filter( | ||
| (_, i) => i !== index && i !== index + 1 | ||
| ); | ||
|
|
||
| expect((await cli(withoutObjective)).code).toBe(0); | ||
| expect(requests).toHaveLength(1); | ||
| expect(requests[0].body).not.toHaveProperty('objective'); | ||
| }); | ||
|
|
||
| it('rejects a blank objective before sending feedback', async () => { | ||
| const blank = [...feedbackArgs]; | ||
| blank[feedbackArgs.indexOf('--objective') + 1] = ' '; | ||
|
|
||
| expect((await cli(blank)).code).not.toBe(0); | ||
| expect(requests).toHaveLength(0); | ||
| }); | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -5,7 +5,7 @@ | |
| * Entry point for the CLI application | ||
| */ | ||
|
|
||
| import { Command, Option } from 'commander'; | ||
| import { Command, InvalidArgumentError, Option } from 'commander'; | ||
| import { createSqlCommand } from './commands/sql'; | ||
| import { addFormatsAlias } from './utils/format-option'; | ||
| import { | ||
|
|
@@ -1401,6 +1401,13 @@ Examples: | |
| return researchCmd; | ||
| } | ||
|
|
||
| function searchFeedbackObjective(value: string): string { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P3: Prompt for AI agents |
||
| const text = value.trim(); | ||
| if (!text || text.length > 2000) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P3: The >2000 rejection branch of Prompt for AI agents |
||
| throw new InvalidArgumentError('Use 1–2000 characters.'); | ||
| return text; | ||
| } | ||
|
|
||
| /** | ||
| * Create the search-feedback command. Used by agents (CLI, MCP, skills) to | ||
| * report search-result quality after a `firecrawl search` call. The first | ||
|
|
@@ -1429,6 +1436,11 @@ function createSearchFeedbackCommand(): Command { | |
| '--query-suggestions <text>', | ||
| 'How the query or result set could be improved' | ||
| ) | ||
| .option( | ||
| '--objective <text>', | ||
| 'The underlying goal: what you or your user were ultimately trying to accomplish', | ||
| searchFeedbackObjective | ||
| ) | ||
| .option( | ||
| '-k, --api-key <key>', | ||
| 'Firecrawl API key (overrides global --api-key)' | ||
|
|
@@ -1471,6 +1483,7 @@ function createSearchFeedbackCommand(): Command { | |
| valuableSources, | ||
| missingContent, | ||
| querySuggestions: options.querySuggestions, | ||
| objective: options.objective, | ||
| apiKey: options.apiKey, | ||
| apiUrl: options.apiUrl, | ||
| output: options.output, | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
P3: The child process inherits
process.env, so a contributor withFIRECRAWL_NO_SEARCH_FEEDBACKorFIRECRAWL_DISABLE_SEARCH_FEEDBACKset locally gets the CLI exiting 0 without sending any request, and the first test fails onrequestsbeing empty (seeisSearchFeedbackDisabledLocallyin src/commands/search-feedback.ts). Neutralize both opt-out vars in the child env so the test is deterministic.Prompt for AI agents