diff --git a/docs/extensions/extensions-and-skills-guide.md b/docs/extensions/extensions-and-skills-guide.md index 9b8a21d51..0b28d11c1 100644 --- a/docs/extensions/extensions-and-skills-guide.md +++ b/docs/extensions/extensions-and-skills-guide.md @@ -107,6 +107,8 @@ Marketplace installs never ask you for a file. **Extensions** → **Browse Exten The **Add Extension** button beside it is the local route, and that one does take a file: drag a `.brxt` onto it, or browse for one. +Both marketplace browsers — **Browse Extensions**, and **Browse Skills** on the **Skills** page — search word by word: a phrase such as `R scripting ggplot visualization` lists every entry that matches any of its words, best match first, ranked the way the agent's own marketplace search ranks them. Clear the search box to browse the whole catalog again. + If an extension needs an API key, passcode or token, Biorouter asks for it in its own dialog and stores it in your operating system's credential store. **Never type a credential into the chat** — it cannot configure anything from there, and it would be visible to every model that reads the conversation. The same is true of the command line: `biorouter extension install` prompts with echo off rather than taking a value as an argument. See [Installing an extension, and where its credentials go](installing-an-extension.md). ### Developing a custom extension diff --git a/ui/desktop/src/components/baam/BrowseExtensionsModal.test.tsx b/ui/desktop/src/components/baam/BrowseExtensionsModal.test.tsx index 6c103749e..81b180d93 100644 --- a/ui/desktop/src/components/baam/BrowseExtensionsModal.test.tsx +++ b/ui/desktop/src/components/baam/BrowseExtensionsModal.test.tsx @@ -2,6 +2,7 @@ import { cleanup, render, screen, waitFor, within } from '@testing-library/react import userEvent from '@testing-library/user-event'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import BrowseExtensionsModal from './BrowseExtensionsModal'; +import { MARKETPLACE_EXTENSIONS } from './marketplace.fixture'; import type { BrxtEnvVar, BrxtManifest } from '../../types/brxt'; /** @@ -20,12 +21,12 @@ const loadRegistry = vi.hoisted(() => vi.fn()); // Spread the real module rather than listing members: a partial factory means // every export this component newly reaches for (`effectivePrivacy`, // `catalogFreshnessLine`) arrives `undefined` and the modal dies at render, in a -// test that has nothing to say about either. Only the two seams the test -// actually controls are replaced. +// test that has nothing to say about either. Only the seam the test actually +// controls is replaced; the search is the real one, and no test here types a +// query, so every entry is listed. vi.mock('./registry', async (importOriginal) => ({ ...(await importOriginal()), loadRegistry, - extensionMatches: () => true, })); vi.mock('../ConfigContext', () => ({ @@ -353,3 +354,37 @@ describe('BrowseExtensionsModal — installed rows (issue #116)', () => { expect(screen.queryByRole('button', { name: 'Configure' })).toBeNull(); }); }); + +/** The extension names the list shows, top to bottom. */ +function shownExtensionNames(): string[] { + return Array.from(document.querySelectorAll('div.biorouter-modal-row')).map( + (row) => row.querySelector('span')?.textContent ?? '' + ); +} + +/// Finding F5 in the extensions catalog: the phrase occurs in no field +/// verbatim, so the whole-phrase matcher this replaced listed nothing for it. +/// The ranking itself is pinned in `search.test.ts`; this pins that the list +/// on screen is the ranked one. +describe('BrowseExtensionsModal — a multi-word search (finding F5)', () => { + it('lists what the phrase matches, best match first', async () => { + loadRegistry.mockResolvedValue({ + live: true, + registry: { extensions: MARKETPLACE_EXTENSIONS, skills: [] }, + }); + const user = userEvent.setup(); + renderModal(); + + await screen.findByText('SPOKEAgent'); + expect(shownExtensionNames()).toEqual([ + 'CDWAgent', + 'SPOKEAgent', + 'CodeGraph Agent', + 'PrimeKGAgent', + ]); + + await user.type(screen.getByPlaceholderText(/Search extensions/), 'SPOKE knowledge graph'); + + expect(shownExtensionNames()).toEqual(['SPOKEAgent', 'PrimeKGAgent', 'CodeGraph Agent']); + }); +}); diff --git a/ui/desktop/src/components/baam/BrowseExtensionsModal.tsx b/ui/desktop/src/components/baam/BrowseExtensionsModal.tsx index 29c175501..cd1b2be2c 100644 --- a/ui/desktop/src/components/baam/BrowseExtensionsModal.tsx +++ b/ui/desktop/src/components/baam/BrowseExtensionsModal.tsx @@ -4,7 +4,7 @@ import { Package } from '../icons/app-icons'; import { BrxtInstallModal } from '../BrxtInstallModal'; import { loadRegistry, - extensionMatches, + rankExtensions, effectivePrivacy, catalogFreshnessLine, type BaamRegistry, @@ -73,9 +73,10 @@ export default function BrowseExtensionsModal({ const isInstalled = (e: RegistryExtension) => installedNames.has(e.name.toLowerCase()) || installedNames.has(e.id.toLowerCase()); + /** Best match first under a query; registry order when there is none. */ const filtered = useMemo(() => { if (!registry) return []; - return registry.extensions.filter((e) => extensionMatches(e, search)); + return rankExtensions(registry.extensions, search).hits.map((hit) => hit.entry); }, [registry, search]); /** diff --git a/ui/desktop/src/components/baam/BrowseSkillsModal.test.tsx b/ui/desktop/src/components/baam/BrowseSkillsModal.test.tsx index 0fd933c2d..b4a692550 100644 --- a/ui/desktop/src/components/baam/BrowseSkillsModal.test.tsx +++ b/ui/desktop/src/components/baam/BrowseSkillsModal.test.tsx @@ -1,5 +1,7 @@ import { render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { MARKETPLACE_SKILLS } from './marketplace.fixture'; const mocks = vi.hoisted(() => ({ loadRegistry: vi.fn() })); @@ -93,3 +95,119 @@ describe('BrowseSkillsModal — the install label has one space between words', expect(button.textContent).toBe('Install skills'); }); }); + +/** The skill names the list shows, top to bottom: one checkbox per row. */ +function shownSkillNames(): string[] { + return screen + .queryAllByRole('checkbox') + .map((box) => box.closest('label')?.querySelector('span')?.textContent ?? ''); +} + +async function openWithMarketplaceSkills() { + mocks.loadRegistry.mockResolvedValue({ + registry: { version: 2, source: 'test', extensions: [], skills: MARKETPLACE_SKILLS }, + live: true, + fetchedAt: '2026-09-10T00:00:00Z', + }); + const user = userEvent.setup(); + render(); + await screen.findByText('R Scripting'); + return { user, searchBox: screen.getByPlaceholderText(/Search skills/) }; +} + +/// Finding F5, in the desktop modal. The model-facing search (#242) and this one +/// were the same defect in two languages: the WHOLE query had to occur inside a +/// single field, so every word of `R scripting ggplot visualization` finds a +/// skill on its own and the phrase found none. +describe('BrowseSkillsModal — a multi-word search (finding F5)', () => { + it('shows the union of what each word finds, best match first', async () => { + const { user, searchBox } = await openWithMarketplaceSkills(); + + await user.type(searchBox, 'R scripting ggplot visualization'); + + expect(shownSkillNames()).toEqual([ + 'ggplot2 Visualization', + 'R Scripting', + 'Data Visualization', + 'Python Scripting', + 'Clinical Biostatistics', + ]); + }); + + /// The two single-word controls the QA run measured beside the phrase. + it('finds exactly the two ggplot skills for `ggplot`', async () => { + const { user, searchBox } = await openWithMarketplaceSkills(); + + await user.type(searchBox, 'ggplot'); + + expect(shownSkillNames()).toEqual(['ggplot2 Visualization', 'Data Visualization']); + }); + + it('puts the skill a query names first', async () => { + const { user, searchBox } = await openWithMarketplaceSkills(); + + await user.type(searchBox, 'r-scripting'); + + expect(shownSkillNames()[0]).toBe('R Scripting'); + }); +}); + +/** The section headings above the list, top to bottom. */ +function shownHeadings(): string[] { + return screen.queryAllByRole('heading', { level: 3 }).map((heading) => heading.textContent ?? ''); +} + +describe('BrowseSkillsModal — browsing is grouped, a search is ranked', () => { + it('groups the catalog under its category headings, in registry order', async () => { + await openWithMarketplaceSkills(); + + expect(shownHeadings()).toEqual(['Core skills (4)', 'Biomedical analysis (3)']); + expect(shownSkillNames()).toEqual([ + 'Scientific Visual Communication', + 'ggplot2 Visualization', + 'R Scripting', + 'Python Scripting', + 'Clinical Biostatistics', + 'Data Visualization', + 'Single-cell', + ]); + }); + + /// Under the category headings, Python Scripting (one term matched) would sit + /// above Data Visualization (two) only because Core is listed before + /// Biomedical — the ranking would be computed and then not shown. + it('shows a search as one list in rank order', async () => { + const { user, searchBox } = await openWithMarketplaceSkills(); + + await user.type(searchBox, 'R scripting ggplot visualization'); + + expect(shownHeadings()).toEqual(['Matches (5)']); + }); + + it('treats a query of only spaces as browsing', async () => { + const { user, searchBox } = await openWithMarketplaceSkills(); + + await user.type(searchBox, ' '); + + expect(shownHeadings()).toEqual(['Core skills (4)', 'Biomedical analysis (3)']); + }); + + it('keeps the category filter under a search', async () => { + const { user, searchBox } = await openWithMarketplaceSkills(); + + await user.click(screen.getByRole('button', { name: 'Biomedical analysis' })); + await user.type(searchBox, 'R scripting ggplot visualization'); + + expect(shownSkillNames()).toEqual(['Data Visualization', 'Clinical Biostatistics']); + }); + + it('says so when nothing matches', async () => { + const { user, searchBox } = await openWithMarketplaceSkills(); + + await user.type(searchBox, 'xylophone'); + + expect(shownSkillNames()).toEqual([]); + expect(shownHeadings()).toEqual([]); + expect(screen.getByText('No skills match your search.')).toBeInTheDocument(); + }); +}); diff --git a/ui/desktop/src/components/baam/BrowseSkillsModal.tsx b/ui/desktop/src/components/baam/BrowseSkillsModal.tsx index 84953acf8..80e8d23fd 100644 --- a/ui/desktop/src/components/baam/BrowseSkillsModal.tsx +++ b/ui/desktop/src/components/baam/BrowseSkillsModal.tsx @@ -3,12 +3,13 @@ import { Button } from '../ui/button'; import { toastSuccess, toastError } from '../../toasts'; import { loadRegistry, - skillMatches, + rankSkills, catalogFreshnessLine, type BaamRegistry, type RegistrySkill, type SkillCategory, } from './registry'; +import { isBrowseQuery } from './search'; import { installRegistrySkill } from './installSkill'; import { Dialog, DialogContent, DialogDescription, DialogTitle } from '../ui/dialog'; @@ -76,19 +77,28 @@ export default function BrowseSkillsModal({ onClose, onInstalled, installedIds } const isInstalled = (s: RegistrySkill) => installedIds.has(s.id.toLowerCase()) || installedIds.has(s.name.toLowerCase()); + /** Best match first under a query; registry order when there is none. */ const filtered = useMemo(() => { if (!registry) return []; - return registry.skills.filter( - (s) => (filter === 'All' || s.category === filter) && skillMatches(s, search) - ); + const inCategory = registry.skills.filter((s) => filter === 'All' || s.category === filter); + return rankSkills(inCategory, search).hits.map((hit) => hit.entry); }, [registry, filter, search]); - const grouped = useMemo(() => { - const map = new Map(); - for (const cat of CATEGORY_ORDER) map.set(cat, []); - for (const s of filtered) map.get(s.category)?.push(s); - return map; - }, [filtered]); + /** + * Browsing groups the catalog under its category headings. A search is one + * list in rank order instead: under the headings, a Core skill matching one + * word of the query would sit above a Biomedical skill matching all of them. + */ + const sections = useMemo(() => { + if (!isBrowseQuery(search)) { + return filtered.length > 0 ? [{ key: 'matches', label: 'Matches', items: filtered }] : []; + } + return CATEGORY_ORDER.map((cat) => ({ + key: cat, + label: CATEGORY_LABELS[cat], + items: filtered.filter((s) => s.category === cat), + })).filter((section) => section.items.length > 0); + }, [filtered, search]); const selectableFiltered = filtered.filter((s) => !isInstalled(s)); const allFilteredSelected = @@ -237,13 +247,11 @@ export default function BrowseSkillsModal({ onClose, onInstalled, installedIds }

)} {registry && - CATEGORY_ORDER.map((cat) => { - const items = grouped.get(cat) ?? []; - if (items.length === 0) return null; + sections.map(({ key, label, items }) => { return ( -
+

- {CATEGORY_LABELS[cat]} ({items.length}) + {label} ({items.length})

{items.map((skill) => { diff --git a/ui/desktop/src/components/baam/marketplace.fixture.ts b/ui/desktop/src/components/baam/marketplace.fixture.ts new file mode 100644 index 000000000..911c07abb --- /dev/null +++ b/ui/desktop/src/components/baam/marketplace.fixture.ts @@ -0,0 +1,191 @@ +// Shared test fixture, deliberately NOT a `.test.ts` file: importing one test +// file from another re-registers its suites in the importer. +import type { RegistryExtension, RegistrySkill } from './registry'; + +/** + * Seven skill rows copied VERBATIM from `landing/registry.json` at 7c96d796 — + * the registry the 2026-09-10 composer QA run measured finding F5 against — and + * kept in that document's order, because registry order is the tie-break a + * ranked search falls back to and the order an empty query browses in. + * + * Frozen here rather than read from `registry.fallback.json`, so the exact + * rankings the tests pin cannot move when the registry gains a skill. PR #242 + * pins the model-facing Rust matcher against the same seven rows, so the two + * searches can be compared row for row. + */ +export const MARKETPLACE_SKILLS: RegistrySkill[] = [ + { + id: 'scientific-visual-communication', + name: 'Scientific Visual Communication', + category: 'Core', + type: 'User-invocable · /scientific-visual-communication', + description: + 'Plans schematics, posters, slides, figure panels, infographics, visual abstracts, and source-to-visual traceability.', + tags: ['Visuals', 'Posters', 'Apache-2.0'], + keywords: [ + 'scientific-visual-communication', + 'schematics', + 'infographics', + 'posters', + 'slides', + 'visual', + 'abstracts', + 'apache', + ], + download: + 'https://github.com/BaranziniLab/biorouter-skills/releases/download/skill-scientific-visual-communication/scientific-visual-communication.zip', + filename: 'scientific-visual-communication.zip', + license: 'Apache-2.0', + }, + { + id: 'ggplot-visualization', + name: 'ggplot2 Visualization', + category: 'Core', + type: 'Auto-applied · R plotting', + description: 'Applies ggplot2 best-practice style when writing R plotting code.', + tags: ['R', 'ggplot2'], + keywords: [], + download: + 'https://github.com/BaranziniLab/biorouter-skills/releases/download/skill-ggplot-visualization/ggplot-visualization.zip', + filename: 'ggplot-visualization.zip', + license: 'Apache-2.0', + }, + { + id: 'r-scripting', + name: 'R Scripting', + category: 'Core', + type: 'Auto-applied · R code', + description: + 'Applies tidyverse conventions and documentation standards when writing or reviewing R code.', + tags: ['R', 'Tidyverse'], + keywords: [], + download: + 'https://github.com/BaranziniLab/biorouter-skills/releases/download/skill-r-scripting/r-scripting.zip', + filename: 'r-scripting.zip', + license: 'Apache-2.0', + }, + { + id: 'python-scripting', + name: 'Python Scripting', + category: 'Core', + type: 'Auto-applied · Python code', + description: + 'Applies Python naming, typing, error handling, and project structure conventions when writing Python code.', + tags: ['Python'], + keywords: [], + download: + 'https://github.com/BaranziniLab/biorouter-skills/releases/download/skill-python-scripting/python-scripting.zip', + filename: 'python-scripting.zip', + license: 'Apache-2.0', + }, + { + id: 'clinical-biostatistics', + name: 'Clinical Biostatistics', + category: 'Biomedical', + type: '6 skills · auto-applied', + description: 'Survival, mixed models, and clinical-trial statistical analysis.', + tags: ['survival', 'R', 'lme4'], + keywords: ['clinical-biostatistics', 'survival', 'r', 'lme4'], + download: + 'https://github.com/BaranziniLab/biorouter-skills/releases/download/skill-clinical-biostatistics/clinical-biostatistics.zip', + filename: 'clinical-biostatistics.zip', + license: 'Apache-2.0', + }, + { + id: 'data-visualization', + name: 'Data Visualization', + category: 'Biomedical', + type: '13 skills · auto-applied', + description: 'Publication-quality plots: heatmaps, volcano, Manhattan, dimplots.', + tags: ['ggplot2', 'matplotlib', 'ComplexHeatmap'], + keywords: ['data-visualization', 'ggplot2', 'matplotlib', 'complexheatmap'], + download: + 'https://github.com/BaranziniLab/biorouter-skills/releases/download/skill-data-visualization/data-visualization.zip', + filename: 'data-visualization.zip', + license: 'Apache-2.0', + }, + { + id: 'single-cell', + name: 'Single-cell', + category: 'Biomedical', + type: '14 skills · auto-applied', + description: 'scRNA-seq clustering, annotation, trajectory, and integration.', + tags: ['Scanpy', 'Seurat', 'scVI'], + keywords: ['single-cell', 'scanpy', 'seurat', 'scvi'], + download: + 'https://github.com/BaranziniLab/biorouter-skills/releases/download/skill-single-cell/single-cell.zip', + filename: 'single-cell.zip', + license: 'Apache-2.0', + }, +]; + +/** + * Four extension rows copied VERBATIM from `landing/registry.json` at 7c96d796, + * in that document's order: three about knowledge graphs, and one private row + * that is about none. + */ +export const MARKETPLACE_EXTENSIONS: RegistryExtension[] = [ + { + id: 'cdwagent', + name: 'CDWAgent', + organization: 'BaranziniLab · UCSF', + version: 'v0.5.1', + description: + 'Multimodal access to the UCSF Clinical Data Warehouse through natural language. One-call cohort building across diagnoses, medications, procedures, labs, radiology/imaging, immunizations, allergies, and vitals; clinical-notes/NLP search; read-only queries, schema discovery, and structured results. Requires UCSF network credentials (CAMPUS\\username and password).', + tags: ['UCSF', 'MCP', 'CDW', 'Clinical'], + github: 'https://github.com/BaranziniLab/CDWAgent', + download: + 'https://github.com/BaranziniLab/CDWAgent/releases/download/v0.5.1-brxt/cdwagent.brxt', + filename: 'cdwagent.brxt', + license: 'Apache-2.0', + privacy: 'private', + extension_name: 'cdwagent', + affiliation: ['ucsf'], + }, + { + id: 'spokeagent', + name: 'SPOKEAgent', + organization: 'BaranziniLab · UCSF', + version: 'v0.4.1', + description: + 'Structure-aware access to the SPOKE biomedical knowledge graph (43M nodes): live schema introspection, entity/identifier resolution, node profiling, shortest-path finding, and guarded read-only Cypher across diseases, genes, proteins, drugs, and pathways. Includes a bundled spoke-knowledge-graph skill. Requires a SPOKEAGENT_PASSCODE (see credentials page).', + tags: ['UCSF', 'MCP', 'Knowledge Graph'], + github: 'https://github.com/BaranziniLab/SPOKEAgent', + download: + 'https://github.com/BaranziniLab/SPOKEAgent/releases/download/v0.4.1/spokeagent-0.4.1.brxt', + filename: 'spokeagent-0.4.1.brxt', + license: 'Apache-2.0', + privacy: 'public', + extension_name: 'spokeagent', + }, + { + id: 'codegraphagent', + name: 'CodeGraph Agent', + organization: 'Broccolito · UCSF', + version: 'v0.1.0', + description: + 'Pre-indexed code knowledge graph. Ask "who calls X?", "what does Y call?", or "what breaks if I change Z?" across 23 languages including R, Julia, MATLAB, Perl. Vendored fork of CodeGraph on tree-sitter.', + tags: ['MCP', 'Code Intelligence', 'R', 'Tree-sitter'], + github: 'https://github.com/Broccolito/CodeGraphAgent', + download: + 'https://github.com/Broccolito/CodeGraphAgent/releases/download/v0.1.0/codegraphagent.brxt', + filename: 'codegraphagent.brxt', + license: 'Apache-2.0', + privacy: 'public', + }, + { + id: 'primekgagent', + name: 'PrimeKGAgent', + organization: 'BaranziniLab · BRXT', + version: 'v0.1.0', + description: + 'MCP adapter for PrimeKG-style biomedical knowledge graph files and APIs, with guarded metadata lookup and graph-resource summaries.', + tags: ['MCP', 'PrimeKG', 'Knowledge Graph', 'Biomedical', 'Apache-2.0'], + github: 'https://github.com/BaranziniLab/PrimeKGAgent', + download: + 'https://github.com/BaranziniLab/PrimeKGAgent/releases/download/v0.1.0-brxt/primekgagent.brxt', + filename: 'primekgagent.brxt', + license: 'Apache-2.0', + privacy: 'public', + }, +]; diff --git a/ui/desktop/src/components/baam/registry.ts b/ui/desktop/src/components/baam/registry.ts index 6d7f72330..d00e3ddb2 100644 --- a/ui/desktop/src/components/baam/registry.ts +++ b/ui/desktop/src/components/baam/registry.ts @@ -8,6 +8,14 @@ import { nameToKey } from '../settings/extensions/utils'; import { rememberPrivateExtensionKeys } from './privateSet'; import fallback from './registry.fallback.json'; import { classifyExtension } from '../settings/extensions/extensionPrivacy'; +import { + EXTENSION_NOISE, + rankEntries, + SKILL_NOISE, + Weight, + type SearchField, + type SearchResult, +} from './search'; export interface RegistryExtension { id: string; @@ -369,29 +377,56 @@ export function catalogFreshnessLine(load: { live: boolean; fetchedAt?: string } return `catalog last updated ${when.toLocaleDateString()}`; } -/** Case-insensitive match of a query against a skill's searchable fields. */ -export function skillMatches(skill: RegistrySkill, q: string): boolean { - if (!q) return true; - const needle = q.toLowerCase(); - return ( - skill.name.toLowerCase().includes(needle) || - skill.description.toLowerCase().includes(needle) || - skill.category.toLowerCase().includes(needle) || - skill.tags.some((t) => t.toLowerCase().includes(needle)) || - skill.keywords.some((t) => t.toLowerCase().includes(needle)) || - (skill.license?.toLowerCase().includes(needle) ?? false) - ); +/** + * Every label in a list, as a search field. Total, because an entry can omit + * any field — `isRegistryDocument` checks only that an entry is an object — and + * a search that throws in render takes the whole modal with it. + */ +function labelFields(labels: readonly string[] | undefined): SearchField[] { + return Array.isArray(labels) ? labels.map((label): SearchField => [label, Weight.Label]) : []; } -/** Case-insensitive match of a query against an extension's searchable fields. */ -export function extensionMatches(ext: RegistryExtension, q: string): boolean { - if (!q) return true; - const needle = q.toLowerCase(); - return ( - ext.name.toLowerCase().includes(needle) || - ext.description.toLowerCase().includes(needle) || - ext.organization.toLowerCase().includes(needle) || - ext.tags.some((t) => t.toLowerCase().includes(needle)) || - (ext.license?.toLowerCase().includes(needle) ?? false) - ); +/** + * Rank skills against what the user typed in Browse skills, best first; an + * empty query returns them all in registry order. How a query is matched — and + * why a phrase is a union of its words rather than one substring (finding F5) — + * is documented in `search.ts`. + * + * The fields are exactly the Rust catalog's (`MarketplaceCatalog::search_skills`), + * so this modal and the model's search tool agree. ⚠ That excludes the license, + * which the whole-phrase matcher searched: every skill and extension in the + * registry is Apache-2.0, so it separates nothing, and under word matching it + * made `PACS` list every skill — a plural's singular, `pac`, is inside `apache`. + */ +export function rankSkills( + skills: readonly RegistrySkill[], + query: string +): SearchResult { + return rankEntries(query, SKILL_NOISE, skills, (skill) => [ + [skill.id, Weight.Name], + [skill.name, Weight.Name], + [skill.category, Weight.Label], + [skill.description, Weight.Prose], + ...labelFields(skill.tags), + ...labelFields(skill.keywords), + ]); +} + +/** + * Rank extensions against what the user typed in Browse extensions, matched as + * {@link rankSkills} is, over exactly the Rust catalog's fields + * (`MarketplaceCatalog::search_extensions`). + */ +export function rankExtensions( + extensions: readonly RegistryExtension[], + query: string +): SearchResult { + return rankEntries(query, EXTENSION_NOISE, extensions, (ext) => [ + [ext.id, Weight.Name], + [ext.extension_name, Weight.Name], + [ext.name, Weight.Name], + [ext.organization, Weight.Label], + [ext.description, Weight.Prose], + ...labelFields(ext.tags), + ]); } diff --git a/ui/desktop/src/components/baam/search.test.ts b/ui/desktop/src/components/baam/search.test.ts new file mode 100644 index 000000000..b555f9784 --- /dev/null +++ b/ui/desktop/src/components/baam/search.test.ts @@ -0,0 +1,519 @@ +import { describe, expect, it } from 'vitest'; +import { MARKETPLACE_EXTENSIONS, MARKETPLACE_SKILLS } from './marketplace.fixture'; +import { rankExtensions, rankSkills, type RegistryExtension, type RegistrySkill } from './registry'; +import { + EXTENSION_NOISE, + isBrowseQuery, + parseQuery, + rankEntries, + scoreEntry, + searchTerms, + SKILL_NOISE, + Weight, + writtenIn, + type SearchField, + type SearchResult, +} from './search'; + +/** + * The marketplace matcher, ported from `crates/biorouter/src/catalog_search.rs` + * (PR #242; moved there from `marketplace/search.rs`, and given the word-boundary + * rule, by PR #266). The first half ports that file's own tests against the same + * three synthetic entries, so the two languages are pinned by one set of cases; + * the second half runs finding F5's queries against real registry rows. + */ + +interface Entry { + id: string; + name: string; + description: string; + tags: string[]; +} + +const ENTRIES: Entry[] = [ + { + id: 'complex-plots', + name: 'Complex Plots', + description: 'Draws annotated heat maps with the ComplexHeatmap package.', + tags: ['ComplexHeatmap'], + }, + { + id: 'prose-only', + name: 'Prose Only', + description: 'Mentions scripting in passing.', + tags: [], + }, + { + id: 'r-scripting', + name: 'R Scripting', + description: 'Tidyverse conventions for R code.', + tags: ['R'], + }, +]; + +function fields(entry: Entry): SearchField[] { + return [ + [entry.id, Weight.Name], + [entry.name, Weight.Name], + [entry.description, Weight.Prose], + ...entry.tags.map((tag): SearchField => [tag, Weight.Label]), + ]; +} + +function rank(query: string): SearchResult { + return rankEntries(query, [], ENTRIES, fields); +} + +function ids(search: SearchResult): string[] { + return search.hits.map((hit) => hit.entry.id); +} + +describe('marketplace search — reading a query', () => { + it('splits a query at whitespace and punctuation, lowercased and de-duplicated', () => { + expect(searchTerms('R scripting, ggplot/Visualization')).toEqual([ + 'r', + 'scripting', + 'ggplot', + 'visualization', + ]); + expect(searchTerms('r-scripting')).toEqual(['r', 'scripting']); + expect(searchTerms('ggplot2 ggplot2')).toEqual(['ggplot2']); + }); + + /// A letter is a letter in any script, as Rust's `char::is_alphanumeric` + /// has it: an `\w`-style ASCII class would cut `naïve` in two. + it('keeps letters and digits of any script inside a word', () => { + expect(searchTerms('scRNA-seq, naïve B-cells')).toEqual([ + 'scrna', + 'seq', + 'naïve', + 'b', + 'cells', + ]); + }); + + it('drops filler words unless they are all there is', () => { + expect(searchTerms('a skill about R scripting or ggplot', SKILL_NOISE)).toEqual([ + 'r', + 'scripting', + 'ggplot', + ]); + expect(searchTerms('I need something for R', SKILL_NOISE)).toEqual(['something', 'r']); + expect(searchTerms('skills', SKILL_NOISE)).toEqual(['skills']); + expect(searchTerms('the', SKILL_NOISE)).toEqual(['the']); + // A catalog's own noise words are its own. + expect(searchTerms('skills', EXTENSION_NOISE)).toEqual(['skills']); + expect(searchTerms('an extension for skills', EXTENSION_NOISE)).toEqual(['skills']); + }); + + it('treats an empty or all-whitespace query as browsing', () => { + expect(isBrowseQuery('')).toBe(true); + expect(isBrowseQuery(' \t ')).toBe(true); + expect(isBrowseQuery(' r ')).toBe(false); + }); +}); + +describe('marketplace search — matching and ranking', () => { + /// `r` as a substring is in nearly every word of prose; as a term it must + /// mean the R language. + it('matches a term under three characters to whole words only', () => { + const search = rank('R scripting'); + const prose = search.hits.find((hit) => hit.entry.id === 'prose-only'); + // `scripting` is in its description, and the `r` inside it is not the R language. + expect(prose?.matchedTerms).toEqual(['scripting']); + // Nor is the `r` inside `Draws`. + expect(ids(search)).not.toContain('complex-plots'); + }); + + it('matches a longer term inside a word, and a plural by its singular', () => { + // `heatmap` inside `complexheatmap`. + expect(ids(rank('heatmap'))).toEqual(['complex-plots']); + // No field says `heatmaps`; its singular is inside `complexheatmap`. + expect(ids(rank('heatmaps'))).toEqual(['complex-plots']); + // `scripts` is not in `scripting`, but `script` starts it. + expect(ids(rank('scripts'))).toEqual(['r-scripting', 'prose-only']); + }); + + it('returns the union, ranked by terms matched and then by where they matched', () => { + const search = rank('R scripting'); + expect(ids(search)).toEqual(['r-scripting', 'prose-only']); + expect(search.hits[0].matchedTerms).toEqual(['r', 'scripting']); + expect(search.hits[1].matchedTerms).toEqual(['scripting']); + + // A match in the name outranks the same match in the description. + expect(ids(rank('scripting'))).toEqual(['r-scripting', 'prose-only']); + }); + + /// The query as written is held to the same edges as a short term. Until PR + /// #266 the whole-query check was a plain substring test, so a query that IS + /// one short term came back in through every word containing it: `r` alone + /// found `complex-plots` through "Draws" and `prose-only` through its own + /// name, each with no matched term at all. + it('reads a one-letter query as a whole word, not a letter inside one', () => { + const search = rank('R'); + expect(ids(search)).toEqual(['r-scripting']); + expect(search.hits[0].matchedTerms).toEqual(['r']); + }); + + /// The query as written outranks any count of separate words, so it has to be + /// written there: `r scripting` inside "for scripting" is the tail of `for` + /// and then a word. Read as a substring it ranked the entry matching one of + /// the two words above the entry matching both. + it('does not read a phrase found only inside longer words as written', () => { + const entries: Entry[] = [ + { + id: 'shell-snippets', + name: 'Shell Snippets', + description: 'Snippets for scripting the shell.', + tags: [], + }, + { + id: 'tidy-style', + name: 'Tidy Style', + description: 'Scripting conventions for R.', + tags: ['R'], + }, + ]; + const search = rankEntries('R scripting', [], entries, fields); + + expect(ids(search)).toEqual(['tidy-style', 'shell-snippets']); + expect(search.hits[0].matchedTerms).toEqual(['r', 'scripting']); + expect(search.hits[1].matchedTerms).toEqual(['scripting']); + }); + + /// A fragment of a word is not the query as written, so a query of nothing but + /// short terms finds nothing at all — `dy` is inside "Tidyverse", not a word of + /// it. The `s p` half held under the substring rule; the `dy` half is the leak + /// that rule asserted as a feature. + it('finds nothing for a query written only inside longer words', () => { + expect(rank('dy').hits).toEqual([]); + expect(rank('s p').hits).toEqual([]); + }); + + it('browses every entry in registry order for an empty query', () => { + const search = rank(' '); + expect(search.terms).toEqual([]); + expect(ids(search)).toEqual(['complex-plots', 'prose-only', 'r-scripting']); + }); + + /// Like the matcher this replaced, so an entry too malformed to read a field + /// from is still listed while the user is only browsing. + it('reads no field at all to browse', () => { + const unreadable = (): SearchField[] => { + throw new Error('a field was read'); + }; + expect(ids(rankEntries('', [], ENTRIES, unreadable))).toEqual([ + 'complex-plots', + 'prose-only', + 'r-scripting', + ]); + }); +}); + +/// The ranking is packed into one number, and the packing is where a subtle +/// defect would hide: a scale too small lets a better placement leak into the +/// tier above it. Each case pits a stronger lower criterion against a weaker +/// higher one. +describe('marketplace search — the score', () => { + it('is 0 for an entry that matches nothing, with no terms matched', () => { + expect(scoreEntry(parseQuery('ggplot'), [['Data Visualization', Weight.Name]])).toEqual({ + score: 0, + matchedTerms: [], + }); + }); + + it('ranks more terms matched above better placement', () => { + const query = parseQuery('heatmap volcano'); + const oneTermInTheName = scoreEntry(query, [['Heatmap', Weight.Name]]); + const twoTermsInProse = scoreEntry(query, [['complexheatmap and volcanoes', Weight.Prose]]); + + expect(oneTermInTheName.matchedTerms).toEqual(['heatmap']); + expect(twoTermsInProse.matchedTerms).toEqual(['heatmap', 'volcano']); + expect(twoTermsInProse.score).toBeGreaterThan(oneTermInTheName.score); + }); + + /// The query as written is a tier of its own, not one more term. It can no + /// longer be pitted against a HIGHER term count, as the substring rule allowed + /// — a phrase written in a field has every one of its words in that field as a + /// whole word, so it always matches every term — so the weaker criterion it is + /// pitted against here is placement: prose against two names. + it('ranks the query as written above the same words scattered in weightier fields', () => { + const query = parseQuery('tidy code'); + const writtenInProse = scoreEntry(query, [['Writes tidy code.', Weight.Prose]]); + const scatteredInNames = scoreEntry(query, [ + ['Code Tidy', Weight.Name], + ['code-tidy', Weight.Name], + ]); + + expect(writtenInProse.matchedTerms).toEqual(['tidy', 'code']); + expect(scatteredInNames.matchedTerms).toEqual(['tidy', 'code']); + expect(writtenInProse.score).toBeGreaterThan(scatteredInNames.score); + + // The same pair ranked, as `catalog_search.rs` asserts it. + const entries: Entry[] = [ + { id: 'code-tidy', name: 'Code Tidy', description: 'Formatting rules.', tags: [] }, + { id: 'styler', name: 'Styler', description: 'Writes tidy code.', tags: [] }, + ]; + expect(ids(rankEntries('tidy code', [], entries, fields))).toEqual(['styler', 'code-tidy']); + }); + + it('skips a field with no text rather than throwing', () => { + expect( + scoreEntry(parseQuery('ggplot'), [ + [undefined, Weight.Label], + ['ggplot2', Weight.Label], + ]).matchedTerms + ).toEqual(['ggplot']); + }); +}); + +/// Finding F5, measured by the 2026-09-10 composer QA run against the live +/// registry and reproduced here against seven of its rows. Before this port, +/// the desktop modal showed no skill for the phrase, and none for `r-scripting` +/// either: the id was not a searched field. +describe('rankSkills — the QA queries (finding F5)', () => { + it('returns the union for `R scripting ggplot visualization`, ggplot-visualization first', () => { + const search = rankSkills(MARKETPLACE_SKILLS, 'R scripting ggplot visualization'); + + expect(search.terms).toEqual(['r', 'scripting', 'ggplot', 'visualization']); + // Three of the four terms, then two, then one. + expect(ids(search)).toEqual([ + 'ggplot-visualization', + 'r-scripting', + 'data-visualization', + 'python-scripting', + 'clinical-biostatistics', + ]); + expect(search.hits[0].matchedTerms).toEqual(['r', 'ggplot', 'visualization']); + // `visual` is not `visualization`: a long term must be found in the entry, + // not the other way round. + expect(ids(search)).not.toContain('scientific-visual-communication'); + }); + + it('finds exactly the two ggplot skills for `ggplot`', () => { + expect(ids(rankSkills(MARKETPLACE_SKILLS, 'ggplot'))).toEqual([ + 'ggplot-visualization', + 'data-visualization', + ]); + }); + + it('puts the skill `r-scripting` names first', () => { + const search = rankSkills(MARKETPLACE_SKILLS, 'r-scripting'); + + expect(search.hits[0].entry.id).toBe('r-scripting'); + expect(search.hits[0].matchedTerms).toEqual(['r', 'scripting']); + // Then one term each: in a name before in a tag, and a tie in registry order. + expect(ids(search)).toEqual([ + 'r-scripting', + 'python-scripting', + 'ggplot-visualization', + 'clinical-biostatistics', + ]); + }); +}); + +describe('rankSkills — the rules that keep a union useful', () => { + /// Without the filler rule `and` is a term, and it is a whole word in the + /// description of every skill below that matches nothing else: single-cell, + /// python-scripting, scientific-visual-communication. + it('drops filler, so a skill that only says `and` is not a hit', () => { + const search = rankSkills(MARKETPLACE_SKILLS, 'skills for R and ggplot'); + + expect(search.terms).toEqual(['r', 'ggplot']); + expect(ids(search)).toEqual([ + 'ggplot-visualization', + 'r-scripting', + 'clinical-biostatistics', + 'data-visualization', + ]); + }); + + it('reads `R` in a phrase as the R language, not the r inside `writing`', () => { + // python-scripting's description says `error`, `structure` and `writing`. + expect(ids(rankSkills(MARKETPLACE_SKILLS, 'R ggplot'))).toEqual([ + 'ggplot-visualization', + 'r-scripting', + 'clinical-biostatistics', + 'data-visualization', + ]); + }); + + /// A one-letter query is the letter as a word: the three skills that are about + /// R, and nothing that merely contains an r. Until PR #266 this returned six — + /// these three, then scientific-visual-communication, python-scripting and + /// single-cell, each matching no term at all and kept only by the substring + /// test. Measured against the live registry rather than this fixture, that rule + /// returned 125 of 129 skills, 117 of them with no matched term. + it('finds only the R skills for `R` alone, not what merely contains an r', () => { + const search = rankSkills(MARKETPLACE_SKILLS, 'R'); + + expect(ids(search)).toEqual(['r-scripting', 'ggplot-visualization', 'clinical-biostatistics']); + expect(search.hits.map((hit) => hit.matchedTerms)).toEqual([['r'], ['r'], ['r']]); + }); + + it('finds a singular field word for a plural term', () => { + // No field says `visualizations`, and `visuals` is not a visualization. + expect(ids(rankSkills(MARKETPLACE_SKILLS, 'visualizations'))).toEqual([ + 'ggplot-visualization', + 'data-visualization', + ]); + expect(ids(rankSkills(MARKETPLACE_SKILLS, 'scripts'))).toEqual([ + 'r-scripting', + 'python-scripting', + ]); + }); + + it('browses every skill in registry order for an empty query', () => { + expect(ids(rankSkills(MARKETPLACE_SKILLS, ''))).toEqual( + MARKETPLACE_SKILLS.map((skill) => skill.id) + ); + }); + + /// `isRegistryDocument` admits any object as an entry, so a cached v1 or + /// hand-edited document can omit fields. The ranker must not be what throws on + /// one: browsing reads no field at all, and a search skips what is missing. + /// The matcher this replaced threw on the missing `name` the moment a query + /// was typed. + it('lists a malformed entry and searches past it without throwing', () => { + const malformed = { id: 'r-scripting' } as unknown as RegistrySkill; + + expect(ids(rankSkills([malformed], ''))).toEqual(['r-scripting']); + expect(ids(rankSkills([malformed], 'R scripting'))).toEqual(['r-scripting']); + }); +}); + +/// Which fields a catalog searches is part of the port. In the registry's own +/// rows, keywords repeat the id and the tags, so a field dropped from the list +/// changes no ranking the QA queries pin; each field is asserted on its own +/// instead, carrying a word no other field holds. +describe('the fields each catalog searches, and what a match in each is worth', () => { + const blankSkill: RegistrySkill = { + id: 'blank', + name: 'Blank', + category: 'Core', + type: '', + description: '', + tags: [], + keywords: [], + download: '', + filename: '', + }; + const blankExtension: RegistryExtension = { + id: 'blank', + name: 'Blank', + organization: '', + version: '', + description: '', + tags: [], + github: '', + download: '', + filename: '', + }; + + it.each<[string, Partial, string]>([ + // The matcher this replaced never searched the id. + ['id', { id: 'zebrafish-imaging' }, 'zebrafish'], + ['name', { name: 'Zebrafish Imaging' }, 'zebrafish'], + ['category', { category: 'Biomedical' }, 'biomedical'], + ['description', { description: 'Segments zebrafish embryos.' }, 'zebrafish'], + ['tag', { tags: ['Zebrafish'] }, 'zebrafish'], + ['keyword', { keywords: ['zebrafish'] }, 'zebrafish'], + ])('finds a skill by its %s', (_field, override, query) => { + expect(rankSkills([{ ...blankSkill, ...override }], query).hits).toHaveLength(1); + }); + + /// The one field the whole-phrase matcher searched and the Rust catalog does + /// not. Every registry entry is Apache-2.0, so it separates nothing — and + /// under word matching it listed every skill for `PACS`, whose singular `pac` + /// is inside `apache`. + it('does not search the license, in either catalog', () => { + expect(rankSkills([{ ...blankSkill, license: 'Apache-2.0' }], 'PACS').hits).toEqual([]); + expect(rankExtensions([{ ...blankExtension, license: 'Apache-2.0' }], 'PACS').hits).toEqual([]); + }); + + it('ranks a skill matched by id or name above a label, and a label above prose', () => { + const skills: RegistrySkill[] = [ + { ...blankSkill, id: 'in-the-description', description: 'Segments zebrafish embryos.' }, + { ...blankSkill, id: 'in-a-keyword', keywords: ['zebrafish'] }, + { ...blankSkill, id: 'in-a-tag', tags: ['Zebrafish'] }, + { ...blankSkill, id: 'in-the-name', name: 'Zebrafish Imaging' }, + ]; + + expect(ids(rankSkills(skills, 'zebrafish'))).toEqual([ + 'in-the-name', + // Two labels tie, and a tie keeps registry order. + 'in-a-keyword', + 'in-a-tag', + 'in-the-description', + ]); + }); + + it.each<[string, Partial]>([ + ['id', { id: 'zebrafish-0.1.0' }], + // The installed config name, which can differ from the id and the display name. + ['extension name', { extension_name: 'zebrafishagent' }], + ['name', { name: 'Zebrafish Agent' }], + ['organization', { organization: 'Zebrafish Lab' }], + ['description', { description: 'Segments zebrafish embryos.' }], + ['tag', { tags: ['Zebrafish'] }], + ])('finds an extension by its %s', (_field, override) => { + expect(rankExtensions([{ ...blankExtension, ...override }], 'zebrafish').hits).toHaveLength(1); + }); +}); + +/** + * The word-boundary test the whole-query rank runs on, ported case for case from + * `written_in` in `catalog_search.rs`, and then pushed at the characters where + * JavaScript's own `\b` disagrees with Rust's `char::is_alphanumeric`. Those + * cases are why this is a hand-written scan over code points and not a regular + * expression — and every one of them is invisible to an ASCII fixture. + */ +describe('marketplace search — the query as written', () => { + it('reads a phrase as written only between word boundaries', () => { + expect(writtenIn('r scripting', 'r scripting')).toBe(true); + // The second `r`, the one that is a word of its own. + expect(writtenIn('tidy code for r.', 'r')).toBe(true); + expect(writtenIn('snippets for scripting', 'r scripting')).toBe(false); + expect(writtenIn('tidyverse', 'dy')).toBe(false); + // Written at the second `a`, which overlaps the refused first occurrence — + // so every position is tried, not only the first one a search would find. + expect(writtenIn('ba a a', 'a a')).toBe(true); + // An edge that is not a letter or digit is a boundary itself. + expect(writtenIn('c++ code', '++')).toBe(true); + }); + + it('reads a word character as Rust does, not as `\b` does', () => { + // `_` is a boundary here and a word character to `\b`, which would refuse + // this. + expect(writtenIn('snake_case', 'case')).toBe(true); + // `ï` is a word character here and a boundary to `\b`, which would accept + // this. + expect(writtenIn('naïve', 'naï')).toBe(false); + expect(writtenIn('naïve bayes', 'naïve')).toBe(true); + // A letter outside the BMP is ONE character, so it closes a word. Compared + // as UTF-16 units its trailing half is a lone surrogate, which matches no + // letter, and the phrase would be read as written. + expect(writtenIn('𝐚rna', 'rna')).toBe(false); + // A digit is a word character, the same way `words` keeps `ggplot2` whole. + expect(writtenIn('ggplot2 plots', 'ggplot')).toBe(false); + }); +}); + +describe('rankExtensions — the same matcher over the extensions catalog', () => { + /// No field holds the phrase as written — SPOKEAgent says "SPOKE biomedical + /// knowledge graph" and "spoke-knowledge-graph" — so the matcher this replaced + /// showed nothing for it. + it('ranks a multi-word query by the terms each extension matches', () => { + const search = rankExtensions(MARKETPLACE_EXTENSIONS, 'SPOKE knowledge graph'); + + expect(ids(search)).toEqual(['spokeagent', 'primekgagent', 'codegraphagent']); + expect(search.hits[0].matchedTerms).toEqual(['spoke', 'knowledge', 'graph']); + }); + + it('drops its own catalog noise: every entry is an extension', () => { + expect(rankExtensions(MARKETPLACE_EXTENSIONS, 'a knowledge graph extension').terms).toEqual([ + 'knowledge', + 'graph', + ]); + }); +}); diff --git a/ui/desktop/src/components/baam/search.ts b/ui/desktop/src/components/baam/search.ts new file mode 100644 index 000000000..834c6530f --- /dev/null +++ b/ui/desktop/src/components/baam/search.ts @@ -0,0 +1,360 @@ +/** + * Free-text search over the marketplace catalog — the matcher behind the Browse + * skills and Browse extensions modals. + * + * A port of the model-facing matcher, `crates/biorouter/src/catalog_search.rs` + * (PR #242, moved there from `marketplace/search.rs` by PR #266 when the + * installed-skill search became its second caller), and it has to stay one: a + * user typing into the modal and a model calling + * `skills__searchMarketplaceSkills` on that user's behalf read the same catalog, + * and the same words must find the same entries, ranked the same way. Only a tie + * can fall differently, because each side breaks ties by its own registry order — + * the document's here, the id's in Rust. **A change to a rule below is a change + * to both files**, which is how the word-boundary rule PR #266 added arrived + * here; the types it returns are `CatalogSearch` / `CatalogSearchHit` there and + * {@link SearchResult} / {@link SearchHit} here. + * + * ⚠ **A query is a set of words, not a substring.** The matcher this replaced + * asked whether the WHOLE lowercased query occurred inside a single field — the + * defect the 2026-09-10 composer QA run measured as finding F5, present here in + * TypeScript as well as in Rust. Measured in this modal against seven rows of + * `landing/registry.json`: `R scripting ggplot visualization` showed no skill + * at all while `ggplot` alone showed two, and `r-scripting` — a skill's own id — + * showed none, because the id was not searched. + * + * So a query is split into terms and an entry is a hit when it matches ANY of + * them. The union is deliberate: no single entry has to contain every word a + * user happened to type, and an AND over a phrase is the same empty list with a + * different cause. Precision comes from the ranking instead, best first: + * + * 1. an entry holding the query **as written** — its words, in that order, as + * whole words, so `r scripting` is in "R scripting" but not in "for + * scripting" — which no scatter of the same words outranks; + * 2. then by **how many terms** it matched, so an entry matching every term + * precedes one matching some; + * 3. then by **where** each term matched — the id or name outweighs a tag, + * which outweighs the description — and how exactly (the whole word, the + * start of one, or inside one); + * 4. then registry order, so a result never reshuffles. + * + * Two rules keep the union from drowning the useful hits, both needed by the + * measured query itself: + * + * - **A term under three characters matches whole words only.** `r` has to find + * the R language; as a substring it matched nearly every entry. The query as + * written is held to the same edges (see {@link writtenIn}) — it counts only + * where it starts and ends at a word boundary. Tested as a plain substring, + * which is what this file did until PR #266, a query that IS one short term + * came back in through every word containing it: measured over the live + * `landing/registry.json`, `R` returned 125 of 129 skills, 117 of them + * matching no term at all, and ranked `empirical-paper-submission-rr` above + * `r-scripting`. Refusing that leak drops 198 hits across 76 searches and adds + * none, and only a query whose every term is short changes at all. + * - **Filler words are dropped** ("a skill about R" is `r`), because in a union + * a word like `for` or `and` inflates the term count of every entry whose + * prose happens to use it, which ranks noise above the real hit. + */ + +/** How much a match in one field says about an entry. */ +export const Weight = { + /** Free prose: a description. */ + Prose: 1, + /** Curated labels: tags, keywords, a category, an organization. */ + Label: 2, + /** What the entry is called: its registry id and names. */ + Name: 3, +} as const; + +/** One of the {@link Weight}s. */ +export type FieldWeight = (typeof Weight)[keyof typeof Weight]; + +/** One searchable field of an entry, and what a match there counts for. */ +export type SearchField = readonly [text: string | undefined, weight: FieldWeight]; + +/** + * Words that say how a request is phrased, not what it is for. Dropped from a + * query unless nothing else is left, so a query made only of them (`agent`, + * `or`) still searches for what it says. The Rust list, word for word. + */ +const FILLER: ReadonlySet = new Set([ + 'a', + 'about', + 'an', + 'and', + 'any', + 'are', + 'baam', + 'be', + 'by', + 'can', + 'do', + 'does', + 'find', + 'for', + 'from', + 'help', + 'how', + 'i', + 'in', + 'into', + 'is', + 'it', + 'its', + 'looking', + 'marketplace', + 'me', + 'my', + 'need', + 'of', + 'on', + 'or', + 'please', + 'search', + 'some', + 'that', + 'the', + 'this', + 'to', + 'use', + 'using', + 'via', + 'want', + 'what', + 'which', + 'with', +]); + +/** Filler specific to the skills catalog: every entry in it is a skill. */ +export const SKILL_NOISE: readonly string[] = ['skill', 'skills']; + +/** Filler specific to the extensions catalog: every entry in it is an extension. */ +export const EXTENSION_NOISE: readonly string[] = ['extension', 'extensions']; + +/** Below this many characters a term matches whole words only. */ +const MIN_PARTIAL_CHARS = 3; + +/** The best a single term can score: a whole-word match (3) in a name (3). */ +const MAX_TERM_QUALITY = 3 * Weight.Name; + +/** + * Anything that is not a letter or a digit, in any script — the complement of + * Rust's `char::is_alphanumeric`, which is Unicode `Alphabetic` or `Numeric`. + */ +const WORD_BREAK = /[^\p{Alphabetic}\p{N}]+/u; + +/** + * One letter or digit: Rust's `char::is_alphanumeric` itself. The complement of + * {@link WORD_BREAK}, and it has to stay the complement — {@link words} splits a + * field on one and {@link writtenIn} tests the other, so a term and the query + * around it are held to the same edges. + * + * ⚠ **Not `\b`**, which JavaScript defines over `[A-Za-z0-9_]` alone. It reads + * `_` as a word character where Rust does not, and every non-ASCII letter — `é`, + * `π`, `中` — as a boundary where Rust reads a word character. On `-` and `.` the + * two agree, which is exactly why an ASCII fixture would pass over the + * disagreement. + */ +const WORD_CHAR = /[\p{Alphabetic}\p{N}]/u; + +/** + * Lowercase words, split at every character that is not a letter or digit — + * whitespace and punctuation alike, so `r-scripting` is `r` + `scripting` and + * `ggplot2` stays one word. + */ +function words(text: string): string[] { + return text + .split(WORD_BREAK) + .filter((word) => word !== '') + .map((word) => word.toLowerCase()); +} + +/** Length in characters rather than UTF-16 code units, like Rust's `chars().count()`. */ +function charCount(text: string): number { + return Array.from(text).length; +} + +/** Is `char` a letter or digit? The text's edge — `undefined` — is not. */ +function isWordChar(char: string | undefined): boolean { + return char !== undefined && WORD_CHAR.test(char); +} + +/** + * Does `text` hold `phrase` as written — starting and ending at a word boundary, + * not inside a longer word? `r scripting` is in "R scripting" but not in "for + * scripting", where its `r` is the tail of `for`. Both are lowercase already. + * + * An edge of `phrase` that is not a letter or digit needs no boundary: it is + * one, so `++` is written in "c++". + * + * Every character position is tried, not only the occurrences `indexOf` would + * step through, because a refused occurrence can overlap an accepted one: + * `a a` in "ba a a" is written only from the second `a`. + * + * Compared as code points rather than UTF-16 units, like Rust's `char_indices`, + * so an astral character is one character on both sides of the port. + * + * Exported only so the boundary cases can be asserted directly, the way Rust + * asserts them from inside the module. + */ +export function writtenIn(text: string, phrase: string): boolean { + const chars = Array.from(text); + const wanted = Array.from(phrase); + const startsWord = isWordChar(wanted[0]); + const endsWord = isWordChar(wanted[wanted.length - 1]); + for (let start = 0; start < chars.length; start += 1) { + if (!wanted.every((want, offset) => chars[start + offset] === want)) continue; + const opens = !startsWord || !isWordChar(chars[start - 1]); + const closes = !endsWord || !isWordChar(chars[start + wanted.length]); + if (opens && closes) return true; + } + return false; +} + +/** + * An empty or all-whitespace query is the browse case: every entry, in + * registry order. + */ +export function isBrowseQuery(query: string): boolean { + return query.trim() === ''; +} + +/** The distinct terms of `query`, in the order written, without filler. */ +export function searchTerms(query: string, noise: readonly string[] = []): string[] { + const all: string[] = []; + for (const word of words(query)) { + if (!all.includes(word)) all.push(word); + } + const meaningful = all.filter((term) => !FILLER.has(term) && !noise.includes(term)); + return meaningful.length > 0 ? meaningful : all; +} + +/** + * How well `term` matches one field word: 3 for the whole word, 2 for its + * start, 1 for anywhere inside it (`heatmap` in `complexheatmap`), 0 for no + * match. A short term matches whole words only. + */ +function strength(term: string, word: string): number { + if (word === term) return 3; + if (charCount(term) < MIN_PARTIAL_CHARS) return 0; + if (word.startsWith(term)) return 2; + if (word.includes(term)) return 1; + return 0; +} + +/** + * `term`'s strength against `word`, falling back to its singular so + * `visualizations` finds `visualization` and `heatmaps` finds `heatmap`. + * `class` and `gis` are left alone. + */ +function termStrength(term: string, word: string): number { + const direct = strength(term, word); + if (direct > 0 || !term.endsWith('s')) return direct; + const stem = term.slice(0, -1); + return charCount(stem) >= MIN_PARTIAL_CHARS && !stem.endsWith('s') ? strength(stem, word) : 0; +} + +/** A query as the matcher reads it. */ +export interface SearchQuery { + /** The whole query, trimmed and lowercased: what {@link writtenIn} looks for. */ + phrase: string; + /** Its distinct words, without filler. See {@link searchTerms}. */ + terms: string[]; +} + +export function parseQuery(query: string, noise: readonly string[] = []): SearchQuery { + return { phrase: query.trim().toLowerCase(), terms: searchTerms(query, noise) }; +} + +/** How one entry matched one query. */ +export interface EntryMatch { + /** + * 0 is no match; otherwise higher ranks first. It packs the ranking — the + * query as written, then terms matched, then where and how well they matched — + * into one number whose scale depends on the query's term count, so it + * compares entries scored against the SAME query and means nothing across two. + */ + score: number; + /** + * The terms this entry matched, in query order. Empty only when the query holds + * no word at all (`++`), so that nothing but the query as written could have + * found the entry — a query written in a field has every one of its words in + * that field as a whole word, and so matches every term. + */ + matchedTerms: string[]; +} + +/** + * Score one entry's fields against a query. Pure, and total over absent field + * text. Under the browse query every entry matches, equally. + */ +export function scoreEntry(query: SearchQuery, fields: readonly SearchField[]): EntryMatch { + if (query.phrase === '') return { score: 1, matchedTerms: [] }; + + const present = fields.filter( + (field): field is readonly [string, FieldWeight] => typeof field[0] === 'string' + ); + const written = present.some(([text]) => writtenIn(text.toLowerCase(), query.phrase)); + const entryWords = present.flatMap(([text, weight]) => + words(text).map((word) => ({ word, weight })) + ); + + const matchedTerms: string[] = []; + let quality = 0; + for (const term of query.terms) { + let best = 0; + for (const { word, weight } of entryWords) { + best = Math.max(best, termStrength(term, word) * weight); + } + if (best > 0) { + matchedTerms.push(term); + quality += best; + } + } + + // Lexicographic (written, terms matched, quality) as one number — the same key + // Rust sorts on. `quality` is at most MAX_TERM_QUALITY per term, so it stays + // below `scale`; the terms matched never exceed the term count, so the query + // as written outranks any scatter of its words. + const termCount = query.terms.length; + const scale = MAX_TERM_QUALITY * termCount + 1; + const tier = (written ? termCount + 1 : 0) + matchedTerms.length; + return { score: tier * scale + quality, matchedTerms }; +} + +/** One entry a search returned. */ +export interface SearchHit extends EntryMatch { + entry: T; +} + +/** A ranked search: what the query was read as, and every entry that matched it, best first. */ +export interface SearchResult { + /** The query's terms, after filler was dropped. Empty for the browse query. */ + terms: string[]; + /** Best first; equal scores keep registry order. The browse query returns every entry. */ + hits: SearchHit[]; +} + +/** + * Rank `entries` against `query`. `fields` names the text of one entry that is + * searched, and what a match there counts for; `noise` is the catalog's own + * filler (every entry in the skills catalog is a skill). + */ +export function rankEntries( + query: string, + noise: readonly string[], + entries: readonly T[], + fields: (entry: T) => readonly SearchField[] +): SearchResult { + const parsed = parseQuery(query, noise); + if (parsed.phrase === '') { + // Browsing reads no field at all — nor did the matcher this replaced — so + // an entry too malformed to search is still listed. + return { terms: [], hits: entries.map((entry) => ({ entry, score: 1, matchedTerms: [] })) }; + } + const hits = entries + .map((entry) => ({ entry, ...scoreEntry(parsed, fields(entry)) })) + .filter((hit) => hit.score > 0); + // `Array.prototype.sort` is stable, so equal scores keep registry order. + hits.sort((left, right) => right.score - left.score); + return { terms: parsed.terms, hits }; +}